From 0f92636b5e8c61f07d568c7c25406ef2d71fb7c9 Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Fri, 13 Nov 2026 15:32:56 -0700 Subject: [PATCH 1/7] feat(observability): correlate HTTP receipts, audit records and traces Closes #5393 Refs #5210 Signed-off-by: Alex Fournier --- AGENTS.md | 2 +- Cargo.lock | 2 + crates/openshell-ocsf/src/events/mod.rs | 28 + crates/openshell-otel/src/ocsf.rs | 27 +- .../src/headers.rs | 77 +- .../openshell-supervisor-network/Cargo.toml | 3 + crates/openshell-supervisor-network/README.md | 30 + .../src/l7/correlation.rs | 640 +++++++ .../src/l7/correlation/tests.rs | 684 ++++++++ .../src/l7/middleware.rs | 457 ++++- .../src/l7/mod.rs | 2 + .../src/l7/relay.rs | 1505 ++++++++++++++--- .../l7/relay/correlation_dispatch_tests.rs | 745 ++++++++ .../l7/relay/token_grant_ownership_tests.rs | 79 +- .../src/l7/request_trace.rs | 212 +++ .../src/l7/request_trace/tests.rs | 245 +++ .../src/l7/rest.rs | 604 ++++++- .../src/l7/rest/http_response.rs | 1150 ++++++++++++- .../src/l7/rest/http_response/http_hooks.rs | 279 ++- .../src/l7/rest/request_body.rs | 74 +- .../src/l7/token_grant_injection.rs | 88 +- .../openshell-supervisor-network/src/proxy.rs | 812 +++++++-- .../src/proxy/tests/compatibility.rs | 35 +- .../src/proxy/tests/request_trace.rs | 453 +++++ .../src/proxy/tests/token_grants.rs | 2 + .../src/proxy/trace_tests.rs | 5 +- .../src/telemetry.rs | 24 +- .../src/telemetry/tests.rs | 83 + .../src/test_support.rs | 39 +- .../src/test_support/request_tracing.rs | 86 + docs/observability/ocsf-json-export.mdx | 92 +- docs/observability/traces.mdx | 6 +- 32 files changed, 7902 insertions(+), 668 deletions(-) create mode 100644 crates/openshell-supervisor-network/src/l7/correlation.rs create mode 100644 crates/openshell-supervisor-network/src/l7/correlation/tests.rs create mode 100644 crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs create mode 100644 crates/openshell-supervisor-network/src/l7/request_trace.rs create mode 100644 crates/openshell-supervisor-network/src/l7/request_trace/tests.rs create mode 100644 crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs create mode 100644 crates/openshell-supervisor-network/src/test_support/request_tracing.rs diff --git a/AGENTS.md b/AGENTS.md index 012969da9f..b12dd0ede7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -153,7 +153,7 @@ ocsf_emit!(event); - `crate::ocsf_ctx()` returns the process-wide `EventContext`. It is always available (falls back to defaults in tests). - `ocsf_emit!()` is non-blocking and cannot panic. It stores the event in a thread-local and emits via `tracing::info!()`. -- In `openshell-supervisor-network`, emit with `crate::telemetry::egress_emit!` instead, or `crate::telemetry::emit_decision` for the records of one decision, so denied HTTP decisions get a deny span. Clippy rejects `ocsf_emit!` there. +- In `openshell-supervisor-network`, route inspected request records through `RequestCorrelation`, which uses their existing request span without creating a denial-only span. Other egress decisions use `crate::telemetry::egress_emit!` or `crate::telemetry::emit_decision`. Clippy rejects direct `ocsf_emit!` calls there. - The shorthand layer and JSONL layer extract the event from the thread-local. The shorthand format is derived automatically from the builder fields. - For security findings, **dual-emit**: one domain event (e.g., `SshActivityBuilder`) AND one `DetectionFindingBuilder` for the same incident. - Never log secrets, credentials, or query parameters in OCSF messages. The OCSF JSONL file may be shipped to external systems. diff --git a/Cargo.lock b/Cargo.lock index d8c338339d..dfb1a7c140 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5341,6 +5341,7 @@ dependencies = [ "openshell-sandbox-backend", "openshell-supervisor-middleware", "openshell-supervisor-middleware-builtins", + "opentelemetry", "opentelemetry-proto", "opentelemetry_sdk", "prost-types", @@ -5365,6 +5366,7 @@ dependencies = [ "tonic", "tower-mcp-types", "tracing", + "tracing-opentelemetry", "tracing-subscriber", "uuid", "webpki-roots", diff --git a/crates/openshell-ocsf/src/events/mod.rs b/crates/openshell-ocsf/src/events/mod.rs index 66038cd9cb..22ecbb6dfc 100644 --- a/crates/openshell-ocsf/src/events/mod.rs +++ b/crates/openshell-ocsf/src/events/mod.rs @@ -169,6 +169,11 @@ impl OcsfEvent { } } + /// Associate this event with other events produced for the same operation. + pub fn set_correlation_uid(&mut self, uid: impl Into) { + self.base_mut().metadata.correlation_uid = Some(uid.into()); + } + /// Returns mutable base data common to all event classes. pub(crate) fn base_mut(&mut self) -> &mut BaseEventData { match self { @@ -281,6 +286,29 @@ mod tests { assert_eq!(deserialized.class_uid(), 2004); } + #[test] + fn correlation_uid_can_be_set_on_any_event_class() { + let ctx = test_sandbox_context(); + let mut event = AuthenticationBuilder::new( + &ctx, + User::other("user-1", "User"), + Service::new("gateway"), + ) + .status(StatusId::Success) + .build(); + + event.set_correlation_uid("request-123"); + + assert_eq!( + event.base().metadata.correlation_uid.as_deref(), + Some("request-123") + ); + assert_eq!( + serde_json::to_value(&event).unwrap()["metadata"]["correlation_uid"], + "request-123" + ); + } + #[test] fn test_roundtrip_application_lifecycle() { let ctx = test_sandbox_context(); diff --git a/crates/openshell-otel/src/ocsf.rs b/crates/openshell-otel/src/ocsf.rs index 63489fde1d..86ff38de6a 100644 --- a/crates/openshell-otel/src/ocsf.rs +++ b/crates/openshell-otel/src/ocsf.rs @@ -34,6 +34,7 @@ const MAX_EXPORT_ATTEMPTS: usize = 4; const INITIAL_BACKOFF: Duration = Duration::from_millis(250); const MAX_BACKOFF: Duration = Duration::from_secs(2); const GAP_EVENT: &str = "openshell.ocsf.gap"; +const CORRELATION_UID_ATTRIBUTE: &str = "ocsf.metadata.correlation_uid"; /// Exports OCSF events to an OTLP collector as log records. pub struct OcsfOtlp { @@ -391,6 +392,9 @@ fn log_record( if let Some(uid) = &base.metadata.uid { record.add_attribute("ocsf.metadata.uid", uid.clone()); } + if let Some(uid) = &base.metadata.correlation_uid { + record.add_attribute(CORRELATION_UID_ATTRIBUTE, uid.clone()); + } if let Some(sandbox_id) = event.sandbox_id() { record.add_attribute("openshell.sandbox.id", sandbox_id.to_string()); } @@ -629,6 +633,16 @@ mod tests { assert_eq!(native(&event)["resources"][0]["uid"], "requested-sandbox"); } + #[test] + fn correlation_uid_is_a_stable_otlp_attribute() { + let mut event = event("sandbox-1", SeverityId::Informational); + event.set_correlation_uid("request-123"); + assert_eq!( + attribute(&record(&event), CORRELATION_UID_ATTRIBUTE), + Some(AnyValue::String("request-123".into())) + ); + } + #[test] fn events_without_a_sandbox_carry_no_sandbox_attribute() { let record = record(&event("", SeverityId::Informational)); @@ -788,12 +802,23 @@ mod tests { .with(otlp.layer()); let span_context = tracing::subscriber::with_default(subscriber, || { let span = tracing::info_span!("decision"); - span.in_scope(|| ocsf_emit!(event("sandbox-1", SeverityId::Medium))); + let mut event = event("sandbox-1", SeverityId::Medium); + event.set_correlation_uid("request-with-or-without-sampling"); + span.in_scope(|| ocsf_emit!(event)); span.context().span().span_context().clone() }); otlp.shutdown().await; let records = exporter.exported(); + assert_eq!(records.len(), 1); + assert_eq!( + attribute(&records[0].record, CORRELATION_UID_ATTRIBUTE), + Some(AnyValue::String("request-with-or-without-sampling".into())) + ); + assert_eq!( + body(&records[0].record)["metadata"]["correlation_uid"], + "request-with-or-without-sampling" + ); let trace = records[0].record.trace_context(); if span_context.is_sampled() { let trace = trace.expect("sampled span context is attached"); diff --git a/crates/openshell-supervisor-middleware/src/headers.rs b/crates/openshell-supervisor-middleware/src/headers.rs index 883bc010c3..f926f16921 100644 --- a/crates/openshell-supervisor-middleware/src/headers.rs +++ b/crates/openshell-supervisor-middleware/src/headers.rs @@ -366,25 +366,31 @@ fn is_safe_value(value: &str) -> bool { } fn is_request_protected(name: &str) -> bool { - matches!( - name, - "authorization" - | "proxy-authorization" - | "proxy-authenticate" - | "cookie" - | "host" - | "content-length" - | "transfer-encoding" - | "connection" - | "proxy-connection" - | "keep-alive" - | "te" - | "trailer" - | "upgrade" - ) || name.starts_with("x-amz-") + is_platform_correlation_header(name) + || matches!( + name, + "authorization" + | "proxy-authorization" + | "proxy-authenticate" + | "cookie" + | "host" + | "content-length" + | "transfer-encoding" + | "connection" + | "proxy-connection" + | "keep-alive" + | "te" + | "trailer" + | "upgrade" + ) + || name.starts_with("x-amz-") || name.starts_with("x-openshell-credential") } +fn is_platform_correlation_header(name: &str) -> bool { + name == "x-openshell-request-id" +} + /// Return whether a response header can carry authentication material and must /// never be exposed to middleware. #[must_use] @@ -402,7 +408,8 @@ pub fn is_response_credential_header(name: &str) -> bool { } fn is_response_protected(name: &str) -> bool { - is_response_credential_header(name) + is_platform_correlation_header(name) + || is_response_credential_header(name) || matches!( name, "connection" @@ -488,6 +495,42 @@ mod tests { ); } + #[test] + fn platform_request_id_is_immutable_for_all_http_phases() { + for authority in [ + HeaderAuthority::Request, + HeaderAuthority::HookRequest, + HeaderAuthority::RequestTrailers, + HeaderAuthority::Response, + HeaderAuthority::HookResponse, + HeaderAuthority::ResponseTrailers, + ] { + let name = "X-OpenShell-Request-ID"; + let write_error = apply( + authority, + &[], + &[], + &[write(name, "forged", ExistingHeaderAction::Overwrite)], + ) + .expect_err("platform request ID write"); + assert_eq!( + write_error, + HeaderMutationError::Protected { + name: name.to_string() + } + ); + + let remove_error = apply(authority, &[], &[], &[remove(name)]) + .expect_err("platform request ID removal"); + assert_eq!( + remove_error, + HeaderMutationError::Protected { + name: name.to_string() + } + ); + } + } + #[test] fn unsafe_header_value_is_rejected() { let error = apply( diff --git a/crates/openshell-supervisor-network/Cargo.toml b/crates/openshell-supervisor-network/Cargo.toml index e51f282cd5..8f000adb7c 100644 --- a/crates/openshell-supervisor-network/Cargo.toml +++ b/crates/openshell-supervisor-network/Cargo.toml @@ -38,6 +38,8 @@ hex = "0.4" hickory-proto = "0.26.1" ipnet = "2" miette = { workspace = true } +opentelemetry = { workspace = true } +opentelemetry_sdk = { workspace = true } prost-types = { workspace = true } rcgen = { workspace = true } regorus = { version = "0.9", default-features = false, features = ["std", "arc", "glob"] } @@ -56,6 +58,7 @@ tokio-rustls = { workspace = true } tonic = { workspace = true } tower-mcp-types = { workspace = true } tracing = { workspace = true } +tracing-opentelemetry = { workspace = true } uuid = { workspace = true } webpki-roots = { workspace = true, optional = true } diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index 92257fb24a..da8cfc19b2 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -55,3 +55,33 @@ domain event and a detection finding without credential values or placeholders. Explicit body and WebSocket rewrite flags authorize those locations for ordinary static credentials. SigV4 signing retains the endpoint-scoped resolver's existing trusted access. + +## HTTP enforcement correlation + +Each inspected HTTP request has one UUID shared by middleware, OCSF +`metadata.correlation_uid`, and the protected `x-openshell-request-id` response +header. Application correlation must use another header. The receipt is only a +join key: it does not establish identity, authorization or complete delivery. +L4 and passthrough traffic do not provide this header guarantee. + +The audit state commits one primary HTTP Activity record when the final response +head is written, with an early-termination fallback. It retains the original +request and process context. Middleware invocation records remain separate, +request-correlated observations, including after the primary is committed. They +retain attribution and transformation details without changing the final +response status or consuming the late-failure budget. Security findings remain +separate. A process crash or indefinitely hung request cannot be finalized by +the fallback. + +This includes HTTP-hook preflight and completion invocation records. + +Request IDs and audit outcomes do not depend on OpenTelemetry, sampling or +collector availability. Inspected requests have a supervisor-owned request span +and children around stages that actually run, with valid workload W3C context +only as an untrusted link. Their records never create an extra denial-only span, +even if the request span is disabled or unsampled. Other egress paths retain the +legacy bounded deny-span behavior. End HTTP tracing before an upgraded session. +The request span ends after its owned stages, not when response headers are logged. +The `openshell.upstream.send` stage measures request writes and flush; response +handling is measured separately. Receipt filtering preserves raw HTTP bytes +and borrows unchanged header blocks. diff --git a/crates/openshell-supervisor-network/src/l7/correlation.rs b/crates/openshell-supervisor-network/src/l7/correlation.rs new file mode 100644 index 0000000000..d36bbabcd2 --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/correlation.rs @@ -0,0 +1,640 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Per-request receipts and consolidated HTTP audit records. + +use std::sync::Mutex; +use std::{ + pin::Pin, + task::{Context, Poll}, +}; + +use openshell_ocsf::{ + ActionId, DispositionId, Endpoint, HttpActivityBuilder, HttpActivityEvent, HttpRequest, + HttpResponse, OcsfEvent, SeverityId, StatusId, +}; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + +use crate::l7::provider::L7Request; +use crate::l7::relay::L7EvalContext; +pub use crate::l7::request_trace::RequestStage; +use crate::l7::request_trace::RequestTrace; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HttpOutcome { + Allowed, + Denied, + PlatformError, + UpstreamError, + Upgraded, +} + +impl HttpOutcome { + fn as_str(self) -> &'static str { + match self { + Self::Allowed => "allowed", + Self::Denied => "denied", + Self::PlatformError => "platform_error", + Self::UpstreamError => "upstream_error", + Self::Upgraded => "upgraded", + } + } +} + +#[derive(Debug)] +pub struct RequestCorrelation { + request_id: String, + inspected: bool, + // Keep the audit payload off the async state machine. + state: Box>, +} + +#[derive(Debug, Default)] +struct RequestState { + trace: Option, + primary: Option, + seeded: bool, + status: Option, + upstream_status: Option, + outcome: Option, + blocked: bool, + failure_reason: Option<&'static str>, + late_failure_emitted: bool, +} + +impl RequestState { + fn record_outcome(&mut self, outcome: HttpOutcome) { + self.outcome = Some(outcome); + if let Some(trace) = &mut self.trace { + trace.record_outcome(outcome); + } + } + + fn record_status(&mut self, status: u16) { + self.status = Some(status); + if let Some(trace) = &self.trace { + trace.record_status(status); + } + } + + fn record_failure_reason(&mut self, reason: &'static str) { + self.failure_reason = Some(reason); + if let Some(trace) = &self.trace { + trace.record_failure_reason(reason); + } + } +} + +impl RequestCorrelation { + #[must_use] + pub(crate) fn new(request: &L7Request, ctx: &L7EvalContext) -> Self { + Self::build(request, ctx, uuid::Uuid::new_v4().to_string()) + } + + pub(crate) fn uninspected() -> Self { + Self { + request_id: uuid::Uuid::new_v4().to_string(), + inspected: false, + state: Box::default(), + } + } + + #[cfg(test)] + #[must_use] + pub(crate) fn with_request_id( + request: &L7Request, + ctx: &L7EvalContext, + request_id: String, + ) -> Self { + Self::build(request, ctx, request_id) + } + + fn build(request: &L7Request, ctx: &L7EvalContext, request_id: String) -> Self { + let target = request.target.split(['?', '#']).next().unwrap_or("/"); + let target = if target.starts_with('/') { target } else { "/" }; + let path = openshell_core::secrets::redact_target_for_policy(target) + .unwrap_or_else(|_| "/".to_string()); + let fallback = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .http_request(HttpRequest::new(&request.action, ctx.ocsf_url(&path))) + .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) + .message("HTTP request ended before a response was selected") + .build(); + let trace = RequestTrace::new( + &request_id, + &request.action, + &ctx.host, + ctx.port, + &request.raw_header, + ); + Self { + request_id, + inspected: true, + state: Box::new(Mutex::new(RequestState { + trace: Some(trace), + primary: Some(fallback), + ..RequestState::default() + })), + } + } + + #[must_use] + pub(crate) fn request_id(&self) -> &str { + &self.request_id + } + + pub(crate) fn stage(&self, stage: RequestStage) -> tracing::Span { + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .trace + .as_ref() + .map_or_else(tracing::Span::none, |trace| trace.stage(stage)) + } + + #[cfg(test)] + pub(crate) fn trace_span(&self) -> tracing::Span { + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .trace + .as_ref() + .map_or_else(tracing::Span::none, RequestTrace::span) + } + + pub(crate) fn finish_http_trace(&self) { + let (mut trace, status) = { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + (state.trace.take(), state.status) + }; + if status == Some(101) + && let Some(trace) = &mut trace + { + trace.record_outcome(HttpOutcome::Upgraded); + } + drop(trace); + } + + pub(crate) fn observe_response<'a, C>(&'a self, client: &'a mut C) -> AuditedClient<'a, C> { + AuditedClient { + inner: client, + correlation: self, + head: ResponseHead::default(), + } + } + + pub(crate) fn record_enforcement_block(&self) { + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .blocked = true; + } + + pub(crate) fn record_outcome(&self, outcome: HttpOutcome) { + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .record_outcome(outcome); + } + + pub(crate) fn record_response_status(&self, status: u16) { + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .record_status(status); + } + + pub(crate) fn record_upstream_status(&self, status: u16) { + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .upstream_status = Some(status); + } + + pub(crate) fn record_rejection( + &self, + status: u16, + outcome: HttpOutcome, + reason: Option<&'static str>, + ) { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.blocked = true; + state.record_outcome(outcome); + state.record_status(status); + if let Some(reason) = reason { + state.record_failure_reason(reason); + } + } + + pub(crate) fn record_failure(&self, outcome: HttpOutcome, reason: &'static str) { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.record_outcome(outcome); + state.record_failure_reason(reason); + } + + pub(crate) fn emit(&self, event: OcsfEvent) { + if !self.inspected { + crate::telemetry::emit_decision([event], None); + return; + } + if let OcsfEvent::HttpActivity(mut incoming) = event { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if is_middleware_invocation(&incoming) { + if incoming.action == Some(ActionId::Denied) + && let Some(OcsfEvent::HttpActivity(primary)) = &mut state.primary + { + merge_middleware_denial(primary, &incoming); + } + drop(state); + self.emit_correlated(OcsfEvent::HttpActivity(incoming)); + return; + } + if !state.seeded && state.primary.is_some() { + state.primary = Some(OcsfEvent::HttpActivity(incoming)); + state.seeded = true; + return; + } + if let Some(OcsfEvent::HttpActivity(primary)) = &mut state.primary { + merge_http_outcome(primary, incoming); + return; + } + let fail_open = incoming.action == Some(ActionId::Other) + && (incoming.disposition == Some(DispositionId::Allowed) + || incoming + .base + .unmapped + .as_ref() + .and_then(|fields| fields.get("response_middleware_outcome")) + .and_then(serde_json::Value::as_str) + == Some("failopen")); + if !is_failure(&incoming) || fail_open || state.late_failure_emitted { + return; + } + state.late_failure_emitted = true; + if let Some(code) = state.status { + incoming.http_response = Some(HttpResponse { code }); + } + drop(state); + self.emit_correlated(OcsfEvent::HttpActivity(incoming)); + } else { + if let OcsfEvent::NetworkActivity(network) = &event { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !state.seeded + && (network.action == Some(ActionId::Denied) + || network.base.status == Some(StatusId::Failure)) + && let Some(OcsfEvent::HttpActivity(primary)) = &mut state.primary + { + primary.action = network.action; + primary.disposition = network.disposition; + primary.base.status = Some(StatusId::Failure); + primary + .base + .status_detail + .clone_from(&network.base.status_detail); + primary.base.severity = network.base.severity; + primary.base.message = + Some("HTTP request rejected before policy evaluation".into()); + primary.src_endpoint.clone_from(&network.src_endpoint); + primary.actor.clone_from(&network.actor); + } + } + self.emit_correlated(event); + } + } + + pub(crate) fn response_selected(&self, status: u16) { + self.record_response_status(status); + self.emit_primary(false); + } + + fn emit_primary(&self, incomplete: bool) { + if !self.inspected { + return; + } + let (mut event, status, upstream_status, outcome, blocked, failure_reason) = { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(primary) = state.primary.take() else { + return; + }; + ( + primary, + state.status, + state.upstream_status, + state.outcome, + state.blocked, + state.failure_reason, + ) + }; + if let OcsfEvent::HttpActivity(http) = &mut event { + let has_failure_reason = is_failure(http) && http.base.status_detail.is_some(); + if let Some(code) = status { + http.http_response = Some(HttpResponse { code }); + } + if let serde_json::Value::Object(unmapped) = http + .base + .unmapped + .get_or_insert_with(|| serde_json::json!({})) + { + unmapped.insert("http_event_kind".into(), "request".into()); + if let Some(upstream) = upstream_status.filter(|upstream| Some(*upstream) != status) + { + unmapped.insert("upstream_response_code".into(), serde_json::json!(upstream)); + } + } + if blocked || outcome == Some(HttpOutcome::Denied) { + if http.action != Some(ActionId::Denied) { + http.base.message = Some("HTTP request blocked by supervisor".into()); + } + http.action = Some(ActionId::Denied); + http.disposition = Some(DispositionId::Blocked); + http.base.status = Some(StatusId::Failure); + raise_severity(http, SeverityId::Medium); + if !has_failure_reason { + http.base.status_detail = Some( + failure_reason + .or_else(|| outcome.map(HttpOutcome::as_str)) + .unwrap_or("supervisor_blocked") + .into(), + ); + } + } else if matches!( + outcome, + Some(HttpOutcome::PlatformError | HttpOutcome::UpstreamError) + ) { + http.base.status = Some(StatusId::Failure); + raise_severity(http, SeverityId::Low); + if !has_failure_reason { + http.base.status_detail = failure_reason + .or_else(|| outcome.map(HttpOutcome::as_str)) + .map(str::to_string); + } + } + if incomplete && outcome.is_none() { + http.base.status = Some(StatusId::Failure); + if !has_failure_reason { + http.base.status_detail = Some("request_incomplete".into()); + } + raise_severity(http, SeverityId::Low); + } + } + self.emit_correlated(event); + } + + fn emit_correlated(&self, mut event: OcsfEvent) { + event.set_correlation_uid(self.request_id.clone()); + let parent = { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.trace.as_mut().map(|trace| { + if let OcsfEvent::HttpActivity(http) = &event + && !is_middleware_invocation(http) + { + trace.observe_http(http); + } + trace.span() + }) + }; + crate::telemetry::emit_inspected_decision([event], parent.as_ref()); + } +} + +pub fn emit_request_events( + correlation: Option<&RequestCorrelation>, + events: impl IntoIterator, +) { + if let Some(correlation) = correlation { + for event in events { + correlation.emit(event); + } + } else { + crate::telemetry::emit_decision(events, None); + } +} + +fn is_failure(event: &HttpActivityEvent) -> bool { + event.action == Some(ActionId::Denied) || event.base.status == Some(StatusId::Failure) +} + +fn is_middleware_invocation(event: &HttpActivityEvent) -> bool { + event + .base + .unmapped + .as_ref() + .and_then(|fields| fields.get("http_event_kind")) + .and_then(serde_json::Value::as_str) + == Some("middleware_invocation") +} + +fn merge_middleware_denial(primary: &mut HttpActivityEvent, invocation: &HttpActivityEvent) { + primary.action = invocation.action; + primary.disposition = invocation.disposition; + primary.base.status = Some(StatusId::Failure); + raise_severity(primary, invocation.base.severity); + if invocation.base.message.is_some() { + primary.base.message.clone_from(&invocation.base.message); + } + if invocation.base.status_detail.is_some() { + primary + .base + .status_detail + .clone_from(&invocation.base.status_detail); + } +} + +fn raise_severity(event: &mut HttpActivityEvent, severity: SeverityId) { + if event.base.severity.as_u8() < severity.as_u8() { + event.base.severity = severity; + } +} + +fn merge_http_outcome(primary: &mut HttpActivityEvent, incoming: HttpActivityEvent) { + let failed = is_failure(&incoming); + if primary.actor.is_none() { + primary.actor = incoming.actor; + } + if primary.src_endpoint.is_none() { + primary.src_endpoint = incoming.src_endpoint; + } + if primary.dst_endpoint.is_none() { + primary.dst_endpoint = incoming.dst_endpoint; + } + if primary.firewall_rule.is_none() { + primary.firewall_rule = incoming.firewall_rule; + } + if primary + .http_request + .as_ref() + .and_then(|request| request.url.as_ref()) + .is_none() + && incoming + .http_request + .as_ref() + .and_then(|request| request.url.as_ref()) + .is_some() + { + primary.http_request = incoming.http_request; + } + if !failed { + if incoming + .base + .unmapped + .as_ref() + .and_then(|fields| fields.get("transformed_body_policy")) + .and_then(serde_json::Value::as_str) + == Some("audit") + { + if let serde_json::Value::Object(fields) = primary + .base + .unmapped + .get_or_insert_with(|| serde_json::json!({})) + { + fields.insert("transformed_body_policy".into(), "audit".into()); + } + if !is_failure(primary) && incoming.base.status_detail.is_some() { + primary + .base + .status_detail + .clone_from(&incoming.base.status_detail); + } + } + if primary.action.is_none() { + primary.action = incoming.action; + primary.disposition = incoming.disposition; + primary.base.status = incoming.base.status; + primary.base.message = incoming.base.message; + primary.base.status_detail = incoming.base.status_detail; + } + return; + } + if incoming.action == Some(ActionId::Denied) { + primary.action = incoming.action; + primary.disposition = incoming.disposition; + } + primary.base.status = Some(StatusId::Failure); + raise_severity(primary, incoming.base.severity); + if incoming.base.message.is_some() { + primary.base.message = incoming.base.message; + } + if incoming.base.status_detail.is_some() { + primary.base.status_detail = incoming.base.status_detail; + } + if incoming.http_response.is_some() { + primary.http_response = incoming.http_response; + } + if let Some(serde_json::Value::Object(fields)) = incoming.base.unmapped + && let serde_json::Value::Object(existing) = primary + .base + .unmapped + .get_or_insert_with(|| serde_json::json!({})) + { + existing.extend(fields); + } +} + +/// Observe accepted response bytes without buffering or changing delivery. +pub struct AuditedClient<'a, C> { + inner: &'a mut C, + correlation: &'a RequestCorrelation, + head: ResponseHead, +} + +#[derive(Default)] +struct ResponseHead { + prefix: [u8; 12], + prefix_len: usize, + delimiter: u32, + complete: bool, +} + +impl ResponseHead { + fn accept(&mut self, bytes: &[u8], correlation: &RequestCorrelation) { + for &byte in bytes { + if self.complete { + break; + } + if self.prefix_len < self.prefix.len() { + self.prefix[self.prefix_len] = byte; + self.prefix_len += 1; + } + self.delimiter = (self.delimiter << 8) | u32::from(byte); + if self.delimiter != u32::from_be_bytes(*b"\r\n\r\n") { + continue; + } + let status = std::str::from_utf8(&self.prefix[9..12]) + .ok() + .and_then(|code| code.parse::().ok()); + if status.is_some_and(|code| (100..200).contains(&code) && code != 101) { + *self = Self::default(); + } else { + self.complete = true; + if let Some(status) = status { + correlation.response_selected(status); + } + } + } + } +} + +impl AsyncRead for AuditedClient<'_, C> { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut *self.get_mut().inner).poll_read(cx, buf) + } +} + +impl AsyncWrite for AuditedClient<'_, C> { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + bytes: &[u8], + ) -> Poll> { + let this = self.get_mut(); + let result = Pin::new(&mut *this.inner).poll_write(cx, bytes); + if let Poll::Ready(Ok(accepted)) = result + && this.correlation.inspected + { + this.head.accept(&bytes[..accepted], this.correlation); + } + result + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut *self.get_mut().inner).poll_flush(cx) + } + + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut *self.get_mut().inner).poll_shutdown(cx) + } +} + +impl Drop for RequestCorrelation { + fn drop(&mut self) { + self.emit_primary(true); + self.finish_http_trace(); + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/openshell-supervisor-network/src/l7/correlation/tests.rs b/crates/openshell-supervisor-network/src/l7/correlation/tests.rs new file mode 100644 index 0000000000..c6b00b2b18 --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/correlation/tests.rs @@ -0,0 +1,684 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use crate::test_support::request_tracing::{RequestTracing, span_attribute}; +use openshell_ocsf::{ActivityId, NetworkActivityBuilder, Process}; +use tokio::io::AsyncWriteExt as _; + +fn request() -> L7Request { + L7Request { + action: "GET".into(), + target: "/data".into(), + query_params: std::collections::HashMap::new(), + raw_header: b"GET /data HTTP/1.1\r\nHost: example.test\r\n\r\n".to_vec(), + body_length: crate::l7::provider::BodyLength::None, + } +} + +fn context() -> L7EvalContext { + L7EvalContext { + host: "example.test".into(), + port: 443, + policy_name: "protected-api".into(), + ..Default::default() + } +} + +fn allowed() -> OcsfEvent { + HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .http_request(HttpRequest::new("GET", context().ocsf_url("/data"))) + .actor_process(Process::new("curl", 42).with_cmd_line("curl /data")) + .firewall_rule("protected-api", "l7") + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .status(StatusId::Success) + .message("HTTP request admitted by policy") + .status_detail("matched_rule") + .build() +} + +fn denied() -> OcsfEvent { + HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .http_request(HttpRequest { + http_method: "GET".parse().unwrap(), + url: None, + }) + .http_response(HttpResponse { code: 403 }) + .firewall_rule("protected-api", "credential-binding") + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .status(StatusId::Failure) + .severity(SeverityId::High) + .message("Credential use denied") + .status_detail("credential_endpoint_mismatch") + .build() +} + +fn http_events(capture: &openshell_ocsf::test_support::OcsfCapture) -> Vec { + capture + .json_events() + .into_iter() + .filter(|event| event["class_uid"] == 4002) + .collect() +} + +#[derive(Clone, Copy)] +enum InvocationOutcome { + Pass, + Transform, + FailOpen, + Deny, +} + +fn invocation(name: &str, phase: &str, outcome: InvocationOutcome) -> OcsfEvent { + let (action, disposition, status) = match outcome { + InvocationOutcome::Pass | InvocationOutcome::Transform => { + (ActionId::Allowed, DispositionId::Allowed, StatusId::Success) + } + InvocationOutcome::FailOpen => (ActionId::Other, DispositionId::Error, StatusId::Failure), + InvocationOutcome::Deny => (ActionId::Denied, DispositionId::Blocked, StatusId::Failure), + }; + HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .http_request(HttpRequest::new("POST", context().ocsf_url("/observed"))) + .http_response(HttpResponse { code: 200 }) + .firewall_rule("middleware-policy", "supervisor-middleware") + .action(action) + .disposition(disposition) + .status(status) + .severity(SeverityId::Medium) + .status_detail(name) + .message(format!("middleware invocation {name}")) + .unmapped("http_event_kind", "middleware_invocation") + .unmapped("middleware_phase", phase) + .unmapped("middleware_config", name) + .unmapped( + "transformed", + matches!(outcome, InvocationOutcome::Transform), + ) + .unmapped("failed", matches!(outcome, InvocationOutcome::FailOpen)) + .build() +} + +#[test] +fn middleware_invocations_are_retained_without_seeding_or_poisoning_the_request() { + use opentelemetry_sdk::trace::Sampler; + + for sampler in [Some(Sampler::AlwaysOn), Some(Sampler::AlwaysOff), None] { + let sampled = matches!(sampler, Some(Sampler::AlwaysOn)); + for phase in ["request", "response"] { + let capture = RequestTracing::new(sampler.clone()); + let mut expected = Vec::new(); + tracing::subscriber::with_default(capture.subscriber(), || { + let correlation = RequestCorrelation::new(&request(), &context()); + for (name, outcome) in [ + ("pass", InvocationOutcome::Pass), + ("redact", InvocationOutcome::Transform), + ("optional", InvocationOutcome::FailOpen), + ] { + let event = invocation(name, phase, outcome); + expected.push(event.to_json().unwrap()); + correlation.emit(event); + } + correlation.emit(allowed()); + correlation.response_selected(200); + }); + let events = http_events(&capture.events); + assert_eq!(events.len(), 4); + for (event, expected) in events[..3].iter().zip(&expected) { + for field in [ + "action", + "disposition", + "status", + "message", + "status_detail", + "http_request", + "http_response", + "firewall_rule", + "unmapped", + ] { + assert_eq!(event[field], expected[field], "{field}"); + } + assert_eq!(event["trace"]["uid"].is_string(), sampled); + } + let primary = &events[3]; + assert_eq!(primary["unmapped"]["http_event_kind"], "request"); + assert_eq!(primary["action"], "Allowed"); + assert_eq!(primary["status"], "Success"); + assert_eq!(primary["http_request"]["url"]["path"], "/data"); + assert_eq!(primary["actor"]["process"]["pid"], 42); + assert_eq!(primary["firewall_rule"]["type"], "l7"); + assert!(events.iter().all(|event| { + event["metadata"]["correlation_uid"] == primary["metadata"]["correlation_uid"] + })); + assert!( + !capture + .created_span_names() + .contains(&"supervisor.egress.deny") + ); + let spans = capture.spans(); + assert_eq!(spans.len(), usize::from(sampled)); + if sampled { + assert_eq!( + span_attribute(&spans[0], "openshell.request.outcome"), + Some(&"allowed".into()) + ); + assert_eq!( + span_attribute(&spans[0], "http.response.status_code"), + Some(&200_i64.into()) + ); + } + capture.shutdown(); + } + } +} + +#[test] +fn denied_middleware_invocation_only_overlays_the_preheader_denial_summary() { + for seeded in [false, true] { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = RequestCorrelation::new(&request(), &context()); + if seeded { + correlation.emit(allowed()); + } + correlation.record_upstream_status(200); + correlation.emit(invocation("guard", "response", InvocationOutcome::Deny)); + correlation.response_selected(403); + drop(correlation); + let events = http_events(&capture); + assert_eq!(events.len(), 2); + let observation = &events[0]; + let primary = &events[1]; + assert_eq!( + observation["unmapped"]["http_event_kind"], + "middleware_invocation" + ); + assert_eq!(observation["http_response"]["code"], 200); + assert_eq!(observation["http_request"]["url"]["path"], "/observed"); + assert_eq!(primary["unmapped"]["http_event_kind"], "request"); + assert_eq!(primary["http_response"]["code"], 403); + assert_eq!(primary["unmapped"]["upstream_response_code"], 200); + assert_eq!(primary["http_request"]["url"]["path"], "/data"); + assert_eq!(primary["action"], "Denied"); + assert_eq!(primary["status"], "Failure"); + assert_eq!(primary["message"], "middleware invocation guard"); + assert_eq!(primary["status_detail"], "guard"); + if seeded { + assert_eq!(primary["firewall_rule"]["type"], "l7"); + assert_eq!(primary["actor"]["process"]["pid"], 42); + } else { + assert!(primary.get("firewall_rule").is_none()); + assert!(primary.get("actor").is_none()); + } + } +} + +#[test] +fn late_middleware_invocations_do_not_consume_the_late_failure_budget() { + use opentelemetry_sdk::trace::Sampler; + + for sampler in [Some(Sampler::AlwaysOn), Some(Sampler::AlwaysOff), None] { + let sampled = matches!(sampler, Some(Sampler::AlwaysOn)); + for late_failure in [false, true] { + let capture = RequestTracing::new(sampler.clone()); + tracing::subscriber::with_default(capture.subscriber(), || { + let correlation = RequestCorrelation::new(&request(), &context()); + correlation.emit(allowed()); + correlation.response_selected(200); + for (name, outcome) in [ + ("optional", InvocationOutcome::FailOpen), + ("guard", InvocationOutcome::Deny), + ("pass", InvocationOutcome::Pass), + ] { + correlation.emit(invocation(name, "response", outcome)); + } + if late_failure { + correlation.emit(denied()); + correlation.emit(denied()); + } + }); + let events = http_events(&capture.events); + assert_eq!(events.len(), 4 + usize::from(late_failure)); + assert_eq!(events[0]["unmapped"]["http_event_kind"], "request"); + assert_eq!(events[0]["action"], "Allowed"); + for event in &events[1..4] { + assert_eq!( + event["unmapped"]["http_event_kind"], + "middleware_invocation" + ); + assert_eq!(event["http_response"]["code"], 200); + } + if late_failure { + assert!(events[4]["unmapped"].get("http_event_kind").is_none()); + assert_eq!(events[4]["status_detail"], "credential_endpoint_mismatch"); + assert_eq!(events[4]["http_response"]["code"], 200); + } + assert!( + events + .iter() + .all(|event| event["metadata"]["correlation_uid"] + == events[0]["metadata"]["correlation_uid"]) + ); + assert!( + !capture + .created_span_names() + .contains(&"supervisor.egress.deny") + ); + let spans = capture.spans(); + assert_eq!(spans.len(), usize::from(sampled)); + if sampled { + assert_eq!( + span_attribute(&spans[0], "openshell.request.outcome"), + Some(&if late_failure { "denied" } else { "allowed" }.into()) + ); + assert_eq!( + span_attribute(&spans[0], "http.response.status_code"), + Some(&200_i64.into()) + ); + } + capture.shutdown(); + } + } +} + +#[test] +fn audit_receipt_and_denial_are_independent_of_sampling_or_exporter() { + use opentelemetry_sdk::trace::Sampler; + + for sampler in [Some(Sampler::AlwaysOn), Some(Sampler::AlwaysOff), None] { + let sampled = matches!(sampler, Some(Sampler::AlwaysOn)); + let capture = RequestTracing::new(sampler); + tracing::subscriber::with_default(capture.subscriber(), || { + let correlation = RequestCorrelation::with_request_id( + &request(), + &context(), + "00000000-0000-4000-8000-000000000001".into(), + ); + correlation.emit(allowed()); + correlation.emit(denied()); + correlation.record_outcome(HttpOutcome::Denied); + correlation.response_selected(403); + }); + let events = http_events(&capture.events); + assert_eq!(events.len(), 1); + let event = &events[0]; + assert_eq!( + event["metadata"]["correlation_uid"], + "00000000-0000-4000-8000-000000000001" + ); + assert_eq!(event["action"], "Denied"); + assert_eq!(event["status"], "Failure"); + assert_eq!(event["http_response"]["code"], 403); + assert_eq!(event["http_request"]["url"]["path"], "/data"); + assert_eq!(event["trace"]["uid"].is_string(), sampled); + let spans = capture.spans(); + assert_eq!(spans.len(), usize::from(sampled)); + if sampled { + assert_eq!(spans[0].name, "openshell.http.request"); + assert_eq!( + event["trace"]["uid"], + spans[0].span_context.trace_id().to_string() + ); + } + capture.shutdown(); + } +} + +#[test] +fn finished_http_trace_is_not_retained_by_an_upgraded_session() { + use opentelemetry_sdk::trace::Sampler; + + let capture = RequestTracing::new(Some(Sampler::AlwaysOn)); + tracing::subscriber::with_default(capture.subscriber(), || { + let correlation = RequestCorrelation::new(&request(), &context()); + correlation.emit(allowed()); + correlation.response_selected(101); + correlation.finish_http_trace(); + assert_eq!(capture.spans().len(), 1); + assert!(correlation.trace_span().is_disabled()); + assert!(correlation.stage(RequestStage::UpstreamSend).is_disabled()); + correlation.finish_http_trace(); + drop(correlation); + assert_eq!(capture.spans().len(), 1); + }); + capture.shutdown(); +} + +#[test] +fn a_provider_without_an_exporter_still_emits_the_request_id_join() { + use opentelemetry::trace::TracerProvider as _; + use opentelemetry_sdk::trace::{Sampler, SdkTracerProvider}; + use tracing_subscriber::layer::SubscriberExt as _; + + crate::telemetry::tests::init_trace_extractor(); + let provider = SdkTracerProvider::builder() + .with_sampler(Sampler::AlwaysOn) + .build(); + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry() + .with(capture.clone()) + .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("no-exporter-test"))); + let mut id = None; + tracing::subscriber::with_default(subscriber, || { + let correlation = RequestCorrelation::new(&request(), &context()); + id = Some(correlation.request_id().to_owned()); + correlation + .stage(RequestStage::PolicyEvaluate) + .in_scope(|| correlation.emit(allowed())); + correlation.record_outcome(HttpOutcome::Allowed); + correlation.response_selected(200); + }); + let events = http_events(&capture); + assert_eq!(events.len(), 1); + assert_eq!(events[0]["metadata"]["correlation_uid"], id.unwrap()); + assert_eq!(events[0]["http_response"]["code"], 200); + assert_eq!(events[0]["action"], "Allowed"); + provider.shutdown().unwrap(); +} + +#[tokio::test] +async fn primary_record_waits_for_final_response_headers_and_is_emitted_once_without_otel() { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = RequestCorrelation::new(&request(), &context()); + let id = correlation.request_id().to_string(); + correlation.emit(allowed()); + let mut output = Vec::new(); + { + let mut client = correlation.observe_response(&mut output); + client + .write_all(b"HTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 4\r\n") + .await + .unwrap(); + assert!(capture.events().is_empty()); + client.write_all(b"\r\nbody").await.unwrap(); + } + drop(correlation); + let events = http_events(&capture); + assert_eq!(events.len(), 1); + assert_eq!(events[0]["http_response"]["code"], 200); + assert_eq!(events[0]["http_request"]["url"]["path"], "/data"); + assert_eq!(events[0]["action"], "Allowed"); + assert_eq!(events[0]["status"], "Success"); + assert_eq!(events[0]["actor"]["process"]["pid"], 42); + assert_eq!(events[0]["metadata"]["correlation_uid"], id); + assert!(events[0].get("trace").is_none()); + assert!(output.ends_with(b"\r\n\r\nbody")); +} + +#[test] +fn later_credential_denial_preserves_policy_request_context() { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = RequestCorrelation::new(&request(), &context()); + let id = correlation.request_id().to_string(); + correlation.emit(allowed()); + correlation.emit(denied()); + // A routine later middleware result cannot undo the denial. + correlation.emit(allowed()); + correlation.response_selected(403); + drop(correlation); + let events = http_events(&capture); + assert_eq!(events.len(), 1); + let event = &events[0]; + assert_eq!(event["action"], "Denied"); + assert_eq!(event["disposition"], "Blocked"); + assert_eq!(event["status"], "Failure"); + assert_eq!(event["severity"], "High"); + assert_eq!(event["status_detail"], "credential_endpoint_mismatch"); + assert_eq!(event["http_response"]["code"], 403); + assert_eq!(event["http_request"]["url"]["path"], "/data"); + assert_eq!(event["firewall_rule"]["type"], "l7"); + assert_eq!(event["actor"]["process"]["pid"], 42); + assert_eq!(event["metadata"]["correlation_uid"], id); +} + +#[test] +fn local_refusals_and_upstream_failures_are_distinct() { + for (outcome, blocked, code, expected_action, expected_disposition) in [ + (HttpOutcome::PlatformError, true, 503, "Denied", "Blocked"), + (HttpOutcome::UpstreamError, false, 502, "Allowed", "Allowed"), + (HttpOutcome::PlatformError, false, 502, "Allowed", "Allowed"), + (HttpOutcome::Allowed, false, 500, "Allowed", "Allowed"), + ] { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = RequestCorrelation::new(&request(), &context()); + correlation.emit(allowed()); + if blocked { + correlation.record_enforcement_block(); + } + correlation.record_outcome(outcome); + correlation.response_selected(code); + drop(correlation); + let events = http_events(&capture); + assert_eq!(events.len(), 1); + assert_eq!(events[0]["action"], expected_action); + assert_eq!(events[0]["disposition"], expected_disposition); + assert_eq!( + events[0]["status"], + if outcome == HttpOutcome::Allowed { + "Success" + } else { + "Failure" + } + ); + assert_eq!(events[0]["http_response"]["code"], code); + } +} + +#[test] +fn pre_policy_rejection_retains_reason_without_claiming_policy_admission() { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = RequestCorrelation::new(&request(), &context()); + let id = correlation.request_id().to_string(); + correlation.emit( + NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Other) + .dst_endpoint(Endpoint::from_domain("example.test", 443)) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .status(StatusId::Failure) + .severity(SeverityId::Medium) + .status_detail("invalid_jsonrpc_request") + .actor_process(Process::new("curl", 42)) + .build(), + ); + correlation.record_enforcement_block(); + correlation.record_outcome(HttpOutcome::Denied); + correlation.response_selected(400); + drop(correlation); + assert_eq!(capture.events().len(), 2); + let events = http_events(&capture); + assert_eq!(events.len(), 1); + assert_eq!(events[0]["http_request"]["url"]["path"], "/data"); + assert_eq!(events[0]["status_detail"], "invalid_jsonrpc_request"); + assert_eq!( + events[0]["message"], + "HTTP request rejected before policy evaluation" + ); + assert_eq!(events[0]["action"], "Denied"); + assert_eq!(events[0]["actor"]["process"]["pid"], 42); + assert!(events[0].get("firewall_rule").is_none()); + assert!( + capture + .json_events() + .iter() + .all(|event| event["metadata"]["correlation_uid"] == id) + ); +} + +#[test] +fn late_failure_is_preserved_once_but_routine_late_records_are_not_duplicated() { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = RequestCorrelation::new(&request(), &context()); + let id = correlation.request_id().to_string(); + correlation.emit(allowed()); + correlation.response_selected(200); + correlation.emit(allowed()); + correlation.emit(denied()); + correlation.emit(denied()); + drop(correlation); + let events = http_events(&capture); + assert_eq!(events.len(), 2); + assert_eq!(events[0]["action"], "Allowed"); + assert_eq!(events[0]["http_response"]["code"], 200); + assert_eq!(events[1]["action"], "Denied"); + assert_eq!(events[1]["status_detail"], "credential_endpoint_mismatch"); + assert_eq!(events[1]["http_response"]["code"], 200); + assert!( + events + .iter() + .all(|event| event["metadata"]["correlation_uid"] == id) + ); +} + +#[test] +fn unfinished_requests_preserve_admission_context_and_omit_uncommitted_response() { + for admitted in [false, true] { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let mut request = request(); + request.target = "/data?secret=SENTINEL-QUERY".into(); + let correlation = RequestCorrelation::new(&request, &context()); + let id = correlation.request_id().to_string(); + if admitted { + correlation.emit(allowed()); + } else { + let mut head = ResponseHead::default(); + head.accept(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n", &correlation); + } + assert!(capture.events().is_empty()); + drop(correlation); + let events = http_events(&capture); + assert_eq!(events.len(), 1); + assert_eq!(events[0]["metadata"]["correlation_uid"], id); + assert_eq!(events[0]["status"], "Failure"); + assert_eq!(events[0]["status_detail"], "request_incomplete"); + assert_eq!(events[0]["http_request"]["url"]["path"], "/data"); + assert!(events[0].get("http_response").is_none()); + assert!(!events[0].to_string().contains("SENTINEL-QUERY")); + if admitted { + assert_eq!(events[0]["action"], "Allowed"); + assert_eq!(events[0]["actor"]["process"]["pid"], 42); + } else { + assert!(events[0].get("firewall_rule").is_none()); + } + } +} + +#[test] +fn workload_traceparent_does_not_supply_audit_trace_identity() { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let mut request = request(); + request.raw_header = b"GET /data HTTP/1.1\r\nHost: example.test\r\ntraceparent: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\r\n\r\n".to_vec(); + let correlation = RequestCorrelation::new(&request, &context()); + correlation.emit(allowed()); + correlation.response_selected(200); + drop(correlation); + let events = http_events(&capture); + assert_eq!(events.len(), 1); + assert!(events[0].get("trace").is_none()); + assert!( + !events[0] + .to_string() + .contains("4bf92f3577b34da6a3ce929d0e0e4736") + ); +} + +#[tokio::test] +async fn uninspected_response_and_events_preserve_legacy_behavior() { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = RequestCorrelation::uninspected(); + assert!(uuid::Uuid::parse_str(correlation.request_id()).is_ok()); + let mut output = Vec::new(); + { + let mut client = correlation.observe_response(&mut output); + client + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .await + .unwrap(); + } + assert!(capture.events().is_empty()); + let expected = allowed(); + correlation.emit(expected.clone()); + drop(correlation); + assert_eq!(capture.events(), vec![expected]); + assert_eq!(output, b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"); +} + +#[test] +fn policy_classification_fills_an_existing_forward_request_context() { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = RequestCorrelation::new(&request(), &context()); + let seed = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .http_request(HttpRequest::new("GET", context().ocsf_url("/forward"))) + .actor_process(Process::new("python3", 77)) + .message("HTTP request awaiting inspection") + .build(); + correlation.emit(seed); + correlation.emit(allowed()); + correlation.response_selected(200); + drop(correlation); + let events = http_events(&capture); + assert_eq!(events.len(), 1); + assert_eq!(events[0]["http_request"]["url"]["path"], "/forward"); + assert_eq!(events[0]["actor"]["process"]["pid"], 77); + assert_eq!(events[0]["action"], "Allowed"); + assert_eq!(events[0]["status"], "Success"); + assert_eq!(events[0]["firewall_rule"]["type"], "l7"); + assert_eq!(events[0]["message"], "HTTP request admitted by policy"); +} + +#[test] +fn failure_reason_replaces_only_a_prior_success_reason() { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = RequestCorrelation::new(&request(), &context()); + correlation.emit(allowed()); + correlation.record_rejection( + 503, + HttpOutcome::PlatformError, + Some("middleware_admission"), + ); + correlation.response_selected(503); + drop(correlation); + let events = http_events(&capture); + assert_eq!(events.len(), 1); + assert_eq!(events[0]["status_detail"], "middleware_admission"); + assert_eq!(events[0]["action"], "Denied"); + assert_eq!(events[0]["status"], "Failure"); +} + +#[test] +fn late_fail_open_does_not_hide_a_later_blocking_failure() { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = RequestCorrelation::new(&request(), &context()); + let id = correlation.request_id().to_string(); + correlation.emit(allowed()); + correlation.response_selected(200); + correlation.emit( + HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .http_request(HttpRequest::new("GET", context().ocsf_url("/data"))) + .action(ActionId::Other) + .disposition(DispositionId::Allowed) + .status(StatusId::Failure) + .status_detail("middleware_fail_open") + .build(), + ); + correlation.emit(denied()); + drop(correlation); + let events = http_events(&capture); + assert_eq!(events.len(), 2); + assert_eq!(events[1]["action"], "Denied"); + assert_eq!(events[1]["status_detail"], "credential_endpoint_mismatch"); + assert_eq!(events[1]["metadata"]["correlation_uid"], id); + assert_eq!(events[1]["http_response"]["code"], 200); +} diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index e688711ad1..e5d5b8641a 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -24,6 +24,9 @@ use std::sync::{Arc, OnceLock}; use std::time::Duration; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tokio::sync::mpsc; +use tracing::Instrument as _; + +use super::correlation::{RequestStage, emit_request_events}; /// How long the sandbox may stop sending a request body while a DUPLEX /// request middleware session is held. DUPLEX has no total deadline, so this @@ -210,6 +213,7 @@ impl HttpMiddlewareExchange { } } + #[cfg(test)] pub async fn apply_request( &self, request: crate::l7::provider::L7Request, @@ -221,18 +225,36 @@ impl HttpMiddlewareExchange { where C: AsyncRead + AsyncWrite + Unpin + Send, { - Box::pin(self.apply_request_with_delivery( + self.apply_request_correlated(request, client, ctx, scheme, transformed_body_policy, None) + .await + } + + pub(crate) async fn apply_request_correlated( + &self, + request: crate::l7::provider::L7Request, + client: &mut C, + ctx: &L7EvalContext, + scheme: &str, + transformed_body_policy: openshell_supervisor_middleware::TransformedBodyPolicy<'_>, + correlation: Option<&super::correlation::RequestCorrelation>, + ) -> Result + where + C: AsyncRead + AsyncWrite + Unpin + Send, + { + Box::pin(self.apply_request_with_delivery_correlated( request, client, ctx, scheme, transformed_body_policy, RequestBodyDelivery::Incremental, + correlation, )) .await } - pub async fn apply_request_with_delivery( + #[allow(clippy::too_many_arguments)] + pub(crate) async fn apply_request_with_delivery_correlated( &self, request: crate::l7::provider::L7Request, client: &mut C, @@ -240,12 +262,13 @@ impl HttpMiddlewareExchange { scheme: &str, transformed_body_policy: openshell_supervisor_middleware::TransformedBodyPolicy<'_>, delivery: RequestBodyDelivery, + correlation: Option<&super::correlation::RequestCorrelation>, ) -> Result where C: AsyncRead + AsyncWrite + Unpin + Send, { let result = Box::pin( - apply_middleware_chain_for_scheme_with_request_id_and_delivery( + apply_middleware_chain_for_scheme_with_request_id_and_delivery_correlated( request, client, ctx, @@ -256,6 +279,7 @@ impl HttpMiddlewareExchange { transformed_body_policy, &self.request_id, delivery, + correlation, ), ) .await?; @@ -389,20 +413,35 @@ pub fn emit_middleware_uninspectable(ctx: &L7EvalContext, detail: &str, denied: egress_emit!(event.build()); } +#[cfg(test)] pub fn emit_websocket_preflight_events( ctx: &L7EvalContext, outcome: &openshell_supervisor_middleware::WebSocketPreflightResult, ) { - emit_websocket_invocations(ctx, &outcome.invocations); - emit_websocket_coverage(ctx, &outcome.coverage); - for event in websocket_preflight_finding_events(outcome) { - egress_emit!(event); - } + emit_websocket_preflight_events_correlated(ctx, outcome, None); +} + +pub fn emit_websocket_preflight_events_correlated( + ctx: &L7EvalContext, + outcome: &openshell_supervisor_middleware::WebSocketPreflightResult, + correlation: Option<&super::correlation::RequestCorrelation>, +) { + emit_websocket_invocations_correlated(ctx, &outcome.invocations, correlation); + emit_request_events( + correlation, + websocket_coverage_events(ctx, &outcome.coverage), + ); + emit_request_events(correlation, websocket_preflight_finding_events(outcome)); if outcome.saturated { - emit_websocket_saturation(ctx); + emit_websocket_saturation_correlated(ctx, correlation); } if outcome.session_capacity_exhausted { - emit_middleware_session_capacity_exhausted(ctx, "websocket_message", !outcome.allowed); + emit_middleware_session_capacity_exhausted( + ctx, + "websocket_message", + !outcome.allowed, + correlation, + ); } } @@ -532,6 +571,14 @@ pub(super) fn middleware_finding_events( fn emit_websocket_invocations( ctx: &L7EvalContext, invocations: &[openshell_supervisor_middleware::WebSocketInvocation], +) { + emit_websocket_invocations_correlated(ctx, invocations, None); +} + +fn emit_websocket_invocations_correlated( + ctx: &L7EvalContext, + invocations: &[openshell_supervisor_middleware::WebSocketInvocation], + correlation: Option<&super::correlation::RequestCorrelation>, ) { for invocation in invocations { use openshell_supervisor_middleware::WebSocketInvocationOutcome as Outcome; @@ -603,7 +650,7 @@ fn emit_websocket_invocations( invocation.transformed, )) .build(); - egress_emit!(event); + emit_request_events(correlation, [event]); if invocation.failed { let fail_closed = invocation.outcome == Outcome::FailClosed; let mut finding = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()); @@ -629,7 +676,7 @@ fn emit_websocket_invocations( ]) .message("WebSocket middleware stage failed") .build(); - egress_emit!(event); + emit_request_events(correlation, [event]); } if invocation.stage_disabled { let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) @@ -649,12 +696,19 @@ fn emit_websocket_invocations( "WebSocket middleware stage stream became unusable and was disabled for this session", ) .build(); - egress_emit!(event); + emit_request_events(correlation, [event]); } } } fn emit_websocket_saturation(ctx: &L7EvalContext) { + emit_websocket_saturation_correlated(ctx, None); +} + +fn emit_websocket_saturation_correlated( + ctx: &L7EvalContext, + correlation: Option<&super::correlation::RequestCorrelation>, +) { let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) .severity(SeverityId::Medium) .finding_info(FindingInfo::new( @@ -668,10 +722,15 @@ fn emit_websocket_saturation(ctx: &L7EvalContext) { ]) .message("WebSocket middleware work waited for admission capacity") .build(); - egress_emit!(event); + emit_request_events(correlation, [event]); } -fn emit_middleware_session_capacity_exhausted(ctx: &L7EvalContext, operation: &str, denied: bool) { +fn emit_middleware_session_capacity_exhausted( + ctx: &L7EvalContext, + operation: &str, + denied: bool, + correlation: Option<&super::correlation::RequestCorrelation>, +) { let mut event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) .severity(SeverityId::Medium) .finding_info(FindingInfo::new( @@ -687,7 +746,7 @@ fn emit_middleware_session_capacity_exhausted(ctx: &L7EvalContext, operation: &s if denied { event = event.action(ActionId::Denied); } - egress_emit!(event.build()); + emit_request_events(correlation, [event.build()]); } fn middleware_admission_exhausted_event(ctx: &L7EvalContext) -> openshell_ocsf::OcsfEvent { @@ -708,10 +767,6 @@ fn middleware_admission_exhausted_event(ctx: &L7EvalContext) -> openshell_ocsf:: .build() } -fn emit_middleware_admission_exhausted(ctx: &L7EvalContext) { - egress_emit!(middleware_admission_exhausted_event(ctx)); -} - /// Largest body-buffering limit across the entries that actually resolved to a /// registered binding. Buffering for the most capable stage lets every stage /// that can handle the body run; stages whose own limit is smaller are failed @@ -731,6 +786,7 @@ pub(super) fn middleware_chain_body_limit( .max() } +#[cfg(test)] #[allow(clippy::too_many_arguments)] pub async fn apply_middleware_chain_with_request_id( req: crate::l7::provider::L7Request, @@ -756,6 +812,7 @@ pub async fn apply_middleware_chain_with_request_id, request_id: &str, +) -> Result { + apply_middleware_chain_for_scheme_with_request_id_correlated( + req, + client, + ctx, + scheme, + chain, + runner, + generation_guard, + transformed_body_policy, + request_id, + None, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +pub async fn apply_middleware_chain_with_request_id_correlated< + C: AsyncRead + AsyncWrite + Unpin + Send, +>( + req: crate::l7::provider::L7Request, + client: &mut C, + ctx: &L7EvalContext, + chain: Vec, + runner: &openshell_supervisor_middleware::ChainRunner, + generation_guard: &PolicyGenerationGuard, + transformed_body_policy: openshell_supervisor_middleware::TransformedBodyPolicy<'_>, + request_id: &str, + correlation: Option<&super::correlation::RequestCorrelation>, +) -> Result { + apply_middleware_chain_for_scheme_with_request_id_correlated( + req, + client, + ctx, + ctx.request_scheme(), + chain, + runner, + generation_guard, + transformed_body_policy, + request_id, + correlation, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +pub async fn apply_middleware_chain_for_scheme_with_request_id_correlated< + C: AsyncRead + AsyncWrite + Unpin + Send, +>( + req: crate::l7::provider::L7Request, + client: &mut C, + ctx: &L7EvalContext, + scheme: &str, + chain: Vec, + runner: &openshell_supervisor_middleware::ChainRunner, + generation_guard: &PolicyGenerationGuard, + transformed_body_policy: openshell_supervisor_middleware::TransformedBodyPolicy<'_>, + request_id: &str, + correlation: Option<&super::correlation::RequestCorrelation>, ) -> Result { Box::pin( - apply_middleware_chain_for_scheme_with_request_id_and_delivery( + apply_middleware_chain_for_scheme_with_request_id_and_delivery_correlated( req, client, ctx, @@ -813,14 +930,46 @@ pub async fn apply_middleware_chain_for_scheme_with_request_id< transformed_body_policy, request_id, RequestBodyDelivery::Incremental, + correlation, + ), + ) + .await +} + +#[allow(clippy::too_many_arguments)] +pub async fn apply_middleware_chain_with_request_id_and_delivery_correlated< + C: AsyncRead + AsyncWrite + Unpin + Send, +>( + req: crate::l7::provider::L7Request, + client: &mut C, + ctx: &L7EvalContext, + chain: Vec, + runner: &openshell_supervisor_middleware::ChainRunner, + generation_guard: &PolicyGenerationGuard, + transformed_body_policy: openshell_supervisor_middleware::TransformedBodyPolicy<'_>, + request_id: &str, + delivery: RequestBodyDelivery, + correlation: Option<&super::correlation::RequestCorrelation>, +) -> Result { + Box::pin( + apply_middleware_chain_for_scheme_with_request_id_and_delivery_correlated( + req, + client, + ctx, + ctx.request_scheme(), + chain, + runner, + generation_guard, + transformed_body_policy, + request_id, + delivery, + correlation, ), ) .await } -/// Describe the request chain and run it on the engine of its HTTP hook version. -/// A chain that mixes entries of both HTTP hook versions fails -/// closed. +#[cfg(test)] #[allow(clippy::too_many_arguments)] pub async fn apply_middleware_chain_for_scheme_with_request_id_and_delivery< C: AsyncRead + AsyncWrite + Unpin + Send, @@ -835,11 +984,52 @@ pub async fn apply_middleware_chain_for_scheme_with_request_id_and_delivery< transformed_body_policy: openshell_supervisor_middleware::TransformedBodyPolicy<'_>, request_id: &str, delivery: RequestBodyDelivery, +) -> Result { + Box::pin( + apply_middleware_chain_for_scheme_with_request_id_and_delivery_correlated( + req, + client, + ctx, + scheme, + chain, + runner, + generation_guard, + transformed_body_policy, + request_id, + delivery, + None, + ), + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn apply_middleware_chain_for_scheme_with_request_id_and_delivery_correlated< + C: AsyncRead + AsyncWrite + Unpin + Send, +>( + req: crate::l7::provider::L7Request, + client: &mut C, + ctx: &L7EvalContext, + scheme: &str, + chain: Vec, + runner: &openshell_supervisor_middleware::ChainRunner, + generation_guard: &PolicyGenerationGuard, + transformed_body_policy: openshell_supervisor_middleware::TransformedBodyPolicy<'_>, + request_id: &str, + delivery: RequestBodyDelivery, + correlation: Option<&super::correlation::RequestCorrelation>, ) -> Result { if chain.is_empty() { return Ok(MiddlewareApplyResult::Allowed(req)); } let chain = runner.describe_chain(&chain).await?; + let span = if chain.is_empty() { + tracing::Span::none() + } else { + correlation.map_or_else(tracing::Span::none, |correlation| { + correlation.stage(RequestStage::RequestMiddleware) + }) + }; match openshell_supervisor_middleware::chain_http_hook_version(&chain) { ChainHttpHookVersion::Legacy => { Box::pin(apply_protocol1_middleware_chain( @@ -852,11 +1042,13 @@ pub async fn apply_middleware_chain_for_scheme_with_request_id_and_delivery< generation_guard, transformed_body_policy, request_id, + correlation, + span.clone(), )) .await } ChainHttpHookVersion::Mixed => { - emit_mixed_protocol_denial(ctx, &req, &chain); + emit_mixed_protocol_denial(ctx, &req, &chain, correlation); Ok(MiddlewareApplyResult::Denied { denial: None, failure: Some(MiddlewareFailureDetail { @@ -888,6 +1080,8 @@ pub async fn apply_middleware_chain_for_scheme_with_request_id_and_delivery< transformed_body_policy, request_id, delivery, + correlation, + span.clone(), )) .await } @@ -906,17 +1100,22 @@ async fn apply_protocol1_middleware_chain, request_id: &str, + correlation: Option<&super::correlation::RequestCorrelation>, + span: tracing::Span, ) -> Result { let admission = if chain.is_empty() { None } else { - let outcome = runner.reserve_middleware_work().await?; + let outcome = runner + .reserve_middleware_work() + .instrument(span.clone()) + .await?; match outcome { openshell_supervisor_middleware::MiddlewareWorkAdmissionOutcome::Admitted( admission, ) => Some(admission), openshell_supervisor_middleware::MiddlewareWorkAdmissionOutcome::QueueExhausted => { - emit_middleware_admission_exhausted(ctx); + emit_request_events(correlation, [middleware_admission_exhausted_event(ctx)]); return Ok(MiddlewareApplyResult::AdmissionExhausted); } } @@ -944,8 +1143,9 @@ async fn apply_protocol1_middleware_chain buffered, crate::l7::rest::BufferResult::OverCapacity { recoverable } => { - return Ok(resolve_unbuffered_body(ctx, req, &chain, recoverable)); + return Ok(resolve_unbuffered_body_correlated( + ctx, + req, + &chain, + recoverable, + correlation, + )); } }; let headers = safe_middleware_headers(&buffered.headers)?; @@ -995,8 +1202,9 @@ async fn apply_protocol1_middleware_chain, request_id: &str, delivery: RequestBodyDelivery, + correlation: Option<&super::correlation::RequestCorrelation>, + span: tracing::Span, ) -> Result { let head = &req.raw_header[..request_header_end(&req.raw_header)]; // Output that cannot stream to the upstream is withheld up to the route's @@ -1039,9 +1249,10 @@ async fn apply_pipeline_middleware_chain { @@ -1134,6 +1348,7 @@ async fn apply_pipeline_middleware_chain return events.body_failed(error), + (Err(error), _) => return events.body_failed(error, correlation), (Ok(finish), Some(collected)) => (finish, collected), }; generation_guard.ensure_current()?; @@ -1156,7 +1371,13 @@ async fn apply_pipeline_middleware_chain, + span: Option, } impl RequestEvents { @@ -1351,6 +1573,7 @@ impl RequestEvents { reason: &str, denial: Option<&openshell_supervisor_middleware::MiddlewareDenial>, body: HttpStageDiagnostics, + correlation: Option<&super::correlation::RequestCorrelation>, ) { if self.terminal_reason.is_some() { return; @@ -1358,7 +1581,7 @@ impl RequestEvents { self.terminal_reason = Some(reason.to_string()); let mut diagnostics = std::mem::take(&mut self.preflight); diagnostics.extend(body); - for event in request_middleware_events( + let events = request_middleware_events( &self.context, &self.action, &self.target, @@ -1366,13 +1589,17 @@ impl RequestEvents { reason, denial, &diagnostics, - ) { - egress_emit!(event); - } + ); + let span = self.span.take().unwrap_or_else(tracing::Span::none); + span.in_scope(|| emit_request_events(correlation, events)); } /// Record a failed body run and return the result the relay acts on. - fn body_failed(&mut self, error: BodyRunError) -> Result { + fn body_failed( + &mut self, + error: BodyRunError, + correlation: Option<&super::correlation::RequestCorrelation>, + ) -> Result { match error { BodyRunError::Middleware(failure) => { self.emit( @@ -1380,6 +1607,7 @@ impl RequestEvents { &failure.reason, failure.denial.as_ref(), *failure.diagnostics, + correlation, ); Ok(MiddlewareApplyResult::Denied { denial: failure.denial, @@ -1387,16 +1615,20 @@ impl RequestEvents { }) } BodyRunError::ClientTimeout => { - egress_emit!(request_client_timeout_event( - &self.context, - &self.action, - &self.target - )); + emit_request_events( + correlation, + [request_client_timeout_event( + &self.context, + &self.action, + &self.target, + )], + ); self.emit( false, "middleware_cancelled: client_progress_timeout", None, HttpStageDiagnostics::default(), + correlation, ); Ok(MiddlewareApplyResult::RequestTimeout) } @@ -1406,6 +1638,7 @@ impl RequestEvents { "middleware_cancelled: request_body_read_failed", None, HttpStageDiagnostics::default(), + correlation, ); Err(error) } @@ -1461,6 +1694,7 @@ impl RequestBodyStream { client: &mut C, output: mpsc::Sender, upstream_failed: &AtomicBool, + correlation: Option<&super::correlation::RequestCorrelation>, ) -> Result { let session = self .session @@ -1478,11 +1712,17 @@ impl RequestBodyStream { output, openshell_supervisor_middleware::TransformedBodyPolicy::NotPolicyRelevant, ) + .instrument(self.events.span.clone().unwrap_or_else(tracing::Span::none)) .await { Ok(mut finish) => { - self.events - .emit(true, "", None, std::mem::take(&mut finish.diagnostics)); + self.events.emit( + true, + "", + None, + std::mem::take(&mut finish.diagnostics), + correlation, + ); return Ok(finish); } Err(error) => error, @@ -1497,6 +1737,7 @@ impl RequestBodyStream { "middleware_cancelled: upstream_write_failed", None, *failure.diagnostics, + correlation, ); return Err(miette!("upstream stopped accepting the request body")); } @@ -1510,20 +1751,28 @@ impl RequestBodyStream { }, BodyRunError::ClientTimeout => crate::l7::rest::LiveRequestFailure::ClientTimeout, BodyRunError::Client(_) => { - return self.events.body_failed(error).map(|_| unreachable!()); + return self + .events + .body_failed(error, correlation) + .map(|_| unreachable!()); } }; - self.events.body_failed(error)?; + self.events.body_failed(error, correlation)?; Err(miette::Report::new(failure)) } /// Record that the middleware stages were cancelled for `reason`. - pub(crate) fn cancelled(&mut self, reason: &str) { + pub(crate) fn cancelled( + &mut self, + reason: &str, + correlation: Option<&super::correlation::RequestCorrelation>, + ) { self.events.emit( false, &format!("middleware_cancelled: {reason}"), None, HttpStageDiagnostics::default(), + correlation, ); } } @@ -1593,10 +1842,12 @@ fn emit_mixed_protocol_denial( ctx: &L7EvalContext, req: &crate::l7::provider::L7Request, chain: &[openshell_supervisor_middleware::DescribedChainEntry], + correlation: Option<&super::correlation::RequestCorrelation>, ) { - for event in mixed_protocol_events(ctx, &req.action, &req.target, chain) { - egress_emit!(event); - } + emit_request_events( + correlation, + mixed_protocol_events(ctx, &req.action, &req.target, chain), + ); } pub(super) fn mixed_protocol_events( @@ -1649,8 +1900,11 @@ pub async fn send_middleware_rejection_response, failure: Option<&MiddlewareFailureDetail>, redacted_target: &str, + request_id: Option<&str>, ) -> Result<()> { - let context = Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)); + let context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx); + let context = request_id.map_or(context, |request_id| context.with_request_id(request_id)); + let context = Some(context); if let Some(denial) = denial { crate::l7::rest::send_middleware_deny_response( req, @@ -1684,13 +1938,16 @@ pub async fn send_middleware_admission_exhausted_response< client: &mut C, ctx: &L7EvalContext, redacted_target: &str, + request_id: Option<&str>, ) -> Result<()> { + let context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx); + let context = request_id.map_or(context, |request_id| context.with_request_id(request_id)); crate::l7::rest::send_middleware_unavailable_response( req, &ctx.policy_name, client, Some(redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some(context), ) .await?; close_after_early_response(client).await @@ -1703,7 +1960,15 @@ pub async fn send_request_timeout_response( client: &mut C, ctx: &L7EvalContext, redacted_target: &str, + correlation: Option<&super::correlation::RequestCorrelation>, ) -> Result<()> { + if let Some(correlation) = correlation { + correlation.record_rejection( + 408, + super::correlation::HttpOutcome::PlatformError, + Some("request_timeout"), + ); + } let req = crate::l7::provider::L7Request { action: action.to_string(), target: redacted_target.to_string(), @@ -1711,12 +1976,16 @@ pub async fn send_request_timeout_response( raw_header: Vec::new(), body_length: crate::l7::provider::BodyLength::None, }; + let context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx); + let context = correlation.map_or(context, |correlation| { + context.with_request_id(correlation.request_id()) + }); crate::l7::rest::send_request_timeout_response( &req, &ctx.policy_name, client, Some(redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some(context), ) .await?; close_after_early_response(client).await @@ -1799,27 +2068,42 @@ pub(super) fn raw_query_from_request_headers(headers: &[u8]) -> Result { /// would individually fail with `request_body_over_capacity`; the aggregate is /// a deny unless every attached middleware is `fail_open`, and passing the /// body through is only safe when no bytes were consumed. +#[cfg(test)] pub(super) fn resolve_unbuffered_body( ctx: &L7EvalContext, req: crate::l7::provider::L7Request, chain: &[openshell_supervisor_middleware::DescribedChainEntry], recoverable: bool, +) -> MiddlewareApplyResult { + resolve_unbuffered_body_correlated(ctx, req, chain, recoverable, None) +} + +fn resolve_unbuffered_body_correlated( + ctx: &L7EvalContext, + req: crate::l7::provider::L7Request, + chain: &[openshell_supervisor_middleware::DescribedChainEntry], + recoverable: bool, + correlation: Option<&super::correlation::RequestCorrelation>, ) -> MiddlewareApplyResult { let all_fail_open = chain .iter() .all(|entry| entry.on_error() == openshell_supervisor_middleware::OnError::FailOpen); if recoverable && all_fail_open { - emit_middleware_body_unavailable(ctx, false); + emit_middleware_body_unavailable(ctx, false, correlation); return MiddlewareApplyResult::Allowed(req); } - emit_middleware_body_unavailable(ctx, true); + emit_middleware_body_unavailable(ctx, true, correlation); MiddlewareApplyResult::Denied { denial: None, failure: None, } } -fn emit_middleware_body_unavailable(ctx: &L7EvalContext, denied: bool) { +fn emit_middleware_body_unavailable( + ctx: &L7EvalContext, + denied: bool, + correlation: Option<&super::correlation::RequestCorrelation>, +) { let mut event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) .severity(if denied { SeverityId::High @@ -1843,7 +2127,7 @@ fn emit_middleware_body_unavailable(ctx: &L7EvalContext, denied: bool) { if denied { event = event.action(ActionId::Denied); } - egress_emit!(event.build()); + emit_request_events(correlation, [event.build()]); } /// Parse the raw header block into middleware-visible headers, preserving @@ -1977,6 +2261,8 @@ pub(super) fn middleware_events( .http_request(HttpRequest::new(&req.action, ctx.ocsf_url(&req.target))) .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) .firewall_rule(&ctx.policy_name, "middleware") + .unmapped("http_event_kind", "middleware_invocation") + .unmapped("middleware_phase", "request") .unmapped("transformed", invocation.transformed) .unmapped("failed", invocation.failed) .message(format!( @@ -2056,6 +2342,7 @@ pub(super) fn middleware_events( /// Emit the OCSF events describing a middleware chain outcome through the /// tracing pipeline. +#[cfg(test)] fn emit_middleware_events( ctx: &L7EvalContext, req: &crate::l7::provider::L7Request, @@ -2163,13 +2450,41 @@ mod trace_tests { } } + #[test] + fn request_invocation_records_are_explicitly_tagged() { + let events = middleware_events( + &L7EvalContext::default(), + &request(), + &outcome( + true, + "", + vec![MiddlewareInvocation { + name: "redact".into(), + implementation: "test/redact".into(), + decision: openshell_core::proto::Decision::Allow, + transformed: true, + failed: false, + }], + vec![], + ), + ); + assert_eq!(events.len(), 1); + let event = events[0].to_json().unwrap(); + assert_eq!( + event["unmapped"]["http_event_kind"], + "middleware_invocation" + ); + assert_eq!(event["unmapped"]["middleware_phase"], "request"); + assert_eq!(event["unmapped"]["transformed"], true); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn inspection_refusals_record_denial_but_fail_open_findings_do_not() { for denied in [false, true] { let (events, received) = exported("warn", || { let ctx = L7EvalContext::default(); emit_middleware_uninspectable(&ctx, "unknown protocol", denied); - emit_middleware_body_unavailable(&ctx, denied); + emit_middleware_body_unavailable(&ctx, denied, None); }) .await; assert_eq!(events.len(), 2); @@ -2605,14 +2920,19 @@ mod tests { Some(&denial), None, "/v1/messages", + Some("00000000-0000-4000-8000-000000000010"), ), read ); sent.expect("send denial"); let response = String::from_utf8(response).expect("UTF-8 response"); + assert!( + response.contains("x-openshell-request-id: 00000000-0000-4000-8000-000000000010\r\n") + ); let (_, body) = response.split_once("\r\n\r\n").expect("HTTP response"); let body: serde_json::Value = serde_json::from_str(body).expect("JSON response"); assert_eq!(body["error"], "middleware_denied"); + assert_eq!(body["request_id"], "00000000-0000-4000-8000-000000000010"); assert_eq!(body["middleware"], "prototype-content-guard"); assert_eq!(body["reason_code"], "content_match"); assert!(body.get("rule_missing").is_none()); @@ -2713,14 +3033,19 @@ mod tests { None, Some(&failure), "/v1/messages", + Some("00000000-0000-4000-8000-000000000011"), ), read ); sent.expect("send failure"); let response = String::from_utf8(response).expect("UTF-8 response"); + assert!( + response.contains("x-openshell-request-id: 00000000-0000-4000-8000-000000000011\r\n") + ); let (_, body) = response.split_once("\r\n\r\n").expect("HTTP response"); let body: serde_json::Value = serde_json::from_str(body).expect("JSON response"); assert_eq!(body["error"], "middleware_failed"); + assert_eq!(body["request_id"], "00000000-0000-4000-8000-000000000011"); assert_eq!( body["detail"], "Request could not be processed by configured middleware" @@ -2765,7 +3090,15 @@ mod tests { response }; let (sent, response) = tokio::join!( - send_middleware_rejection_response(&req, &mut server, &ctx, None, None, "/upload"), + send_middleware_rejection_response( + &req, + &mut server, + &ctx, + None, + None, + "/upload", + None + ), upload ); sent.expect("send failure"); diff --git a/crates/openshell-supervisor-network/src/l7/mod.rs b/crates/openshell-supervisor-network/src/l7/mod.rs index c24777826e..0647b8b2fe 100644 --- a/crates/openshell-supervisor-network/src/l7/mod.rs +++ b/crates/openshell-supervisor-network/src/l7/mod.rs @@ -8,6 +8,7 @@ //! doing a raw `copy_bidirectional`. Each request within the tunnel is parsed, //! evaluated against OPA policy, and either forwarded or denied. +pub(crate) mod correlation; pub mod graphql; pub(crate) mod http; pub mod jsonrpc; @@ -16,6 +17,7 @@ pub(crate) mod middleware; pub mod path; pub mod provider; pub mod relay; +mod request_trace; pub mod rest; pub mod tls; pub(crate) mod token_grant_injection; diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 361895db5e..1b14041d85 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -7,13 +7,16 @@ //! Parses each request within the tunnel, evaluates it against OPA policy, //! and either forwards or denies the request. +use crate::l7::correlation::{HttpOutcome, RequestCorrelation, RequestStage}; use crate::l7::middleware::{ MiddlewareApplyResult, RequestBodyDelivery, UninspectableTrafficGate, - apply_middleware_chain_with_request_id, apply_middleware_chain_with_request_id_and_delivery, - emit_middleware_uninspectable, middleware_network_input, uninspectable_traffic_gate, + apply_middleware_chain_with_request_id_and_delivery_correlated, + apply_middleware_chain_with_request_id_correlated, emit_middleware_uninspectable, + middleware_network_input, uninspectable_traffic_gate, }; #[cfg(test)] use crate::l7::middleware::{ + apply_middleware_chain_with_request_id, apply_middleware_chain_with_request_id_and_delivery, middleware_chain_body_limit, middleware_events, middleware_request_input, raw_query_from_request_headers, resolve_unbuffered_body, }; @@ -35,8 +38,10 @@ use std::collections::BTreeMap; use std::collections::HashSet; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; -use tracing::{debug, warn}; +use tracing::{Instrument as _, debug, warn}; +#[cfg(test)] +pub(crate) mod correlation_dispatch_tests; #[cfg(test)] mod token_grant_ownership_tests; @@ -163,6 +168,7 @@ fn credential_generation_guard( /// /// A refused grant returns a [`TokenGrantInjectionError`](crate::l7::token_grant_injection::TokenGrantInjectionError) /// inside the report, so callers can tell it apart from other failures. +#[allow(clippy::too_many_arguments)] pub(crate) async fn prepare_inspected_request( req: crate::l7::provider::L7Request, ctx: &L7EvalContext, @@ -171,6 +177,7 @@ pub(crate) async fn prepare_inspected_request( request_info: &L7RequestInfo, request_id: &str, observation_context: Option<&openshell_core::endpoint_status::EndpointObservationContext>, + correlation: Option<&RequestCorrelation>, ) -> Result<( crate::l7::provider::L7Request, L7EvalContext, @@ -217,7 +224,7 @@ pub(crate) async fn prepare_inspected_request( L7Protocol::Rest | L7Protocol::Websocket | L7Protocol::Sql => {} } let owners = admitted_token_grant_owners(engine, &scoped, ¤t_info)?; - crate::l7::token_grant_injection::inject_for_admitted_owners( + crate::l7::token_grant_injection::inject_for_admitted_owners_correlated( req, &scoped, snapshot, @@ -226,6 +233,7 @@ pub(crate) async fn prepare_inspected_request( observation_context.filter(|context| { context.matches_config(&config.policy_hash, Some(snapshot.revision)) }), + correlation, ) .await? } else { @@ -280,35 +288,46 @@ async fn reject_request_authority_mismatch( client: &mut W, ctx: &L7EvalContext, method: &str, + correlation: Option<&RequestCorrelation>, ) -> Result<()> where W: AsyncWrite + Unpin, { - let body = r#"{"error":"request_authority_mismatch","message":"HTTP request authority does not match the authorized tunnel endpoint"}"#; - let response = format!( - "HTTP/1.1 403 Forbidden\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - client - .write_all(response.as_bytes()) - .await - .into_diagnostic()?; - client.flush().await.into_diagnostic()?; - - crate::telemetry::emit_decision( - [ - build_request_authority_mismatch_event(ctx, method), - build_request_authority_mismatch_finding(ctx), - ], - None, - ); + let activity = build_request_authority_mismatch_event(ctx, method); + let finding = build_request_authority_mismatch_finding(ctx); + if let Some(correlation) = correlation { + correlation.record_rejection(403, HttpOutcome::Denied, None); + correlation.emit(activity); + correlation.emit(finding); + crate::l7::rest::send_json_response_with_request_id( + &ctx.policy_name, + serde_json::json!({ + "error": "request_authority_mismatch", + "message": "HTTP request authority does not match the authorized tunnel endpoint", + }), + client, + "403 Forbidden", + correlation.request_id(), + ) + .await?; + } else { + crate::telemetry::emit_decision([activity, finding], None); + crate::l7::rest::send_json_response( + &ctx.policy_name, + serde_json::json!({ + "error": "request_authority_mismatch", + "message": "HTTP request authority does not match the authorized tunnel endpoint", + }), + client, + "403 Forbidden", + ) + .await?; + } Ok(()) } -/// Return the selected revision's inspection for policy evaluation, or emit a -/// rejection and return `None`. Non-MCP adapters retain their original inspection. -/// Record local rejections before client response delivery can fail. -pub(crate) async fn enforce_mcp_protocol_version( +#[allow(clippy::too_many_arguments)] +pub(crate) async fn enforce_mcp_protocol_version_correlated( config: &L7EndpointConfig, request: &crate::l7::provider::L7Request, mut info: crate::l7::jsonrpc::JsonRpcRequestInfo, @@ -316,6 +335,7 @@ pub(crate) async fn enforce_mcp_protocol_version( ctx: &L7EvalContext, redacted_target: &str, observer: Option<&EndpointObserver>, + correlation: Option<&RequestCorrelation>, ) -> Result> where W: AsyncWrite + Unpin, @@ -351,7 +371,7 @@ where crate::l7::mcp::selected_revision_context(request, version) ); let summary = l7_protocol_log_summary(None, Some(&info)); - egress_emit!(build_l7_request_event( + let event = build_l7_request_event( ctx, &request.action, redacted_target, @@ -359,7 +379,13 @@ where "l7-mcp", &reason, summary.as_deref(), - )); + ); + if let Some(correlation) = correlation { + correlation.record_rejection(400, HttpOutcome::Denied, None); + correlation.emit(event); + } else { + egress_emit!(event); + } emit_activity(ctx, true, "l7_parse_rejection"); let body = serde_json::json!({ "error": "invalid_mcp_request", @@ -370,13 +396,24 @@ where "method": request.action, "path": redacted_target, }); - crate::l7::rest::send_json_response( - &ctx.policy_name, - body, - client, - "400 Bad Request", - ) - .await?; + if let Some(correlation) = correlation { + crate::l7::rest::send_json_response_with_request_id( + &ctx.policy_name, + body, + client, + "400 Bad Request", + correlation.request_id(), + ) + .await?; + } else { + crate::l7::rest::send_json_response( + &ctx.policy_name, + body, + client, + "400 Bad Request", + ) + .await?; + } return Ok(None); } @@ -389,6 +426,7 @@ where ctx, redacted_target, observer, + correlation, ) .await?; return Ok(None); @@ -405,6 +443,7 @@ where ctx, redacted_target, observer, + correlation, ) .await?; Ok(None) @@ -413,6 +452,7 @@ where } /// Emit the same transport or policy rejection at initial and final inspection. +#[allow(clippy::too_many_arguments)] async fn reject_mcp_protocol_version( error: crate::l7::mcp::McpProtocolVersionError, request: &crate::l7::provider::L7Request, @@ -421,6 +461,7 @@ async fn reject_mcp_protocol_version( ctx: &L7EvalContext, redacted_target: &str, observer: Option<&EndpointObserver>, + correlation: Option<&RequestCorrelation>, ) -> Result<()> where W: AsyncWrite + Unpin, @@ -432,7 +473,7 @@ where } let reason = error.rejection_detail(request); let summary = l7_protocol_log_summary(None, Some(info)); - egress_emit!(build_l7_request_event( + let event = build_l7_request_event( ctx, &request.action, redacted_target, @@ -440,7 +481,20 @@ where "l7-mcp", &reason, summary.as_deref(), - )); + ); + if let Some(correlation) = correlation { + let status = match error { + crate::l7::mcp::McpProtocolVersionError::InvalidHeader + | crate::l7::mcp::McpProtocolVersionError::UnsupportedHeaderValue + | crate::l7::mcp::McpProtocolVersionError::InvalidRequestMetadata => 400, + crate::l7::mcp::McpProtocolVersionError::MethodNotAllowed => 405, + crate::l7::mcp::McpProtocolVersionError::NotAllowed(_) => 403, + }; + correlation.record_rejection(status, HttpOutcome::Denied, None); + correlation.emit(event); + } else { + egress_emit!(event); + } let deny_group = match error { crate::l7::mcp::McpProtocolVersionError::NotAllowed(_) => "l7_policy", crate::l7::mcp::McpProtocolVersionError::InvalidHeader @@ -460,19 +514,32 @@ where }); let allowed_methods = (error == crate::l7::mcp::McpProtocolVersionError::MethodNotAllowed).then_some("POST"); - crate::l7::rest::send_json_response_with_allow( - &ctx.policy_name, - body, - client, - error.http_status(), - allowed_methods, - ) - .await?; + if let Some(correlation) = correlation { + crate::l7::rest::send_json_response_with_allow_and_request_id( + &ctx.policy_name, + body, + client, + error.http_status(), + allowed_methods, + correlation.request_id(), + ) + .await?; + } else { + crate::l7::rest::send_json_response_with_allow( + &ctx.policy_name, + body, + client, + error.http_status(), + allowed_methods, + ) + .await?; + } Ok(()) } /// Reinspect the buffered outgoing MCP request after request transformations. /// The forwarding adapter must call this before any upstream request write. +#[cfg(test)] pub(crate) async fn enforce_final_mcp_protocol_version( config: &L7EndpointConfig, request: &crate::l7::provider::L7Request, @@ -481,6 +548,31 @@ pub(crate) async fn enforce_final_mcp_protocol_version( redacted_target: &str, observer: Option<&EndpointObserver>, ) -> Result +where + W: AsyncWrite + Unpin, +{ + enforce_final_mcp_protocol_version_correlated( + config, + request, + client, + ctx, + redacted_target, + observer, + None, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn enforce_final_mcp_protocol_version_correlated( + config: &L7EndpointConfig, + request: &crate::l7::provider::L7Request, + client: &mut W, + ctx: &L7EvalContext, + redacted_target: &str, + observer: Option<&EndpointObserver>, + correlation: Option<&RequestCorrelation>, +) -> Result where W: AsyncWrite + Unpin, { @@ -491,7 +583,7 @@ where request, crate::l7::jsonrpc::JsonRpcInspectionOptions::for_config(config), )?; - Ok(enforce_mcp_protocol_version( + Ok(enforce_mcp_protocol_version_correlated( config, request, info, @@ -499,6 +591,7 @@ where ctx, redacted_target, observer, + correlation, ) .await? .is_some()) @@ -601,6 +694,7 @@ pub(crate) async fn reject_credential_resolution( ctx: &L7EvalContext, method: &str, error: &secrets::UnresolvedPlaceholderError, + correlation: Option<&RequestCorrelation>, ) -> Result<()> where W: AsyncWrite + Unpin, @@ -613,22 +707,21 @@ where "500 Internal Server Error" }; let body = if placement_mismatch { - r#"{"error":"credential_placement_mismatch","message":"Credential is not authorized for this request location"}"# + serde_json::json!({ + "error": "credential_placement_mismatch", + "message": "Credential is not authorized for this request location", + }) } else if endpoint_mismatch { - r#"{"error":"credential_endpoint_mismatch","message":"Credential is not authorized for this request endpoint"}"# + serde_json::json!({ + "error": "credential_endpoint_mismatch", + "message": "Credential is not authorized for this request endpoint", + }) } else { - r#"{"error":"credential_unavailable","message":"Credential placeholder could not be resolved"}"# + serde_json::json!({ + "error": "credential_unavailable", + "message": "Credential placeholder could not be resolved", + }) }; - let response = format!( - "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - client - .write_all(response.as_bytes()) - .await - .into_diagnostic()?; - client.flush().await.into_diagnostic()?; - let mut events = if placement_mismatch { vec![ HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -660,28 +753,55 @@ where if endpoint_mismatch { events.push(build_credential_endpoint_mismatch_finding(ctx)); } - crate::telemetry::emit_decision(events, None); + if let Some(correlation) = correlation { + let (response_code, outcome) = if endpoint_mismatch || placement_mismatch { + (403, HttpOutcome::Denied) + } else { + (500, HttpOutcome::PlatformError) + }; + correlation.record_rejection(response_code, outcome, None); + for event in events { + correlation.emit(event); + } + crate::l7::rest::send_json_response_with_request_id( + &ctx.policy_name, + body, + client, + status, + correlation.request_id(), + ) + .await?; + } else { + crate::telemetry::emit_decision(events, None); + crate::l7::rest::send_json_response(&ctx.policy_name, body, client, status).await?; + } Ok(()) } pub(crate) async fn reject_body_credential( client: &mut C, error: secrets::body::BodyCredentialError, + ctx: &L7EvalContext, + correlation: Option<&RequestCorrelation>, ) -> Result<()> { let body = serde_json::json!({"error": { "code": "credential_placeholder_in_request_body", "reason": error.reason(), "message": "A credential placeholder in the request body cannot be forwarded. Remove the reference from conversation history or restore provider access; body credential rewriting is disabled." - }}).to_string(); - let response = format!( - "HTTP/1.1 403 Forbidden\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - client - .write_all(response.as_bytes()) + }}); + if let Some(correlation) = correlation { + correlation.record_rejection(403, HttpOutcome::Denied, None); + crate::l7::rest::send_json_response_with_request_id( + &ctx.policy_name, + body, + client, + "403 Forbidden", + correlation.request_id(), + ) .await - .into_diagnostic()?; - client.flush().await.into_diagnostic() + } else { + crate::l7::rest::send_json_response(&ctx.policy_name, body, client, "403 Forbidden").await + } } struct InspectedForwarding<'a> { @@ -689,6 +809,7 @@ struct InspectedForwarding<'a> { engine: &'a TunnelPolicyEngine, request_info: &'a L7RequestInfo, request_id: &'a str, + correlation: Option<&'a RequestCorrelation>, response_chain: &'a [openshell_supervisor_middleware::ChainEntry], websocket_middleware: bool, observation_context: Option<&'a openshell_core::endpoint_status::EndpointObservationContext>, @@ -715,6 +836,7 @@ where engine, request_info, request_id, + correlation, response_chain, websocket_middleware, observation_context, @@ -732,11 +854,15 @@ where request_info, request_id, observation_context, + correlation, ) .await { Ok(prepared) => prepared, Err(error) => { + if let Some(correlation) = correlation { + correlation.record_rejection(502, HttpOutcome::PlatformError, None); + } if let Some(refusal) = error.downcast_ref::() { @@ -755,7 +881,8 @@ where write_bad_gateway_response(client, Some(request_id)).await?; } else { warn!(error = %error, "Token grant failed before forwarding"); - write_bad_gateway_response(client, None).await?; + write_bad_gateway_response(client, correlation.map(RequestCorrelation::request_id)) + .await?; } return Ok(None); } @@ -813,7 +940,10 @@ where engine.middleware_runner(), Some(engine.generation_guard()), )), - observer.as_ref(), + RelayHttpInstrumentation { + observer: observer.as_ref(), + correlation, + }, )) .await } @@ -838,11 +968,17 @@ where options, ctx, response_middleware, - None, + RelayHttpInstrumentation::default(), ) .await } +#[derive(Clone, Copy, Default)] +struct RelayHttpInstrumentation<'a> { + observer: Option<&'a EndpointObserver>, + correlation: Option<&'a RequestCorrelation>, +} + #[cfg(test)] async fn relay_http_request_with_credential_rejection_observed( request: &crate::l7::provider::L7Request, @@ -851,7 +987,7 @@ async fn relay_http_request_with_credential_rejection_observed( options: crate::l7::rest::RelayRequestOptions<'_>, ctx: &L7EvalContext, response_middleware: Option>, - observer: Option<&EndpointObserver>, + instrumentation: RelayHttpInstrumentation<'_>, ) -> Result> where C: AsyncRead + AsyncWrite + Unpin, @@ -865,7 +1001,7 @@ where ctx, None, response_middleware, - observer, + instrumentation, )) .await } @@ -882,14 +1018,18 @@ async fn relay_http_request_with_body_observed( ctx: &L7EvalContext, live_body: Option<&mut crate::l7::middleware::RequestBodyStream>, response_middleware: Option>, - observer: Option<&EndpointObserver>, + instrumentation: RelayHttpInstrumentation<'_>, ) -> Result> where C: AsyncRead + AsyncWrite + Unpin, U: AsyncRead + AsyncWrite + Unpin, { + let RelayHttpInstrumentation { + observer, + correlation, + } = instrumentation; match Box::pin( - crate::l7::rest::relay_http_request_with_body_guarded_observed( + crate::l7::rest::relay_http_request_with_body_guarded_observed_correlated( request, client, upstream, @@ -897,6 +1037,7 @@ where live_body, response_middleware, observer, + correlation, ), ) .await @@ -905,7 +1046,8 @@ where Err(report) => { if let Some(failure) = report.downcast_ref::() { let _ = upstream.shutdown().await; - reject_live_request_body(client, ctx, request, failure, observer).await?; + reject_live_request_body(client, ctx, request, failure, observer, correlation) + .await?; return Ok(None); } if let Some(error) = report.downcast_ref::() { @@ -916,14 +1058,15 @@ where observer.observe(EndpointResult::PolicyDenied); } let _ = upstream.shutdown().await; - reject_body_credential(client, *error).await?; + reject_body_credential(client, *error, ctx, correlation).await?; Ok(None) } else if let Some(error) = report.downcast_ref::() { if let Some(observer) = observer { observer.observe_credential_failure(error.is_endpoint_mismatch()); } - reject_credential_resolution(client, ctx, &request.action, error).await?; + reject_credential_resolution(client, ctx, &request.action, error, correlation) + .await?; Ok(None) } else if report .downcast_ref::() @@ -932,6 +1075,15 @@ where if let Some(observer) = observer { observer.observe_credential_failure(false); } + if let Some(correlation) = correlation { + correlation.record_enforcement_block(); + correlation.record_outcome(HttpOutcome::PlatformError); + let mut event = build_credential_resolution_event(ctx, &request.action, false); + if let openshell_ocsf::OcsfEvent::HttpActivity(http) = &mut event { + http.http_response = None; + } + correlation.emit(event); + } Err(report) } else { Err(report) @@ -948,6 +1100,7 @@ async fn reject_live_request_body( request: &crate::l7::provider::L7Request, failure: &crate::l7::rest::LiveRequestFailure, observer: Option<&EndpointObserver>, + correlation: Option<&RequestCorrelation>, ) -> Result<()> { let redacted_target = secrets::redact_target_for_policy(&request.target) .unwrap_or_else(|_| "[REDACTED]".to_string()); @@ -958,11 +1111,25 @@ async fn reject_live_request_body( raw_header: Vec::new(), body_length: crate::l7::provider::BodyLength::None, }; - let context = Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)); + let context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx); + let context = Some(correlation.map_or(context, |correlation| { + context.with_request_id(correlation.request_id()) + })); match failure { crate::l7::rest::LiveRequestFailure::Middleware { denial, failure, .. } => { + if let Some(correlation) = correlation { + correlation.record_rejection( + 403, + if denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + None, + ); + } if let Some(observer) = observer { observer.observe(EndpointResult::PolicyDenied); } @@ -989,6 +1156,13 @@ async fn reject_live_request_body( } } crate::l7::rest::LiveRequestFailure::ClientTimeout => { + if let Some(correlation) = correlation { + correlation.record_rejection( + 408, + HttpOutcome::PlatformError, + Some("request_timeout"), + ); + } crate::l7::rest::send_request_timeout_response( &rejected, &ctx.policy_name, @@ -1170,6 +1344,15 @@ fn parse_rejection_detail(error: &str, mode: ParseRejectionMode) -> String { } fn emit_parse_rejection(ctx: &L7EvalContext, detail: &str, engine_type: &str) { + emit_parse_rejection_correlated(ctx, detail, engine_type, None); +} + +fn emit_parse_rejection_correlated( + ctx: &L7EvalContext, + detail: &str, + engine_type: &str, + correlation: Option<&RequestCorrelation>, +) { let policy_name = if ctx.policy_name.is_empty() { "-" } else { @@ -1189,7 +1372,11 @@ fn emit_parse_rejection(ctx: &L7EvalContext, detail: &str, engine_type: &str) { )) .status_detail(detail) .build(); - egress_emit!(event); + if let Some(correlation) = correlation { + correlation.emit(event); + } else { + egress_emit!(event); + } emit_activity(ctx, true, "l7_parse_rejection"); } @@ -1218,6 +1405,7 @@ async fn deny_unsupported_upgrade_if_requested( ctx: &L7EvalContext, observer: Option<&EndpointObserver>, client: &mut C, + correlation: Option<&RequestCorrelation>, ) -> Result where C: AsyncRead + AsyncWrite + Unpin + Send, @@ -1231,17 +1419,32 @@ where if let Some(observer) = observer { observer.observe(EndpointResult::PolicyDenied); } - emit_parse_rejection(ctx, detail, engine_type_for_protocol(config.protocol)); - crate::l7::rest::send_json_response( - &ctx.policy_name, - serde_json::json!({ - "error": "unsupported_l7_protocol", - "detail": detail, - }), - client, - "403 Forbidden", - ) - .await?; + if let Some(correlation) = correlation { + correlation.record_rejection(403, HttpOutcome::Denied, None); + } + emit_parse_rejection_correlated( + ctx, + detail, + engine_type_for_protocol(config.protocol), + correlation, + ); + let body = serde_json::json!({ + "error": "unsupported_l7_protocol", + "detail": detail, + }); + if let Some(correlation) = correlation { + crate::l7::rest::send_json_response_with_request_id( + &ctx.policy_name, + body, + client, + "403 Forbidden", + correlation.request_id(), + ) + .await?; + } else { + crate::l7::rest::send_json_response(&ctx.policy_name, body, client, "403 Forbidden") + .await?; + } Ok(true) } @@ -1378,21 +1581,35 @@ where return Ok(()); } }; + let correlation = RequestCorrelation::new(&req, ctx); + let mut audited_client = correlation.observe_response(client); + let client = &mut audited_client; if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)).await?; return Ok(()); } let route_target = match secrets::redact_target_for_policy(&req.target) { Ok(target) => target, Err(error) => { - reject_credential_resolution(client, ctx, &req.action, &error).await?; + reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) + .await?; return Ok(()); } }; let Some(config) = select_l7_config_for_path(configs, &route_target) else { let reason = "no L7 endpoint path matched request"; - emit_l7_request_log(ctx, &req.action, &route_target, "deny", "l7", reason, None); + correlation.record_rejection(403, HttpOutcome::Denied, None); + emit_l7_request_log_correlated( + ctx, + &req.action, + &route_target, + "deny", + "l7", + reason, + None, + Some(&correlation), + ); crate::l7::rest::RestProvider::default() .deny_with_redacted_target( &req, @@ -1400,7 +1617,10 @@ where reason, client, None, - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some( + crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()), + ), ) .await?; return Ok(()); @@ -1428,7 +1648,15 @@ where if let Some(observer) = observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } - emit_parse_rejection(ctx, detail, engine_type_for_protocol(config.protocol)); + emit_parse_rejection_correlated( + ctx, + detail, + engine_type_for_protocol(config.protocol), + Some(&correlation), + ); + correlation.record_rejection(403, HttpOutcome::Denied, None); + let deny_context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()); crate::l7::rest::RestProvider::default() .deny_with_redacted_target( &req, @@ -1436,13 +1664,20 @@ where detail, client, None, - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some(deny_context), ) .await?; return Ok(()); } - if deny_unsupported_upgrade_if_requested(&req, config, ctx, observer.as_ref(), client) - .await? + if deny_unsupported_upgrade_if_requested( + &req, + config, + ctx, + observer.as_ref(), + client, + Some(&correlation), + ) + .await? { return Ok(()); } @@ -1467,7 +1702,16 @@ where } else { let detail = parse_rejection_detail(&e.to_string(), ParseRejectionMode::L7Endpoint); - emit_parse_rejection(ctx, &detail, "l7-graphql"); + { + correlation.record_enforcement_block(); + correlation.record_outcome(HttpOutcome::Denied); + } + emit_parse_rejection_correlated( + ctx, + &detail, + "l7-graphql", + Some(&correlation), + ); } return Ok(()); } @@ -1503,7 +1747,16 @@ where &e.to_string(), ParseRejectionMode::L7Endpoint, ); - emit_parse_rejection(ctx, &detail, "l7-jsonrpc"); + { + correlation.record_enforcement_block(); + correlation.record_outcome(HttpOutcome::Denied); + } + emit_parse_rejection_correlated( + ctx, + &detail, + "l7-jsonrpc", + Some(&correlation), + ); } return Ok(()); } @@ -1523,13 +1776,14 @@ where if let Some(observer) = observer.as_ref() { observer.observe_credential_failure(error.is_endpoint_mismatch()); } - reject_credential_resolution(client, ctx, &req.action, &error).await?; + reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) + .await?; return Ok(()); } }; if let Some(info) = jsonrpc_info.take() { - let Some(inspected) = enforce_mcp_protocol_version( + let Some(inspected) = enforce_mcp_protocol_version_correlated( config, &req, info, @@ -1537,6 +1791,7 @@ where ctx, &redacted_target, observer.as_ref(), + Some(&correlation), ) .await? else { @@ -1553,14 +1808,28 @@ where }; let websocket_request = crate::l7::rest::request_is_websocket_upgrade(&req.raw_header); if config.protocol == L7Protocol::Websocket && !websocket_request { + let reason = "websocket endpoint requires a valid WebSocket upgrade request"; + correlation.record_rejection(403, HttpOutcome::Denied, None); + emit_l7_request_log_correlated( + ctx, + &request_info.action, + &redacted_target, + "deny", + "l7-websocket", + reason, + None, + Some(&correlation), + ); + let deny_context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()); crate::l7::rest::RestProvider::default() .deny_with_redacted_target( &req, &ctx.policy_name, - "websocket endpoint requires a valid WebSocket upgrade request", + reason, client, Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some(deny_context), ) .await?; return Ok(()); @@ -1571,7 +1840,9 @@ where let (allowed, reason) = if let Some(reason) = hard_deny_reason { (false, reason) } else { - evaluate_l7_request(&engine, ctx, &request_info)? + correlation + .stage(RequestStage::PolicyEvaluate) + .in_scope(|| evaluate_l7_request(&engine, ctx, &request_info))? }; if close_if_stale(engine.generation_guard(), ctx) { @@ -1584,10 +1855,15 @@ where (false, EnforcementMode::Audit) => "audit", (false, EnforcementMode::Enforce) => "deny", }; + { + if !allowed && (config.enforcement != EnforcementMode::Audit || force_deny) { + correlation.record_enforcement_block(); + } + } let engine_type = engine_type_for_protocol(config.protocol); let protocol_summary = l7_protocol_log_summary(graphql_info.as_ref(), jsonrpc_info.as_ref()); - emit_l7_request_log( + emit_l7_request_log_correlated( ctx, &request_info.action, &redacted_target, @@ -1595,18 +1871,20 @@ where engine_type, &reason, protocol_summary.as_deref(), + Some(&correlation), ); if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; let response_chain = chain.clone(); - let request_id = uuid::Uuid::new_v4().to_string(); + let request_id = correlation.request_id(); let websocket_chain = websocket_request.then(|| chain.clone()); // Route selection resolved `config` per request, so re-check the // body against that protocol's policy after every transforming // stage (a no-op for REST and websocket, whose policy inputs the // chain cannot mutate). - let validate = transformed_body_validator(config, &engine, ctx, &request_info); + let validate = + transformed_body_validator(config, &engine, ctx, &request_info, Some(&correlation)); let transformed_body_policy = if matches!( config.protocol, L7Protocol::Rest | L7Protocol::Websocket | L7Protocol::Sql @@ -1615,7 +1893,7 @@ where } else { openshell_supervisor_middleware::TransformedBodyPolicy::Reevaluate(&validate) }; - let middleware_result = apply_middleware_chain_with_request_id_and_delivery( + let middleware_result = apply_middleware_chain_with_request_id_and_delivery_correlated( req, client, ctx, @@ -1623,8 +1901,9 @@ where engine.middleware_runner(), engine.generation_guard(), transformed_body_policy, - &request_id, + request_id, request_body_delivery(config), + Some(&correlation), ) .await; let middleware_result = middleware_result?; @@ -1637,6 +1916,7 @@ where client, ctx, &redacted_target, + Some(&correlation), ) .await?; return Ok(()); @@ -1645,6 +1925,15 @@ where if let Some(observer) = observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } + correlation.record_rejection( + 403, + if denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + None, + ); let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -1659,11 +1948,13 @@ where denial.as_ref(), failure.as_ref(), &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); } MiddlewareApplyResult::AdmissionExhausted => { + correlation.record_rejection(503, HttpOutcome::PlatformError, None); let unavailable_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -1676,12 +1967,18 @@ where client, ctx, &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); } }; let mut middleware_session = if let Some(chain) = websocket_chain.as_deref() { + let preflight_span = if chain.is_empty() { + tracing::Span::none() + } else { + correlation.stage(RequestStage::WebsocketPreflight) + }; let preflight = websocket_middleware_preflight( &req, chain, @@ -1689,17 +1986,32 @@ where ctx, ctx.websocket_scheme(), ) + .instrument(preflight_span) .await; let preflight = match preflight { Ok(preflight) => preflight, Err(error) => { warn!(error = %error, "WebSocket middleware preflight failed"); - write_bad_gateway_response(client, None).await?; + correlation.record_rejection(502, HttpOutcome::PlatformError, None); + write_bad_gateway_response(client, Some(correlation.request_id())).await?; return Ok(()); } }; - crate::l7::middleware::emit_websocket_preflight_events(ctx, &preflight); + crate::l7::middleware::emit_websocket_preflight_events_correlated( + ctx, + &preflight, + Some(&correlation), + ); if preflight.terminal_reason.is_some() { + correlation.record_rejection( + 403, + if preflight.denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + None, + ); crate::l7::middleware::send_middleware_rejection_response( &req, client, @@ -1707,6 +2019,7 @@ where preflight.denial.as_ref(), None, &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); @@ -1726,7 +2039,8 @@ where config, engine: &engine, request_info: &request_info, - request_id: &request_id, + request_id, + correlation: Some(&correlation), response_chain: &response_chain, websocket_middleware: middleware_session.is_some(), observation_context: observation_context.as_ref(), @@ -1809,6 +2123,7 @@ where options.websocket.permessage_deflate = websocket_permessage_deflate; options.middleware_session = middleware_session.take(); options.selected_subprotocol = websocket_subprotocol; + correlation.finish_http_trace(); return handle_upgrade( client, upstream, overflow, &ctx.host, ctx.port, options, ) @@ -1819,6 +2134,12 @@ where if let Some(observer) = observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } + { + correlation.record_response_status(403); + correlation.record_outcome(HttpOutcome::Denied); + } + let deny_context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()); crate::l7::rest::RestProvider::default() .deny_with_redacted_target( &req, @@ -1826,7 +2147,7 @@ where &reason, client, Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some(deny_context), ) .await?; return Ok(()); @@ -1844,7 +2165,8 @@ fn select_l7_config_for_path<'a>( .max_by_key(|config| config.path_specificity()) } -fn emit_l7_request_log( +#[allow(clippy::too_many_arguments)] +fn emit_l7_request_log_correlated( ctx: &L7EvalContext, action: &str, redacted_target: &str, @@ -1852,6 +2174,7 @@ fn emit_l7_request_log( engine_type: &str, reason: &str, protocol_summary: Option<&str>, + correlation: Option<&RequestCorrelation>, ) { let event = build_l7_request_event( ctx, @@ -1862,7 +2185,11 @@ fn emit_l7_request_log( reason, protocol_summary, ); - egress_emit!(event); + if let Some(correlation) = correlation { + correlation.emit(event); + } else { + egress_emit!(event); + } emit_activity(ctx, decision_str == "deny", "l7_policy"); } @@ -2270,11 +2597,23 @@ where return Ok(()); // Close connection on parse error } }; + let correlation = RequestCorrelation::new(&req, ctx); + let mut audited_client = correlation.observe_response(client); + let client = &mut audited_client; if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)).await?; return Ok(()); } - if deny_unsupported_upgrade_if_requested(&req, config, ctx, None, client).await? { + if deny_unsupported_upgrade_if_requested( + &req, + config, + ctx, + None, + client, + Some(&correlation), + ) + .await? + { return Ok(()); } @@ -2287,7 +2626,8 @@ where let redacted_target = match secrets::redact_target_for_policy(&req.target) { Ok(target) => target, Err(error) => { - reject_credential_resolution(client, ctx, &req.action, &error).await?; + reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) + .await?; return Ok(()); } }; @@ -2301,21 +2641,38 @@ where }; let websocket_request = crate::l7::rest::request_is_websocket_upgrade(&req.raw_header); if config.protocol == L7Protocol::Websocket && !websocket_request { + let reason = "websocket endpoint requires a valid WebSocket upgrade request"; + correlation.record_rejection(403, HttpOutcome::Denied, None); + emit_l7_request_log_correlated( + ctx, + &request_info.action, + &redacted_target, + "deny", + "l7-websocket", + reason, + None, + Some(&correlation), + ); provider .deny_with_redacted_target( &req, &ctx.policy_name, - "websocket endpoint requires a valid WebSocket upgrade request", + reason, client, Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some( + crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()), + ), ) .await?; return Ok(()); } // Evaluate L7 policy via Rego (using redacted target) - let (allowed, reason) = evaluate_l7_request(engine, ctx, &request_info)?; + let (allowed, reason) = correlation + .stage(RequestStage::PolicyEvaluate) + .in_scope(|| evaluate_l7_request(engine, ctx, &request_info))?; if close_if_stale(engine.generation_guard(), ctx) { return Ok(()); @@ -2340,6 +2697,9 @@ where (false, EnforcementMode::Audit, _) => "audit", (false, EnforcementMode::Enforce, _) => "deny", }; + if !allowed && config.enforcement != EnforcementMode::Audit { + correlation.record_enforcement_block(); + } // Log every L7 decision as an OCSF HTTP Activity event. // Uses redacted_target (path only, no query params) to avoid logging secrets. @@ -2373,18 +2733,18 @@ where request_info.action, ctx.host, ctx.port, redacted_target, reason, )) .build(); - egress_emit!(event); + correlation.emit(event); } if allowed || config.enforcement == EnforcementMode::Audit { let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; let response_chain = chain.clone(); - let request_id = uuid::Uuid::new_v4().to_string(); + let request_id = correlation.request_id(); let websocket_chain = websocket_request.then(|| chain.clone()); // REST and websocket-upgrade policy evaluates only the method, // path, and query, which a middleware result cannot mutate, so no // per-stage body re-check is needed. - let middleware_result = apply_middleware_chain_with_request_id_and_delivery( + let middleware_result = apply_middleware_chain_with_request_id_and_delivery_correlated( req, client, ctx, @@ -2392,8 +2752,9 @@ where engine.middleware_runner(), engine.generation_guard(), openshell_supervisor_middleware::TransformedBodyPolicy::NotPolicyRelevant, - &request_id, + request_id, request_body_delivery(config), + Some(&correlation), ) .await; let middleware_result = middleware_result?; @@ -2406,11 +2767,21 @@ where client, ctx, &redacted_target, + Some(&correlation), ) .await?; return Ok(()); } MiddlewareApplyResult::Denied { denial, failure } => { + correlation.record_rejection( + 403, + if denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + None, + ); let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -2425,11 +2796,13 @@ where denial.as_ref(), failure.as_ref(), &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); } MiddlewareApplyResult::AdmissionExhausted => { + correlation.record_rejection(503, HttpOutcome::PlatformError, None); let unavailable_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -2442,12 +2815,18 @@ where client, ctx, &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); } }; let mut middleware_session = if let Some(chain) = websocket_chain.as_deref() { + let preflight_span = if chain.is_empty() { + tracing::Span::none() + } else { + correlation.stage(RequestStage::WebsocketPreflight) + }; let preflight = websocket_middleware_preflight( &req, chain, @@ -2455,17 +2834,32 @@ where ctx, ctx.websocket_scheme(), ) + .instrument(preflight_span) .await; let preflight = match preflight { Ok(preflight) => preflight, Err(error) => { warn!(error = %error, "WebSocket middleware preflight failed"); - write_bad_gateway_response(client, None).await?; + correlation.record_rejection(502, HttpOutcome::PlatformError, None); + write_bad_gateway_response(client, Some(correlation.request_id())).await?; return Ok(()); } }; - crate::l7::middleware::emit_websocket_preflight_events(ctx, &preflight); + crate::l7::middleware::emit_websocket_preflight_events_correlated( + ctx, + &preflight, + Some(&correlation), + ); if preflight.terminal_reason.is_some() { + correlation.record_rejection( + 403, + if preflight.denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + None, + ); crate::l7::middleware::send_middleware_rejection_response( &req, client, @@ -2473,6 +2867,7 @@ where preflight.denial.as_ref(), None, &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); @@ -2492,7 +2887,8 @@ where config, engine, request_info: &request_info, - request_id: &request_id, + request_id, + correlation: Some(&correlation), response_chain: &response_chain, websocket_middleware: middleware_session.is_some(), observation_context: observation_context.as_ref(), @@ -2565,6 +2961,7 @@ where options.websocket.permessage_deflate = websocket_permessage_deflate; options.middleware_session = middleware_session.take(); options.selected_subprotocol = websocket_subprotocol; + correlation.finish_http_trace(); return handle_upgrade( client, upstream, overflow, &ctx.host, ctx.port, options, ) @@ -2573,6 +2970,8 @@ where } } else { // Enforce mode: deny with 403 and close connection (use redacted target) + correlation.record_response_status(403); + correlation.record_outcome(HttpOutcome::Denied); provider .deny_with_redacted_target( &req, @@ -2580,7 +2979,10 @@ where &reason, client, Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some( + crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()), + ), ) .await?; return Ok(()); @@ -2715,6 +3117,9 @@ where let req = parsed.request; let jsonrpc_info = parsed.info; + let correlation = RequestCorrelation::new(&req, ctx); + let mut audited_client = correlation.observe_response(client); + let client = &mut audited_client; let observer = EndpointObserver::begin_captured( ctx.endpoint_observation_tx.as_ref(), config, @@ -2726,11 +3131,18 @@ where if let Some(observer) = observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } - reject_request_authority_mismatch(client, ctx, &req.action).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)).await?; return Ok(()); } - if deny_unsupported_upgrade_if_requested(&req, config, ctx, observer.as_ref(), client) - .await? + if deny_unsupported_upgrade_if_requested( + &req, + config, + ctx, + observer.as_ref(), + client, + Some(&correlation), + ) + .await? { return Ok(()); } @@ -2744,12 +3156,13 @@ where if let Some(observer) = observer.as_ref() { observer.observe_credential_failure(error.is_endpoint_mismatch()); } - reject_credential_resolution(client, ctx, &req.action, &error).await?; + reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) + .await?; return Ok(()); } }; - let Some(jsonrpc_info) = enforce_mcp_protocol_version( + let Some(jsonrpc_info) = enforce_mcp_protocol_version_correlated( config, &req, jsonrpc_info, @@ -2757,6 +3170,7 @@ where ctx, &redacted_target, observer.as_ref(), + Some(&correlation), ) .await? else { @@ -2776,8 +3190,11 @@ where let (allowed, reason, jsonrpc_log_info) = if let Some(reason) = hard_deny_reason { (false, reason, jsonrpc_info.clone()) } else { - let evaluation = - evaluate_jsonrpc_l7_request_for_log(engine, ctx, &request_info, &jsonrpc_info)?; + let evaluation = correlation + .stage(RequestStage::PolicyEvaluate) + .in_scope(|| { + evaluate_jsonrpc_l7_request_for_log(engine, ctx, &request_info, &jsonrpc_info) + })?; (evaluation.allowed, evaluation.reason, evaluation.log_info) }; @@ -2791,6 +3208,9 @@ where (false, EnforcementMode::Audit) => "audit", (false, EnforcementMode::Enforce) => "deny", }; + if !allowed && (config.enforcement != EnforcementMode::Audit || force_deny) { + correlation.record_enforcement_block(); + } { let (action_id, disposition_id, severity) = match decision_str { @@ -2823,19 +3243,20 @@ where &reason, )) .build(); - egress_emit!(event); + correlation.emit(event); } if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; let response_chain = chain.clone(); - let request_id = uuid::Uuid::new_v4().to_string(); + let request_id = correlation.request_id(); // Policy admitted the original body above; re-check the body // against the same body-aware policy after every transforming // stage so a middleware cannot smuggle a denied operation to the // upstream or the next stage. - let validate = transformed_body_validator(config, engine, ctx, &request_info); - let middleware_result = apply_middleware_chain_with_request_id( + let validate = + transformed_body_validator(config, engine, ctx, &request_info, Some(&correlation)); + let middleware_result = apply_middleware_chain_with_request_id_correlated( req, client, ctx, @@ -2843,7 +3264,8 @@ where engine.middleware_runner(), engine.generation_guard(), openshell_supervisor_middleware::TransformedBodyPolicy::Reevaluate(&validate), - &request_id, + request_id, + Some(&correlation), ) .await?; let req = match middleware_result { @@ -2861,6 +3283,7 @@ where client, ctx, &redacted_target, + Some(&correlation), ) .await?; return Ok(()); @@ -2869,6 +3292,15 @@ where if let Some(observer) = observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } + correlation.record_rejection( + 403, + if denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + None, + ); let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -2883,11 +3315,13 @@ where denial.as_ref(), failure.as_ref(), &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); } MiddlewareApplyResult::AdmissionExhausted => { + correlation.record_rejection(503, HttpOutcome::PlatformError, None); let unavailable_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -2900,6 +3334,7 @@ where client, ctx, &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); @@ -2915,7 +3350,8 @@ where config, engine, request_info: &request_info, - request_id: &request_id, + request_id, + correlation: Some(&correlation), response_chain: &response_chain, websocket_middleware: false, observation_context: observation_context.as_ref(), @@ -2944,6 +3380,8 @@ where if let Some(observer) = observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } + correlation.record_response_status(403); + correlation.record_outcome(HttpOutcome::Denied); crate::l7::rest::RestProvider::default() .deny_with_redacted_target( &req, @@ -2951,7 +3389,10 @@ where &reason, client, Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some( + crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()), + ), ) .await?; return Ok(()); @@ -3008,12 +3449,24 @@ where return Ok(()); } }; + let correlation = RequestCorrelation::new(&req, ctx); + let mut audited_client = correlation.observe_response(client); + let client = &mut audited_client; if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)).await?; return Ok(()); } - if deny_unsupported_upgrade_if_requested(&req, config, ctx, None, client).await? { + if deny_unsupported_upgrade_if_requested( + &req, + config, + ctx, + None, + client, + Some(&correlation), + ) + .await? + { return Ok(()); } @@ -3036,7 +3489,9 @@ where } else { let detail = parse_rejection_detail(&e.to_string(), ParseRejectionMode::L7Endpoint); - emit_parse_rejection(ctx, &detail, "l7-graphql"); + correlation.record_enforcement_block(); + correlation.record_outcome(HttpOutcome::Denied); + emit_parse_rejection_correlated(ctx, &detail, "l7-graphql", Some(&correlation)); } return Ok(()); } @@ -3046,7 +3501,7 @@ where // without Host must still be denied if that body contains reserved // credential markers, so repeat the authority check on the full request. if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)).await?; return Ok(()); } @@ -3057,7 +3512,8 @@ where let redacted_target = match secrets::redact_target_for_policy(&req.target) { Ok(target) => target, Err(error) => { - reject_credential_resolution(client, ctx, &req.action, &error).await?; + reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) + .await?; return Ok(()); } }; @@ -3079,7 +3535,9 @@ where let (allowed, reason) = if let Some(reason) = hard_deny_reason { (false, reason) } else { - evaluate_l7_request(engine, ctx, &request_info)? + correlation + .stage(RequestStage::PolicyEvaluate) + .in_scope(|| evaluate_l7_request(engine, ctx, &request_info))? }; if close_if_stale(engine.generation_guard(), ctx) { @@ -3092,6 +3550,9 @@ where (false, EnforcementMode::Audit) => "audit", (false, EnforcementMode::Enforce) => "deny", }; + if !allowed && (config.enforcement != EnforcementMode::Audit || force_deny) { + correlation.record_enforcement_block(); + } { let (action_id, disposition_id, severity) = match decision_str { @@ -3124,19 +3585,20 @@ where request_info.action, ctx.host, ctx.port, redacted_target, reason, )) .build(); - egress_emit!(event); + correlation.emit(event); } if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; let response_chain = chain.clone(); - let request_id = uuid::Uuid::new_v4().to_string(); + let request_id = correlation.request_id(); // Policy admitted the original body above; re-check the body // against the same body-aware policy after every transforming // stage so a middleware cannot smuggle a denied operation to the // upstream or the next stage. - let validate = transformed_body_validator(config, engine, ctx, &request_info); - let middleware_result = apply_middleware_chain_with_request_id( + let validate = + transformed_body_validator(config, engine, ctx, &request_info, Some(&correlation)); + let middleware_result = apply_middleware_chain_with_request_id_correlated( req, client, ctx, @@ -3144,7 +3606,8 @@ where engine.middleware_runner(), engine.generation_guard(), openshell_supervisor_middleware::TransformedBodyPolicy::Reevaluate(&validate), - &request_id, + request_id, + Some(&correlation), ) .await?; let req = match middleware_result { @@ -3162,11 +3625,21 @@ where client, ctx, &redacted_target, + Some(&correlation), ) .await?; return Ok(()); } MiddlewareApplyResult::Denied { denial, failure } => { + correlation.record_rejection( + 403, + if denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + None, + ); let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -3181,11 +3654,13 @@ where denial.as_ref(), failure.as_ref(), &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); } MiddlewareApplyResult::AdmissionExhausted => { + correlation.record_rejection(503, HttpOutcome::PlatformError, None); let unavailable_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -3198,6 +3673,7 @@ where client, ctx, &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); @@ -3213,7 +3689,8 @@ where config, engine, request_info: &request_info, - request_id: &request_id, + request_id, + correlation: Some(&correlation), response_chain: &response_chain, websocket_middleware: false, observation_context: observation_context.as_ref(), @@ -3251,6 +3728,8 @@ where } } } else { + correlation.record_response_status(403); + correlation.record_outcome(HttpOutcome::Denied); crate::l7::rest::RestProvider::default() .deny_with_redacted_target( &req, @@ -3258,7 +3737,10 @@ where &reason, client, Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some( + crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()), + ), ) .await?; return Ok(()); @@ -3504,6 +3986,7 @@ fn reevaluate_transformed_body( ctx: &L7EvalContext, request_info: &L7RequestInfo, body: &[u8], + correlation: Option<&RequestCorrelation>, ) -> Result> { let (engine_type, transformed_info) = match config.protocol { // REST and websocket-upgrade policy evaluates only the method, path, @@ -3551,20 +4034,38 @@ fn reevaluate_transformed_body( if let Some(reason) = l7_request_hard_deny_reason(config.protocol, &transformed_info) { let reason = format!("middleware transformation rejected: {reason}"); - emit_transformed_body_decision(ctx, request_info, engine_type, "deny", &reason); + emit_transformed_body_decision( + ctx, + request_info, + engine_type, + "deny", + &reason, + correlation, + ); return Ok(Some(reason)); } - let (allowed, reason) = evaluate_l7_request(engine, ctx, &transformed_info)?; + let (allowed, reason) = correlation + .map_or_else(tracing::Span::none, |correlation| { + correlation.stage(RequestStage::PolicyEvaluate) + }) + .in_scope(|| evaluate_l7_request(engine, ctx, &transformed_info))?; if allowed { return Ok(None); } let reason = format!("middleware transformation denied by policy: {reason}"); if config.enforcement == EnforcementMode::Audit { - emit_transformed_body_decision(ctx, request_info, engine_type, "audit", &reason); + emit_transformed_body_decision( + ctx, + request_info, + engine_type, + "audit", + &reason, + correlation, + ); return Ok(None); } - emit_transformed_body_decision(ctx, request_info, engine_type, "deny", &reason); + emit_transformed_body_decision(ctx, request_info, engine_type, "deny", &reason, correlation); Ok(Some(reason)) } @@ -3576,8 +4077,11 @@ pub(crate) fn transformed_body_validator<'a>( engine: &'a TunnelPolicyEngine, ctx: &'a L7EvalContext, request_info: &'a L7RequestInfo, + correlation: Option<&'a RequestCorrelation>, ) -> impl Fn(&[u8]) -> Result> + Send + Sync + 'a { - move |body: &[u8]| reevaluate_transformed_body(config, engine, ctx, request_info, body) + move |body: &[u8]| { + reevaluate_transformed_body(config, engine, ctx, request_info, body, correlation) + } } /// Log the post-transformation policy decision as an OCSF HTTP Activity @@ -3589,6 +4093,7 @@ fn emit_transformed_body_decision( engine_type: &str, decision_str: &str, reason: &str, + correlation: Option<&RequestCorrelation>, ) { let (action_id, disposition_id, severity) = match decision_str { "deny" => (ActionId::Denied, DispositionId::Blocked, SeverityId::Medium), @@ -3598,7 +4103,7 @@ fn emit_transformed_body_decision( SeverityId::Informational, ), }; - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + let mut event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Other) .action(action_id) .disposition(disposition_id) @@ -3609,12 +4114,18 @@ fn emit_transformed_body_decision( )) .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) .firewall_rule(&ctx.policy_name, engine_type) + .status_detail(reason) .message(format!( "L7_REQUEST_TRANSFORMED {decision_str} {} {}:{}{} reason={}", request_info.action, ctx.host, ctx.port, request_info.target, reason )) .build(); - egress_emit!(event); + if decision_str == "audit" + && let openshell_ocsf::OcsfEvent::HttpActivity(http) = &mut event + { + http.base.unmapped = Some(serde_json::json!({"transformed_body_policy": "audit"})); + } + crate::l7::correlation::emit_request_events(correlation, [event]); } fn jsonrpc_policy_input(info: &crate::l7::jsonrpc::JsonRpcRequestInfo) -> serde_json::Value { @@ -3795,8 +4306,20 @@ where .as_ref() .filter(|_| ctx.token_grant_resolver.is_some()) .and_then(EndpointObservationSender::capture); + let middleware_selection = middleware_engine.map(|engine| { + engine.query_middleware_chain_with_generation(&middleware_network_input(ctx)) + }); + let inspected = matches!( + &middleware_selection, + Some(Ok((chain, _))) if !chain.is_empty() + ); // Read next request from client. - let req = match provider.parse_request(client).await { + let parsed = if inspected { + provider.parse_request(client).await + } else { + provider.parse_uninspected_request(client).await + }; + let req = match parsed { Ok(Some(req)) => req, Ok(None) => break, // Client closed connection. Err(e) => { @@ -3809,8 +4332,16 @@ where return Ok(()); } }; + let correlation = if inspected { + RequestCorrelation::new(&req, ctx) + } else { + RequestCorrelation::uninspected() + }; + let receipt = inspected.then_some(&correlation); + let mut audited_client = correlation.observe_response(client); + let client = &mut audited_client; if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action).await?; + reject_request_authority_mismatch(client, ctx, &req.action, receipt).await?; return Ok(()); } if close_if_stale(generation_guard, ctx) { @@ -3823,7 +4354,7 @@ where let redacted_target = match secrets::redact_target_for_policy(&req.target) { Ok(target) => target, Err(error) => { - reject_credential_resolution(client, ctx, &req.action, &error).await?; + reject_credential_resolution(client, ctx, &req.action, &error, receipt).await?; return Ok(()); } }; @@ -3847,15 +4378,14 @@ where req.action, ctx.host, ctx.port, redacted_target, )) .build(); - egress_emit!(event); + correlation.emit(event); } - let request_id = uuid::Uuid::new_v4().to_string(); + let request_id = correlation.request_id().to_string(); let mut response_selection = None; let mut live_body = None; - let req = if let Some(engine) = middleware_engine { - let input = middleware_network_input(ctx); - let (chain, generation) = engine.query_middleware_chain_with_generation(&input)?; + let req = if let Some((engine, selection)) = middleware_engine.zip(middleware_selection) { + let (chain, generation) = selection?; if generation != generation_guard.captured_generation() { return Ok(()); } @@ -3869,12 +4399,13 @@ where // The passthrough path enforces no L7 policy, so there is no // body-aware decision to re-check after a transformation. let result = exchange - .apply_request( + .apply_request_correlated( req, client, ctx, ctx.request_scheme(), openshell_supervisor_middleware::TransformedBodyPolicy::NotPolicyRelevant, + receipt, ) .await?; let request = match result { @@ -3889,11 +4420,21 @@ where client, ctx, &redacted_target, + receipt, ) .await?; return Ok(()); } MiddlewareApplyResult::Denied { denial, failure } => { + correlation.record_rejection( + 403, + if denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + None, + ); let denied_request = crate::l7::provider::L7Request { action: "HTTP".into(), target: redacted_target.clone(), @@ -3908,11 +4449,13 @@ where denial.as_ref(), failure.as_ref(), &redacted_target, + receipt.map(RequestCorrelation::request_id), ) .await?; return Ok(()); } MiddlewareApplyResult::AdmissionExhausted => { + correlation.record_rejection(503, HttpOutcome::PlatformError, None); let unavailable_request = crate::l7::provider::L7Request { action: "HTTP".into(), target: redacted_target.clone(), @@ -3925,6 +4468,7 @@ where client, ctx, &redacted_target, + receipt.map(RequestCorrelation::request_id), ) .await?; return Ok(()); @@ -3939,12 +4483,13 @@ where let injected = match crate::l7::token_grant_injection::pin_live_credentials(&mut grant_ctx) { Some(snapshot) => { - crate::l7::token_grant_injection::inject_for_selectors( + crate::l7::token_grant_injection::inject_for_selectors_correlated( req, &grant_ctx, &snapshot, &request_id, observation_context.as_ref(), + receipt, ) .await } @@ -3954,6 +4499,7 @@ where )), }; let Ok((req_with_auth, injected)) = injected else { + correlation.record_rejection(502, HttpOutcome::PlatformError, None); // The refusal event names the credential and request ID. // The sandbox-level grant record does not require an MCP endpoint identity. write_bad_gateway_response(client, Some(&request_id)).await?; @@ -3986,7 +4532,10 @@ where ctx, live_body.as_deref_mut(), response_middleware, - None, + RelayHttpInstrumentation { + observer: None, + correlation: receipt, + }, ) .await? else { @@ -3997,6 +4546,7 @@ where RelayOutcome::Reusable => {} // continue loop RelayOutcome::Consumed => break, RelayOutcome::Upgraded { overflow, .. } => { + correlation.finish_http_trace(); return handle_upgrade( client, upstream, @@ -4057,12 +4607,12 @@ mod trace_tests { ..Default::default() }; let mut response = Vec::new(); - // The in-memory span processor exports with a futures executor. tokio::task::block_in_place(|| { tokio::runtime::Handle::current().block_on(reject_request_authority_mismatch( &mut response, &ctx, "POST", + None, )) }) .unwrap(); @@ -4108,6 +4658,7 @@ mod trace_tests { mod tests { use super::*; use crate::opa::{NetworkInput, OpaEngine}; + use crate::test_support::{assert_http_request_records, response_request_id}; use openshell_core::proto::{StaticCredentialBinding, StaticCredentialEndpointBinding}; use openshell_core::provider_credentials::ProviderCredentialState; use openshell_core::token_grant_failure::{ @@ -4143,6 +4694,7 @@ mod tests { #[tokio::test] async fn body_denial_returns_actionable_local_json() { + const REQUEST_ID: &str = "8b02c179-9e6b-4eb0-948b-d9f40582fc44"; let req = crate::l7::provider::L7Request { action: "POST".into(), target: "/v1/responses".into(), query_params: TestHashMap::new(), raw_header: b"POST /v1/responses HTTP/1.1\r\nHost: api.openai.com\r\nContent-Length: 25\r\n\r\nopenshell:resolve:env:KEY".to_vec(), @@ -4150,7 +4702,12 @@ mod tests { }; let (mut client, mut caller) = tokio::io::duplex(4096); let (mut upstream, mut server) = tokio::io::duplex(4096); - let outcome = relay_http_request_with_credential_rejection( + let ctx = L7EvalContext { + policy_name: "body-policy".into(), + ..Default::default() + }; + let correlation = RequestCorrelation::with_request_id(&req, &ctx, REQUEST_ID.into()); + let outcome = relay_http_request_with_credential_rejection_observed( &req, &mut client, &mut upstream, @@ -4158,8 +4715,12 @@ mod tests { deny_uninspected_credentials: true, ..Default::default() }, - &L7EvalContext::default(), + &ctx, None, + RelayHttpInstrumentation { + observer: None, + correlation: Some(&correlation), + }, ) .await .unwrap(); @@ -4168,6 +4729,10 @@ mod tests { let mut response = String::new(); caller.read_to_string(&mut response).await.unwrap(); assert!(response.starts_with("HTTP/1.1 403 Forbidden")); + assert!( + response.contains(&format!("x-openshell-request-id: {REQUEST_ID}\r\n")), + "{response}" + ); let body: serde_json::Value = serde_json::from_str(response.split_once("\r\n\r\n").unwrap().1).unwrap(); assert_eq!( @@ -4175,6 +4740,7 @@ mod tests { "credential_placeholder_in_request_body" ); assert_eq!(body["error"]["reason"], "classification_unavailable"); + assert_eq!(body["request_id"], REQUEST_ID); let mut sent = String::new(); server.read_to_string(&mut sent).await.unwrap(); assert!(!sent.contains("openshell:resolve:")); @@ -4248,7 +4814,10 @@ mod tests { }, &L7EvalContext::default(), None, - Some(&observer), + RelayHttpInstrumentation { + observer: Some(&observer), + correlation: None, + }, ) .await; if disconnect_client { @@ -4586,6 +5155,7 @@ mod tests { resolver: &SecretResolver, options: crate::l7::rest::RelayRequestOptions<'_>, ) { + const REQUEST_ID: &str = "1329fced-3bb8-4810-bc25-45a1282997ad"; let (mut client_peer, mut client) = tokio::io::duplex(8192); let (mut upstream, mut upstream_peer) = tokio::io::duplex(8192); let ctx = L7EvalContext { @@ -4596,8 +5166,10 @@ mod tests { secret_resolver: Some(Arc::new(resolver.clone())), ..Default::default() }; + let correlation = + RequestCorrelation::with_request_id(&request, &ctx, REQUEST_ID.to_string()); - let outcome = relay_http_request_with_credential_rejection( + let outcome = relay_http_request_with_credential_rejection_observed( &request, &mut client, &mut upstream, @@ -4607,6 +5179,10 @@ mod tests { }, &ctx, None, + RelayHttpInstrumentation { + observer: None, + correlation: Some(&correlation), + }, ) .await .expect("typed credential denial"); @@ -4621,6 +5197,13 @@ mod tests { response.contains("credential_endpoint_mismatch"), "{response}" ); + assert!( + response.contains(&format!("x-openshell-request-id: {REQUEST_ID}\r\n")), + "{response}" + ); + let body: serde_json::Value = + serde_json::from_str(response.split_once("\r\n\r\n").unwrap().1).unwrap(); + assert_eq!(body["request_id"], REQUEST_ID); let mut forwarded = Vec::new(); upstream_peer.read_to_end(&mut forwarded).await.unwrap(); assert!( @@ -5275,6 +5858,68 @@ mod tests { .await; } + #[tokio::test] + async fn correlated_credential_unavailable_preserves_transport_failure() { + const REQUEST_ID: &str = "250b9326-7a4f-4aa0-a6a9-7e0a1d96beaa"; + let request = crate::l7::provider::L7Request { + action: "GET".to_string(), + target: "/v1/resource".to_string(), + query_params: TestHashMap::new(), + raw_header: b"GET /v1/resource HTTP/1.1\r\nHost: api.example.test\r\n\r\n".to_vec(), + body_length: crate::l7::provider::BodyLength::None, + }; + let ctx = L7EvalContext { + host: "api.example.test".into(), + port: 443, + policy_name: "sigv4-policy".into(), + ..Default::default() + }; + let correlation = + RequestCorrelation::with_request_id(&request, &ctx, REQUEST_ID.to_string()); + let (mut client_peer, mut client) = tokio::io::duplex(8192); + let (mut upstream, mut upstream_peer) = tokio::io::duplex(8192); + + let error = relay_http_request_with_credential_rejection_observed( + &request, + &mut client, + &mut upstream, + crate::l7::rest::RelayRequestOptions { + credential_signing: crate::l7::CredentialSigning::SigV4NoBody, + signing_service: "execute-api", + signing_region: "us-west-2", + host: "api.example.test", + port: 443, + ..Default::default() + }, + &ctx, + None, + RelayHttpInstrumentation { + observer: None, + correlation: Some(&correlation), + }, + ) + .await + .expect_err("credential availability failure remains a transport failure"); + assert!( + error + .downcast_ref::() + .is_some() + ); + drop(client); + drop(upstream); + + let mut response = String::new(); + client_peer.read_to_string(&mut response).await.unwrap(); + assert!( + response.is_empty(), + "no new synthetic HTTP response: {response}" + ); + + let mut forwarded = Vec::new(); + upstream_peer.read_to_end(&mut forwarded).await.unwrap(); + assert!(forwarded.is_empty()); + } + fn install_builtin_middleware(engine: &OpaEngine) { engine.set_middleware_runner_for_tests(openshell_supervisor_middleware::ChainRunner::new( openshell_supervisor_middleware_builtins::services() @@ -7817,22 +8462,26 @@ network_policies: async fn run_jsonrpc_transform_case( enforcement: &str, - replacement: &'static [u8], - ) -> (String, Option) { - let (config, tunnel_engine, ctx) = - jsonrpc_transforming_relay_parts(enforcement, replacement); - let (mut app, mut relay_client) = tokio::io::duplex(8192); - let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); - let relay = tokio::spawn(async move { - relay_jsonrpc( - &config, - &tunnel_engine, - &mut relay_client, - &mut relay_upstream, - &ctx, - ) - .await - }); + replacement: &'static [u8], + ) -> (String, Option) { + use tracing::instrument::WithSubscriber as _; + let (config, tunnel_engine, ctx) = + jsonrpc_transforming_relay_parts(enforcement, replacement); + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = tokio::spawn( + async move { + relay_jsonrpc( + &config, + &tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + } + .with_current_subscriber(), + ); let body = br#"{"jsonrpc":"2.0","id":1,"method":"reports.list"}"#; let request = format!( @@ -7881,32 +8530,51 @@ network_policies: } #[tokio::test] - async fn transformed_jsonrpc_body_is_reevaluated_and_denied() { - // Policy allows reports.list; the middleware replaces the body with a - // method the policy denies. The transformed body must be re-evaluated - // and the request denied before anything reaches the upstream. - let (response, upstream_seen) = run_jsonrpc_transform_case( - "enforce", - br#"{"jsonrpc":"2.0","id":1,"method":"admin.delete"}"#, - ) - .await; - assert_middleware_failure_response(&response, "jsonrpc_api"); - assert!(upstream_seen.is_none(), "upstream must not see the request"); - } + async fn transformed_jsonrpc_policy_outcome_has_one_primary_and_shared_receipt() { + use tracing::instrument::WithSubscriber as _; + use tracing_subscriber::layer::SubscriberExt as _; - #[tokio::test] - async fn transformed_jsonrpc_body_policy_deny_forwards_under_audit() { - // Under enforcement: audit a policy deny of the transformed body is - // logged but forwarded, mirroring audit semantics for original - // bodies. - let (response, upstream_seen) = run_jsonrpc_transform_case( - "audit", - br#"{"jsonrpc":"2.0","id":1,"method":"admin.delete"}"#, - ) - .await; - assert!(response.contains("204 No Content"), "{response}"); - let upstream_seen = upstream_seen.expect("audited request reaches upstream"); - assert!(upstream_seen.contains("admin.delete"), "{upstream_seen}"); + for enforcement in ["enforce", "audit"] { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let (response, upstream_seen) = run_jsonrpc_transform_case( + enforcement, + br#"{"jsonrpc":"2.0","id":1,"method":"admin.delete"}"#, + ) + .with_subscriber(subscriber) + .await; + let request_id = response_request_id(response.as_bytes()); + let events = capture.json_events(); + let (primary, companions) = assert_http_request_records(&events, 1); + assert_eq!(companions[0]["unmapped"]["middleware_phase"], "request"); + assert_eq!(companions[0]["unmapped"]["transformed"], true); + assert_eq!(companions[0]["metadata"]["correlation_uid"], request_id); + assert_eq!(primary["metadata"]["correlation_uid"], request_id); + assert_eq!(primary["http_request"]["url"]["path"], "/rpc"); + assert_eq!(primary["firewall_rule"]["type"], "l7-jsonrpc"); + if enforcement == "enforce" { + assert_middleware_failure_response(&response, "jsonrpc_api"); + assert!(upstream_seen.is_none(), "upstream must not see the request"); + assert_eq!(primary["action"], "Denied"); + assert_eq!(primary["http_response"]["code"], 403); + assert!( + primary["status_detail"] + .as_str() + .unwrap() + .contains("middleware transformation denied") + ); + } else { + assert!(response.contains("204 No Content"), "{response}"); + assert!( + upstream_seen + .expect("audited request reaches upstream") + .contains("admin.delete") + ); + assert_eq!(primary["action"], "Allowed"); + assert_eq!(primary["http_response"]["code"], 204); + assert_eq!(primary["unmapped"]["transformed_body_policy"], "audit"); + } + } } #[tokio::test] @@ -8775,7 +9443,7 @@ network_policies: denied_http.format_shorthand(), "HTTP:POST [MED] DENIED POST https://api.example.test/v1/messages \ [policy:rest_api engine:middleware] \ - [failed:false transformed:false \ + [failed:false http_event_kind:middleware_invocation middleware_phase:request transformed:false \ reason:middleware_denied:content-guard:content_match]" ); @@ -8794,6 +9462,163 @@ network_policies: assert!(!serialized.contains(RAW_SECRET)); } + #[tokio::test] + async fn passthrough_receipts_require_selected_middleware_and_keep_one_primary() { + use tracing::instrument::WithSubscriber as _; + use tracing_subscriber::layer::SubscriberExt as _; + + for selected in [false, true] { + let middleware = if selected { + "network_middlewares:\n scheme:\n middleware: test/scheme\n on_error: fail_closed\n endpoints:\n include: [api.example.test]\n" + } else { + "" + }; + let engine = OpaEngine::from_strings( + TEST_POLICY, + &format!("{middleware}network_policies: {{}}\n"), + ) + .unwrap(); + engine.set_middleware_runner_for_tests( + openshell_supervisor_middleware::ChainRunner::new(Arc::new(SchemeRecorder( + Arc::new(std::sync::Mutex::new(Vec::new())), + ))), + ); + let generation_guard = engine + .generation_guard(engine.current_generation()) + .unwrap(); + let state = ProviderCredentialState::from_bound_environment( + 1, + TestHashMap::from([("API_TOKEN".into(), "synthetic-live-credential".into())]), + TestHashMap::new(), + TestHashMap::new(), + TestHashMap::from([( + "API_TOKEN".into(), + StaticCredentialBinding { + endpoints: vec![StaticCredentialEndpointBinding { + host: "api.example.test".into(), + port: 8080, + path: "/v1/**".into(), + }], + credential_identity: "provider-a:API_TOKEN".into(), + workload_credential_handle: String::new(), + }, + )]), + Vec::new(), + ) + .unwrap(); + let placeholder = state.snapshot().child_env["API_TOKEN"].clone(); + let ctx = L7EvalContext { + host: "api.example.test".into(), + port: 8080, + request_default_port: Some(8080), + policy_name: "passthrough_api".into(), + binary_path: "/usr/bin/curl".into(), + provider_credentials: Some(state), + ..Default::default() + }; + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = async move { + relay_passthrough_with_credentials( + &mut relay_client, + &mut relay_upstream, + &ctx, + &generation_guard, + Some(&engine), + ) + .await + } + .with_subscriber(subscriber); + let request = format!( + "GET /v1/messages HTTP/1.1\r\nHost: api.example.test:8080\r\nAuthorization: Bearer {placeholder}\r\nX-OpenShell-Request-ID: application-in\r\nConnection: close\r\n\r\n" + ); + let exchange = async { + tokio::join!( + relay, + async { + app.write_all(request.as_bytes()).await.unwrap(); + let mut response = String::new(); + app.read_to_string(&mut response).await.unwrap(); + response + }, + async { + let forwarded = read_http_headers(&mut upstream).await; + upstream.write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nX-OpenShell-Request-ID: application-out\r\nConnection: close\r\n\r\n").await.unwrap(); + forwarded + }, + ) + }; + let (relay, response, forwarded) = + tokio::time::timeout(std::time::Duration::from_secs(5), Box::pin(exchange)) + .await + .expect("credential-only CONNECT exchange"); + relay.unwrap(); + let forwarded = String::from_utf8(forwarded).unwrap(); + assert!( + forwarded.contains("Authorization: Bearer synthetic-live-credential"), + "{forwarded}" + ); + let records = capture.json_events(); + let http: Vec<_> = records + .iter() + .filter(|event| event["class_uid"] == 4002) + .collect(); + let primary: Vec<_> = records + .iter() + .filter(|event| { + event["class_uid"] == 4002 + && if selected { + event["unmapped"]["http_event_kind"] == "request" + } else { + event["unmapped"].get("http_event_kind").is_none() + } + }) + .collect(); + assert_eq!(primary.len(), 1, "selected={selected}: {records:?}"); + let companions: Vec<_> = http + .iter() + .filter(|event| event["unmapped"]["http_event_kind"] == "middleware_invocation") + .collect(); + assert_eq!(companions.len(), usize::from(selected)); + assert_eq!(http.len(), primary.len() + companions.len()); + if selected { + assert_eq!(companions[0]["unmapped"]["middleware_phase"], "request"); + assert_eq!(companions[0]["unmapped"]["failed"], false); + assert_eq!( + companions[0]["metadata"]["correlation_uid"], + primary[0]["metadata"]["correlation_uid"] + ); + } + assert_eq!(primary[0]["action"], "Allowed"); + assert_eq!(primary[0]["http_request"]["url"]["path"], "/v1/messages"); + assert!( + !serde_json::to_string(&records) + .unwrap() + .contains("synthetic-live-credential") + ); + if selected { + assert!(!forwarded.contains("application-in")); + assert!(!response.contains("application-out")); + let receipt = response_request_id(response.as_bytes()); + assert!(uuid::Uuid::parse_str(receipt).is_ok()); + assert_eq!(primary[0]["metadata"]["correlation_uid"], receipt); + assert_eq!(primary[0]["http_response"]["code"], 204); + } else { + assert!( + forwarded.contains("X-OpenShell-Request-ID: application-in"), + "{forwarded}" + ); + assert!( + response.contains("X-OpenShell-Request-ID: application-out"), + "{response}" + ); + assert!(primary[0]["metadata"].get("correlation_uid").is_none()); + } + } + } + #[tokio::test] async fn passthrough_relay_runs_middleware_redaction() { // A no-protocol endpoint takes the credential-injection passthrough path; @@ -9079,6 +9904,117 @@ network_policies: .unwrap(); } + async fn assert_websocket_endpoint_rejects_plain_http(route_selected: bool) { + let data = r" +network_policies: + ws_api: + name: ws_api + endpoints: + - host: gateway.example.test + port: 443 + path: /ws + protocol: websocket + enforcement: enforce + rules: + - allow: + method: GET + path: /ws + binaries: + - { path: /usr/bin/node } +"; + let engine = OpaEngine::from_strings(TEST_POLICY, data).unwrap(); + let input = NetworkInput { + host: "gateway.example.test".into(), + port: 443, + binary_path: PathBuf::from("/usr/bin/node"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (endpoints, generation) = engine + .query_endpoint_configs_with_generation(&input) + .unwrap(); + let configs: Vec<_> = endpoints + .iter() + .map(|endpoint| crate::l7::parse_l7_config(endpoint).unwrap()) + .collect(); + let tunnel_engine = engine.clone_engine_for_tunnel(generation).unwrap(); + let ctx = L7EvalContext { + host: "gateway.example.test".into(), + port: 443, + request_default_port: Some(443), + policy_name: "ws_api".into(), + binary_path: "/usr/bin/node".into(), + ..Default::default() + }; + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = tokio::spawn(async move { + if route_selected { + relay_with_route_selection( + &configs, + tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + } else { + relay_with_inspection( + &configs[0], + tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + } + }); + + app.write_all( + b"GET /ws HTTP/1.1\r\nHost: gateway.example.test\r\nConnection: close\r\n\r\n", + ) + .await + .unwrap(); + let response = tokio::time::timeout( + std::time::Duration::from_secs(1), + read_http_headers(&mut app), + ) + .await + .expect("denial should reach client"); + let response = String::from_utf8_lossy(&response); + assert!(response.contains("403 Forbidden"), "{response}"); + let request_id = response_request_id(response.as_bytes()); + assert!(uuid::Uuid::parse_str(request_id).is_ok()); + + let mut upstream_bytes = [0u8; 16]; + assert!(matches!( + tokio::time::timeout( + std::time::Duration::from_millis(100), + upstream.read(&mut upstream_bytes) + ) + .await, + Err(_) | Ok(Ok(0)) + )); + + drop(app); + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("relay should finish") + .unwrap() + .unwrap(); + } + + #[tokio::test] + async fn websocket_endpoint_plain_http_denial_has_receipt() { + assert_websocket_endpoint_rejects_plain_http(false).await; + } + + #[tokio::test] + async fn route_selected_websocket_plain_http_denial_has_receipt() { + assert_websocket_endpoint_rejects_plain_http(true).await; + } + #[tokio::test] async fn passthrough_relay_injects_token_grant_authorization_header() { let (generation_guard, ctx, fixture) = @@ -10142,12 +11078,16 @@ network_policies: .await .unwrap(); - let mut response = [0u8; 1024]; - let n = tokio::time::timeout(std::time::Duration::from_secs(1), app.read(&mut response)) - .await - .expect("denial should reach client") - .unwrap(); - assert!(String::from_utf8_lossy(&response[..n]).contains("403 Forbidden")); + let response = tokio::time::timeout( + std::time::Duration::from_secs(1), + read_http_headers(&mut app), + ) + .await + .expect("denial should reach client"); + let response = String::from_utf8_lossy(&response); + assert!(response.contains("403 Forbidden")); + let request_id = response_request_id(response.as_bytes()); + assert!(uuid::Uuid::parse_str(request_id).is_ok()); let mut upstream_bytes = [0u8; 16]; assert!(matches!( @@ -11318,7 +12258,7 @@ network_policies: let _guard = capture.install(); let (_app, mut client) = tokio::io::duplex(8192); assert!( - deny_unsupported_upgrade_if_requested(&request, config, &ctx, None, &mut client) + deny_unsupported_upgrade_if_requested(&request, config, &ctx, None, &mut client, None) .await .unwrap() ); @@ -12906,6 +13846,10 @@ network_policies: #[tokio::test] async fn live_request_body_is_completed_only_after_the_stage_finishes() { + use crate::test_support::request_tracing::RequestTracing; + use opentelemetry_sdk::trace::Sampler; + use tracing::instrument::WithSubscriber as _; + let data = r#" network_middlewares: echo: @@ -12937,12 +13881,14 @@ network_policies: ("emptied, finished", true, false, Some("")), ("emptied, rejected", true, true, None), ] { + let capture = RequestTracing::new(Some(Sampler::AlwaysOn)); + let preflights = Arc::new(std::sync::Mutex::new(Vec::new())); let engine = OpaEngine::from_strings(TEST_POLICY, data).expect("test policy"); engine.set_middleware_runner_for_tests( openshell_supervisor_middleware::ChainRunner::new(Arc::new(HookEchoRequestStage { drop_body, reject, - preflights: Arc::default(), + preflights: Arc::clone(&preflights), })), ); let input = NetworkInput { @@ -12961,28 +13907,42 @@ network_policies: let tunnel_engine = engine .clone_engine_for_tunnel(generation) .expect("tunnel engine"); + let (child_env, resolver) = SecretResolver::from_provider_env(TestHashMap::from([( + "API_TOKEN".to_string(), + "synthetic-live-token".to_string(), + )])); + let credentialed = !reject && !drop_body; let ctx = L7EvalContext { host: "api.example.test".into(), port: 8080, request_default_port: Some(8080), policy_name: "rest_api".into(), binary_path: "/usr/bin/curl".into(), + secret_resolver: credentialed.then(|| Arc::new(resolver.unwrap())), ..Default::default() }; let (mut app, mut relay_client) = tokio::io::duplex(8192); let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); - let relay = tokio::spawn(async move { - relay_rest( - &config, - &tunnel_engine, - &mut relay_client, - &mut relay_upstream, - &ctx, - ) - .await - }); + let relay = tokio::spawn( + async move { + relay_rest( + &config, + &tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + } + .with_subscriber(capture.subscriber()), + ); + let authorization = if credentialed { + format!("Authorization: Bearer {}\r\n", child_env["API_TOKEN"]) + } else { + String::new() + }; app.write_all( - b"POST /v1/items HTTP/1.1\r\nHost: api.example.test\r\nContent-Length: 5\r\n\r\nhello", + format!("POST /v1/items HTTP/1.1\r\nHost: api.example.test\r\n{authorization}Content-Length: 5\r\n\r\nhello").as_bytes(), ) .await .expect("send request"); @@ -13024,6 +13984,51 @@ network_policies: let forwarded = String::from_utf8(forwarded).unwrap(); let response = String::from_utf8(response).unwrap(); + let request_id = response_request_id(response.as_bytes()); + assert_eq!( + preflights.lock().unwrap()[0] + .context + .as_ref() + .unwrap() + .request_id, + request_id + ); + let events = capture.events.json_events(); + let (primary, companions) = assert_http_request_records(&events, 1); + assert_eq!(companions[0]["unmapped"]["middleware_phase"], "request"); + assert_eq!(companions[0]["metadata"]["correlation_uid"], request_id); + assert_eq!( + companions[0]["action"], + if reject { "Denied" } else { "Allowed" } + ); + assert_eq!(primary["metadata"]["correlation_uid"], request_id); + let spans = capture.spans(); + let stage = |stage_name: &str| { + spans + .iter() + .filter(|span| span.name == stage_name) + .collect::>() + }; + assert_eq!( + stage("openshell.middleware.request").len(), + 1, + "{name}: {spans:?}" + ); + assert_eq!( + stage("openshell.credential.prepare").len(), + usize::from(credentialed), + "{name}: {spans:?}" + ); + if !reject && !drop_body { + let sent = stage("openshell.upstream.send"); + let received = stage("openshell.http.response"); + assert_eq!(sent.len(), 1); + assert_eq!(received.len(), 1); + assert!(sent[0].end_time <= received[0].end_time); + } + assert!(!format!("{spans:?}").contains("synthetic-live-token")); + assert!(!format!("{events:?}").contains("synthetic-live-token")); + capture.shutdown(); match forwarded_body { Some(body) => { let (head, forwarded_body) = forwarded @@ -14709,6 +15714,7 @@ network_policies: #[tokio::test] async fn final_mcp_version_check_reclassifies_a_rewritten_initialize_body() { + const REQUEST_ID: &str = "36a15952-692c-44d8-a373-eeca9735fb6b"; let (config, _, ctx) = mcp_test_relay_context(); let final_body = br#"{"jsonrpc":"2.0","id":2,"method":"tools/list"}"#; let mut raw_header = format!( @@ -14725,14 +15731,17 @@ network_policies: body_length: crate::l7::provider::BodyLength::ContentLength(final_body.len() as u64), }; let (mut client, mut relay_client) = tokio::io::duplex(2048); + let correlation = + RequestCorrelation::with_request_id(&request, &ctx, REQUEST_ID.to_string()); - let allowed = enforce_final_mcp_protocol_version( + let allowed = enforce_final_mcp_protocol_version_correlated( &config, &request, &mut relay_client, &ctx, "/mcp", None, + Some(&correlation), ) .await .expect("final request inspection"); @@ -14740,6 +15749,7 @@ network_policies: !allowed, "rewritten non-initialize request must require a version" ); + drop(correlation); drop(relay_client); let mut response = Vec::new(); @@ -14750,6 +15760,13 @@ network_policies: response.contains("mcp_protocol_version_not_allowed"), "{response}" ); + assert!( + response.contains(&format!("x-openshell-request-id: {REQUEST_ID}\r\n")), + "{response}" + ); + let response_body: serde_json::Value = + serde_json::from_str(response.split_once("\r\n\r\n").unwrap().1).unwrap(); + assert_eq!(response_body["request_id"], REQUEST_ID); } #[tokio::test] diff --git a/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs new file mode 100644 index 0000000000..c2c4a9e6df --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs @@ -0,0 +1,745 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Correlation contracts for the production single-config relay dispatch. + +use super::L7EvalContext; +use crate::l7::L7EndpointConfig; +use crate::opa::{NetworkInput, OpaEngine, TunnelPolicyEngine}; +use crate::test_support::request_tracing::RequestTracing; +use crate::test_support::{assert_http_request_records, response_request_id as request_id}; +use opentelemetry_sdk::trace::{Sampler, SpanData}; +use std::future::Future; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; +use tracing::instrument::WithSubscriber as _; +use tracing_subscriber::layer::SubscriberExt as _; + +const POLICY: &str = include_str!("../../../data/sandbox-policy.rego"); + +pub struct TracedOperation { + pub(crate) output: T, + pub(crate) events: Vec, + pub(crate) spans: Vec, + created: Vec<&'static str>, +} + +impl TracedOperation { + pub(crate) fn assert_no_deny_span(&self, sampled: bool) { + assert!(!self.created.contains(&"supervisor.egress.deny")); + assert_eq!( + self.created + .iter() + .filter(|name| **name == "openshell.http.request") + .count(), + 1 + ); + let roots: Vec<_> = self + .spans + .iter() + .filter(|span| span.name == "openshell.http.request") + .collect(); + assert_eq!(roots.len(), usize::from(sampled)); + if let Some(root) = roots.first() { + let root_id = root.span_context.span_id(); + assert_eq!(root.parent_span_id, opentelemetry::trace::SpanId::INVALID); + let by_id = self + .spans + .iter() + .map(|span| (span.span_context.span_id(), span)) + .collect::>(); + for span in &self.spans { + assert_eq!(span.span_context.trace_id(), root.span_context.trace_id()); + if span.span_context.span_id() == root_id { + continue; + } + let mut parent = span.parent_span_id; + for _ in 0..self.spans.len() { + if parent == root_id { + break; + } + parent = by_id + .get(&parent) + .expect("span parent is captured") + .parent_span_id; + } + assert_eq!( + parent, root_id, + "span parent graph reaches the request root" + ); + if matches!( + span.name.as_ref(), + "openshell.policy.evaluate" + | "openshell.middleware.request" + | "openshell.middleware.websocket.preflight" + | "openshell.credential.prepare" + | "openshell.upstream.resolve" + | "openshell.upstream.connect" + | "openshell.upstream.send" + | "openshell.http.response" + | "openshell.middleware.response" + ) { + assert_eq!(span.parent_span_id, root_id, "request stage parent"); + } + } + for event in self + .events + .iter() + .filter(|event| event["class_uid"] == 4002) + { + assert_eq!( + event["trace"]["uid"], + root.span_context.trace_id().to_string() + ); + } + } else { + assert!(self.spans.is_empty()); + assert!(self.events.iter().all(|event| event.get("trace").is_none())); + } + } +} + +pub async fn capture_request_spans( + sampled: bool, + operation: impl Future, +) -> TracedOperation { + capture_request( + Some(if sampled { + Sampler::AlwaysOn + } else { + Sampler::AlwaysOff + }), + operation, + ) + .await +} + +async fn capture_request( + sampler: Option, + operation: impl Future, +) -> TracedOperation { + let _lock = openshell_otel_test_support::tracing_test_lock().await; + let capture = RequestTracing::new(sampler); + let output = Box::pin(operation.with_subscriber(capture.subscriber())).await; + let spans = capture.spans(); + capture.shutdown(); + TracedOperation { + output, + events: capture.events.json_events(), + spans, + created: capture.created_span_names(), + } +} + +#[tokio::test] +async fn request_trace_capture_accepts_rpc_grandchildren() { + let evidence = capture_request_spans(true, async { + let request = tracing::info_span!(parent: None, "openshell.http.request"); + request.in_scope(|| { + tracing::info_span!("openshell.middleware.request").in_scope(|| { + tracing::info_span!("middleware.rpc").in_scope(|| {}); + }); + }); + }) + .await; + assert_eq!(evidence.spans.len(), 3); + evidence.assert_no_deny_span(true); +} + +struct Fixture { + config: L7EndpointConfig, + tunnel_engine: TunnelPolicyEngine, + middleware_engine: OpaEngine, + ctx: L7EvalContext, +} + +struct Exchange { + response: Vec, + forwarded: Vec, +} + +fn fixture(data: &str, host: &str, policy_name: &str) -> Fixture { + let middleware_engine = OpaEngine::from_strings(POLICY, data).expect("valid test policy"); + let input = NetworkInput { + host: host.into(), + port: 8000, + binary_path: PathBuf::from("/usr/bin/python3"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (endpoint_config, generation) = middleware_engine + .query_endpoint_config_with_generation(&input) + .expect("query endpoint config"); + let config = crate::l7::parse_l7_config( + &endpoint_config.expect("single configured endpoint should match"), + ) + .expect("parse endpoint config"); + let tunnel_engine = middleware_engine + .clone_engine_for_tunnel(generation) + .expect("pin test policy generation"); + let ctx = L7EvalContext { + host: host.into(), + port: 8000, + request_default_port: Some(8000), + policy_name: policy_name.into(), + binary_path: "/usr/bin/python3".into(), + ..Default::default() + }; + Fixture { + config, + tunnel_engine, + middleware_engine, + ctx, + } +} + +async fn read_http_message(reader: &mut R) -> Vec { + let mut message = Vec::new(); + let mut byte = [0_u8; 1]; + while !message.ends_with(b"\r\n\r\n") { + reader + .read_exact(&mut byte) + .await + .expect("complete HTTP headers"); + message.push(byte[0]); + } + let headers = std::str::from_utf8(&message).expect("ASCII test headers"); + let body_len = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length").then(|| { + value + .trim() + .parse::() + .expect("numeric content length") + }) + }) + .unwrap_or_default(); + let header_len = message.len(); + message.resize(header_len + body_len, 0); + reader + .read_exact(&mut message[header_len..]) + .await + .expect("complete HTTP body"); + message +} + +async fn run_allowed( + fixture: Fixture, + request: Vec, + upstream_response: &'static [u8], +) -> TracedOperation { + capture_request( + None, + Box::pin(run_exchange(fixture, request, Some(upstream_response))), + ) + .await +} + +async fn run_denied(fixture: Fixture, request: Vec) -> TracedOperation { + capture_request(None, Box::pin(run_exchange(fixture, request, None))).await +} + +async fn run_exchange( + fixture: Fixture, + request: Vec, + upstream_response: Option<&'static [u8]>, +) -> Exchange { + let (mut app, mut relay_client) = tokio::io::duplex(32 * 1024); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(32 * 1024); + let Fixture { + config, + tunnel_engine, + middleware_engine, + ctx, + } = fixture; + + let exchange = async { + tracing::callsite::rebuild_interest_cache(); + tokio::join!( + async { + let result = crate::proxy::relay_inspected_http_stream_for_test( + &mut relay_client, + &mut relay_upstream, + vec![config], + tunnel_engine, + &middleware_engine, + &ctx, + ) + .await; + drop(relay_upstream); + result + }, + async { + app.write_all(&request).await.expect("send request"); + app.flush().await.expect("flush request"); + read_http_message(&mut app).await + }, + async { + if let Some(response) = upstream_response { + let forwarded = read_http_message(&mut upstream).await; + upstream + .write_all(response) + .await + .expect("send upstream response"); + upstream.flush().await.expect("flush upstream response"); + forwarded + } else { + let mut forwarded = Vec::new(); + upstream + .read_to_end(&mut forwarded) + .await + .expect("relay closes upstream"); + forwarded + } + }, + ) + }; + let (relay, response, forwarded) = + tokio::time::timeout(std::time::Duration::from_secs(5), Box::pin(exchange)) + .await + .expect("single-config relay exchange completes"); + relay.expect("relay succeeds"); + Exchange { + response, + forwarded, + } +} + +fn assert_primary_record<'a>( + exchange: &'a TracedOperation, + engine_type: &str, + status: u16, + action: &str, + path: &str, +) -> &'a serde_json::Value { + let receipt = request_id(&exchange.output.response); + assert!( + uuid::Uuid::parse_str(receipt).is_ok(), + "canonical receipt: {receipt}" + ); + let primary: Vec<_> = exchange + .events + .iter() + .filter(|event| event["class_uid"] == 4002) + .collect(); + assert_eq!(primary.len(), 1, "one HTTP Activity: {:?}", exchange.events); + let event = primary[0]; + assert_eq!(event["metadata"]["correlation_uid"], receipt); + assert_eq!(event["firewall_rule"]["type"], engine_type); + assert_eq!(event["http_response"]["code"], status); + assert_eq!(event["action"], action); + assert_eq!(event["http_request"]["url"]["path"], path); + assert!( + event.get("trace").is_none(), + "receipts must work without tracing" + ); + event +} + +fn mcp_fixture() -> Fixture { + fixture( + r#" +network_policies: + mcp_api: + name: mcp_api + endpoints: + - host: mcp.example.test + port: 8000 + path: /mcp + protocol: mcp + enforcement: enforce + mcp: + versions: ["2025-11-25"] + rules: + - allow: + method: initialize + binaries: + - { path: /usr/bin/python3 } +"#, + "mcp.example.test", + "mcp_api", + ) +} + +fn request(host: &str, path: &str, extra_headers: &str, body: &str) -> Vec { + format!( + "POST {path} HTTP/1.1\r\nHost: {host}:8000\r\nContent-Type: application/json\r\n{extra_headers}Content-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len(), + ) + .into_bytes() +} + +struct BlockingRequestMiddleware; + +#[tonic::async_trait] +impl openshell_supervisor_middleware::InProcessMiddleware for BlockingRequestMiddleware { + async fn describe(&self) -> openshell_core::proto::MiddlewareManifest { + use openshell_core::proto::{ + MiddlewareBinding, SupervisorMiddlewareOperation, SupervisorMiddlewarePhase, + }; + openshell_core::proto::MiddlewareManifest { + name: "test/block-request".into(), + service_version: "test".into(), + bindings: vec![MiddlewareBinding { + operation: SupervisorMiddlewareOperation::HttpRequest as i32, + phase: SupervisorMiddlewarePhase::PreCredentials as i32, + max_payload_bytes: 4096, + request_timeout: None, + }], + extension: Some(openshell_core::extension_protocol::extension_metadata( + openshell_core::extension_protocol::ExtensionFamily::SupervisorMiddleware, + "openshell/test-block-request", + "test", + [], + )), + ..Default::default() + } + } + + async fn validate_config(&self, _: &str, _: &prost_types::Struct) -> miette::Result<()> { + Ok(()) + } + + async fn evaluate_http_request( + &self, + _: openshell_supervisor_middleware::HttpRequestView<'_>, + ) -> miette::Result { + Ok(openshell_core::proto::HttpRequestResult { + decision: openshell_core::proto::Decision::Deny as i32, + reason_code: "test_block".into(), + ..Default::default() + }) + } +} + +fn rest_fixture(method: &str, block_middleware: bool) -> Fixture { + let middleware = if block_middleware { + "network_middlewares:\n request:\n middleware: test/block-request\n on_error: fail_closed\n endpoints:\n include: [rest.example.test]\n" + } else { + "" + }; + let fixture = fixture( + &format!( + r"{middleware} +network_policies: + rest_api: + name: rest_api + endpoints: + - host: rest.example.test + port: 8000 + path: /item + protocol: rest + enforcement: enforce + rules: + - allow: {{ method: {method}, path: /item }} + binaries: + - {{ path: /usr/bin/python3 }} +" + ), + "rest.example.test", + "rest_api", + ); + if block_middleware { + fixture.middleware_engine.set_middleware_runner_for_tests( + openshell_supervisor_middleware::ChainRunner::new(Arc::new(BlockingRequestMiddleware)), + ); + } + fixture +} + +#[tokio::test] +async fn inspected_allow_creates_only_the_stages_that_run() { + for sampled in [true, false] { + let evidence = capture_request_spans( + sampled, + Box::pin(run_exchange( + rest_fixture("POST", false), + request("rest.example.test", "/item", "", "{}"), + Some(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"), + )), + ) + .await; + evidence.assert_no_deny_span(sampled); + assert!( + evidence + .output + .forwarded + .starts_with(b"POST /item HTTP/1.1\r\n") + ); + if sampled { + let names: Vec<_> = evidence + .spans + .iter() + .map(|span| span.name.as_ref()) + .collect(); + for (name, expected) in [ + ("openshell.policy.evaluate", true), + ("openshell.upstream.send", true), + ("openshell.http.response", true), + ("openshell.middleware.request", false), + ("openshell.middleware.response", false), + ("openshell.credential.prepare", false), + ] { + assert_eq!(names.contains(&name), expected, "{name}: {names:?}"); + } + } + let (record, _) = assert_http_request_records(&evidence.events, 0); + assert_eq!( + record["metadata"]["correlation_uid"], + request_id(&evidence.output.response) + ); + assert_eq!(record["action"], "Allowed"); + } +} + +#[tokio::test] +async fn request_middleware_block_never_creates_a_denial_only_span() { + for sampled in [true, false] { + let evidence = capture_request_spans( + sampled, + Box::pin(run_exchange( + rest_fixture("POST", true), + request("rest.example.test", "/item", "", "{}"), + None, + )), + ) + .await; + evidence.assert_no_deny_span(sampled); + assert!(evidence.output.forwarded.is_empty()); + assert!( + evidence + .output + .response + .starts_with(b"HTTP/1.1 403 Forbidden\r\n") + ); + if sampled { + assert_eq!( + evidence + .spans + .iter() + .filter(|span| span.name == "openshell.middleware.request") + .count(), + 1 + ); + assert!( + evidence + .spans + .iter() + .all(|span| span.name != "openshell.upstream.send") + ); + } + let (record, companions) = assert_http_request_records(&evidence.events, 1); + assert_eq!(companions[0]["unmapped"]["middleware_phase"], "request"); + assert_eq!(companions[0]["action"], "Denied"); + assert!(companions[0].get("http_response").is_none()); + assert_eq!( + record["metadata"]["correlation_uid"], + request_id(&evidence.output.response) + ); + assert_eq!(record["action"], "Denied"); + assert_eq!(record["http_response"]["code"], 403); + } +} + +#[tokio::test] +async fn inspected_keep_alive_requests_have_distinct_receipts_without_tracing() { + let Fixture { + config, + tunnel_engine, + middleware_engine, + ctx, + } = rest_fixture("GET", false); + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let exchange = async { + tokio::join!( + crate::proxy::relay_inspected_http_stream_for_test( + &mut relay_client, + &mut relay_upstream, + vec![config], + tunnel_engine, + &middleware_engine, + &ctx, + ), + async { + let mut responses = Vec::new(); + for (sampled, connection) in [("01", "keep-alive"), ("00", "close")] { + let request = format!( + "GET /item HTTP/1.1\r\nHost: rest.example.test:8000\r\ntraceparent: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-{sampled}\r\nConnection: {connection}\r\n\r\n", + ); + app.write_all(request.as_bytes()).await.expect("send request"); + app.flush().await.expect("flush request"); + responses.push(read_http_message(&mut app).await); + } + responses + }, + async { + for connection in ["keep-alive", "close"] { + let request = read_http_message(&mut upstream).await; + assert!(request.starts_with(b"GET /item HTTP/1.1\r\n")); + let response = format!("HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: {connection}\r\n\r\n"); + upstream.write_all(response.as_bytes()).await.expect("upstream response"); + upstream.flush().await.expect("flush response"); + } + }, + ) + }.with_subscriber(subscriber); + let (relay, responses, ()) = + tokio::time::timeout(std::time::Duration::from_secs(5), Box::pin(exchange)) + .await + .expect("two inspected requests complete"); + relay.expect("inspected relay"); + let receipts: Vec<_> = responses + .iter() + .map(|response| request_id(response)) + .collect(); + assert_eq!(receipts.len(), 2); + assert_ne!(receipts[0], receipts[1]); + let records: Vec<_> = capture + .json_events() + .into_iter() + .filter(|event| event["class_uid"] == 4002) + .collect(); + assert_eq!( + records.len(), + 2, + "one primary per inspected request: {records:?}" + ); + for (record, receipt) in records.iter().zip(receipts) { + assert!(uuid::Uuid::parse_str(receipt).is_ok()); + assert_eq!(record["metadata"]["correlation_uid"], receipt); + assert_eq!(record["action"], "Allowed"); + assert_eq!(record["http_response"]["code"], 200); + assert_eq!(record["http_request"]["url"]["path"], "/item"); + assert!(record.get("trace").is_none()); + assert!( + !record + .to_string() + .contains("4bf92f3577b34da6a3ce929d0e0e4736") + ); + } +} + +#[tokio::test] +async fn production_single_mcp_allow_correlates_record_and_response_receipt() { + let body = r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"test","version":"1"}}}"#; + let exchange = Box::pin(run_allowed( + mcp_fixture(), + request("mcp.example.test", "/mcp", "", body), + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}", + )) + .await; + + assert!( + exchange + .output + .forwarded + .starts_with(b"POST /mcp HTTP/1.1\r\n") + ); + assert!(exchange.output.response.starts_with(b"HTTP/1.1 200 OK\r\n")); + assert_primary_record(&exchange, "l7-mcp", 200, "Allowed", "/mcp"); +} + +#[tokio::test] +async fn production_single_mcp_invalid_version_correlates_denial_and_receipt() { + let body = r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"test","version":"1"}}}"#; + let exchange = Box::pin(run_denied( + mcp_fixture(), + request( + "mcp.example.test", + "/mcp", + "MCP-Protocol-Version: 1900-01-01\r\n", + body, + ), + )) + .await; + + assert!(exchange.output.forwarded.is_empty()); + assert!( + exchange + .output + .response + .starts_with(b"HTTP/1.1 400 Bad Request\r\n") + ); + assert!( + String::from_utf8_lossy(&exchange.output.response) + .contains("unsupported_mcp_protocol_version") + ); + assert_primary_record(&exchange, "l7-mcp", 400, "Denied", "/mcp"); +} + +#[tokio::test] +async fn production_single_mcp_policy_deny_correlates_record_and_receipt() { + let body = r#"{"jsonrpc":"2.0","id":1,"method":"tools/list"}"#; + let exchange = Box::pin(run_denied( + mcp_fixture(), + request( + "mcp.example.test", + "/mcp", + "MCP-Protocol-Version: 2025-11-25\r\n", + body, + ), + )) + .await; + + assert!(exchange.output.forwarded.is_empty()); + assert!( + exchange + .output + .response + .starts_with(b"HTTP/1.1 403 Forbidden\r\n") + ); + let event = assert_primary_record(&exchange, "l7-mcp", 403, "Denied", "/mcp"); + assert!( + event["message"] + .as_str() + .is_some_and(|message| message.starts_with("JSONRPC_L7_REQUEST decision=deny")) + ); +} + +#[tokio::test] +async fn production_single_graphql_allow_correlates_record_and_response_receipt() { + let graphql = fixture( + r" +network_policies: + graphql_api: + name: graphql_api + endpoints: + - host: graphql.example.test + port: 8000 + path: /graphql + protocol: graphql + enforcement: enforce + rules: + - allow: + operation_type: query + fields: [viewer] + binaries: + - { path: /usr/bin/python3 } +", + "graphql.example.test", + "graphql_api", + ); + let body = r#"{"query":"query { viewer }"}"#; + let exchange = Box::pin(run_allowed( + graphql, + request("graphql.example.test", "/graphql", "", body), + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 22\r\nConnection: close\r\n\r\n{\"data\":{\"viewer\":{}}}", + )) + .await; + + assert!( + exchange + .output + .forwarded + .starts_with(b"POST /graphql HTTP/1.1\r\n") + ); + assert!(exchange.output.response.starts_with(b"HTTP/1.1 200 OK\r\n")); + assert_primary_record(&exchange, "l7-graphql", 200, "Allowed", "/graphql"); +} diff --git a/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs index 19a9633fb0..1c1d7f2977 100644 --- a/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs +++ b/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs @@ -31,6 +31,7 @@ use tokio::io::{AsyncReadExt, DuplexStream, ReadBuf}; use tokio::sync::Notify; use tokio::task::JoinHandle; use tokio::time::timeout; +use tracing::instrument::WithSubscriber as _; const POLICY: &str = include_str!("../../../data/sandbox-policy.rego"); const HOST: &str = "api.example.test"; @@ -304,17 +305,20 @@ impl Fixture { notified: false, }; let (mut relay_upstream, upstream) = tokio::io::duplex(16 * 1024); - let relay = tokio::spawn(async move { - crate::proxy::relay_inspected_http_stream_for_test( - &mut client, - &mut relay_upstream, - configs, - engine, - &middleware_engine, - &ctx, - ) - .await - }); + let relay = tokio::spawn( + async move { + crate::proxy::relay_inspected_http_stream_for_test( + &mut client, + &mut relay_upstream, + configs, + engine, + &middleware_engine, + &ctx, + ) + .await + } + .with_current_subscriber(), + ); Connection { app, relay, @@ -1003,6 +1007,58 @@ async fn failure_then_reconnect(route_selection: bool) { ]); } +#[tokio::test] +async fn token_grant_refusal_never_creates_a_denial_only_span() { + use super::correlation_dispatch_tests::capture_request_spans; + + for sampled in [true, false] { + let evidence = capture_request_spans(sampled, async { + let fixture = Fixture::new(&[route("a", "/a/**")], &[("/a/**", "a", "aud-a")], 1); + fixture.resolver.fail_once.store(true, Ordering::SeqCst); + let mut connection = fixture.connect(); + let response = connection.exchange("GET", "/a/item", true).await; + assert!(connection.observes_eof().await); + let capture = connection.finish().await; + assert_eq!(capture.bytes_seen, 0); + response + }) + .await; + evidence.assert_no_deny_span(sampled); + assert!(evidence.output.starts_with("HTTP/1.1 502")); + if sampled { + assert_eq!( + evidence + .spans + .iter() + .filter(|span| span.name == "openshell.credential.prepare") + .count(), + 1 + ); + assert!( + evidence + .spans + .iter() + .all(|span| span.name != "openshell.upstream.send") + ); + } + let records: Vec<_> = evidence + .events + .iter() + .filter(|event| event["class_uid"] == 4002) + .collect(); + assert_eq!(records.len(), 1, "{:?}", evidence.events); + assert_eq!(records[0]["action"], "Denied"); + assert_eq!(records[0]["http_response"]["code"], 502); + let receipt = records[0]["metadata"]["correlation_uid"].as_str().unwrap(); + assert!( + evidence + .output + .to_ascii_lowercase() + .contains(&format!("x-openshell-request-id: {receipt}\r\n")) + ); + } +} + #[tokio::test] async fn single_endpoint_recovers_after_one_grant_failure() { failure_then_reconnect(false).await; @@ -1418,6 +1474,7 @@ async fn transformed_jsonrpc_body_selects_final_operation_owner() { &info, "request-final-body", None, + None, ), ) .await diff --git a/crates/openshell-supervisor-network/src/l7/request_trace.rs b/crates/openshell-supervisor-network/src/l7/request_trace.rs new file mode 100644 index 0000000000..5e5065e553 --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/request_trace.rs @@ -0,0 +1,212 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Supervisor-owned request traces with untrusted workload links. + +use opentelemetry::propagation::{Extractor, TextMapPropagator as _}; +use opentelemetry::trace::{SpanContext, TraceContextExt as _}; +use opentelemetry::{Context, KeyValue}; +use opentelemetry_sdk::propagation::TraceContextPropagator; +use tracing::Span; +use tracing_opentelemetry::OpenTelemetrySpanExt as _; + +use super::correlation::HttpOutcome; +use openshell_ocsf::{ActionId, HttpActivityEvent, StatusId}; + +const MAX_TRACEPARENT_BYTES: usize = 256; + +#[derive(Debug)] +pub(super) struct RequestTrace { + span: Span, + outcome: &'static str, +} + +#[derive(Debug, Clone, Copy)] +pub enum RequestStage { + PolicyEvaluate, + RequestMiddleware, + WebsocketPreflight, + CredentialPrepare, + UpstreamResolve, + UpstreamDial, + UpstreamSend, + Response, + ResponseMiddleware, +} + +impl RequestStage { + fn name(self) -> &'static str { + match self { + Self::PolicyEvaluate => "openshell.policy.evaluate", + Self::RequestMiddleware => "openshell.middleware.request", + Self::WebsocketPreflight => "openshell.middleware.websocket.preflight", + Self::CredentialPrepare => "openshell.credential.prepare", + Self::UpstreamResolve => "openshell.upstream.resolve", + Self::UpstreamDial => "openshell.upstream.connect", + Self::UpstreamSend => "openshell.upstream.send", + Self::Response => "openshell.http.response", + Self::ResponseMiddleware => "openshell.middleware.response", + } + } +} + +impl RequestTrace { + pub(super) fn new( + request_id: &str, + method: &str, + host: &str, + port: u16, + raw_request: &[u8], + ) -> Self { + let span = tracing::info_span!( + parent: None, + "openshell.http.request", + openshell.request.id = request_id, + http.request.method = normalized_method(method), + server.address = host, + server.port = i64::from(port), + openshell.request.outcome = tracing::field::Empty, + openshell.request.failure_reason = tracing::field::Empty, + http.response.status_code = tracing::field::Empty, + otel.kind = "server", + otel.status_code = tracing::field::Empty, + ); + if let Some(context) = workload_link(raw_request) { + span.add_link_with_attributes( + context, + vec![ + KeyValue::new("openshell.correlation.source", "http.traceparent"), + KeyValue::new("openshell.correlation.trust", "workload"), + ], + ); + } + Self { + span, + outcome: "incomplete", + } + } + + pub(super) fn span(&self) -> Span { + self.span.clone() + } + + pub(super) fn stage(&self, stage: RequestStage) -> Span { + if self.span.is_disabled() || !self.span.context().span().span_context().is_valid() { + return Span::none(); + } + tracing::info_span!( + parent: &self.span, + "openshell.http.stage", + otel.name = stage.name(), + otel.status_code = tracing::field::Empty, + ) + } + + pub(super) fn record_status(&self, status: u16) { + self.span + .record("http.response.status_code", i64::from(status)); + } + + pub(super) fn record_outcome(&mut self, outcome: HttpOutcome) { + let outcome = match outcome { + HttpOutcome::Allowed if self.outcome == "denied" => "denied", + HttpOutcome::Allowed => "allowed", + HttpOutcome::Denied => "denied", + HttpOutcome::PlatformError | HttpOutcome::UpstreamError => "failed", + HttpOutcome::Upgraded => "upgraded", + }; + self.outcome = outcome; + self.span.record("openshell.request.outcome", outcome); + } + + pub(super) fn record_failure_reason(&self, reason: &'static str) { + self.span.record("openshell.request.failure_reason", reason); + } + + pub(super) fn observe_http(&mut self, event: &HttpActivityEvent) { + if let Some(response) = &event.http_response { + self.record_status(response.code); + } + if event.action == Some(ActionId::Denied) { + if self.outcome != "failed" { + self.record_outcome(HttpOutcome::Denied); + } + } else if self.outcome == "incomplete" { + self.record_outcome(if event.base.status == Some(StatusId::Failure) { + HttpOutcome::PlatformError + } else { + HttpOutcome::Allowed + }); + } + } +} + +impl Drop for RequestTrace { + fn drop(&mut self) { + // Finalize exported timing after the request's child stages. + self.span.in_scope(|| { + self.span.record("openshell.request.outcome", self.outcome); + if matches!(self.outcome, "failed" | "incomplete") { + self.span.record("otel.status_code", "ERROR"); + } + }); + } +} + +fn normalized_method(method: &str) -> &'static str { + match method { + "GET" => "GET", + "HEAD" => "HEAD", + "POST" => "POST", + "PUT" => "PUT", + "DELETE" => "DELETE", + "CONNECT" => "CONNECT", + "OPTIONS" => "OPTIONS", + "TRACE" => "TRACE", + "PATCH" => "PATCH", + _ => "_OTHER", + } +} + +fn workload_link(raw_request: &[u8]) -> Option { + let end = raw_request + .windows(4) + .position(|bytes| bytes == b"\r\n\r\n")?; + let mut value = None; + for line in raw_request[..end].split(|byte| *byte == b'\n').skip(1) { + let line = line.strip_suffix(b"\r").unwrap_or(line); + let Some(colon) = line.iter().position(|byte| *byte == b':') else { + continue; + }; + if !line[..colon].eq_ignore_ascii_case(b"traceparent") { + continue; + } + if value.is_some() { + return None; + } + let candidate = std::str::from_utf8(&line[colon + 1..]).ok()?.trim(); + if candidate.len() > MAX_TRACEPARENT_BYTES { + return None; + } + value = Some(candidate); + } + let context = TraceContextPropagator::new() + .extract_with_context(&Context::new(), &OnlyTraceparent(value?)); + let span_context = context.span().span_context().clone(); + span_context.is_valid().then_some(span_context) +} + +struct OnlyTraceparent<'a>(&'a str); + +impl Extractor for OnlyTraceparent<'_> { + fn get(&self, key: &str) -> Option<&str> { + (key == "traceparent").then_some(self.0) + } + + fn keys(&self) -> Vec<&str> { + vec!["traceparent"] + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/openshell-supervisor-network/src/l7/request_trace/tests.rs b/crates/openshell-supervisor-network/src/l7/request_trace/tests.rs new file mode 100644 index 0000000000..b0818f4f3e --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/request_trace/tests.rs @@ -0,0 +1,245 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use crate::l7::correlation::HttpOutcome; +use crate::test_support::request_tracing::{RequestTracing, span_attribute}; +use opentelemetry::trace::TracerProvider as _; +use opentelemetry_sdk::trace::{InMemorySpanExporter, Sampler, SdkTracerProvider, SpanData}; +use tracing_subscriber::layer::SubscriberExt as _; + +const TRACE_ID: &str = "4bf92f3577b34da6a3ce929d0e0e4736"; +const PARENT_ID: &str = "00f067aa0ba902b7"; + +fn request(flags: &str) -> Vec { + format!( + "GET /account?secret=DO-NOT-RECORD HTTP/1.1\r\nHost: example.test\r\ntraceparent: 00-{TRACE_ID}-{PARENT_ID}-{flags}\r\ntracestate: vendor=DO-NOT-RECORD\r\nbaggage: secret=DO-NOT-RECORD\r\n\r\n" + ) + .into_bytes() +} + +fn capture(sampler: Sampler, emit: impl FnOnce()) -> Vec { + let capture = RequestTracing::new(Some(sampler)); + tracing::subscriber::with_default(capture.subscriber(), emit); + let spans = capture.spans(); + capture.shutdown(); + spans +} + +fn root(spans: &[SpanData]) -> &SpanData { + spans + .iter() + .find(|span| span.name == "openshell.http.request") + .expect("one supervisor request root") +} + +#[test] +fn workload_context_is_linked_not_parented_for_both_sampled_flags() { + for flags in ["00", "01"] { + let spans = capture(Sampler::ParentBased(Box::new(Sampler::AlwaysOn)), || { + let unrelated = tracing::info_span!("unrelated-host-operation"); + unrelated.in_scope(|| { + let mut trace = RequestTrace::new( + "test-request-id", + "GET", + "example.test", + 443, + &request(flags), + ); + trace.stage(RequestStage::PolicyEvaluate).in_scope(|| {}); + trace.record_status(403); + trace.record_outcome(HttpOutcome::Denied); + }); + }); + let root = root(&spans); + assert_eq!(root.parent_span_id, opentelemetry::trace::SpanId::INVALID); + assert_ne!(root.span_context.trace_id().to_string(), TRACE_ID); + assert_eq!(root.links.links.len(), 1); + let link = &root.links.links[0]; + assert_eq!(link.span_context.trace_id().to_string(), TRACE_ID); + assert_eq!(link.span_context.span_id().to_string(), PARENT_ID); + assert_eq!(link.span_context.is_sampled(), flags == "01"); + assert!(link.span_context.trace_state().header().is_empty()); + assert!(link.attributes.iter().any(|field| { + field.key.as_str() == "openshell.correlation.trust" + && field.value.as_str() == "workload" + })); + let policy = spans + .iter() + .find(|span| span.name == "openshell.policy.evaluate") + .unwrap(); + assert_eq!(policy.parent_span_id, root.span_context.span_id()); + assert_eq!(policy.span_context.trace_id(), root.span_context.trace_id()); + assert_eq!( + span_attribute(root, "openshell.request.id") + .unwrap() + .as_str(), + "test-request-id" + ); + assert_eq!(root.status, opentelemetry::trace::Status::Unset); + assert!(!format!("{spans:?}").contains("DO-NOT-RECORD")); + } +} + +#[test] +fn explicit_request_root_does_not_adopt_ambient_otel_context() { + let remote = workload_link(&request("01")).unwrap(); + let spans = capture(Sampler::AlwaysOn, || { + let _guard = Context::new().with_remote_span_context(remote).attach(); + let mut trace = RequestTrace::new("local", "GET", "example.test", 443, &request("00")); + trace.record_outcome(HttpOutcome::Allowed); + }); + let root = root(&spans); + assert_eq!(root.parent_span_id, opentelemetry::trace::SpanId::INVALID); + assert_ne!(root.span_context.trace_id().to_string(), TRACE_ID); + assert_eq!(root.links.links.len(), 1); +} + +#[test] +fn request_completion_encloses_stages_after_an_early_root_exit() { + let spans = capture(Sampler::AlwaysOn, || { + let mut trace = RequestTrace::new("local", "GET", "example.test", 443, &request("01")); + trace.span().in_scope(|| trace.record_status(200)); + trace.stage(RequestStage::Response).in_scope(|| {}); + trace.record_outcome(HttpOutcome::Allowed); + }); + let root = root(&spans); + let response = spans + .iter() + .find(|span| span.name == RequestStage::Response.name()) + .unwrap(); + assert_eq!(response.parent_span_id, root.span_context.span_id()); + assert!(response.start_time >= root.start_time); + assert!(response.end_time <= root.end_time); +} + +#[test] +fn remote_sampled_flag_cannot_override_local_always_off() { + for flags in ["00", "01"] { + let spans = capture(Sampler::ParentBased(Box::new(Sampler::AlwaysOff)), || { + let mut trace = RequestTrace::new("local", "GET", "example.test", 443, &request(flags)); + trace.stage(RequestStage::PolicyEvaluate).in_scope(|| {}); + trace.record_outcome(HttpOutcome::Denied); + }); + assert!(spans.is_empty()); + } +} + +#[test] +fn every_outcome_has_the_same_root_shape_and_stages_have_fixed_names() { + let stages = [ + RequestStage::PolicyEvaluate, + RequestStage::RequestMiddleware, + RequestStage::WebsocketPreflight, + RequestStage::CredentialPrepare, + RequestStage::UpstreamResolve, + RequestStage::UpstreamDial, + RequestStage::UpstreamSend, + RequestStage::Response, + RequestStage::ResponseMiddleware, + ]; + for (outcome, failed) in [ + (HttpOutcome::Allowed, false), + (HttpOutcome::Denied, false), + (HttpOutcome::UpstreamError, true), + ] { + let spans = capture(Sampler::AlwaysOn, || { + let mut trace = RequestTrace::new("local", "GET", "example.test", 443, &request("01")); + for stage in stages { + trace.stage(stage).in_scope(|| {}); + } + trace.record_outcome(outcome); + }); + let root = root(&spans); + assert_eq!(spans.len(), stages.len() + 1); + for stage in stages { + let child = spans.iter().find(|span| span.name == stage.name()).unwrap(); + assert_eq!(child.parent_span_id, root.span_context.span_id()); + assert_eq!(child.span_context.trace_id(), root.span_context.trace_id()); + } + assert_eq!( + matches!(root.status, opentelemetry::trace::Status::Error { .. }), + failed + ); + } +} + +#[test] +fn malformed_duplicate_oversized_and_zero_context_is_ignored() { + let valid = String::from_utf8(request("01")).unwrap(); + let duplicate = valid.replace( + "\r\ntracestate:", + &format!("\r\nTraceparent: 00-{TRACE_ID}-{PARENT_ID}-00\r\ntracestate:"), + ); + for headers in [ + b"GET / HTTP/1.1\r\nHost: example.test\r\n\r\n".to_vec(), + b"GET / HTTP/1.1\r\ntraceparent: invalid\r\n\r\n".to_vec(), + duplicate.into_bytes(), + valid + .replace(TRACE_ID, "00000000000000000000000000000000") + .into_bytes(), + valid.replace(PARENT_ID, "0000000000000000").into_bytes(), + format!("GET / HTTP/1.1\r\ntraceparent: {}\r\n\r\n", "a".repeat(257)).into_bytes(), + ] { + assert!(workload_link(&headers).is_none()); + } +} + +#[test] +fn unrelated_opaque_header_values_do_not_discard_valid_trace_context() { + let mut headers = b"GET / HTTP/1.1\r\nX-Opaque: \xff\r\n".to_vec(); + headers + .extend_from_slice(format!("traceparent: 00-{TRACE_ID}-{PARENT_ID}-01\r\n\r\n").as_bytes()); + assert!(workload_link(&headers).is_some()); +} + +#[test] +fn custom_method_values_are_not_exported() { + let spans = capture(Sampler::AlwaysOn, || { + let mut trace = RequestTrace::new( + "local", + "DO-NOT-RECORD", + "example.test", + 443, + &request("01"), + ); + trace.record_outcome(HttpOutcome::Allowed); + }); + assert_eq!( + span_attribute(root(&spans), "http.request.method") + .unwrap() + .as_str(), + "_OTHER" + ); + assert!(!format!("{spans:?}").contains("DO-NOT-RECORD")); +} + +#[test] +fn a_filtered_request_root_does_not_create_orphan_stage_spans() { + use tracing_subscriber::Layer as _; + + let exporter = InMemorySpanExporter::default(); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture).with( + tracing_opentelemetry::layer() + .with_tracer(provider.tracer("filtered-root-test")) + .with_filter(tracing_subscriber::filter::filter_fn(|metadata| { + metadata.name() != "openshell.http.request" + })), + ); + tracing::subscriber::with_default(subscriber, || { + let mut trace = RequestTrace::new("local", "GET", "example.test", 443, &request("01")); + assert!( + !trace.span().is_disabled(), + "another layer still enables the root" + ); + assert!(trace.stage(RequestStage::PolicyEvaluate).is_disabled()); + trace.record_outcome(HttpOutcome::Allowed); + }); + provider.force_flush().unwrap(); + assert!(exporter.get_finished_spans().unwrap().is_empty()); + provider.shutdown().unwrap(); +} diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 8bc198b9ee..3e5636193e 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -27,6 +27,7 @@ pub(crate) use request_body::{ }; use crate::l7::EndpointObserver; +use crate::l7::correlation::{RequestCorrelation, RequestStage}; use crate::l7::http::{parse_chunk_size_line, validate_chunk_trailer_line}; use crate::l7::provider::{BodyLength, L7Provider, L7Request, RelayOutcome}; use crate::opa::PolicyGenerationGuard; @@ -44,10 +45,11 @@ use openshell_core::secrets::{ }; use openshell_ocsf::ctx::ctx as ocsf_ctx; use sha1::{Digest, Sha1}; +use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::fmt::{self, Write as _}; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; -use tracing::debug; +use tracing::{Instrument as _, debug}; const MAX_HEADER_BYTES: usize = 16384; // 16 KiB for HTTP headers pub(crate) const MAX_REWRITE_BODY_BYTES: usize = 256 * 1024; @@ -164,6 +166,7 @@ pub(crate) struct DenyResponseContext<'a> { pub(crate) port: Option, pub(crate) binary: Option<&'a str>, pub(crate) agent_proposals_enabled: bool, + pub(crate) request_id: Option<&'a str>, } impl<'a> DenyResponseContext<'a> { @@ -174,11 +177,25 @@ impl<'a> DenyResponseContext<'a> { port: Some(ctx.port), binary: Some(&ctx.binary_path), agent_proposals_enabled: ctx.agent_proposals.enabled(), + request_id: None, } } + + #[must_use] + pub(crate) fn with_request_id(mut self, request_id: &'a str) -> Self { + self.request_id = Some(request_id); + self + } } impl RestProvider { + pub(crate) async fn parse_uninspected_request( + &self, + client: &mut C, + ) -> Result> { + parse_http_request_with_receipt_protection(client, &self.canonicalize_options, false).await + } + /// Deny with a redacted target for the response body. pub(crate) async fn deny_with_redacted_target( &self, @@ -204,6 +221,14 @@ impl RestProvider { async fn parse_http_request( client: &mut C, canonicalize_options: &crate::l7::path::CanonicalizeOptions, +) -> Result> { + parse_http_request_with_receipt_protection(client, canonicalize_options, true).await +} + +async fn parse_http_request_with_receipt_protection( + client: &mut C, + canonicalize_options: &crate::l7::path::CanonicalizeOptions, + protect_receipt: bool, ) -> Result> { let mut buf = Vec::with_capacity(4096); @@ -308,6 +333,10 @@ async fn parse_http_request( )?; } + if protect_receipt && let Cow::Owned(sanitized) = strip_platform_request_id_from_head(&buf) { + buf = sanitized; + } + Ok(Some(L7Request { action: method, target: canonical.path, @@ -327,6 +356,20 @@ pub(crate) fn request_from_buffered_http( target: impl Into, query_target: &str, raw_header: Vec, +) -> Result { + let mut request = + request_from_buffered_http_unprotected(action, target, query_target, raw_header)?; + if let Cow::Owned(sanitized) = strip_platform_request_id_from_head(&request.raw_header) { + request.raw_header = sanitized; + } + Ok(request) +} + +pub(crate) fn request_from_buffered_http_unprotected( + action: impl Into, + target: impl Into, + query_target: &str, + raw_header: Vec, ) -> Result { let header_end = raw_header .windows(4) @@ -914,7 +957,32 @@ where C: AsyncRead + AsyncWrite + Unpin, U: AsyncRead + AsyncWrite + Unpin, { - Box::pin(relay_http_request_with_body_guarded_observed( + relay_http_request_with_response_middleware_guarded_observed_correlated( + req, + client, + upstream, + options, + response_middleware, + observer, + None, + ) + .await +} + +pub(crate) async fn relay_http_request_with_response_middleware_guarded_observed_correlated( + req: &L7Request, + client: &mut C, + upstream: &mut U, + options: RelayRequestOptions<'_>, + response_middleware: Option>, + observer: Option<&EndpointObserver>, + correlation: Option<&RequestCorrelation>, +) -> Result +where + C: AsyncRead + AsyncWrite + Unpin, + U: AsyncRead + AsyncWrite + Unpin, +{ + Box::pin(relay_http_request_with_body_guarded_observed_correlated( req, client, upstream, @@ -922,13 +990,15 @@ where None, response_middleware, observer, + correlation, )) .await } /// Relay one request and its response. `live_body` carries a body that HTTP /// hook request middleware streams; `req` then holds only its head. -pub(crate) async fn relay_http_request_with_body_guarded_observed( +#[allow(clippy::too_many_arguments)] +pub(crate) async fn relay_http_request_with_body_guarded_observed_correlated( req: &L7Request, client: &mut C, upstream: &mut U, @@ -936,12 +1006,19 @@ pub(crate) async fn relay_http_request_with_body_guarded_observed( live_body: Option<&mut crate::l7::middleware::RequestBodyStream>, response_middleware: Option>, observer: Option<&EndpointObserver>, + correlation: Option<&RequestCorrelation>, ) -> Result where C: AsyncRead + AsyncWrite + Unpin, U: AsyncRead + AsyncWrite + Unpin, { - let mut observed_upstream = ObservedUpstream { upstream, observer }; + let mut observed_upstream = ObservedUpstream { + upstream, + observer, + correlation, + request_span: None, + track_request_span: live_body.is_none(), + }; let upstream = &mut observed_upstream; let mut options = options; if let Some(middleware) = response_middleware.as_ref() { @@ -978,6 +1055,7 @@ where websocket: None, client_requested_upgrade, observer, + correlation, }, response_middleware, ) @@ -1025,8 +1103,18 @@ where offered_subprotocols: request.subprotocols.clone(), }); - let rewrite_result = - rewrite_http_header_block(&header_bytes, options.resolver).map_err(miette::Report::new)?; + let rewrite_span = + if options.resolver.is_some() || contains_reserved_credential_marker_bytes(&header_bytes) { + correlation.map_or_else(tracing::Span::none, |correlation| { + correlation.stage(RequestStage::CredentialPrepare) + }) + } else { + tracing::Span::none() + }; + let rewrite_result = rewrite_span + .in_scope(|| rewrite_http_header_block(&header_bytes, options.resolver)) + .map_err(miette::Report::new)?; + drop(rewrite_span); if let Some(validation) = options.mcp_request_validation { // Header credential resolution and hop-by-hop cleanup must finish @@ -1041,13 +1129,14 @@ where raw_header, body_length: req.body_length, }; - if !crate::l7::relay::enforce_final_mcp_protocol_version( + if !crate::l7::relay::enforce_final_mcp_protocol_version_correlated( validation.config, &outgoing, client, validation.ctx, validation.redacted_target, observer, + correlation, ) .await? { @@ -1180,15 +1269,22 @@ where guard.ensure_current()?; } - let signed = crate::sigv4::apply_sigv4_to_request( - &full_request, - options.host, - ®ion, - service, - access_key, - secret_key, - session_token, - )?; + let signing_span = correlation + .map_or_else(tracing::Span::none, |correlation| { + correlation.stage(RequestStage::CredentialPrepare) + }); + let signed = signing_span.in_scope(|| { + crate::sigv4::apply_sigv4_to_request( + &full_request, + options.host, + ®ion, + service, + access_key, + secret_key, + session_token, + ) + })?; + drop(signing_span); ensure_credential_generation_current(options)?; upstream.write_all(&signed).await.into_diagnostic()?; } else { @@ -1199,16 +1295,23 @@ where } _ => SignableBody::UnsignedPayload, }; - let signed_headers = crate::sigv4::apply_sigv4_headers_only_with_body( - &rewrite_result.rewritten, - options.host, - ®ion, - service, - access_key, - secret_key, - session_token, - signable_body, - )?; + let signing_span = correlation + .map_or_else(tracing::Span::none, |correlation| { + correlation.stage(RequestStage::CredentialPrepare) + }); + let signed_headers = signing_span.in_scope(|| { + crate::sigv4::apply_sigv4_headers_only_with_body( + &rewrite_result.rewritten, + options.host, + ®ion, + service, + access_key, + secret_key, + session_token, + signable_body, + ) + })?; + drop(signing_span); ensure_credential_generation_current(options)?; upstream .write_all(&signed_headers) @@ -1234,8 +1337,14 @@ where } } BodyLength::Chunked => { - relay_chunked(client, upstream, overflow, options.generation_guard) - .await?; + relay_chunked_with_receipt_protection( + client, + upstream, + overflow, + options.generation_guard, + correlation.is_some(), + ) + .await?; } BodyLength::None => { if !overflow.is_empty() { @@ -1301,6 +1410,7 @@ where &rewrite_result.rewritten, &req.raw_header[header_end..], options, + correlation.is_some(), ) .await { @@ -1329,7 +1439,14 @@ where } } BodyLength::Chunked => { - relay_chunked(client, upstream, overflow, options.generation_guard).await?; + relay_chunked_with_receipt_protection( + client, + upstream, + overflow, + options.generation_guard, + correlation.is_some(), + ) + .await?; } BodyLength::None => { if !overflow.is_empty() { @@ -1339,7 +1456,11 @@ where } } upstream.flush().await.into_diagnostic()?; + upstream.request_span.take(); + let response_span = correlation.map_or_else(tracing::Span::none, |correlation| { + correlation.stage(RequestStage::Response) + }); let outcome = relay_response( &req.action, upstream, @@ -1349,9 +1470,11 @@ where websocket: websocket_response, client_requested_upgrade, observer, + correlation, }, response_middleware, ) + .instrument(response_span) .await?; Ok(outcome) @@ -1376,6 +1499,9 @@ fn ensure_body_generation_current(options: RelayRequestOptions<'_>) -> Result<() struct ObservedUpstream<'a, U> { upstream: &'a mut U, observer: Option<&'a EndpointObserver>, + correlation: Option<&'a RequestCorrelation>, + request_span: Option, + track_request_span: bool, } impl AsyncRead for ObservedUpstream<'_, U> { @@ -1402,7 +1528,17 @@ impl AsyncWrite for ObservedUpstream<'_, U> { buf: &[u8], ) -> std::task::Poll> { let this = self.get_mut(); - let result = std::pin::Pin::new(&mut *this.upstream).poll_write(cx, buf); + let result = if this.track_request_span { + let span = this.request_span.get_or_insert_with(|| { + this.correlation + .map_or_else(tracing::Span::none, |correlation| { + correlation.stage(RequestStage::UpstreamSend) + }) + }); + span.in_scope(|| std::pin::Pin::new(&mut *this.upstream).poll_write(cx, buf)) + } else { + std::pin::Pin::new(&mut *this.upstream).poll_write(cx, buf) + }; if matches!(result, std::task::Poll::Ready(Err(_))) && let Some(observer) = this.observer { @@ -1416,7 +1552,11 @@ impl AsyncWrite for ObservedUpstream<'_, U> { cx: &mut std::task::Context<'_>, ) -> std::task::Poll> { let this = self.get_mut(); - let result = std::pin::Pin::new(&mut *this.upstream).poll_flush(cx); + let result = if let Some(span) = &this.request_span { + span.in_scope(|| std::pin::Pin::new(&mut *this.upstream).poll_flush(cx)) + } else { + std::pin::Pin::new(&mut *this.upstream).poll_flush(cx) + }; if matches!(result, std::task::Poll::Ready(Err(_))) && let Some(observer) = this.observer { @@ -1468,6 +1608,7 @@ async fn relay_request_body_with_marker_guard( headers: &[u8], already_read: &[u8], options: RelayRequestOptions<'_>, + protect_receipt: bool, ) -> Result<()> where C: AsyncRead + Unpin, @@ -1509,7 +1650,14 @@ where write_body_bytes(upstream, &scanner.finish()?, options).await?; } BodyLength::Chunked => { - relay_chunked_with_marker_guard(client, upstream, already_read, options).await?; + relay_chunked_with_marker_guard( + client, + upstream, + already_read, + options, + protect_receipt, + ) + .await?; } } Ok(()) @@ -1520,6 +1668,7 @@ async fn relay_chunked_with_marker_guard( upstream: &mut U, already_read: &[u8], options: RelayRequestOptions<'_>, + protect_receipt: bool, ) -> Result<()> where C: AsyncRead + Unpin, @@ -1551,11 +1700,19 @@ where if contains_reserved_credential_marker_bytes(&trailer) { return Err(BodyCredentialError::Trailer.into()); } - write_body_bytes(upstream, &trailer, options).await?; - write_body_bytes(upstream, b"\r\n", options).await?; if trailer.is_empty() { + write_body_bytes(upstream, b"\r\n", options).await?; return Ok(()); } + if !protect_receipt + || !http_field_line_has_name( + &trailer, + openshell_core::token_grant_failure::REQUEST_ID_HEADER, + ) + { + write_body_bytes(upstream, &trailer, options).await?; + write_body_bytes(upstream, b"\r\n", options).await?; + } } } @@ -2371,6 +2528,102 @@ fn strip_header(headers: &[u8], strip_name: &str) -> Result> { Ok(out.into_bytes()) } +fn strip_platform_request_id_from_head(headers: &[u8]) -> Cow<'_, [u8]> { + let header_end = headers + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map_or(headers.len(), |position| position + 4); + let (header_block, buffered_suffix) = headers.split_at(header_end); + let protected = openshell_core::token_grant_failure::REQUEST_ID_HEADER.as_bytes(); + let mut out: Option> = None; + let mut position = 0usize; + + while position < header_block.len() { + let line_end = header_block[position..] + .windows(2) + .position(|window| window == b"\r\n") + .map_or(header_block.len(), |end| position + end); + let line = &header_block[position..line_end]; + if line.is_empty() { + if position + 2 < header_block.len() { + out.get_or_insert_with(|| header_block[..position].to_vec()); + } + if let Some(out) = &mut out { + out.extend_from_slice(b"\r\n"); + } + break; + } + if let Some(colon) = line.iter().position(|byte| *byte == b':') { + let name = trim_ascii_ows(&line[..colon]); + let receipt = name.eq_ignore_ascii_case(protected); + let trailer = name.eq_ignore_ascii_case(b"trailer") + && line[colon + 1..] + .split(|byte| *byte == b',') + .any(|value| trim_ascii_ows(value).eq_ignore_ascii_case(protected)); + if receipt || trailer { + let out = out.get_or_insert_with(|| { + let mut out = Vec::with_capacity(headers.len()); + out.extend_from_slice(&header_block[..position]); + out + }); + if trailer { + let mut values = line[colon + 1..] + .split(|byte| *byte == b',') + .filter(|value| !trim_ascii_ows(value).eq_ignore_ascii_case(protected)); + let mut retained = values.clone(); + if retained.any(|value| !trim_ascii_ows(value).is_empty()) { + out.extend_from_slice(&line[..=colon]); + if let Some(first) = values.next() { + out.extend_from_slice(first); + } + for value in values { + out.push(b','); + out.extend_from_slice(value); + } + out.extend_from_slice(b"\r\n"); + } + } + position = line_end.saturating_add(2); + continue; + } + } + if line_end + 2 > header_block.len() { + out.get_or_insert_with(|| header_block[..position].to_vec()); + } + if let Some(out) = &mut out { + out.extend_from_slice(line); + out.extend_from_slice(b"\r\n"); + } + position = line_end.saturating_add(2); + } + out.map_or(Cow::Borrowed(headers), |mut out| { + out.extend_from_slice(buffered_suffix); + Cow::Owned(out) + }) +} + +fn trim_ascii_ows(mut value: &[u8]) -> &[u8] { + while value + .first() + .is_some_and(|byte| matches!(byte, b' ' | b'\t')) + { + value = &value[1..]; + } + while value + .last() + .is_some_and(|byte| matches!(byte, b' ' | b'\t')) + { + value = &value[..value.len() - 1]; + } + value +} + +fn http_field_line_has_name(line: &[u8], expected: &str) -> bool { + line.iter() + .position(|byte| *byte == b':') + .is_some_and(|end| line[..end].eq_ignore_ascii_case(expected.as_bytes())) +} + pub(crate) fn connection_nominated_header_names(headers: &[u8]) -> Result> { validate_http_request_header_block(headers)?; let header_str = @@ -2906,7 +3159,15 @@ async fn send_deny_response( context: Option>, ) -> Result<()> { let body = deny_response_body(req, policy_name, reason, redacted_target, context); - send_forbidden_json(policy_name, body, client).await + send_json_response_internal( + policy_name, + body, + client, + "403 Forbidden", + None, + context.and_then(|context| context.request_id), + ) + .await } /// Send a middleware-specific 403 response without policy-advisor guidance. @@ -2922,7 +3183,15 @@ pub(crate) async fn send_middleware_deny_response( context: Option>, ) -> Result<()> { let body = middleware_deny_response_body(req, policy_name, denial, redacted_target, context); - send_forbidden_json(policy_name, body, client).await + send_json_response_internal( + policy_name, + body_with_request_id(body, context.and_then(|context| context.request_id)), + client, + "403 Forbidden", + None, + context.and_then(|context| context.request_id), + ) + .await } /// Send a platform-owned response for fail-closed middleware failures. @@ -2943,7 +3212,15 @@ pub(crate) async fn send_middleware_failure_response( if let Some(failure) = failure { insert_middleware_failure(&mut body, failure); } - send_forbidden_json(policy_name, body, client).await + send_json_response_internal( + policy_name, + body_with_request_id(body, context.and_then(|context| context.request_id)), + client, + "403 Forbidden", + None, + context.and_then(|context| context.request_id), + ) + .await } /// Add a middleware failure's stage and platform-owned code to a response @@ -2981,7 +3258,25 @@ pub(crate) async fn send_middleware_unavailable_response( code: openshell_supervisor_middleware::HttpFailureCode::Unavailable, }, ); - send_json_response(policy_name, body, client, "503 Service Unavailable").await + send_json_response_internal( + policy_name, + body_with_request_id(body, context.and_then(|context| context.request_id)), + client, + "503 Service Unavailable", + None, + context.and_then(|context| context.request_id), + ) + .await +} + +fn body_with_request_id( + mut body: serde_json::Value, + request_id: Option<&str>, +) -> serde_json::Value { + if let (Some(request_id), serde_json::Value::Object(fields)) = (request_id, &mut body) { + fields.insert("request_id".to_string(), serde_json::json!(request_id)); + } + body } /// Send a platform-owned 408 when the sandbox stops sending a request body @@ -3001,15 +3296,15 @@ pub(crate) async fn send_request_timeout_response( serde_json::json!("Request body was not received in time"), ); } - send_json_response(policy_name, body, client, "408 Request Timeout").await -} - -async fn send_forbidden_json( - policy_name: &str, - body: serde_json::Value, - client: &mut C, -) -> Result<()> { - send_json_response(policy_name, body, client, "403 Forbidden").await + send_json_response_internal( + policy_name, + body_with_request_id(body, context.and_then(|context| context.request_id)), + client, + "408 Request Timeout", + None, + context.and_then(|context| context.request_id), + ) + .await } /// Send a platform-owned JSON response with an explicit HTTP status. @@ -3030,15 +3325,70 @@ pub(crate) async fn send_json_response_with_allow( client: &mut C, status: &str, allowed_methods: Option<&'static str>, +) -> Result<()> { + send_json_response_internal(policy_name, body, client, status, allowed_methods, None).await +} + +pub(crate) async fn send_json_response_with_allow_and_request_id( + policy_name: &str, + body: serde_json::Value, + client: &mut C, + status: &str, + allowed_methods: Option<&'static str>, + request_id: &str, +) -> Result<()> { + send_json_response_internal( + policy_name, + body_with_request_id(body, Some(request_id)), + client, + status, + allowed_methods, + Some(request_id), + ) + .await +} + +pub(crate) async fn send_json_response_with_request_id( + policy_name: &str, + body: serde_json::Value, + client: &mut C, + status: &str, + request_id: &str, +) -> Result<()> { + send_json_response_with_allow_and_request_id( + policy_name, + body, + client, + status, + None, + request_id, + ) + .await +} + +async fn send_json_response_internal( + policy_name: &str, + body: serde_json::Value, + client: &mut C, + status: &str, + allowed_methods: Option<&'static str>, + request_id: Option<&str>, ) -> Result<()> { let body_bytes = body.to_string(); let allow = allowed_methods.map_or_else(String::new, |methods| format!("Allow: {methods}\r\n")); + let request_id = request_id.map_or_else(String::new, |request_id| { + format!( + "{}: {request_id}\r\n", + openshell_core::token_grant_failure::REQUEST_ID_HEADER + ) + }); let response = format!( "HTTP/1.1 {status}\r\n\ Content-Type: application/json\r\n\ Content-Length: {}\r\n\ X-OpenShell-Policy: {}\r\n\ {allow}\ + {request_id}\ Connection: close\r\n\ \r\n\ {}", @@ -3172,6 +3522,9 @@ fn deny_response_body( if let Some(binary) = binary { body.insert("binary".to_string(), serde_json::json!(binary)); } + if let Some(request_id) = context.request_id { + body.insert("request_id".to_string(), serde_json::json!(request_id)); + } body.insert( "rule_missing".to_string(), serde_json::Value::Object(rule_missing), @@ -3365,11 +3718,12 @@ where /// `already_read` are overflow bytes buffered while parsing the headers. They /// are validated here before being written so ambiguous framing cannot reach /// the peer. -async fn relay_chunked( +async fn relay_chunked_with_receipt_protection( reader: &mut R, writer: &mut W, already_read: &[u8], generation_guard: Option<&PolicyGenerationGuard>, + protect_receipt: bool, ) -> Result<()> where R: AsyncRead + Unpin, @@ -3394,6 +3748,7 @@ where let mut trailer_count = 0usize; let mut trailer_bytes = 0usize; loop { + let pending_start = input.pending_len(); let trailer_line = input .read_line( reader, @@ -3419,6 +3774,14 @@ where ); return Ok(()); } + if protect_receipt + && http_field_line_has_name( + &trailer_line, + openshell_core::token_grant_failure::REQUEST_ID_HEADER, + ) + { + input.pending.truncate(pending_start); + } if trailer_count == MAX_CHUNK_TRAILER_FIELDS { return Err(miette!( "Chunk trailers exceed {MAX_CHUNK_TRAILER_FIELDS} fields" @@ -3906,7 +4269,7 @@ mod tests { const TEXT_OPCODE: u8 = 0x1; #[derive(Debug, Clone, Copy)] - enum ResponseRelayScript { + pub(super) enum ResponseRelayScript { HeadersOnly, WholeBody, WholeBodyWithTrailer, @@ -4243,9 +4606,10 @@ mod tests { let mut reader = CountingReader::new(wire); let mut writer = tokio::io::sink(); - let error = relay_chunked(&mut reader, &mut writer, &[], None) - .await - .expect_err("an incomplete chunk must fail"); + let error = + relay_chunked_with_receipt_protection(&mut reader, &mut writer, &[], None, false) + .await + .expect_err("an incomplete chunk must fail"); assert!(error.to_string().contains("ended mid-chunk")); assert_eq!( @@ -4262,9 +4626,10 @@ mod tests { let mut reader = CountingReader::new(wire); let mut writer = tokio::io::sink(); - let error = relay_chunked(&mut reader, &mut writer, &[], None) - .await - .expect_err("an oversized chunk-size line must fail"); + let error = + relay_chunked_with_receipt_protection(&mut reader, &mut writer, &[], None, false) + .await + .expect_err("an oversized chunk-size line must fail"); assert!(error.to_string().contains("Chunk-size line exceeds")); } @@ -4281,9 +4646,10 @@ mod tests { let mut reader = CountingReader::new(wire); let mut writer = tokio::io::sink(); - let error = relay_chunked(&mut reader, &mut writer, &[], None) - .await - .expect_err("an oversized trailer block must fail"); + let error = + relay_chunked_with_receipt_protection(&mut reader, &mut writer, &[], None, false) + .await + .expect_err("an oversized trailer block must fail"); assert!(error.to_string().contains("Chunk trailers exceed")); } @@ -4298,9 +4664,10 @@ mod tests { let mut reader = CountingReader::new(wire); let mut writer = tokio::io::sink(); - let error = relay_chunked(&mut reader, &mut writer, &[], None) - .await - .expect_err("too many trailer fields must fail"); + let error = + relay_chunked_with_receipt_protection(&mut reader, &mut writer, &[], None, false) + .await + .expect_err("too many trailer fields must fail"); assert!( error @@ -4326,9 +4693,15 @@ mod tests { let mut reader = CountingReader::new(input); let mut writer = CountingWriter::default(); - let error = relay_chunked(&mut reader, &mut writer, already_read, None) - .await - .expect_err("ambiguous chunked framing must fail closed"); + let error = relay_chunked_with_receipt_protection( + &mut reader, + &mut writer, + already_read, + None, + false, + ) + .await + .expect_err("ambiguous chunked framing must fail closed"); assert!( error.to_string().contains("chunk"), @@ -4774,6 +5147,7 @@ mod tests { port: Some(443), binary: Some("/usr/bin/gh"), agent_proposals_enabled: true, + request_id: None, }), ); @@ -4838,6 +5212,7 @@ mod tests { port: Some(443), binary: Some("/usr/bin/gh"), agent_proposals_enabled: false, + request_id: None, }), ); @@ -4941,6 +5316,7 @@ mod tests { port: Some(443), binary: Some("/usr/bin/curl"), agent_proposals_enabled: true, + request_id: None, }), ); @@ -4979,6 +5355,7 @@ mod tests { port: Some(443), binary: Some("/usr/bin/gh"), agent_proposals_enabled: true, + request_id: Some("request-123"), }), ) .await @@ -4993,12 +5370,14 @@ mod tests { assert!(response.starts_with("HTTP/1.1 403 Forbidden")); assert!(response.contains("Content-Type: application/json")); assert!(response.contains("X-OpenShell-Policy: github-readonly")); + assert!(response.contains("x-openshell-request-id: request-123")); let (_, body) = response.split_once("\r\n\r\n").unwrap(); let body: serde_json::Value = serde_json::from_str(body).unwrap(); assert_eq!(body["error"], "policy_denied"); assert_eq!(body["method"], "POST"); assert_eq!(body["path"], "/user/repos"); + assert_eq!(body["request_id"], "request-123"); assert_eq!(body["rule_missing"]["host"], "api.github.com"); assert_eq!(body["next_steps"][2]["action"], "inspect_recent_denials"); assert!(body["agent_guidance"].as_str().unwrap().contains("retry")); @@ -5104,9 +5483,15 @@ mod tests { let mut relay_reader = tokio::io::BufReader::with_capacity(4096, relay_reader); let (mut relay_writer, mut upstream_reader) = tokio::io::duplex(4096); - relay_chunked(&mut relay_reader, &mut relay_writer, &[], None) - .await - .expect("chunked body should relay"); + relay_chunked_with_receipt_protection( + &mut relay_reader, + &mut relay_writer, + &[], + None, + false, + ) + .await + .expect("chunked body should relay"); let mut remaining = vec![0; pipelined_request.len()]; relay_reader.read_exact(&mut remaining).await.unwrap(); @@ -5132,9 +5517,15 @@ mod tests { let mut relay_reader = tokio::io::BufReader::with_capacity(4096, relay_reader); let (mut relay_writer, mut upstream_reader) = tokio::io::duplex(4096); - relay_chunked(&mut relay_reader, &mut relay_writer, already_read, None) - .await - .expect("chunked body with trailers should relay"); + relay_chunked_with_receipt_protection( + &mut relay_reader, + &mut relay_writer, + already_read, + None, + false, + ) + .await + .expect("chunked body with trailers should relay"); let mut remaining = vec![0; pipelined_request.len()]; relay_reader.read_exact(&mut remaining).await.unwrap(); @@ -5148,6 +5539,38 @@ mod tests { assert_eq!(forwarded, expected); } + #[tokio::test] + async fn relay_chunked_strips_platform_request_id_trailer() { + let mut reader = + b"3\r\nabc\r\n0\r\nX-OpenShell-Request-ID: forged\r\nX-Checksum: ok\r\n\r\n".as_slice(); + let mut writer = Vec::new(); + + relay_chunked_with_receipt_protection(&mut reader, &mut writer, &[], None, true) + .await + .expect("chunked body should relay"); + + assert_eq!(writer, b"3\r\nabc\r\n0\r\nX-Checksum: ok\r\n\r\n"); + } + + #[tokio::test] + async fn guarded_chunked_request_strips_platform_request_id_trailer() { + let mut reader = + b"3\r\nabc\r\n0\r\nx-openshell-request-id: forged\r\nX-Checksum: ok\r\n\r\n".as_slice(); + let mut writer = Vec::new(); + + relay_chunked_with_marker_guard( + &mut reader, + &mut writer, + &[], + RelayRequestOptions::default(), + true, + ) + .await + .expect("guarded chunked body should relay"); + + assert_eq!(writer, b"3\r\nabc\r\n0\r\nX-Checksum: ok\r\n\r\n"); + } + #[tokio::test] async fn relay_chunked_handles_tiny_chunks_without_an_aggregate_framing_limit() { let mut chunked_body = Vec::new(); @@ -5162,7 +5585,7 @@ mod tests { let mut reader = tokio::io::BufReader::with_capacity(RELAY_BUF_SIZE, CountingReader::new(wire)); let mut writer = CountingWriter::default(); - relay_chunked(&mut reader, &mut writer, &[], None) + relay_chunked_with_receipt_protection(&mut reader, &mut writer, &[], None, false) .await .expect("valid tiny chunks must not be rejected by an aggregate framing limit"); @@ -5190,7 +5613,14 @@ mod tests { let relay = tokio::spawn(async move { let mut relay_reader = tokio::io::BufReader::with_capacity(4096, relay_reader); let mut relay_writer = relay_writer; - relay_chunked(&mut relay_reader, &mut relay_writer, &[], None).await + relay_chunked_with_receipt_protection( + &mut relay_reader, + &mut relay_writer, + &[], + None, + false, + ) + .await }); let first_chunk = b"5\r\nhello\r\n"; @@ -5405,6 +5835,18 @@ mod tests { assert!(parse_body_length(headers).is_err()); } + #[test] + fn platform_request_id_is_stripped_without_touching_body() { + let suffix = b"4\r\nbody\r\n0\r\nX-OpenShell-Request-ID: forged-trailer\r\n\r\n"; + let mut request = b"POST /api HTTP/1.1\r\nHost: x\r\nTransfer-Encoding: chunked\r\nX-Opaque: \xff\xfe\r\nX-OpenShell-Request-ID: forged-head\r\nTrailer: X-OpenShell-Request-ID, X-Checksum\r\n\r\n".to_vec(); + request.extend_from_slice(suffix); + + let stripped = strip_platform_request_id_from_head(&request); + let mut expected = b"POST /api HTTP/1.1\r\nHost: x\r\nTransfer-Encoding: chunked\r\nX-Opaque: \xff\xfe\r\nTrailer: X-Checksum\r\n\r\n".to_vec(); + expected.extend_from_slice(suffix); + assert_eq!(stripped, expected); + } + #[test] fn strip_connection_nominated_headers_before_forwarding() { let raw = b"GET /api HTTP/1.1\r\nHost: x\r\nX-Guard: hidden\r\nConnection: keep-alive, x-guard\r\nKeep-Alive: timeout=5\r\nX-Visible: yes\r\n\r\n"; @@ -6642,6 +7084,9 @@ mod tests { let mut upstream = ObservedUpstream { upstream: &mut response, observer: Some(&observer), + correlation: None, + request_span: None, + track_request_span: true, }; let mut client = Vec::new(); relay_response( @@ -6953,7 +7398,7 @@ mod tests { } } - fn response_middleware_fixture( + pub(super) fn response_middleware_fixture( script: ResponseRelayScript, ) -> ( openshell_supervisor_middleware::ChainRunner, @@ -6989,7 +7434,7 @@ mod tests { (runner, chain) } - fn response_middleware_context<'a>( + pub(super) fn response_middleware_context<'a>( runner: &'a openshell_supervisor_middleware::ChainRunner, chain: &'a [openshell_supervisor_middleware::ChainEntry], method: &str, @@ -8644,7 +9089,13 @@ mod tests { failure_category: Some("timeout".into()), }], ); - let json = events[0].to_json().unwrap().to_string(); + let event = events[0].to_json().unwrap(); + assert_eq!( + event["unmapped"]["http_event_kind"], + "middleware_invocation" + ); + assert_eq!(event["unmapped"]["middleware_phase"], "response"); + let json = event.to_string(); for forbidden in [ "secret-response-body", "authorization", @@ -10895,6 +11346,7 @@ mod tests { )), ..Default::default() }, + false, ) .await .unwrap(); @@ -11040,6 +11492,7 @@ mod tests { &mut upstream_writer, wire, RelayRequestOptions::default(), + false, ) .await .expect_err("encoded placeholder must fail closed"); @@ -11060,6 +11513,7 @@ mod tests { &mut output, wire, RelayRequestOptions::default(), + false, ) .await .expect_err("ambiguous trailer framing must fail closed"); diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs index d189963a8b..11ecc60ee5 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs @@ -6,6 +6,7 @@ mod http_hooks; use super::*; +use crate::l7::correlation::{HttpOutcome, RequestCorrelation, RequestStage}; /// Default wall-clock bound shared by whole-body stages in one response. pub const DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT: std::time::Duration = @@ -53,6 +54,7 @@ pub(super) struct RelayResponseOptions<'a> { pub(super) client_requested_upgrade: bool, pub(super) websocket: Option, pub(super) observer: Option<&'a EndpointObserver>, + pub(super) correlation: Option<&'a RequestCorrelation>, } impl Default for RelayResponseOptions<'_> { @@ -62,6 +64,7 @@ impl Default for RelayResponseOptions<'_> { client_requested_upgrade: true, websocket: None, observer: None, + correlation: None, } } } @@ -84,7 +87,7 @@ where // Forward interim responses unchanged, but retain the final response head // until response middleware preflight completes. let mut informational_header_bytes = 0; - let header_end = loop { + let mut header_end = loop { let header_end = buf .windows(4) .position(|w| w == b"\r\n\r\n") @@ -100,16 +103,27 @@ where } if let Some(header_end) = header_end { + if options.correlation.is_some() + && let Err(error) = validate_http_response_header_block(&buf[..header_end]) + { + if let Some(observer) = options.observer.as_ref() { + observer.observe(EndpointResult::TransportFailed); + } + record_upstream_response_error(options.correlation, "upstream_response_headers"); + return Err(error); + } let header_str = String::from_utf8_lossy(&buf[..header_end]); if matches!( parse_observed_http_status_code(&header_str), Some(100 | 102..=199) ) { informational_header_bytes += header_end; - client - .write_all(&buf[..header_end]) - .await - .into_diagnostic()?; + let sanitized = if options.correlation.is_some() { + strip_platform_request_id_from_head(&buf[..header_end]) + } else { + Cow::Borrowed(&buf[..header_end]) + }; + client.write_all(&sanitized).await.into_diagnostic()?; client.flush().await.into_diagnostic()?; buf.drain(..header_end); continue; @@ -122,6 +136,7 @@ where if let Some(observer) = options.observer.as_ref() { observer.observe(EndpointResult::TransportFailed); } + record_upstream_response_error(options.correlation, "upstream_response_incomplete"); if !buf.is_empty() { client.write_all(&buf).await.into_diagnostic()?; } @@ -130,9 +145,21 @@ where buf.extend_from_slice(&tmp[..n]); }; + if options.correlation.is_some() + && let Cow::Owned(sanitized) = strip_platform_request_id_from_head(&buf[..header_end]) + { + let overflow = buf.split_off(header_end); + buf = sanitized; + header_end = buf.len(); + buf.extend_from_slice(&overflow); + } + // Parse response framing let header_str = String::from_utf8_lossy(&buf[..header_end]); let observed_status = parse_observed_http_status_code(&header_str); + if let (Some(correlation), Some(status_code)) = (options.correlation, observed_status) { + correlation.record_upstream_status(status_code); + } if let Some(observer) = options.observer.as_ref() { match observed_status { Some(status_code) if status_code < 400 => { @@ -172,10 +199,8 @@ where options.websocket_extensions, options.websocket.as_ref(), )?; - client - .write_all(&buf[..header_end]) - .await - .into_diagnostic()?; + let final_head = final_response_head(&buf[..header_end], options.correlation); + client.write_all(&final_head).await.into_diagnostic()?; client.flush().await.into_diagnostic()?; let overflow = buf[header_end..].to_vec(); debug!( @@ -183,6 +208,7 @@ where overflow_bytes = overflow.len(), "101 Switching Protocols — signaling protocol upgrade" ); + record_allowed_response(options.correlation); return Ok(RelayOutcome::Upgraded { overflow, websocket_permessage_deflate, @@ -202,6 +228,7 @@ where body_length, server_wants_close || http_10_closes_by_default, event_stream, + options.correlation, )) .await? { @@ -212,13 +239,15 @@ where }; } + let final_head = final_response_head(&buf[..header_end], options.correlation); + // Bodiless responses (HEAD, 1xx, 204, 304): forward headers only, skip body if is_bodiless_response(request_method, status_code) { - client - .write_all(&buf[..header_end]) - .await - .into_diagnostic()?; - return finish_response(client, server_wants_close || http_10_closes_by_default).await; + client.write_all(&final_head).await.into_diagnostic()?; + let outcome = + finish_response(client, server_wants_close || http_10_closes_by_default).await?; + record_allowed_response(options.correlation); + return Ok(outcome); } // No explicit framing (no Content-Length, no Transfer-Encoding). @@ -227,7 +256,7 @@ where if server_wants_close || http_10_closes_by_default || event_stream { // Server indicated it will close, or this is a streaming response // such as SSE where the body is intentionally delimited by EOF. - let before_end = &buf[..header_end - 2]; + let before_end = &final_head[..final_head.len() - 2]; client.write_all(before_end).await.into_diagnostic()?; if server_wants_close || http_10_closes_by_default { client @@ -249,25 +278,21 @@ where } client.flush().await.into_diagnostic()?; client.shutdown().await.into_diagnostic()?; + record_allowed_response(options.correlation); return Ok(RelayOutcome::Consumed); } // No Connection: close — an HTTP/1.1 keep-alive server that omits // framing headers has an empty body. Forward headers and continue // the relay loop instead of blocking on relay_until_eof. debug!("BodyLength::None without Connection: close — treating body as empty"); - client - .write_all(&buf[..header_end]) - .await - .into_diagnostic()?; + client.write_all(&final_head).await.into_diagnostic()?; client.flush().await.into_diagnostic()?; + record_allowed_response(options.correlation); return Ok(RelayOutcome::Reusable); } // Hold buffered chunk framing until relay_chunked validates it. - client - .write_all(&buf[..header_end]) - .await - .into_diagnostic()?; + client.write_all(&final_head).await.into_diagnostic()?; let overflow = &buf[header_end..]; let overflow_len = (buf.len() - header_end) as u64; @@ -283,7 +308,14 @@ where } } BodyLength::Chunked => { - relay_chunked(upstream, client, overflow, None).await?; + relay_chunked_with_receipt_protection( + upstream, + client, + overflow, + None, + options.correlation.is_some(), + ) + .await?; } BodyLength::None => unreachable!(), } @@ -294,7 +326,212 @@ where "relay_response complete (explicit framing)" ); - finish_response(client, server_wants_close || http_10_closes_by_default).await + let outcome = finish_response(client, server_wants_close || http_10_closes_by_default).await?; + record_allowed_response(options.correlation); + Ok(outcome) +} + +fn record_allowed_response(correlation: Option<&RequestCorrelation>) { + if let Some(correlation) = correlation { + correlation.record_outcome(HttpOutcome::Allowed); + } +} + +fn record_denied_response(correlation: Option<&RequestCorrelation>, stage: &'static str) { + if let Some(correlation) = correlation { + correlation.record_rejection(403, HttpOutcome::Denied, Some(stage)); + } +} + +fn record_upstream_response_error(correlation: Option<&RequestCorrelation>, stage: &'static str) { + if let Some(correlation) = correlation { + correlation.record_failure(HttpOutcome::UpstreamError, stage); + } +} + +fn record_committed_response_error( + correlation: Option<&RequestCorrelation>, + outcome: HttpOutcome, + stage: &'static str, +) { + if let Some(correlation) = correlation { + correlation.record_failure(outcome, stage); + } +} + +fn validate_http_response_header_block(headers: &[u8]) -> Result<()> { + if !headers.ends_with(b"\r\n\r\n") { + return Err(miette!("HTTP response header block is incomplete")); + } + + for (index, byte) in headers.iter().copied().enumerate() { + match byte { + b'\r' if headers.get(index + 1) != Some(&b'\n') => { + return Err(miette!("HTTP response headers contain a bare CR")); + } + b'\n' if index == 0 || headers.get(index - 1) != Some(&b'\r') => { + return Err(miette!("HTTP response headers contain a bare LF")); + } + _ => {} + } + } + + let header_lines = &headers[..headers.len() - 2]; + let mut lines = header_lines + .split(|byte| *byte == b'\n') + .map(|line| line.strip_suffix(b"\r").unwrap_or(line)); + let status_line = lines + .next() + .ok_or_else(|| miette!("HTTP response status line is missing"))?; + validate_http_response_status_line(status_line)?; + + while let Some(line) = lines.next() { + if line.is_empty() { + if lines.next().is_some() { + return Err(miette!( + "HTTP response header block contains an unexpected empty line" + )); + } + break; + } + if line + .first() + .is_some_and(|byte| matches!(byte, b' ' | b'\t')) + { + return Err(miette!( + "HTTP response header continuation lines are not supported" + )); + } + let colon = line + .iter() + .position(|byte| *byte == b':') + .ok_or_else(|| miette!("HTTP response header field is malformed"))?; + let name = &line[..colon]; + if name.last().is_some_and(|byte| matches!(byte, b' ' | b'\t')) { + return Err(miette!( + "HTTP response header field contains whitespace before the colon" + )); + } + http::HeaderName::from_bytes(name) + .map_err(|_| miette!("HTTP response field name is malformed"))?; + http::HeaderValue::from_bytes(&line[colon + 1..]) + .map_err(|_| miette!("HTTP response field value is malformed"))?; + } + Ok(()) +} + +fn validate_http_response_status_line(status_line: &[u8]) -> Result<()> { + let remainder = if let Some(remainder) = status_line.strip_prefix(b"HTTP/1.0 ") { + remainder + } else if let Some(remainder) = status_line.strip_prefix(b"HTTP/1.1 ") { + remainder + } else { + return Err(miette!( + "HTTP response status line must use HTTP/1.0 or HTTP/1.1" + )); + }; + if remainder.len() < 3 || (remainder.len() > 3 && remainder[3] != b' ') { + return Err(miette!("HTTP response status line is malformed")); + } + http::StatusCode::from_bytes(&remainder[..3]) + .map_err(|_| miette!("HTTP response status code is invalid"))?; + http::HeaderValue::from_bytes(remainder.get(4..).unwrap_or_default()) + .map_err(|_| miette!("HTTP response reason phrase is malformed"))?; + Ok(()) +} + +fn final_response_head<'a>( + headers: &'a [u8], + correlation: Option<&RequestCorrelation>, +) -> Cow<'a, [u8]> { + let Some(correlation) = correlation else { + return Cow::Borrowed(headers); + }; + let stripped = + strip_platform_request_id_from_connection(strip_platform_request_id_from_head(headers)); + let Some(header_end) = stripped.windows(4).position(|window| window == b"\r\n\r\n") else { + return stripped; + }; + let name = openshell_core::token_grant_failure::REQUEST_ID_HEADER; + let request_id = correlation.request_id(); + let mut output = Vec::with_capacity(stripped.len() + name.len() + request_id.len() + 4); + output.extend_from_slice(&stripped[..header_end + 2]); + output.extend_from_slice(name.as_bytes()); + output.extend_from_slice(b": "); + output.extend_from_slice(request_id.as_bytes()); + output.extend_from_slice(b"\r\n\r\n"); + output.extend_from_slice(&stripped[header_end + 4..]); + Cow::Owned(output) +} + +fn strip_platform_request_id_from_connection(headers: Cow<'_, [u8]>) -> Cow<'_, [u8]> { + let header_end = headers + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map_or(headers.len(), |position| position + 4); + let (header_block, buffered_suffix) = headers.split_at(header_end); + let protected = openshell_core::token_grant_failure::REQUEST_ID_HEADER.as_bytes(); + let mut output: Option> = None; + let mut position = 0usize; + + while position < header_block.len() { + let line_end = header_block[position..] + .windows(2) + .position(|window| window == b"\r\n") + .map_or(header_block.len(), |end| position + end); + let line = &header_block[position..line_end]; + if line.is_empty() { + if position + 2 < header_block.len() { + output.get_or_insert_with(|| header_block[..position].to_vec()); + } + if let Some(output) = &mut output { + output.extend_from_slice(b"\r\n"); + } + break; + } + + if let Some(colon) = line.iter().position(|byte| *byte == b':') + && trim_ascii_ows(&line[..colon]).eq_ignore_ascii_case(b"connection") + && line[colon + 1..] + .split(|byte| *byte == b',') + .any(|value| trim_ascii_ows(value).eq_ignore_ascii_case(protected)) + { + let output = output.get_or_insert_with(|| { + let mut output = Vec::with_capacity(headers.len()); + output.extend_from_slice(&header_block[..position]); + output + }); + let mut retained = line[colon + 1..] + .split(|byte| *byte == b',') + .map(trim_ascii_ows) + .filter(|token| !token.is_empty() && !token.eq_ignore_ascii_case(protected)); + if let Some(first) = retained.next() { + output.extend_from_slice(&line[..=colon]); + output.push(b' '); + output.extend_from_slice(first); + for token in retained { + output.extend_from_slice(b", "); + output.extend_from_slice(token); + } + output.extend_from_slice(b"\r\n"); + } + position = line_end.saturating_add(2); + continue; + } + if line_end + 2 > header_block.len() { + output.get_or_insert_with(|| header_block[..position].to_vec()); + } + if let Some(output) = &mut output { + output.extend_from_slice(line); + output.extend_from_slice(b"\r\n"); + } + position = line_end.saturating_add(2); + } + + if let Some(output) = &mut output { + output.extend_from_slice(buffered_suffix); + } + output.map_or(headers, Cow::Owned) } /// Body framing determines when delivery finishes, not whether another @@ -326,6 +563,7 @@ async fn relay_response_through_middleware( body_length: BodyLength, server_wants_close: bool, event_stream: bool, + response_correlation: Option<&RequestCorrelation>, ) -> Result> where U: AsyncRead + Unpin, @@ -351,6 +589,50 @@ where } Err(error) => Err(error), }; + let span = response_correlation.map_or_else(tracing::Span::none, |correlation| { + correlation.stage(RequestStage::ResponseMiddleware) + }); + relay_described_response_through_middleware( + request_method, + upstream, + client, + middleware, + buffered, + header_end, + status_code, + body_length, + server_wants_close, + event_stream, + response_correlation, + parsed, + ) + .instrument(span) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn relay_described_response_through_middleware( + request_method: &str, + upstream: &mut U, + client: &mut C, + middleware: HttpResponseMiddlewareRelay<'_>, + buffered: &[u8], + header_end: usize, + status_code: u16, + body_length: BodyLength, + server_wants_close: bool, + event_stream: bool, + response_correlation: Option<&RequestCorrelation>, + parsed: Result<( + Vec, + ParsedResponseHead, + )>, +) -> Result> +where + U: AsyncRead + Unpin, + C: AsyncWrite + Unpin, +{ + let header_bytes = &buffered[..header_end]; if let Some(guard) = middleware.generation_guard { guard.ensure_current()?; } @@ -359,6 +641,7 @@ where Err(error) => { debug!(error = %error, "HTTP response head normalization failed"); emit_http_response_middleware_failure( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -370,6 +653,9 @@ where middleware.policy_name, &middleware.target, None, + HttpOutcome::PlatformError, + "response_middleware_headers", + response_correlation, ) .await?; return Ok(Some(RelayOutcome::Consumed)); @@ -391,6 +677,7 @@ where body_length, server_wants_close, event_stream, + response_correlation, )) .await; } @@ -400,6 +687,7 @@ where "HTTP response middleware chain mixes HTTP hook versions" ); emit_http_response_middleware_failure( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -411,6 +699,9 @@ where middleware.policy_name, &middleware.target, None, + HttpOutcome::PlatformError, + "response_middleware_hook_version", + response_correlation, ) .await?; return Ok(Some(RelayOutcome::Consumed)); @@ -448,6 +739,7 @@ where Err(error) => { debug!(error = %error, "HTTP response middleware preflight failed"); emit_http_response_middleware_failure( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -459,6 +751,9 @@ where middleware.policy_name, &middleware.target, None, + HttpOutcome::PlatformError, + "response_middleware_preflight", + response_correlation, ) .await?; return Ok(Some(RelayOutcome::Consumed)); @@ -499,7 +794,7 @@ where )); } } - crate::telemetry::emit_decision(preflight_events, None); + crate::l7::correlation::emit_request_events(response_correlation, preflight_events); if !preflight.allowed { if let Some(denial) = preflight.denial.as_ref() { send_response_middleware_denial( @@ -508,6 +803,8 @@ where middleware.policy_name, &middleware.target, denial, + "response_middleware_preflight", + response_correlation, ) .await?; } else { @@ -517,19 +814,22 @@ where middleware.policy_name, &middleware.target, None, + HttpOutcome::PlatformError, + "response_middleware_preflight", + response_correlation, ) .await?; } return Ok(Some(RelayOutcome::Consumed)); } - crate::telemetry::emit_decision( + crate::l7::correlation::emit_request_events( + response_correlation, http_response_decision_events( middleware.policy_name, &middleware.target, status_code, &preflight.invocations, ), - None, ); let Some(mut session) = preflight.session else { @@ -550,8 +850,10 @@ where body_length, server_wants_close, event_stream, + response_correlation, ) .await?; + record_allowed_response(response_correlation); return Ok(Some(outcome)); }; @@ -577,13 +879,16 @@ where false, true, )); - crate::telemetry::emit_decision(events, None); + crate::l7::correlation::emit_request_events(response_correlation, events); send_response_delivery_failure( client, request_method, middleware.policy_name, &middleware.target, None, + HttpOutcome::PlatformError, + "response_middleware_finish", + response_correlation, ) .await?; return Ok(Some(RelayOutcome::Consumed)); @@ -593,7 +898,8 @@ where guard.ensure_current()?; } let mut headers = preflight.headers; - crate::telemetry::emit_decision( + crate::l7::correlation::emit_request_events( + response_correlation, http_response_diagnostic_events( middleware.policy_name, &middleware.target, @@ -601,7 +907,6 @@ where &finish.invocations, &finish.findings, ), - None, ); if finish.strip_stale_integrity_headers { strip_response_integrity_headers(&mut headers); @@ -614,8 +919,10 @@ where server_wants_close, &[], ); + let head = final_response_head(&head, response_correlation); client.write_all(&head).await.into_diagnostic()?; client.flush().await.into_diagnostic()?; + record_allowed_response(response_correlation); return Ok(Some(if server_wants_close { RelayOutcome::Consumed } else { @@ -648,6 +955,7 @@ where server_wants_close || close_delimited_output, downstream_trailers, ); + let streaming_head = final_response_head(&streaming_head, response_correlation); let mut committed = !whole_body; if committed { if let Err(error) = client.write_all(&streaming_head).await { @@ -681,6 +989,7 @@ where &middleware.target, status_code, &connection_nominated_headers, + response_correlation, ) .await; let trailers = match body_result { @@ -724,8 +1033,21 @@ where middleware_stop.is_some(), )); } - crate::telemetry::emit_decision(events, None); + crate::l7::correlation::emit_request_events(response_correlation, events); if committed { + if middleware_stop.is_some_and(|stop| stop.failure.denial.is_some()) { + record_committed_response_error( + response_correlation, + HttpOutcome::Denied, + "response_middleware_body", + ); + } else { + record_committed_response_error( + response_correlation, + HttpOutcome::PlatformError, + "response_middleware_body", + ); + } return Err(error); } debug!(error = %error, "HTTP response processing failed before commitment"); @@ -736,6 +1058,8 @@ where middleware.policy_name, &middleware.target, denial, + "response_middleware_body", + response_correlation, ) .await?; } else { @@ -745,6 +1069,9 @@ where middleware.policy_name, &middleware.target, None, + HttpOutcome::PlatformError, + "response_middleware_body", + response_correlation, ) .await?; } @@ -780,8 +1107,21 @@ where true, )); } - crate::telemetry::emit_decision(events, None); + crate::l7::correlation::emit_request_events(response_correlation, events); if committed { + if error.denial.is_some() { + record_committed_response_error( + response_correlation, + HttpOutcome::Denied, + "response_middleware_finish", + ); + } else { + record_committed_response_error( + response_correlation, + HttpOutcome::PlatformError, + "response_middleware_finish", + ); + } return Err(miette!( "HTTP response middleware failed after commitment: {error}" )); @@ -794,6 +1134,8 @@ where middleware.policy_name, &middleware.target, denial, + "response_middleware_finish", + response_correlation, ) .await?; } else { @@ -803,6 +1145,9 @@ where middleware.policy_name, &middleware.target, None, + HttpOutcome::PlatformError, + "response_middleware_finish", + response_correlation, ) .await?; } @@ -812,7 +1157,8 @@ where if let Some(guard) = middleware.generation_guard { guard.ensure_current()?; } - crate::telemetry::emit_decision( + crate::l7::correlation::emit_request_events( + response_correlation, http_response_diagnostic_events( middleware.policy_name, &middleware.target, @@ -820,7 +1166,6 @@ where &finish.invocations, &finish.findings, ), - None, ); if whole_body && !committed { @@ -855,6 +1200,7 @@ where server_wants_close, &trailer_names, ); + let head = final_response_head(&head, response_correlation); client.write_all(&head).await.into_diagnostic()?; if matches!(framing, ResponseFraming::Chunked) { for unit in &finish.body_units { @@ -879,6 +1225,7 @@ where } } client.flush().await.into_diagnostic()?; + record_allowed_response(response_correlation); Ok(Some( if server_wants_close || (committed && close_delimited_output) @@ -905,6 +1252,7 @@ async fn relay_headers_only_response( body_length: BodyLength, server_wants_close: bool, event_stream: bool, + response_correlation: Option<&RequestCorrelation>, ) -> Result where U: AsyncRead + Unpin, @@ -918,6 +1266,7 @@ where server_wants_close, declared_trailers, ); + let head = final_response_head(&head, response_correlation); client.write_all(&head).await.into_diagnostic()?; if is_bodiless_response(request_method, status_code) { @@ -939,7 +1288,16 @@ where relay_fixed(upstream, client, remaining, None).await?; } } - BodyLength::Chunked => relay_chunked(upstream, client, overflow, None).await?, + BodyLength::Chunked => { + relay_chunked_with_receipt_protection( + upstream, + client, + overflow, + None, + response_correlation.is_some(), + ) + .await?; + } BodyLength::None if server_wants_close || event_stream => { if !overflow.is_empty() { client.write_all(overflow).await.into_diagnostic()?; @@ -1002,12 +1360,14 @@ fn http_response_diagnostic_events( } fn emit_http_response_diagnostics( + correlation: Option<&RequestCorrelation>, policy_name: &str, target: &HttpRequestTarget, status_code: u16, diagnostics: &openshell_supervisor_middleware::HttpResponseDiagnostics, ) { - crate::telemetry::emit_decision( + crate::l7::correlation::emit_request_events( + correlation, http_response_diagnostic_events( policy_name, target, @@ -1015,7 +1375,6 @@ fn emit_http_response_diagnostics( &diagnostics.invocations, &diagnostics.findings, ), - None, ); } @@ -1035,7 +1394,7 @@ pub(super) fn http_response_middleware_invocation_events( openshell_supervisor_middleware::HttpResponseInvocationOutcome::BlockDelivery | openshell_supervisor_middleware::HttpResponseInvocationOutcome::FailClosed ); - openshell_ocsf::HttpActivityBuilder::new(ocsf_ctx()) + openshell_ocsf::HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(openshell_ocsf::ActivityId::Other) .action(if blocked { openshell_ocsf::ActionId::Denied @@ -1076,6 +1435,8 @@ pub(super) fn http_response_middleware_invocation_events( u16::try_from(target.port).unwrap_or_default(), )) .firewall_rule(policy_name, "supervisor-middleware") + .unmapped("http_event_kind", "middleware_invocation") + .unmapped("middleware_phase", "response") .unmapped("middleware_config", invocation.config_name.as_str()) .unmapped( "middleware_implementation", @@ -1183,18 +1544,22 @@ pub(super) fn http_response_middleware_fail_open_finding_event( } fn emit_http_response_middleware_failure( + correlation: Option<&RequestCorrelation>, policy_name: &str, target: &HttpRequestTarget, status_code: u16, committed: bool, ) { - crate::telemetry::egress_emit!(http_response_middleware_failure_event( - policy_name, - target, - status_code, - committed, - true, - )); + crate::l7::correlation::emit_request_events( + correlation, + [http_response_middleware_failure_event( + policy_name, + target, + status_code, + committed, + true, + )], + ); } fn http_response_middleware_failure_event( @@ -1361,7 +1726,8 @@ fn validate_http_field_value(value: &str) -> Result<()> { } fn is_protected_response_field(name: &str) -> bool { - name.eq_ignore_ascii_case("content-length") + name.eq_ignore_ascii_case(openshell_core::token_grant_failure::REQUEST_ID_HEADER) + || name.eq_ignore_ascii_case("content-length") || is_hidden_response_field(name) || openshell_supervisor_middleware::headers::is_response_credential_header(name) } @@ -1544,6 +1910,7 @@ async fn relay_normalized_response_body( target: &HttpRequestTarget, status_code: u16, connection_nominated_headers: &[String], + correlation: Option<&RequestCorrelation>, ) -> Result> where R: AsyncRead + Unpin, @@ -1558,6 +1925,7 @@ where policy_name, target, status_code, + correlation, }; match body_length { BodyLength::ContentLength(mut remaining) => { @@ -1761,6 +2129,7 @@ struct ResponseOutputState<'a> { policy_name: &'a str, target: &'a HttpRequestTarget, status_code: u16, + correlation: Option<&'a RequestCorrelation>, } #[derive(Debug)] @@ -1824,6 +2193,7 @@ async fn expire_whole_body_deadline( let diagnostics = session.take_diagnostics(); if output.is_ok() { emit_http_response_diagnostics( + framing.correlation, framing.policy_name, framing.target, framing.status_code, @@ -1968,6 +2338,7 @@ async fn process_response_unit( let diagnostics = session.take_diagnostics(); if output.is_ok() { emit_http_response_diagnostics( + framing.correlation, framing.policy_name, framing.target, framing.status_code, @@ -2098,7 +2469,10 @@ async fn send_response_middleware_denial( policy_name: &str, target: &HttpRequestTarget, denial: &openshell_supervisor_middleware::MiddlewareDenial, + stage: &'static str, + correlation: Option<&RequestCorrelation>, ) -> Result<()> { + record_denied_response(correlation, stage); let mut body = serde_json::Map::new(); body.insert("error".into(), serde_json::json!("middleware_denied")); body.insert( @@ -2123,7 +2497,8 @@ async fn send_response_middleware_denial( "HTTP/1.1 403 Forbidden\r\nContent-Type: application/json\r\nContent-Length: {}\r\nX-OpenShell-Policy: {policy_name}\r\nConnection: close\r\n\r\n", body.len() ); - client.write_all(head.as_bytes()).await.into_diagnostic()?; + let head = final_response_head(head.as_bytes(), correlation); + client.write_all(&head).await.into_diagnostic()?; if !request_method.eq_ignore_ascii_case("HEAD") { client.write_all(&body).await.into_diagnostic()?; } @@ -2133,13 +2508,25 @@ async fn send_response_middleware_denial( /// Answer a response that middleware failed before its head was committed. /// An HTTP hook failure carries its platform-owned code and stage. +#[allow(clippy::too_many_arguments)] async fn send_response_delivery_failure( client: &mut C, request_method: &str, policy_name: &str, target: &HttpRequestTarget, failure: Option<&openshell_supervisor_middleware::MiddlewareFailureDetail>, + outcome: HttpOutcome, + stage: &'static str, + correlation: Option<&RequestCorrelation>, ) -> Result<()> { + if let Some(correlation) = correlation { + if outcome == HttpOutcome::UpstreamError { + correlation.record_response_status(502); + correlation.record_failure(outcome, stage); + } else { + correlation.record_rejection(502, outcome, Some(stage)); + } + } let mut body = serde_json::json!({ "error": "response_delivery_failed", "detail": "The upstream request may have completed, but OpenShell could not deliver its response. Retrying may repeat upstream side effects.", @@ -2153,12 +2540,14 @@ async fn send_response_delivery_failure( if let Some(failure) = failure { insert_middleware_failure(&mut body, failure); } + let body = body_with_request_id(body, correlation.map(RequestCorrelation::request_id)); let body = serde_json::to_vec(&body).into_diagnostic()?; let head = format!( "HTTP/1.1 502 Bad Gateway\r\nContent-Type: application/json\r\nContent-Length: {}\r\nX-OpenShell-Policy: {policy_name}\r\nConnection: close\r\n\r\n", body.len() ); - client.write_all(head.as_bytes()).await.into_diagnostic()?; + let head = final_response_head(head.as_bytes(), correlation); + client.write_all(&head).await.into_diagnostic()?; if !request_method.eq_ignore_ascii_case("HEAD") { client.write_all(&body).await.into_diagnostic()?; } @@ -2237,9 +2626,13 @@ pub(super) fn response_is_event_stream(headers: &str) -> bool { #[cfg(test)] mod tests { + use super::{final_response_head, strip_platform_request_id_from_connection}; + use crate::l7::correlation::HttpOutcome; + use crate::l7::rest::strip_platform_request_id_from_head; use openshell_supervisor_middleware::{ HttpResponseInvocation, HttpResponseInvocationOutcome as Outcome, NamespacedFinding, }; + use std::borrow::Cow; fn invocation(outcome: Outcome, failed: bool) -> HttpResponseInvocation { HttpResponseInvocation { @@ -2278,8 +2671,9 @@ mod tests { for committed in [false, true] { let (events, received) = exported("warn", || { super::emit_http_response_middleware_failure( + None, "response-policy", - &super::HttpRequestTarget::default(), + &HttpRequestTarget::default(), 200, committed, ); @@ -2312,8 +2706,9 @@ mod tests { ..Default::default() }; super::emit_http_response_diagnostics( + None, "response-policy", - &super::HttpRequestTarget::default(), + &HttpRequestTarget::default(), 200, &diagnostics, ); @@ -2340,7 +2735,7 @@ mod tests { #[test] fn fail_closed_invocations_and_enforced_failures_record_the_denial() { - let target = super::HttpRequestTarget::default(); + let target = HttpRequestTarget::default(); for (outcome, failed, action_id) in [ (Outcome::BlockDelivery, false, 2), (Outcome::FailClosed, true, 2), @@ -2368,12 +2763,655 @@ mod tests { } } + #[test] + fn late_fail_open_invocation_does_not_hide_block_delivery() { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = test_correlation("late-response-receipt"); + let target = HttpRequestTarget { + method: "GET".into(), + scheme: "https".into(), + host: "example.test".into(), + path: "/data".into(), + port: 443, + ..Default::default() + }; + correlation.emit( + openshell_ocsf::HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .http_request(openshell_ocsf::HttpRequest::new( + "GET", + openshell_ocsf::Url::new("https", "example.test", "/data", 443), + )) + .action(openshell_ocsf::ActionId::Allowed) + .status(openshell_ocsf::StatusId::Success) + .build(), + ); + correlation.response_selected(200); + for (outcome, failed) in [(Outcome::FailOpen, true), (Outcome::BlockDelivery, false)] { + for event in super::http_response_middleware_invocation_events( + "response-policy", + &target, + 200, + &[invocation(outcome, failed)], + ) { + correlation.emit(event); + } + } + drop(correlation); + let events: Vec<_> = capture + .json_events() + .into_iter() + .filter(|event| event["class_uid"] == 4002) + .collect(); + assert_eq!(events.len(), 3); + assert_eq!(events[0]["unmapped"]["http_event_kind"], "request"); + assert_eq!(events[0]["action"], "Allowed"); + assert_eq!(events[1]["action"], "Other"); + assert_eq!( + events[1]["unmapped"]["response_middleware_outcome"], + "failopen" + ); + assert_eq!(events[2]["action"], "Denied"); + for event in &events[1..] { + assert_eq!( + event["unmapped"]["http_event_kind"], + "middleware_invocation" + ); + assert_eq!(event["unmapped"]["middleware_phase"], "response"); + assert_eq!(event["http_response"]["code"], 200); + } + assert_eq!( + events[2]["unmapped"]["response_middleware_outcome"], + "blockdelivery" + ); + assert!( + events + .iter() + .all(|event| event["metadata"]["correlation_uid"] == "late-response-receipt") + ); + } + use super::{ - ResponseFraming, parse_connection_keep_alive, parse_response_head_for_middleware, - response_is_http_10, serialize_response_head, + RelayResponseOptions, ResponseFraming, parse_connection_keep_alive, + parse_response_head_for_middleware, relay_response, response_is_http_10, + serialize_response_head, validate_http_response_header_block, }; - use crate::l7::provider::BodyLength; - use openshell_core::proto::HttpHeader; + use crate::l7::provider::{BodyLength, L7Request, RelayOutcome}; + use crate::l7::relay::L7EvalContext; + use crate::l7::rest::tests::{ + ResponseRelayScript, response_middleware_context, response_middleware_fixture, + }; + use openshell_core::proto::{HttpHeader, HttpRequestTarget}; + use std::collections::HashMap; + + async fn relay_bytes( + mut upstream: &[u8], + correlation: Option<&crate::l7::correlation::RequestCorrelation>, + ) -> (miette::Result, Vec) { + let mut client = Vec::new(); + let outcome = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation, + ..Default::default() + }, + None, + ) + .await; + (outcome, client) + } + + fn test_correlation_for_method( + request_id: &str, + method: &str, + ) -> crate::l7::correlation::RequestCorrelation { + let request = L7Request { + action: method.into(), + target: "/data".into(), + query_params: HashMap::new(), + raw_header: b"GET /data HTTP/1.1\r\nHost: example.test\r\n\r\n".to_vec(), + body_length: BodyLength::None, + }; + crate::l7::correlation::RequestCorrelation::with_request_id( + &request, + &L7EvalContext { + host: "example.test".into(), + port: 443, + ..Default::default() + }, + request_id.into(), + ) + } + + fn test_correlation(request_id: &str) -> crate::l7::correlation::RequestCorrelation { + test_correlation_for_method(request_id, "GET") + } + + #[test] + fn untouched_receipt_heads_remain_borrowed() { + let head = b"HTTP/1.1 200\r\nX-Opaque: \xff\xfe\r\nX-Duplicate: first\r\nX-Duplicate: second\r\nConnection: keep-alive\r\nTrailer: X-Checksum\r\n\r\nbody"; + let uncorrelated = final_response_head(head, None); + assert!(matches!(uncorrelated, Cow::Borrowed(_))); + assert_eq!(uncorrelated.as_ref(), head); + + let stripped = strip_platform_request_id_from_head(head); + assert!(matches!(stripped, Cow::Borrowed(_))); + let stripped = strip_platform_request_id_from_connection(stripped); + assert!(matches!(stripped, Cow::Borrowed(_))); + assert_eq!(stripped.as_ref(), head); + } + + #[test] + fn receipt_head_rewrite_preserves_unrelated_bytes_and_buffered_body() { + let correlation = test_correlation("receipt-cow"); + let head = b"HTTP/1.1 200\r\nX-Opaque: \xff\xfe\r\nX-Duplicate: first\r\nX-Duplicate: second\r\nX-OpenShell-Request-ID: forged\r\nx-openshell-request-id: forged-again\r\nConnection: keep-alive, X-OpenShell-Request-ID, , Upgrade\r\nTrailer: X-OpenShell-Request-ID, X-Checksum\r\n\r\n\x00\xffbody"; + + let rewritten = final_response_head(head, Some(&correlation)); + assert!(matches!(rewritten, Cow::Owned(_))); + assert_eq!( + rewritten.as_ref(), + b"HTTP/1.1 200\r\nX-Opaque: \xff\xfe\r\nX-Duplicate: first\r\nX-Duplicate: second\r\nConnection: keep-alive, Upgrade\r\nTrailer: X-Checksum\r\nx-openshell-request-id: receipt-cow\r\n\r\n\x00\xffbody" + ); + } + + #[test] + fn clean_correlated_head_adds_only_the_receipt() { + let correlation = test_correlation("receipt-clean"); + let head = b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody"; + + let rewritten = final_response_head(head, Some(&correlation)); + assert_eq!( + rewritten.as_ref(), + b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nx-openshell-request-id: receipt-clean\r\n\r\nbody" + ); + } + + fn request_id_header_count(response: &[u8]) -> usize { + String::from_utf8_lossy(response) + .lines() + .filter(|line| { + line.split_once(':').is_some_and(|(name, _)| { + name.eq_ignore_ascii_case( + openshell_core::token_grant_failure::REQUEST_ID_HEADER, + ) + }) + }) + .count() + } + + #[tokio::test] + async fn allowed_final_response_carries_platform_receipt() { + let correlation = test_correlation("receipt-1"); + let (outcome, client) = relay_bytes( + b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody", + Some(&correlation), + ) + .await; + + assert!(matches!( + outcome.expect("relay response"), + RelayOutcome::Reusable + )); + assert_eq!(request_id_header_count(&client), 1); + assert!(String::from_utf8_lossy(&client).contains("x-openshell-request-id: receipt-1\r\n")); + } + + #[tokio::test] + async fn delivery_failure_receipt_distinguishes_upstream_error_from_local_block() { + for outcome in [HttpOutcome::UpstreamError, HttpOutcome::PlatformError] { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = test_correlation("delivery-failure"); + let target = HttpRequestTarget { + scheme: "https".into(), + host: "example.test".into(), + port: 443, + method: "GET".into(), + path: "/data".into(), + query: String::new(), + }; + let mut client = Vec::new(); + { + let mut client = correlation.observe_response(&mut client); + super::send_response_delivery_failure( + &mut client, + "GET", + "policy", + &target, + None, + outcome, + "response_body", + Some(&correlation), + ) + .await + .unwrap(); + } + drop(correlation); + assert_eq!(request_id_header_count(&client), 1); + let events = capture.json_events(); + assert_eq!(events.len(), 1); + assert_eq!(events[0]["http_response"]["code"], 502); + assert_eq!(events[0]["metadata"]["correlation_uid"], "delivery-failure"); + assert_eq!( + events[0]["action"] == "Denied", + outcome == HttpOutcome::PlatformError + ); + } + } + + #[tokio::test] + async fn correlated_upstream_eof_preserves_transport_close_and_audits_failure() { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = test_correlation("eof-request"); + let (outcome, client) = relay_bytes(b"", Some(&correlation)).await; + assert!(matches!( + outcome.expect("upstream close"), + RelayOutcome::Consumed + )); + assert!( + client.is_empty(), + "EOF must not become a synthetic HTTP response" + ); + drop(correlation); + let records = capture.json_events(); + let primary: Vec<_> = records + .iter() + .filter(|event| event["class_uid"] == 4002) + .collect(); + assert_eq!(primary.len(), 1, "{records:?}"); + assert_eq!(primary[0]["status"], "Failure"); + assert_eq!(primary[0]["metadata"]["correlation_uid"], "eof-request"); + assert!(primary[0].get("http_response").is_none()); + } + + #[tokio::test] + async fn correlated_response_without_reason_phrase_retains_legacy_tolerance() { + let correlation = test_correlation("empty-reason"); + let (outcome, client) = relay_bytes( + b"HTTP/1.1 200\r\nContent-Length: 0\r\n\r\n", + Some(&correlation), + ) + .await; + outcome.expect("tolerated status line"); + assert!(client.starts_with(b"HTTP/1.1 200\r\n")); + assert_eq!(request_id_header_count(&client), 1); + } + + #[tokio::test] + async fn uncorrelated_response_preserves_application_owned_request_id() { + let response = + b"HTTP/1.1 200\r\nContent-Length: 0\r\nX-OpenShell-Request-ID: application\r\n\r\n"; + let (outcome, client) = relay_bytes(response, None).await; + outcome.expect("legacy response"); + assert_eq!(client, response); + } + + #[tokio::test] + async fn response_middleware_denial_has_one_correlated_primary_without_tracing() { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + exercise_response_middleware_block().await; + + assert_response_middleware_block_record(&capture.json_events(), false); + } + + async fn exercise_response_middleware_block() { + let correlation = test_correlation_for_method("response-denied", "POST"); + correlation.emit( + openshell_ocsf::HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .http_request(openshell_ocsf::HttpRequest::new( + "POST", + openshell_ocsf::Url::new("https", "example.test", "/mutate", 443), + )) + .firewall_rule("test-policy", "l7-rest") + .action(openshell_ocsf::ActionId::Allowed) + .status(openshell_ocsf::StatusId::Success) + .build(), + ); + let (runner, chain) = response_middleware_fixture(ResponseRelayScript::BlockPreflight); + let mut middleware = response_middleware_context(&runner, &chain, "POST"); + middleware.request_context.request_id = "response-denied".into(); + middleware.target.path = "/mutate".into(); + middleware.request_head = None; + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"[..]; + let mut client = Vec::new(); + + let outcome = relay_response( + "POST", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect("middleware denial response"); + + assert!(matches!(outcome, RelayOutcome::Consumed)); + assert!(client.starts_with(b"HTTP/1.1 403 Forbidden\r\n")); + assert!( + String::from_utf8_lossy(&client) + .contains("x-openshell-request-id: response-denied\r\n") + ); + drop(correlation); + } + + fn assert_response_middleware_block_record(records: &[serde_json::Value], traced: bool) { + let http: Vec<_> = records + .iter() + .filter(|event| event["class_uid"] == 4002) + .collect(); + assert_eq!(http.len(), 2); + let companions: Vec<_> = http + .iter() + .filter(|event| event["unmapped"]["http_event_kind"] == "middleware_invocation") + .collect(); + assert_eq!(companions.len(), 1); + assert_eq!(companions[0]["http_response"]["code"], 200); + assert_eq!(companions[0]["unmapped"]["middleware_phase"], "response"); + assert_eq!( + companions[0]["unmapped"]["response_middleware_outcome"], + "blockdelivery" + ); + assert_eq!(companions[0]["action"], "Denied"); + assert_eq!( + companions[0]["metadata"]["correlation_uid"], + "response-denied" + ); + let primary: Vec<_> = records + .iter() + .filter(|event| { + event["class_uid"] == 4002 && event["unmapped"]["http_event_kind"] == "request" + }) + .collect(); + assert_eq!(primary.len(), 1, "{records:?}"); + assert_eq!(primary[0]["action"], "Denied"); + assert_eq!(primary[0]["status"], "Failure"); + assert_eq!(primary[0]["http_response"]["code"], 403); + assert_eq!(primary[0]["http_request"]["url"]["path"], "/mutate"); + assert_eq!(primary[0]["firewall_rule"]["type"], "l7-rest"); + assert_eq!(primary[0]["metadata"]["correlation_uid"], "response-denied"); + assert_eq!(primary[0].get("trace").is_some(), traced); + } + + #[tokio::test] + async fn response_middleware_block_never_creates_a_denial_only_span() { + use crate::l7::relay::correlation_dispatch_tests::capture_request_spans; + + for sampled in [true, false] { + let evidence = Box::pin(capture_request_spans( + sampled, + exercise_response_middleware_block(), + )) + .await; + evidence.assert_no_deny_span(sampled); + assert_response_middleware_block_record(&evidence.events, sampled); + if sampled { + assert_eq!( + evidence + .spans + .iter() + .filter(|span| span.name == "openshell.middleware.response") + .count(), + 1 + ); + } + } + } + + #[tokio::test] + async fn upstream_receipt_is_replaced_by_platform_receipt() { + let correlation = test_correlation("trusted"); + let response = + &b"HTTP/1.1 200 OK\r\nX-OpenShell-Request-ID: forged\r\nContent-Length: 0\r\n\r\n"[..]; + let (outcome, client) = relay_bytes(response, Some(&correlation)).await; + outcome.expect("relay response"); + + assert!(!String::from_utf8_lossy(&client).contains("forged")); + assert!(String::from_utf8_lossy(&client).contains("x-openshell-request-id: trusted\r\n")); + assert_eq!(request_id_header_count(&client), 1); + } + + #[tokio::test] + async fn ordinary_response_removes_receipt_connection_nomination() { + let correlation = test_correlation("trusted"); + let response = &b"HTTP/1.1 200 OK\r\n\ + Connection: X-OpenShell-Request-ID\r\n\ + cOnNeCtIoN: keep-alive, x-OpEnShElL-rEqUeSt-Id\r\n\ + X-OpenShell-Request-ID: forged-head\r\n\ + Trailer: x-openshell-request-id, X-Checksum\r\n\ + Transfer-Encoding: chunked\r\n\r\n\ + 4\r\nbody\r\n\ + 0\r\n\ + x-OPENshell-request-ID: forged-trailer\r\n\ + X-Checksum: ok\r\n\r\n"[..]; + let (outcome, client) = relay_bytes(response, Some(&correlation)).await; + let outcome = outcome.expect("relay response"); + + assert!(matches!(outcome, RelayOutcome::Reusable)); + let delivered = String::from_utf8(client).expect("HTTP response is UTF-8"); + let head = delivered + .split_once("\r\n\r\n") + .expect("response head terminator") + .0; + let connection_headers: Vec<_> = head + .lines() + .filter(|line| { + line.split_once(':') + .is_some_and(|(name, _)| name.eq_ignore_ascii_case("connection")) + }) + .collect(); + assert_eq!(connection_headers, ["cOnNeCtIoN: keep-alive"]); + assert!(!delivered.contains("forged"), "{delivered}"); + assert!(head.contains("Trailer: X-Checksum"), "{head}"); + assert!(delivered.contains("0\r\nX-Checksum: ok\r\n\r\n")); + assert!(head.contains("x-openshell-request-id: trusted"), "{head}"); + assert_eq!(request_id_header_count(delivered.as_bytes()), 1); + } + + #[tokio::test] + async fn ordinary_response_rejects_folded_receipt_connection_nomination() { + let correlation = test_correlation("trusted"); + let response = &b"HTTP/1.1 200 OK\r\nConnection: keep-alive,\r\n X-OpenShell-Request-ID\r\nContent-Length: 0\r\n\r\n"[..]; + let (outcome, client) = relay_bytes(response, Some(&correlation)).await; + let error = outcome.expect_err("folded response fields must fail closed"); + + assert!( + error + .to_string() + .contains("response header continuation lines are not supported"), + "{error}" + ); + assert!(client.is_empty()); + } + + #[tokio::test] + async fn ordinary_response_rejects_bare_lf_receipt_smuggling() { + let correlation = test_correlation("trusted"); + let response = &b"HTTP/1.1 200 OK\r\nX-Safe: yes\nX-OpenShell-Request-ID: forged\r\nContent-Length: 0\r\n\r\n"[..]; + let (outcome, client) = relay_bytes(response, Some(&correlation)).await; + let error = outcome.expect_err("bare LF response fields must fail closed"); + + assert!(error.to_string().contains("bare LF"), "{error}"); + assert!(client.is_empty()); + } + + #[test] + fn response_head_validation_rejects_malformed_syntax() { + let cases: &[(&str, &[u8])] = &[ + ("incomplete block", b"HTTP/1.1 200 OK\r\nX-Test: yes\r\n"), + ("bare LF", b"HTTP/1.1 200 OK\r\nX-Test: yes\n\r\n"), + ("bare CR", b"HTTP/1.1 200 OK\r\nX-Test: yes\rX: no\r\n\r\n"), + ( + "obsolete fold", + b"HTTP/1.1 200 OK\r\nX-Test: yes\r\n continued\r\n\r\n", + ), + ("unsupported protocol", b"HTTP/2 200 OK\r\n\r\n"), + ("invalid status range", b"HTTP/1.1 099 Nope\r\n\r\n"), + ("non-numeric status", b"HTTP/1.1 2O0 Nope\r\n\r\n"), + ("four-digit status", b"HTTP/1.1 2000 Nope\r\n\r\n"), + ("malformed field", b"HTTP/1.1 200 OK\r\nX-Test\r\n\r\n"), + ("empty field name", b"HTTP/1.1 200 OK\r\n: value\r\n\r\n"), + ( + "non-token field name", + b"HTTP/1.1 200 OK\r\nBad Name: value\r\n\r\n", + ), + ( + "whitespace before colon", + b"HTTP/1.1 200 OK\r\nX-Test : value\r\n\r\n", + ), + ( + "field control byte", + b"HTTP/1.1 200 OK\r\nX-Test: value\x0bmore\r\n\r\n", + ), + ( + "field delete byte", + b"HTTP/1.1 200 OK\r\nX-Test: value\x7f\r\n\r\n", + ), + ("reason control byte", b"HTTP/1.1 200 O\x00K\r\n\r\n"), + ( + "early empty line", + b"HTTP/1.1 200 OK\r\n\r\nX-Smuggled: yes\r\n\r\n", + ), + ]; + + for (name, input) in cases { + assert!( + validate_http_response_header_block(input).is_err(), + "{name} unexpectedly passed validation" + ); + } + } + + #[test] + fn response_head_validation_accepts_supported_informational_and_upgrade_responses() { + for response in [ + &b"HTTP/1.0 200 OK\r\nX-Test: yes\r\n\r\n"[..], + &b"HTTP/1.1 200\r\nContent-Length: 0\r\n\r\n"[..], + &b"HTTP/1.1 103 Early Hints\r\nLink: ; rel=preload\r\n\r\n"[..], + &b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n"[..], + ] { + validate_http_response_header_block(response).expect("valid HTTP/1.x response head"); + } + } + + #[tokio::test] + async fn response_middleware_rejects_forged_request_id_trailer() { + let correlation = test_correlation("trusted"); + let (runner, chain) = response_middleware_fixture(ResponseRelayScript::Stream); + let mut middleware = response_middleware_context(&runner, &chain, "GET"); + middleware.request_context.request_id = "middleware-request".into(); + middleware.request_head = None; + let mut upstream = &b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\nX-OpenShell-Request-ID: forged\r\n\r\n"[..]; + let mut client = Vec::new(); + + let error = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect_err("platform correlation trailers must fail closed"); + + assert!( + error.to_string().contains("protected field name"), + "{error}" + ); + let delivered = String::from_utf8_lossy(&client); + assert!(!delivered.contains("forged"), "{delivered}"); + assert!( + delivered.contains("x-openshell-request-id: trusted\r\n"), + "{delivered}" + ); + assert_eq!(request_id_header_count(&client), 1); + } + + #[test] + fn response_middleware_rejects_declared_request_id_trailer() { + let error = parse_response_head_for_middleware( + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nTrailer: X-OpenShell-Request-ID\r\n\r\n", + ) + .expect_err("platform correlation trailer declarations must fail closed"); + + assert!( + error.to_string().contains("protected trailer field"), + "{error}" + ); + } + + #[tokio::test] + async fn informational_responses_are_not_decorated_but_upgrade_receives_a_receipt() { + let correlation = test_correlation("receipt-1"); + let response = + &b"HTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"[..]; + let (outcome, client) = relay_bytes(response, Some(&correlation)).await; + outcome.expect("relay response"); + assert!(client.starts_with(b"HTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 200 OK\r\n")); + assert_eq!(request_id_header_count(&client), 1); + + let response = &b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nX-OpenShell-Request-ID: forged\r\n\r\n"[..]; + let (outcome, client) = relay_bytes(response, Some(&correlation)).await; + let outcome = outcome.expect("relay response"); + assert!(matches!(outcome, RelayOutcome::Upgraded { .. })); + assert_eq!(request_id_header_count(&client), 1); + assert!(!String::from_utf8_lossy(&client).contains("forged")); + assert!(String::from_utf8_lossy(&client).contains("x-openshell-request-id: receipt-1\r\n")); + } + + #[tokio::test] + async fn switching_protocols_removes_receipt_connection_nomination() { + let correlation = test_correlation("trusted-upgrade"); + let response = &b"HTTP/1.1 101 Switching Protocols\r\n\ + Connection: X-OpenShell-Request-ID\r\n\ + cOnNeCtIoN: keep-alive, x-OpEnShElL-rEqUeSt-Id, UpGrAdE\r\n\ + Upgrade: websocket\r\n\ + X-OpenShell-Request-ID: forged\r\n\r\n"[..]; + let (outcome, client) = relay_bytes(response, Some(&correlation)).await; + let outcome = outcome.expect("relay response"); + + assert!(matches!(outcome, RelayOutcome::Upgraded { .. })); + let delivered = String::from_utf8(client).expect("HTTP response is UTF-8"); + let connection_headers: Vec<_> = delivered + .lines() + .filter(|line| { + line.split_once(':') + .is_some_and(|(name, _)| name.eq_ignore_ascii_case("connection")) + }) + .collect(); + assert_eq!(connection_headers, ["cOnNeCtIoN: keep-alive, UpGrAdE"]); + assert!(!delivered.contains("forged"), "{delivered}"); + assert!( + delivered.contains("x-openshell-request-id: trusted-upgrade\r\n"), + "{delivered}" + ); + assert_eq!(request_id_header_count(delivered.as_bytes()), 1); + } + + #[tokio::test] + async fn switching_protocols_rejects_folded_receipt_connection_nomination() { + let correlation = test_correlation("trusted-upgrade"); + let response = &b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade,\r\n\tX-OpenShell-Request-ID\r\nUpgrade: websocket\r\n\r\n"[..]; + let (outcome, client) = relay_bytes(response, Some(&correlation)).await; + let error = outcome.expect_err("folded upgrade fields must fail closed"); + + assert!( + error + .to_string() + .contains("response header continuation lines are not supported"), + "{error}" + ); + assert!(client.is_empty()); + } #[test] fn http_10_connection_persistence() { diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs index 6928d46b11..7d15476208 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs @@ -76,6 +76,15 @@ impl BodyRelayError { } } + const fn correlation_failure(&self) -> (HttpOutcome, &'static str) { + match self { + Self::Upstream(_) => (HttpOutcome::UpstreamError, "upstream_response_body"), + Self::Downstream(_) => (HttpOutcome::PlatformError, "downstream_write"), + Self::PolicyReload(_) => (HttpOutcome::PlatformError, "policy_generation_changed"), + Self::Output(_) => (HttpOutcome::PlatformError, "middleware_response_output"), + } + } + fn into_report(self) -> miette::Report { match self { Self::Upstream(error) | Self::PolicyReload(error) | Self::Output(error) => error, @@ -99,6 +108,7 @@ pub(super) async fn relay_response_through_pipeline( body_length: BodyLength, server_wants_close: bool, event_stream: bool, + response_correlation: Option<&RequestCorrelation>, ) -> Result> where U: AsyncRead + Unpin, @@ -152,6 +162,7 @@ where Err(error) => { debug!(error = %error, "HTTP response middleware preflight failed"); emit_http_response_middleware_failure( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -163,6 +174,9 @@ where middleware.policy_name, &middleware.target, None, + HttpOutcome::PlatformError, + "middleware_response_preflight", + response_correlation, ) .await?; return Ok(Some(RelayOutcome::Consumed)); @@ -174,9 +188,15 @@ where if let Some(session) = preflight.pipeline.take() { session.end(MiddlewareSessionEndReason::PolicyReload).await; } + record_committed_response_error( + response_correlation, + HttpOutcome::PlatformError, + "policy_generation_changed", + ); return Err(error); } emit_http_response_stage_events( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -191,10 +211,13 @@ where middleware.policy_name, &middleware.target, denial, + "middleware_response_preflight", + response_correlation, ) .await?; } else { emit_http_response_middleware_failure( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -206,6 +229,9 @@ where middleware.policy_name, &middleware.target, preflight.failure.as_ref(), + HttpOutcome::PlatformError, + "middleware_response_preflight", + response_correlation, ) .await?; } @@ -229,8 +255,10 @@ where body_length, server_wants_close, event_stream, + response_correlation, ) .await?; + record_allowed_response(response_correlation); return Ok(Some(outcome)); }; // Eligibility offers a bodyless response no body mode, so no stage can @@ -240,6 +268,7 @@ where .end(MiddlewareSessionEndReason::MiddlewareFailure) .await; emit_http_response_middleware_failure( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -251,6 +280,9 @@ where middleware.policy_name, &middleware.target, None, + HttpOutcome::PlatformError, + "middleware_response_preflight", + response_correlation, ) .await?; return Ok(Some(RelayOutcome::Consumed)); @@ -276,6 +308,7 @@ where chunked, connection_close: server_wants_close, generation_guard: middleware.generation_guard, + correlation: response_correlation, framing: None, committed: false, withheld: Vec::new(), @@ -313,14 +346,22 @@ where let failure = match (finish, relayed) { (Ok(finish), Ok(())) => { emit_http_response_stage_events( + response_correlation, middleware.policy_name, &middleware.target, status_code, &finish.diagnostics, ); if let Some(guard) = middleware.generation_guard { - guard.ensure_current()?; + guard.ensure_current().inspect_err(|_| { + record_committed_response_error( + response_correlation, + HttpOutcome::PlatformError, + "policy_generation_changed", + ); + })?; } + record_allowed_response(response_correlation); return Ok(Some( if server_wants_close || framing == Some(OutputFraming::CloseDelimited) @@ -334,6 +375,7 @@ where } (Err(failure), relayed) => { emit_http_response_stage_events( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -347,6 +389,7 @@ where } (Ok(finish), Err(error)) => { emit_http_response_stage_events( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -357,9 +400,14 @@ where }; let committed = writer.committed; match failure { - Err(error @ BodyRelayError::Downstream(_)) => Err(error.into_report()), + Err(error @ BodyRelayError::Downstream(_)) => { + record_relay_error(response_correlation, &error); + Err(error.into_report()) + } Err(error) if committed => { + record_relay_error(response_correlation, &error); emit_http_response_middleware_abort( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -369,8 +417,10 @@ where Err(error.into_report()) } Err(error) => { + let (outcome, stage) = error.correlation_failure(); debug!(error = %error.into_report(), "HTTP response relay failed before commitment"); emit_http_response_middleware_failure( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -382,12 +432,27 @@ where middleware.policy_name, &middleware.target, None, + outcome, + stage, + response_correlation, ) .await?; Ok(Some(RelayOutcome::Consumed)) } Ok(failure) if committed => { + if let Some(correlation) = response_correlation { + correlation.record_enforcement_block(); + correlation.record_failure( + if failure.denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + "middleware_response_finalize", + ); + } emit_http_response_middleware_abort( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -408,10 +473,13 @@ where middleware.policy_name, &middleware.target, denial, + "middleware_response_finalize", + response_correlation, ) .await?; } else { emit_http_response_middleware_failure( + response_correlation, middleware.policy_name, &middleware.target, status_code, @@ -423,6 +491,9 @@ where middleware.policy_name, &middleware.target, failure.failure.as_deref(), + HttpOutcome::PlatformError, + "middleware_response_finalize", + response_correlation, ) .await?; } @@ -431,6 +502,11 @@ where } } +fn record_relay_error(correlation: Option<&RequestCorrelation>, error: &BodyRelayError) { + let (outcome, reason) = error.correlation_failure(); + record_committed_response_error(correlation, outcome, reason); +} + /// Feed the upstream body to the stages in units of at most `limit` bytes. /// Upstream silence is never a failure: a stream ends when its framing ends. #[allow(clippy::too_many_arguments)] @@ -575,6 +651,7 @@ struct ResponseWriter<'c, 'a, C> { chunked: bool, connection_close: bool, generation_guard: Option<&'a PolicyGenerationGuard>, + correlation: Option<&'a RequestCorrelation>, /// Set on the output `Start`. framing: Option, /// True once any byte reached the client: no canonical error response may @@ -659,6 +736,11 @@ impl ResponseWriter<'_, '_, C> { &[] }, ); + let head = if self.correlation.is_some() { + final_response_head(&head, self.correlation).into_owned() + } else { + head + }; self.framing = Some(framing); // The head alone completes an empty declared body. if framing == (OutputFraming::ContentLength { remaining: 0 }) { @@ -730,14 +812,16 @@ impl ResponseWriter<'_, '_, C> { /// OCSF events for one phase of a response through the stage pipeline. fn emit_http_response_stage_events( + correlation: Option<&RequestCorrelation>, policy_name: &str, target: &HttpRequestTarget, status_code: u16, diagnostics: &HttpStageDiagnostics, ) { - for event in http_response_stage_events(policy_name, target, status_code, diagnostics) { - crate::telemetry::egress_emit!(event); - } + crate::l7::correlation::emit_request_events( + correlation, + http_response_stage_events(policy_name, target, status_code, diagnostics), + ); } fn http_response_stage_events( @@ -831,6 +915,8 @@ fn http_response_stage_event( .http_response(openshell_ocsf::HttpResponse { code: status_code }) .dst_endpoint(openshell_ocsf::Endpoint::from_domain(&target.host, port)) .firewall_rule(policy_name, "supervisor-middleware") + .unmapped("http_event_kind", "middleware_invocation") + .unmapped("middleware_phase", "response") .unmapped("middleware_config", invocation.config_name.as_str()) .unmapped( "middleware_implementation", @@ -882,6 +968,7 @@ fn push_trailer( /// upstream failed, so delivery was aborted. `denied` is true when a stage /// stopped delivery, matching the legacy delivery failure finding. fn emit_http_response_middleware_abort( + correlation: Option<&RequestCorrelation>, policy_name: &str, target: &HttpRequestTarget, status_code: u16, @@ -908,12 +995,192 @@ fn emit_http_response_middleware_abort( ]) .message("HTTP response middleware failed after response commitment; delivery aborted") .build(); - crate::telemetry::egress_emit!(event); + crate::l7::correlation::emit_request_events(correlation, [event]); } #[cfg(test)] mod tests { use super::*; + use crate::test_support::request_tracing::RequestTracing; + use opentelemetry_sdk::trace::Sampler; + use tokio::io::AsyncReadExt as _; + use tracing::instrument::WithSubscriber as _; + + #[test] + fn response_hook_invocations_survive_request_consolidation_without_tracing() { + let capture = RequestTracing::new(None); + tracing::subscriber::with_default(capture.subscriber(), || { + let request = L7Request { + action: "GET".into(), + target: "/data".into(), + query_params: HashMap::default(), + raw_header: b"GET /data HTTP/1.1\r\nHost: example.test\r\n\r\n".to_vec(), + body_length: BodyLength::None, + }; + let correlation = RequestCorrelation::with_request_id( + &request, + &crate::l7::relay::L7EvalContext { + host: "example.test".into(), + port: 443, + ..Default::default() + }, + "response-audit".into(), + ); + correlation.emit( + openshell_ocsf::HttpActivityBuilder::new(ocsf_ctx()) + .http_request(openshell_ocsf::HttpRequest::new( + "GET", + openshell_ocsf::Url::new("https", "example.test", "/data", 443), + )) + .firewall_rule("request-policy", "l7-rest") + .action(openshell_ocsf::ActionId::Allowed) + .status(openshell_ocsf::StatusId::Success) + .build(), + ); + for (outcome, input_bytes) in [ + (HttpStageOutcome::WholeBody, 0), + (HttpStageOutcome::Unchanged, 5), + ] { + emit_http_response_stage_events( + Some(&correlation), + "response-policy", + &HttpRequestTarget::default(), + 200, + &HttpStageDiagnostics { + invocations: vec![HttpStageInvocation { + config_name: "guard".into(), + implementation: "test/guard".into(), + outcome, + input_bytes, + output_bytes: None, + transformed: false, + failed: false, + reason_code: None, + failure_reason: None, + }], + ..Default::default() + }, + ); + if outcome == HttpStageOutcome::WholeBody { + correlation.response_selected(200); + } + } + }); + + let records = capture.events.json_events(); + let (primary, invocations) = crate::test_support::assert_http_request_records(&records, 2); + assert_eq!(primary["firewall_rule"]["type"], "l7-rest"); + assert_eq!(primary["action"], "Allowed"); + assert_eq!(primary["http_response"]["code"], 200); + for (event, (outcome, input_bytes)) in + invocations.iter().zip([("wholebody", 0), ("unchanged", 5)]) + { + assert_eq!(event["unmapped"]["middleware_phase"], "response"); + assert_eq!(event["unmapped"]["middleware_config"], "guard"); + assert_eq!(event["unmapped"]["middleware_implementation"], "test/guard"); + assert_eq!(event["unmapped"]["http_hook_version"], "2"); + assert_eq!(event["unmapped"]["response_middleware_outcome"], outcome); + assert_eq!(event["unmapped"]["input_bytes"], input_bytes); + assert_eq!(event["unmapped"]["failed"], false); + assert_eq!(event["http_response"]["code"], 200); + assert_eq!(event["action"], "Allowed"); + } + for event in &records { + assert_eq!(event["metadata"]["correlation_uid"], "response-audit"); + assert!(event.get("trace").is_none()); + } + capture.shutdown(); + } + + #[tokio::test] + async fn empty_pipeline_response_defers_its_protected_receipt_until_completion() { + for sampler in [None, Some(Sampler::AlwaysOff), Some(Sampler::AlwaysOn)] { + let capture = RequestTracing::new(sampler); + async { + let request = L7Request { + action: "GET".into(), + target: "/data".into(), + query_params: HashMap::default(), + raw_header: b"GET /data HTTP/1.1\r\nHost: example.test\r\n\r\n".to_vec(), + body_length: BodyLength::None, + }; + let correlation = RequestCorrelation::with_request_id( + &request, + &crate::l7::relay::L7EvalContext { + host: "example.test".into(), + port: 443, + ..Default::default() + }, + "pipeline-receipt".into(), + ); + let parsed = ParsedResponseHead { + representable: true, + headers: vec![HttpHeader { + name: "x-openshell-request-id".into(), + value: "upstream-forgery".into(), + }], + preserved_credential_headers: Vec::new(), + connection_nominated: Vec::new(), + declared_trailers: Vec::new(), + }; + let (mut relay, mut app) = tokio::io::duplex(4096); + { + let mut audited = correlation.observe_response(&mut relay); + let mut writer = ResponseWriter { + client: &mut audited, + status_line: "HTTP/1.1 200 OK", + headers: &parsed.headers, + parsed: &parsed, + chunked: true, + connection_close: false, + generation_guard: None, + correlation: Some(&correlation), + framing: None, + committed: false, + withheld: Vec::new(), + }; + writer + .write(HttpBodyOutput::Start { + header_mutations: Vec::new(), + output_body_bytes: Some(0), + }) + .await + .unwrap(); + assert!(!writer.committed); + assert!(capture.events.json_events().is_empty()); + writer + .write(HttpBodyOutput::End { + trailers: Vec::new(), + }) + .await + .unwrap(); + assert!(writer.committed); + } + drop(relay); + let mut response = Vec::new(); + app.read_to_end(&mut response).await.unwrap(); + let response = std::str::from_utf8(&response).unwrap(); + let mut receipts = response.lines().filter(|line| { + line.split_once(':').is_some_and(|(name, _)| { + name.eq_ignore_ascii_case("x-openshell-request-id") + }) + }); + assert_eq!( + receipts.next(), + Some("x-openshell-request-id: pipeline-receipt") + ); + assert_eq!(receipts.next(), None); + assert!(!response.contains("upstream-forgery")); + let events = capture.events.json_events(); + assert_eq!(events.len(), 1); + assert_eq!(events[0]["metadata"]["correlation_uid"], "pipeline-receipt"); + assert_eq!(events[0]["http_response"]["code"], 200); + } + .with_subscriber(capture.subscriber()) + .await; + capture.shutdown(); + } + } #[test] fn denied_stage_emits_blocked_finding_with_denied_action() { diff --git a/crates/openshell-supervisor-network/src/l7/rest/request_body.rs b/crates/openshell-supervisor-network/src/l7/rest/request_body.rs index ab07329575..780d7d7384 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/request_body.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/request_body.rs @@ -563,6 +563,8 @@ pub struct UpstreamWriteFailed { struct UpstreamWrites<'a, W> { inner: &'a mut W, failed: &'a AtomicBool, + correlation: Option<&'a RequestCorrelation>, + request_span: Option, } impl UpstreamWrites<'_, W> { @@ -584,7 +586,13 @@ impl AsyncWrite for UpstreamWrites<'_, W> { buf: &[u8], ) -> std::task::Poll> { let this = self.get_mut(); - let result = std::pin::Pin::new(&mut *this.inner).poll_write(cx, buf); + let span = this.request_span.get_or_insert_with(|| { + this.correlation + .map_or_else(tracing::Span::none, |correlation| { + correlation.stage(RequestStage::UpstreamSend) + }) + }); + let result = span.in_scope(|| std::pin::Pin::new(&mut *this.inner).poll_write(cx, buf)); this.record(result) } @@ -593,7 +601,11 @@ impl AsyncWrite for UpstreamWrites<'_, W> { cx: &mut std::task::Context<'_>, ) -> std::task::Poll> { let this = self.get_mut(); - let result = std::pin::Pin::new(&mut *this.inner).poll_flush(cx); + let result = if let Some(span) = &this.request_span { + span.in_scope(|| std::pin::Pin::new(&mut *this.inner).poll_flush(cx)) + } else { + std::pin::Pin::new(&mut *this.inner).poll_flush(cx) + }; this.record(result) } @@ -664,6 +676,7 @@ where C: AsyncRead + AsyncWrite + Unpin, U: AsyncRead + AsyncWrite + Unpin, { + let correlation = response_options.correlation; let head = &req.raw_header[..request_header_end(&req.raw_header)]; let (mut client_reader, client_writer) = tokio::io::split(&mut *client); let mut client_writer = ResponseCommit { @@ -677,14 +690,22 @@ where &mut client_reader, &mut upstream_writer, options, + correlation, )); - let mut response = Box::pin(relay_response( - &req.action, - &mut upstream_reader, - &mut client_writer, - response_options, - response_middleware, - )); + let mut response = Box::pin(async { + let span = correlation.map_or_else(tracing::Span::none, |correlation| { + correlation.stage(RequestStage::Response) + }); + relay_response( + &req.action, + &mut upstream_reader, + &mut client_writer, + response_options, + response_middleware, + ) + .instrument(span) + .await + }); tokio::select! { biased; @@ -728,7 +749,7 @@ where drop(response); // Dropping the upload cancels every request middleware stage. drop(upload); - body.cancelled("upstream_response"); + body.cancelled("upstream_response", correlation); let _ = upstream_writer.shutdown().await; match responded? { RelayOutcome::Reusable => finish_response(&mut client_writer, true).await, @@ -755,6 +776,7 @@ async fn upload_live_request( client: &mut C, upstream: &mut U, options: RelayRequestOptions<'_>, + correlation: Option<&RequestCorrelation>, ) -> Result<()> where C: AsyncRead + Unpin, @@ -764,8 +786,19 @@ where let mut upstream = UpstreamWrites { inner: upstream, failed: &failed, + correlation, + request_span: None, }; - let uploaded = upload_live_body(head, body, client, &mut upstream, options, &failed).await; + let uploaded = upload_live_body( + head, + body, + client, + &mut upstream, + options, + &failed, + correlation, + ) + .await; match uploaded { Err(error) if failed.load(Ordering::Acquire) => { Err(miette::Report::new(UpstreamWriteFailed { @@ -783,6 +816,7 @@ async fn upload_live_body( upstream: &mut U, options: RelayRequestOptions<'_>, upstream_failed: &AtomicBool, + correlation: Option<&RequestCorrelation>, ) -> Result<()> where C: AsyncRead + Unpin, @@ -793,7 +827,7 @@ where // Chunked input may end with trailers, which only chunked output carries. let input_chunked = body.input_is_chunked(); let (sender, mut receiver) = tokio::sync::mpsc::channel(4); - let run = body.run_to(client, sender, upstream_failed); + let run = body.run_to(client, sender, upstream_failed, correlation); let upstream_for_write = &mut *upstream; let write = async move { let mut scanner = options @@ -826,7 +860,19 @@ where head = require_identity_response_coding(&head) .map_err(LiveWriteError::Failed)?; } - let head = rewrite_http_header_block(&head, options.resolver) + let credential_span = if options.resolver.is_some() + || contains_reserved_credential_marker_bytes(&head) + { + correlation.map_or_else(tracing::Span::none, |correlation| { + correlation.stage(RequestStage::CredentialPrepare) + }) + } else { + tracing::Span::none() + }; + let rewritten = credential_span + .in_scope(|| rewrite_http_header_block(&head, options.resolver)); + drop(credential_span); + let head = rewritten .map_err(|error| LiveWriteError::Failed(miette::Report::new(error)))? .rewritten; fixed_length = Some(output_body_bytes.is_some()); @@ -967,6 +1013,8 @@ mod tests { let mut writes = UpstreamWrites { inner: &mut upstream, failed: &failed, + correlation: None, + request_span: None, }; writes .write_all(b"body") diff --git a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs index 9ae8b02d91..057dfd7104 100644 --- a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs +++ b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs @@ -24,11 +24,11 @@ use openshell_ocsf::{ ActionId, ActivityId, DispositionId, Endpoint, HttpActivityBuilder, HttpRequest, SeverityId, StatusId, ctx::ctx as ocsf_ctx, }; -use tracing::warn; +use tracing::{Instrument as _, warn}; +use crate::l7::correlation::{RequestCorrelation, RequestStage}; use crate::l7::provider::L7Request; use crate::l7::relay::L7EvalContext; -use crate::telemetry::egress_emit; /// Borrowed grant configuration for one endpoint-bound provider credential. pub struct TokenGrantRequest<'a> { @@ -189,15 +189,35 @@ pub fn pin_live_credentials(ctx: &mut L7EvalContext) -> Option, +) -> Result<(L7Request, InjectedHeaders), TokenGrantInjectionError> { + inject_for_selectors_correlated(req, ctx, snapshot, request_id, observation_context, None).await +} + +pub async fn inject_for_selectors_correlated( + req: L7Request, + ctx: &L7EvalContext, + snapshot: &ProviderCredentialSnapshot, + request_id: &str, + observation_context: Option<&EndpointObservationContext>, + correlation: Option<&RequestCorrelation>, ) -> Result<(L7Request, InjectedHeaders), TokenGrantInjectionError> { let selected = select_scoped(snapshot, ctx, request_path(&req.target), None); - inject_or_report(req, ctx, selected, request_id, observation_context).await + inject_or_report( + req, + ctx, + selected, + request_id, + observation_context, + correlation, + ) + .await } /// Inject only grants owned by endpoints that admitted the inspected request. @@ -207,6 +227,7 @@ pub async fn inject_for_selectors( /// Credentials come from the pinned `snapshot`, and each selected key is scoped /// to its revision and installation after selection so the one-grant-per-header /// ambiguity check still compares the original credential identities. +#[cfg(test)] pub(super) async fn inject_for_admitted_owners( req: L7Request, ctx: &L7EvalContext, @@ -214,6 +235,27 @@ pub(super) async fn inject_for_admitted_owners( admitted_owners: &HashSet, request_id: &str, observation_context: Option<&EndpointObservationContext>, +) -> Result<(L7Request, InjectedHeaders), TokenGrantInjectionError> { + inject_for_admitted_owners_correlated( + req, + ctx, + snapshot, + admitted_owners, + request_id, + observation_context, + None, + ) + .await +} + +pub(super) async fn inject_for_admitted_owners_correlated( + req: L7Request, + ctx: &L7EvalContext, + snapshot: &ProviderCredentialSnapshot, + admitted_owners: &HashSet, + request_id: &str, + observation_context: Option<&EndpointObservationContext>, + correlation: Option<&RequestCorrelation>, ) -> Result<(L7Request, InjectedHeaders), TokenGrantInjectionError> { let request_path = request_path(&req.target); let unowned = snapshot @@ -235,7 +277,15 @@ pub(super) async fn inject_for_admitted_owners( } else { select_scoped(snapshot, ctx, request_path, Some(admitted_owners)) }; - inject_or_report(req, ctx, selected, request_id, observation_context).await + inject_or_report( + req, + ctx, + selected, + request_id, + observation_context, + correlation, + ) + .await } fn select_scoped( @@ -266,6 +316,7 @@ async fn inject_or_report( selected: Result, TokenGrantInjectionError>, request_id: &str, observation_context: Option<&EndpointObservationContext>, + correlation: Option<&RequestCorrelation>, ) -> Result<(L7Request, InjectedHeaders), TokenGrantInjectionError> { if selected.as_ref().is_ok_and(Vec::is_empty) { // Most requests need no grant; keep that path free of allocation. @@ -290,12 +341,19 @@ async fn inject_or_report( } else { Vec::new() }; + let span = correlation.map_or_else(tracing::Span::none, |correlation| { + correlation.stage(RequestStage::CredentialPrepare) + }); let result = match selected { - Ok(credentials) => inject_selected(req, ctx, credentials, request_id).await, - Err(error) => Err(error), + Ok(credentials) => { + inject_selected(req, ctx, credentials, request_id, correlation) + .instrument(span) + .await + } + Err(error) => span.in_scope(|| Err(error)), }; if let Err(error) = &result { - report_refused_grant(ctx, &method, &request_path, request_id, error); + report_refused_grant(ctx, &method, &request_path, request_id, error, correlation); } if let Some((sender, context)) = ctx .endpoint_observation_tx @@ -338,6 +396,7 @@ fn report_refused_grant( request_path: &str, request_id: &str, error: &TokenGrantInjectionError, + correlation: Option<&RequestCorrelation>, ) { let credential = error .credential @@ -356,8 +415,9 @@ fn report_refused_grant( .map_or_else(|| "none".to_string(), |status| status.to_string()), "Token grant failed" ); - egress_emit!( - HttpActivityBuilder::new(ocsf_ctx()) + crate::l7::correlation::emit_request_events( + correlation, + [HttpActivityBuilder::new(ocsf_ctx()) .activity(ActivityId::Fail) .action(ActionId::Denied) .disposition(DispositionId::Blocked) @@ -370,7 +430,7 @@ fn report_refused_grant( ctx.host, ctx.port )) .unmapped("request_id", request_id) - .build() + .build()], ); } @@ -382,6 +442,7 @@ async fn inject_selected( ctx: &L7EvalContext, credentials: Vec<(String, ProviderProfileCredential)>, request_id: &str, + correlation: Option<&RequestCorrelation>, ) -> Result<(L7Request, InjectedHeaders), TokenGrantInjectionError> { let resolver = ctx .token_grant_resolver @@ -413,8 +474,9 @@ async fn inject_selected( .apply(&req.raw_header) .map_err(|_| TokenGrantInjectionError::new(injection_configuration_failure()))?; for (provider_key, _) in credentials { - egress_emit!( - HttpActivityBuilder::new(ocsf_ctx()) + crate::l7::correlation::emit_request_events( + correlation, + [HttpActivityBuilder::new(ocsf_ctx()) .activity(ActivityId::Other) .action(ActionId::Allowed) .disposition(DispositionId::Allowed) @@ -432,7 +494,7 @@ async fn inject_selected( ctx.host, ctx.port )) - .build() + .build()], ); } Ok((L7Request { raw_header, ..req }, injected)) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index a72c23c661..e75053e520 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -15,6 +15,7 @@ mod trace_tests; use openshell_sandbox_backend::test_support as rpc_test; use crate::identity::{BinaryIdentityCache, SuppliedIdentityError}; +use crate::l7::correlation::{HttpOutcome, RequestStage}; use crate::l7::tls::ProxyTlsState; use crate::opa::{NetworkAction, OpaEngine, PolicyGenerationGuard}; use crate::policy_dns::{ @@ -253,6 +254,7 @@ fn emit_credential_endpoint_mismatch( port: u16, policy_name: &str, connect_span: &tracing::Span, + correlation: Option<&crate::l7::correlation::RequestCorrelation>, ) { let event = build_credential_endpoint_mismatch_event(method, host, port, policy_name); let finding = crate::l7::build_credential_endpoint_mismatch_finding( @@ -261,7 +263,12 @@ fn emit_credential_endpoint_mismatch( None, "Provider credential endpoint binding mismatch; request denied", ); - crate::telemetry::emit_decision([event, finding], Some(connect_span)); + if let Some(correlation) = correlation { + correlation.emit(event); + correlation.emit(finding); + } else { + crate::telemetry::emit_decision([event, finding], Some(connect_span)); + } } /// Hostnames injected by compute drivers as `/etc/hosts` aliases for the host @@ -1913,6 +1920,7 @@ struct ForwardMiddlewarePipeline<'a> { scheme: &'a str, exchange: &'a crate::l7::middleware::HttpMiddlewareExchange, l7_reevaluation: Option>, + correlation: Option<&'a crate::l7::correlation::RequestCorrelation>, } impl ForwardMiddlewarePipeline<'_> { @@ -1936,6 +1944,7 @@ impl ForwardMiddlewarePipeline<'_> { l7.engine, self.ctx, l7.request_info, + self.correlation, ); openshell_supervisor_middleware::TransformedBodyPolicy::Reevaluate(&validate) } @@ -1945,13 +1954,14 @@ impl ForwardMiddlewarePipeline<'_> { // The forward proxy writes the request as one buffer, so middleware // output is withheld and inlined. self.exchange - .apply_request_with_delivery( + .apply_request_with_delivery_correlated( request, client, self.ctx, self.scheme, transformed_body_policy, crate::l7::middleware::RequestBodyDelivery::WITHHOLD, + self.correlation, ) .await } @@ -2077,6 +2087,24 @@ fn build_forward_allow_ocsf_event( .build() } +fn build_forward_inspection_rejection_ocsf_event( + method: &str, + reason: &'static str, +) -> openshell_ocsf::OcsfEvent { + HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .http_request(HttpRequest { + http_method: method.parse().expect("HTTP method parsing is infallible"), + url: None, + }) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .status_detail(reason) + .message("FORWARD request rejected before policy evaluation") + .build() +} + fn build_forward_rejection_event( host: &str, port: u16, @@ -2123,13 +2151,18 @@ fn emit_forward_l7_decision( decision: openshell_ocsf::OcsfEvent, evaluation_error: Option, connect_span: &tracing::Span, + correlation: Option<&crate::l7::correlation::RequestCorrelation>, ) { // The HTTP decision carries policy context that its evaluation-error // companion lacks, so it must supply the shared deny span's attributes. - crate::telemetry::emit_decision( - std::iter::once(decision).chain(evaluation_error), - Some(connect_span), - ); + let events = std::iter::once(decision).chain(evaluation_error); + if let Some(correlation) = correlation { + for event in events { + correlation.emit(event); + } + return; + } + crate::telemetry::emit_decision(events, Some(connect_span)); } fn build_forward_parse_error_ocsf_event( @@ -2440,17 +2473,21 @@ async fn deny_forward_destination( decision: &EgressDecision, denial_tx: Option<&mpsc::UnboundedSender>, activity_tx: Option<&ActivitySender>, + correlation: Option<&crate::l7::correlation::RequestCorrelation>, ) -> Result<()> where C: TokioAsyncWrite + Unpin, { let detail = destination_denial_detail(denial.kind); - crate::telemetry::emit_decision( - [build_forward_destination_deny_ocsf_event( - denial, peer_addr, method, host, port, path, binary, pid, ancestors, cmdline, policy, - )], - Some(connect_span), + let event = build_forward_destination_deny_ocsf_event( + denial, peer_addr, method, host, port, path, binary, pid, ancestors, cmdline, policy, ); + if let Some(correlation) = correlation { + correlation.record_rejection(403, HttpOutcome::Denied, None); + correlation.emit(event); + } else { + crate::telemetry::emit_decision([event], Some(connect_span)); + } emit_denial_simple( denial_tx, @@ -2468,11 +2505,12 @@ where } respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "ssrf_denied", &format!("{method} {host}:{port} blocked: {detail}"), + correlation.map(crate::l7::correlation::RequestCorrelation::request_id), ), ) .await @@ -5401,6 +5439,7 @@ struct ForwardResponseMiddleware<'a> { exchange: &'a crate::l7::middleware::HttpMiddlewareExchange, } +#[cfg(test)] async fn relay_rewritten_forward_request( method: &str, path: &str, @@ -5409,6 +5448,25 @@ async fn relay_rewritten_forward_request( upstream: &mut U, options: ForwardRelayOptions<'_>, ) -> Result +where + C: TokioAsyncRead + TokioAsyncWrite + Unpin, + U: TokioAsyncRead + TokioAsyncWrite + Unpin, +{ + relay_rewritten_forward_request_correlated( + method, path, rewritten, client, upstream, options, None, + ) + .await +} + +async fn relay_rewritten_forward_request_correlated( + method: &str, + path: &str, + rewritten: Vec, + client: &mut C, + upstream: &mut U, + options: ForwardRelayOptions<'_>, + correlation: Option<&crate::l7::correlation::RequestCorrelation>, +) -> Result where C: TokioAsyncRead + TokioAsyncWrite + Unpin, U: TokioAsyncRead + TokioAsyncWrite + Unpin, @@ -5434,7 +5492,7 @@ where .response_relay(&req, middleware.ctx, middleware.scheme) }); - crate::l7::rest::relay_http_request_with_response_middleware_guarded_observed( + crate::l7::rest::relay_http_request_with_response_middleware_guarded_observed_correlated( &req, client, upstream, @@ -5456,10 +5514,12 @@ where }, response_middleware, options.endpoint_observer, + correlation, ) .await } +#[allow(clippy::too_many_arguments)] async fn inject_token_grant_for_forward_request( method: &str, upstream_target: &str, @@ -5468,14 +5528,24 @@ async fn inject_token_grant_for_forward_request( inspection: Option>, request_id: &str, observation_context: Option<&EndpointObservationContext>, + correlation: Option<&crate::l7::correlation::RequestCorrelation>, ) -> Result> { let (path, _) = crate::l7::rest::parse_target_query(upstream_target)?; - let request = crate::l7::rest::request_from_buffered_http( - method, - path, - upstream_target, - forward_request_bytes, - )?; + let request = if correlation.is_some() { + crate::l7::rest::request_from_buffered_http( + method, + path, + upstream_target, + forward_request_bytes, + )? + } else { + crate::l7::rest::request_from_buffered_http_unprotected( + method, + path, + upstream_target, + forward_request_bytes, + )? + }; if let Some(inspection) = inspection { let (request, prepared_ctx, _) = crate::l7::relay::prepare_inspected_request( request, @@ -5485,6 +5555,7 @@ async fn inject_token_grant_for_forward_request( inspection.request_info, request_id, observation_context, + correlation, ) .await?; *l7_ctx = prepared_ctx; @@ -5496,16 +5567,19 @@ async fn inject_token_grant_for_forward_request( let Some(snapshot) = crate::l7::token_grant_injection::pin_live_credentials(l7_ctx) else { return Ok(request.raw_header); }; - Ok(crate::l7::token_grant_injection::inject_for_selectors( - request, - l7_ctx, - &snapshot, - request_id, - observation_context, + Ok( + crate::l7::token_grant_injection::inject_for_selectors_correlated( + request, + l7_ctx, + &snapshot, + request_id, + observation_context, + correlation, + ) + .await? + .0 + .raw_header, ) - .await? - .0 - .raw_header) } /// Handle a plain HTTP forward proxy request (non-CONNECT). @@ -5872,6 +5946,63 @@ async fn handle_forward_proxy( "https" => Some(443), _ => None, }; + let middleware_input = crate::opa::NetworkInput { + host: host_lc.clone(), + port, + binary_path: decision.binary.clone().unwrap_or_default(), + binary_sha256: String::new(), + ancestors: decision.ancestors.clone(), + cmdline_paths: decision.cmdline_paths.clone(), + }; + let middleware_chain = opa_engine.query_middleware_chain_with_generation(&middleware_input); + let l7_inspected = decision + .endpoint + .l7_route + .as_ref() + .is_some_and(|route| !route.configs.is_empty()); + let inspected = l7_inspected + || middleware_chain + .as_ref() + .is_ok_and(|(chain, _)| !chain.is_empty()); + let correlation_request = crate::l7::rest::request_from_buffered_http_unprotected( + method, + path.clone(), + &upstream_target, + forward_request_bytes, + )?; + let correlation = if inspected { + crate::l7::correlation::RequestCorrelation::new(&correlation_request, &l7_ctx) + } else { + crate::l7::correlation::RequestCorrelation::uninspected() + }; + let request_correlation = inspected.then_some(&correlation); + let receipt_id = + request_correlation.map(crate::l7::correlation::RequestCorrelation::request_id); + let mut audited_client = correlation.observe_response(client); + let client = &mut audited_client; + if inspected { + let mut primary = build_forward_allow_ocsf_event( + workload_addr, + method, + &host_lc, + port, + &telemetry_path, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + policy_str, + ); + if l7_inspected && let openshell_ocsf::OcsfEvent::HttpActivity(event) = &mut primary { + event.action = None; + event.disposition = None; + event.base.status = None; + event.firewall_rule = None; + event.base.message = Some("FORWARD request awaiting inspection".into()); + } + correlation.emit(primary); + } + forward_request_bytes = correlation_request.raw_header; if let Some(route) = decision .endpoint .l7_route @@ -5898,13 +6029,15 @@ async fn handle_forward_proxy( ), ); emit_activity_simple(activity_tx, true, "policy_stale"); + correlation.record_rejection(403, HttpOutcome::Denied, None); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "policy_denied", &format!("{method} {host_lc}:{port}{telemetry_path} not permitted by policy"), + receipt_id, ), ) .await?; @@ -5923,15 +6056,17 @@ async fn handle_forward_proxy( ); emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); emit_activity_simple(activity_tx, true, "policy_stale"); + correlation.record_rejection(403, HttpOutcome::Denied, None); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "policy_denied", &format!( "{method} {host_lc}:{port}{telemetry_path} not permitted by policy" ), + receipt_id, ), ) .await?; @@ -5940,29 +6075,60 @@ async fn handle_forward_proxy( }; let Ok(redacted_path) = secrets::redact_target_for_policy(&path) else { + correlation.record_outcome(HttpOutcome::PlatformError); + correlation.record_response_status(400); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 400, "Bad Request", "invalid_credential_placeholder", "request-target contains an invalid credential placeholder", + receipt_id, ), ) .await?; return Ok(()); }; let Some(l7_config) = select_l7_config_for_path(&route.configs, &redacted_path) else { + let reason = "no L7 endpoint path matched request"; + correlation.record_rejection(403, HttpOutcome::Denied, None); + correlation.emit( + HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::for_http_method(method)) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .http_request(HttpRequest::new( + method, + OcsfUrl::new("http", &host_lc, &telemetry_path, port), + )) + .http_response(HttpResponse { code: 403 }) + .dst_endpoint(Endpoint::from_domain(&host_lc, port)) + .src_endpoint(Endpoint::from_ip(workload_addr.ip(), workload_addr.port())) + .actor_process( + Process::from_bypass(&binary_str, &pid_str, &ancestors_str) + .with_cmd_line(&cmdline_str), + ) + .firewall_rule(policy_str, "l7") + .message(format!( + "FORWARD_L7 denied {method} {host_lc}:{port}{telemetry_path}" + )) + .status_detail(reason) + .build(), + ); emit_activity_simple(activity_tx, true, "l7_policy"); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "policy_denied", &format!( "{method} {host_lc}:{port}{telemetry_path} did not match an L7 endpoint path" ), + receipt_id, ), ) .await?; @@ -5985,30 +6151,33 @@ async fn handle_forward_proxy( if !l7_config.config.allow_encoded_slash && crate::l7::path::canonical_path_has_encoded_slash(&path) { - crate::telemetry::emit_decision( - [build_forward_l7_parse_rejection_ocsf_event( - workload_addr, - method, - &host_lc, - port, - &telemetry_path, - &binary_str, - &pid_str, - &ancestors_str, - &cmdline_str, - policy_str, - FORWARD_ENCODED_SLASH_REJECTION_DETAIL, - )], - Some(&connect_span), - ); + correlation.record_rejection(400, HttpOutcome::Denied, None); + correlation.emit(build_forward_inspection_rejection_ocsf_event( + method, + "invalid_request_target", + )); + correlation.emit(build_forward_l7_parse_rejection_ocsf_event( + workload_addr, + method, + &host_lc, + port, + &telemetry_path, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + policy_str, + FORWARD_ENCODED_SLASH_REJECTION_DETAIL, + )); emit_activity_simple(activity_tx, true, "forward_parse_rejection"); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 400, "Bad Request", "invalid_request_target", "request-target must be canonical", + receipt_id, ), ) .await?; @@ -6020,6 +6189,7 @@ async fn handle_forward_proxy( &forward_request_bytes, l7_config.config.protocol, ) { + correlation.record_rejection(403, HttpOutcome::Denied, None); if let Some(observer) = endpoint_observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } @@ -6045,7 +6215,7 @@ async fn handle_forward_proxy( )) .status_detail(upgrade_detail) .build(); - crate::telemetry::emit_decision([event], Some(&connect_span)); + correlation.emit(event); emit_activity_simple(activity_tx, true, "l7_parse_rejection"); emit_denial_simple( denial_tx, @@ -6058,11 +6228,12 @@ async fn handle_forward_proxy( ); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "unsupported_l7_protocol", upgrade_detail, + receipt_id, ), ) .await?; @@ -6103,20 +6274,26 @@ async fn handle_forward_proxy( { Ok(info) => info, Err(e) => { + correlation.record_rejection(400, HttpOutcome::Denied, None); + correlation.emit(build_forward_inspection_rejection_ocsf_event( + method, + "invalid_graphql_request", + )); let event = build_forward_rejection_event( &host_lc, port, format!("FORWARD_GRAPHQL_L7 request rejected: {e}"), ); - egress_emit!(event); + correlation.emit(event); emit_activity_simple(activity_tx, true, "l7_parse_rejection"); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 400, "Bad Request", "invalid_graphql_request", &format!("GraphQL request rejected before policy evaluation: {e}"), + receipt_id, ), ) .await?; @@ -6155,20 +6332,26 @@ async fn handle_forward_proxy( { Ok(body) => body, Err(e) => { + correlation.record_rejection(400, HttpOutcome::Denied, None); + correlation.emit(build_forward_inspection_rejection_ocsf_event( + method, + "invalid_jsonrpc_request", + )); let event = build_forward_rejection_event( &host_lc, port, format!("FORWARD_JSONRPC_L7 request rejected: {e}"), ); - egress_emit!(event); + correlation.emit(event); emit_activity_simple(activity_tx, true, "l7_parse_rejection"); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 400, "Bad Request", "invalid_jsonrpc_request", &format!("JSON-RPC request rejected before policy evaluation: {e}"), + receipt_id, ), ) .await?; @@ -6182,7 +6365,7 @@ async fn handle_forward_proxy( }; // Policy evaluation must use the selected revision's inspection, // including method classification, just as the CONNECT relays do. - let Some(info) = crate::l7::relay::enforce_mcp_protocol_version( + let Some(info) = crate::l7::relay::enforce_mcp_protocol_version_correlated( &l7_config.config, &jsonrpc_request, info, @@ -6190,6 +6373,7 @@ async fn handle_forward_proxy( &l7_ctx, &telemetry_path, endpoint_observer.as_ref(), + request_correlation, ) .await? else { @@ -6214,16 +6398,22 @@ async fn handle_forward_proxy( let mut evaluation_error = None; let (allowed, reason) = hard_deny_reason.map_or_else( || { - crate::l7::relay::evaluate_l7_request(&tunnel_engine, &l7_ctx, &request_info) - .unwrap_or_else(|e| { - evaluation_error = Some(build_forward_l7_evaluation_error( - &host_lc, - port, - l7_config.config.enforcement, - &e.to_string(), - )); - (false, format!("L7 evaluation error: {e}")) - }) + let stage = correlation.stage(RequestStage::PolicyEvaluate); + let evaluation = stage.in_scope(|| { + crate::l7::relay::evaluate_l7_request(&tunnel_engine, &l7_ctx, &request_info) + }); + if evaluation.is_err() { + egress::mark_error(&stage); + } + evaluation.unwrap_or_else(|e| { + evaluation_error = Some(build_forward_l7_evaluation_error( + &host_lc, + port, + l7_config.config.enforcement, + &e.to_string(), + )); + (false, format!("L7 evaluation error: {e}")) + }) }, |reason| (false, reason), ); @@ -6234,6 +6424,11 @@ async fn handle_forward_proxy( (false, crate::l7::EnforcementMode::Audit) => "audit", (false, crate::l7::EnforcementMode::Enforce) => "deny", }; + if !allowed + && (l7_config.config.enforcement != crate::l7::EnforcementMode::Audit || force_deny) + { + correlation.record_enforcement_block(); + } { let (action_id, disposition_id, severity) = match decision_str { @@ -6297,13 +6492,15 @@ async fn handle_forward_proxy( .firewall_rule(policy_str, engine_type) .message(log_message) .build(); - emit_forward_l7_decision(event, evaluation_error, &connect_span); + emit_forward_l7_decision(event, evaluation_error, &connect_span, request_correlation); } let effectively_denied = force_deny || (!allowed && l7_config.config.enforcement == crate::l7::EnforcementMode::Enforce); if effectively_denied { + correlation.record_outcome(HttpOutcome::Denied); + correlation.record_response_status(403); emit_activity_simple(activity_tx, true, "l7_policy"); emit_denial_simple( denial_tx, @@ -6316,13 +6513,14 @@ async fn handle_forward_proxy( ); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "policy_denied", &format!( "{method} {host_lc}:{port}{telemetry_path} denied by L7 policy: {reason}" ), + receipt_id, ), ) .await?; @@ -6361,6 +6559,7 @@ async fn handle_forward_proxy( &decision, denial_tx, activity_tx, + request_correlation, ) .await?; return Ok(()); @@ -6372,15 +6571,24 @@ async fn handle_forward_proxy( .as_ref() .expect("destination plan hydrated"); - let connector = match validate_destination(DestinationRequest { + let resolve_span = if inspected { + correlation.stage(RequestStage::UpstreamResolve) + } else { + egress::resolve_span(&connect_span) + }; + let resolved = validate_destination(DestinationRequest { host: &raw_host, port, sandbox_entrypoint_pid, plan: destination_plan, }) - .instrument(egress::resolve_span(&connect_span)) - .await - { + .instrument(resolve_span.clone()) + .await; + if inspected && resolved.is_err() { + egress::mark_error(&resolve_span); + } + drop(resolve_span); + let connector = match resolved { Ok(connector) => connector, Err(denial) => { deny_forward_destination( @@ -6400,6 +6608,7 @@ async fn handle_forward_proxy( &decision, denial_tx, activity_tx, + request_correlation, ) .await?; return Ok(()); @@ -6417,13 +6626,15 @@ async fn handle_forward_proxy( ); emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); emit_activity_simple(activity_tx, true, "policy_stale"); + correlation.record_rejection(403, HttpOutcome::Denied, None); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "policy_denied", &format!("{method} {host_lc}:{port}{telemetry_path} not permitted by policy"), + receipt_id, ), ) .await?; @@ -6431,16 +6642,7 @@ async fn handle_forward_proxy( } let middleware_path = path.split_once('?').map_or(path.as_str(), |(path, _)| path); - let middleware_input = crate::opa::NetworkInput { - host: host_lc.clone(), - port, - binary_path: decision.binary.clone().unwrap_or_default(), - binary_sha256: String::new(), - ancestors: decision.ancestors.clone(), - cmdline_paths: decision.cmdline_paths.clone(), - }; - let (chain, generation) = - opa_engine.query_middleware_chain_with_generation(&middleware_input)?; + let (chain, generation) = middleware_chain?; if generation != forward_generation_guard.captured_generation() { emit_l7_tunnel_close_after_policy_change( &host_lc, @@ -6451,19 +6653,21 @@ async fn handle_forward_proxy( generation, ), ); + correlation.record_rejection(403, HttpOutcome::Denied, None); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "policy_denied", &format!("{method} {host_lc}:{port}{telemetry_path} not permitted by policy"), + receipt_id, ), ) .await?; return Ok(()); } - let request_id = uuid::Uuid::new_v4().to_string(); + let request_id = correlation.request_id().to_string(); let websocket_chain = forward_websocket_request.then(|| chain.clone()); let response_selection = if chain.is_empty() { None @@ -6494,8 +6698,8 @@ async fn handle_forward_proxy( scheme: &scheme, exchange: &middleware_exchange, l7_reevaluation, + correlation: request_correlation, }; - // Middleware denials join the connect trace, like other FORWARD denials. let middleware_result = pipeline .apply(request, client) .instrument(connect_span.clone()) @@ -6509,11 +6713,17 @@ async fn handle_forward_proxy( } crate::l7::middleware::MiddlewareApplyResult::RequestTimeout => { emit_activity_simple(activity_tx, true, "middleware"); - let response = build_json_error_response( + correlation.record_rejection( + 408, + HttpOutcome::PlatformError, + Some("request_timeout"), + ); + let response = build_json_error_response_with_request_id( 408, "Request Timeout", "request_timeout", "Request body was not received in time", + receipt_id, ); respond(client, &response).await?; crate::l7::middleware::close_after_early_response(client).await?; @@ -6521,9 +6731,26 @@ async fn handle_forward_proxy( } crate::l7::middleware::MiddlewareApplyResult::Denied { denial, failure } => { emit_activity_simple(activity_tx, true, "middleware"); + correlation.record_rejection( + 403, + if denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + None, + ); let response = denial.as_ref().map_or_else( - || build_middleware_failure_response(&l7_ctx.policy_name, failure.as_ref()), - |denial| build_middleware_deny_response(&l7_ctx.policy_name, denial), + || { + build_middleware_failure_response( + &l7_ctx.policy_name, + failure.as_ref(), + receipt_id, + ) + }, + |denial| { + build_middleware_deny_response(&l7_ctx.policy_name, denial, receipt_id) + }, ); respond(client, &response).await?; crate::l7::middleware::close_after_early_response(client).await?; @@ -6531,7 +6758,9 @@ async fn handle_forward_proxy( } crate::l7::middleware::MiddlewareApplyResult::AdmissionExhausted => { emit_activity_simple(activity_tx, true, "middleware"); - let response = build_middleware_unavailable_response(&l7_ctx.policy_name); + correlation.record_rejection(503, HttpOutcome::PlatformError, None); + let response = + build_middleware_unavailable_response(&l7_ctx.policy_name, receipt_id); respond(client, &response).await?; crate::l7::middleware::close_after_early_response(client).await?; return Ok(()); @@ -6547,6 +6776,11 @@ async fn handle_forward_proxy( forward_request_bytes.clone(), )?; let middleware_runner = opa_engine.middleware_runner()?; + let stage = if chain.is_empty() { + tracing::Span::none() + } else { + correlation.stage(RequestStage::WebsocketPreflight) + }; let preflight = crate::l7::relay::websocket_middleware_preflight( &request, chain, @@ -6554,29 +6788,49 @@ async fn handle_forward_proxy( &l7_ctx, "ws", ) + .instrument(stage.clone()) .await; + if preflight.is_err() { + egress::mark_error(&stage); + } + drop(stage); let preflight = match preflight { Ok(preflight) => preflight, Err(error) => { warn!(error = %error, "Plaintext WebSocket middleware preflight failed"); + correlation.record_rejection(502, HttpOutcome::PlatformError, None); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 502, "Bad Gateway", "middleware_failed", "WebSocket middleware preflight failed", + receipt_id, ), ) .await?; return Ok(()); } }; - crate::l7::middleware::emit_websocket_preflight_events(&l7_ctx, &preflight); + crate::l7::middleware::emit_websocket_preflight_events_correlated( + &l7_ctx, + &preflight, + request_correlation, + ); if preflight.terminal_reason.is_some() { + correlation.record_rejection( + 403, + if preflight.denial.is_some() { + HttpOutcome::Denied + } else { + HttpOutcome::PlatformError + }, + None, + ); let response = preflight.denial.as_ref().map_or_else( - || build_middleware_failure_response(&l7_ctx.policy_name, None), - |denial| build_middleware_deny_response(&l7_ctx.policy_name, denial), + || build_middleware_failure_response(&l7_ctx.policy_name, None, receipt_id), + |denial| build_middleware_deny_response(&l7_ctx.policy_name, denial, receipt_id), ); respond(client, &response).await?; return Ok(()); @@ -6603,6 +6857,7 @@ async fn handle_forward_proxy( }), &request_id, endpoint_observation_context.as_ref(), + request_correlation, ) .await { @@ -6610,6 +6865,7 @@ async fn handle_forward_proxy( Err(e) => { let refusal = e.downcast_ref::(); + correlation.record_rejection(502, HttpOutcome::PlatformError, None); if refusal.is_none() { warn!( dst_host = %host_lc, @@ -6625,15 +6881,15 @@ async fn handle_forward_proxy( } let response = refusal.map_or_else( || { - build_json_error_response( + build_json_error_response_with_request_id( 502, "Bad Gateway", "token_grant_failed", "dynamic token grant failed", + receipt_id, ) }, |refusal| { - // The refusal event already names the credential and request ID. if let Some(observer) = endpoint_observer.as_ref() { observer.observe_token_grant_refusal(refusal, &request_id); } @@ -6648,21 +6904,29 @@ async fn handle_forward_proxy( // asynchronous admission step. Holding an endpoint-scoped resolver across // middleware or token-grant awaits would let a revoked generation reach // the upstream. - let endpoint_credentials = if forward_l7_reeval.is_some() { - ForwardEndpointCredentials { - resolver: l7_ctx.secret_resolver.clone(), - body_classifier: l7_ctx.body_classifier.clone(), - revision: l7_ctx.provider_credential_revision, + let credential_stage = + if l7_ctx.provider_credentials.is_some() || l7_ctx.secret_resolver.is_some() { + correlation.stage(RequestStage::CredentialPrepare) + } else { + tracing::Span::none() + }; + let endpoint_credentials = credential_stage.in_scope(|| { + if forward_l7_reeval.is_some() { + ForwardEndpointCredentials { + resolver: l7_ctx.secret_resolver.clone(), + body_classifier: l7_ctx.body_classifier.clone(), + revision: l7_ctx.provider_credential_revision, + } + } else { + endpoint_credentials_for_request( + l7_ctx.provider_credentials.as_ref(), + l7_ctx.secret_resolver.clone(), + &host_lc, + port, + &path, + ) } - } else { - endpoint_credentials_for_request( - l7_ctx.provider_credentials.as_ref(), - l7_ctx.secret_resolver.clone(), - &host_lc, - port, - &path, - ) - }; + }); let secret_resolver = endpoint_credentials.resolver; let credential_generation = match ( l7_ctx.provider_credentials.as_ref(), @@ -6675,17 +6939,27 @@ async fn handle_forward_proxy( _ => None, }; if let Some(guard) = credential_generation { - guard.ensure_current()?; + guard.ensure_current().inspect_err(|_| { + correlation.record_enforcement_block(); + correlation.record_outcome(HttpOutcome::PlatformError); + })?; } // 9. Rewrite request and forward to upstream - let rewritten = match rewrite_forward_request( - &forward_request_bytes, - forward_request_bytes.len(), - &upstream_target, - &canonical_authority, - secret_resolver.as_deref(), - ) { + let rewrite = credential_stage.in_scope(|| { + rewrite_forward_request( + &forward_request_bytes, + forward_request_bytes.len(), + &upstream_target, + &canonical_authority, + secret_resolver.as_deref(), + ) + }); + if rewrite.is_err() { + egress::mark_error(&credential_stage); + } + drop(credential_stage); + let rewritten = match rewrite { Ok(bytes) => bytes, Err(e) => { warn!( @@ -6700,11 +6974,18 @@ async fn handle_forward_proxy( .await; } if e.is_placement_mismatch() { - crate::l7::relay::reject_credential_resolution(client, &l7_ctx, method, &e) - .instrument(connect_span.clone()) - .await?; + crate::l7::relay::reject_credential_resolution( + client, + &l7_ctx, + method, + &e, + request_correlation, + ) + .instrument(connect_span.clone()) + .await?; client.shutdown().await.into_diagnostic()?; } else if e.is_endpoint_mismatch() { + correlation.record_rejection(403, HttpOutcome::Denied, None); if let Some(observer) = endpoint_observer.as_ref() { observer.observe_credential_failure(true); } @@ -6714,14 +6995,16 @@ async fn handle_forward_proxy( port, policy_str, &connect_span, + request_correlation, ); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "credential_endpoint_mismatch", "credential is not authorized for this request endpoint", + receipt_id, ), ) .await?; @@ -6737,13 +7020,15 @@ async fn handle_forward_proxy( }) .await; } else { + correlation.record_rejection(500, HttpOutcome::PlatformError, None); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 500, "Internal Server Error", "credential_injection_failed", "unresolved credential placeholder in request", + receipt_id, ), ) .await?; @@ -6763,13 +7048,14 @@ async fn handle_forward_proxy( &upstream_target, rewritten, )?; - if !crate::l7::relay::enforce_final_mcp_protocol_version( + if !crate::l7::relay::enforce_final_mcp_protocol_version_correlated( config, &request, client, &l7_ctx, &telemetry_path, endpoint_observer.as_ref(), + request_correlation, ) .await? { @@ -6800,13 +7086,15 @@ async fn handle_forward_proxy( .end(openshell_core::proto::MiddlewareSessionEndReason::PolicyReload) .await; } + correlation.record_rejection(403, HttpOutcome::Denied, None); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "policy_denied", &format!("{method} {host_lc}:{port}{telemetry_path} not permitted by policy"), + receipt_id, ), ) .await?; @@ -6817,7 +7105,11 @@ async fn handle_forward_proxy( // would need absolute-form requests rather than a CONNECT tunnel. Dial // only after every local authorization and transformation step so a // rejected WebSocket preflight cannot contact the destination. - let dial_span = egress::dial_span(&connect_span); + let dial_span = if inspected { + correlation.stage(RequestStage::UpstreamDial) + } else { + egress::dial_span(&connect_span) + }; let dial_result = connector.connect().instrument(dial_span.clone()).await; let mut upstream = match dial_result { Ok(s) => s, @@ -6841,19 +7133,22 @@ async fn handle_forward_proxy( "FORWARD upstream connect failed for {host_lc}:{port}: {e}" )) .build(); - egress_emit!(event); + correlation.emit(event); if let Some(session) = middleware_session.take() { session .end(openshell_core::proto::MiddlewareSessionEndReason::UpstreamFailure) .await; } + correlation.record_response_status(502); + correlation.record_outcome(HttpOutcome::UpstreamError); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 502, "Bad Gateway", "upstream_unreachable", &format!("connection to {host_lc}:{port} failed"), + receipt_id, ), ) .await?; @@ -6878,13 +7173,15 @@ async fn handle_forward_proxy( .end(openshell_core::proto::MiddlewareSessionEndReason::PolicyReload) .await; } + correlation.record_rejection(403, HttpOutcome::Denied, None); respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "policy_denied", &format!("{method} {host_lc}:{port}{path} not permitted by policy"), + receipt_id, ), ) .await?; @@ -6902,7 +7199,7 @@ async fn handle_forward_proxy( let signing_region = forward_upgrade_config .as_ref() .map_or("", |config| config.signing_region.as_str()); - let outcome_result = relay_rewritten_forward_request( + let outcome_result = relay_rewritten_forward_request_correlated( method, &upstream_target, rewritten, @@ -6930,6 +7227,7 @@ async fn handle_forward_proxy( }), endpoint_observer: endpoint_observer.as_ref(), }, + request_correlation, ) .await; let outcome_result = match outcome_result { @@ -6941,7 +7239,13 @@ async fn handle_forward_proxy( .await; } let _ = upstream.shutdown().await; - crate::l7::relay::reject_body_credential(client, *error).await?; + crate::l7::relay::reject_body_credential( + client, + *error, + &l7_ctx, + request_correlation, + ) + .await?; return Ok(()); } if let Some(error) = report.downcast_ref::() { @@ -6950,8 +7254,14 @@ async fn handle_forward_proxy( .end(openshell_core::proto::MiddlewareSessionEndReason::Cancellation) .await; } - crate::l7::relay::reject_credential_resolution(client, &l7_ctx, method, error) - .await?; + crate::l7::relay::reject_credential_resolution( + client, + &l7_ctx, + method, + error, + request_correlation, + ) + .await?; return Ok(()); } Err(report) @@ -6968,21 +7278,20 @@ async fn handle_forward_proxy( ) .await?; - // The request has now survived middleware, token grant, credential - // rewriting, generation checks, and the HTTP relay. Only now record the - // final allowed outcome. - egress_emit!(build_forward_allow_ocsf_event( - workload_addr, - method, - &host_lc, - port, - &telemetry_path, - &binary_str, - &pid_str, - &ancestors_str, - &cmdline_str, - policy_str, - )); + if !inspected { + correlation.emit(build_forward_allow_ocsf_event( + workload_addr, + method, + &host_lc, + port, + &telemetry_path, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + policy_str, + )); + } emit_forward_success_activity(activity_tx, l7_activity_pending); match outcome { @@ -6999,6 +7308,7 @@ async fn handle_forward_proxy( websocket_permessage_deflate, websocket_subprotocol, } => { + correlation.finish_http_trace(); // Protocols whose rules apply to individual HTTP requests never // upgrade (see `upgrade_refusal_for_protocol`). No current path // forwards upgrade headers for them: the request-side refusal @@ -7121,6 +7431,47 @@ fn build_json_error_response(status: u16, status_text: &str, error: &str, detail .into_bytes() } +fn build_json_error_response_with_request_id( + status: u16, + status_text: &str, + error: &str, + detail: &str, + request_id: Option<&str>, +) -> Vec { + request_id.map_or_else( + || build_json_error_response(status, status_text, error, detail), + |id| build_correlated_json_error_response(status, status_text, error, detail, id), + ) +} + +fn build_correlated_json_error_response( + status: u16, + status_text: &str, + error: &str, + detail: &str, + request_id: &str, +) -> Vec { + let body = serde_json::json!({ + "error": error, + "detail": detail, + "request_id": request_id, + }) + .to_string(); + format!( + "HTTP/1.1 {status} {status_text}\r\n\ + Content-Type: application/json\r\n\ + {}: {request_id}\r\n\ + Content-Length: {}\r\n\ + Connection: close\r\n\ + \r\n\ + {}", + openshell_core::token_grant_failure::REQUEST_ID_HEADER, + body.len(), + body, + ) + .into_bytes() +} + /// Build the forward proxy's refusal for a failed dynamic token grant. /// /// This is the generic grant refusal plus a request ID header, which lets the @@ -7175,6 +7526,7 @@ fn build_json_error_response_with_reason( fn build_middleware_deny_response( policy_name: &str, denial: &openshell_supervisor_middleware::MiddlewareDenial, + request_id: Option<&str>, ) -> Vec { let mut body = serde_json::Map::new(); body.insert("error".to_string(), serde_json::json!("middleware_denied")); @@ -7190,11 +7542,21 @@ fn build_middleware_deny_response( if let Some(reason_code) = &denial.reason_code { body.insert("reason_code".to_string(), serde_json::json!(reason_code)); } + if let Some(request_id) = request_id { + body.insert("request_id".to_string(), serde_json::json!(request_id)); + } let body_str = serde_json::Value::Object(body).to_string(); + let request_id_header = request_id.map_or_else(String::new, |request_id| { + format!( + "{}: {request_id}\r\n", + openshell_core::token_grant_failure::REQUEST_ID_HEADER + ) + }); format!( "HTTP/1.1 403 Forbidden\r\n\ Content-Type: application/json\r\n\ Content-Length: {}\r\n\ + {request_id_header}\ Connection: close\r\n\ \r\n\ {}", @@ -7207,11 +7569,12 @@ fn build_middleware_deny_response( fn build_middleware_failure_response( policy_name: &str, failure: Option<&openshell_supervisor_middleware::MiddlewareFailureDetail>, + request_id: Option<&str>, ) -> Vec { - build_middleware_platform_response(policy_name, "403 Forbidden", failure) + build_middleware_platform_response(policy_name, "403 Forbidden", failure, request_id) } -fn build_middleware_unavailable_response(policy_name: &str) -> Vec { +fn build_middleware_unavailable_response(policy_name: &str, request_id: Option<&str>) -> Vec { build_middleware_platform_response( policy_name, "503 Service Unavailable", @@ -7219,6 +7582,7 @@ fn build_middleware_unavailable_response(policy_name: &str) -> Vec { config_name: None, code: openshell_supervisor_middleware::HttpFailureCode::Unavailable, }), + request_id, ) } @@ -7226,20 +7590,31 @@ fn build_middleware_platform_response( policy_name: &str, status: &str, failure: Option<&openshell_supervisor_middleware::MiddlewareFailureDetail>, + request_id: Option<&str>, ) -> Vec { let mut body = serde_json::json!({ "error": "middleware_failed", "detail": "Request could not be processed by configured middleware", "policy": policy_name, }); + if let (Some(request_id), serde_json::Value::Object(fields)) = (request_id, &mut body) { + fields.insert("request_id".to_string(), serde_json::json!(request_id)); + } if let Some(failure) = failure { crate::l7::rest::insert_middleware_failure(&mut body, failure); } let body_str = body.to_string(); + let request_id_header = request_id.map_or_else(String::new, |request_id| { + format!( + "{}: {request_id}\r\n", + openshell_core::token_grant_failure::REQUEST_ID_HEADER + ) + }); format!( "HTTP/1.1 {status}\r\n\ Content-Type: application/json\r\n\ Content-Length: {}\r\n\ + {request_id_header}\ Connection: close\r\n\ \r\n\ {}", @@ -7323,6 +7698,7 @@ fn is_benign_relay_error(err: &miette::Report) -> bool { mod tests { mod destination_policy; mod ipv4_mapped; + mod request_trace; #[test] fn proxy_authorization_rejects_missing_wrong_cross_and_duplicate_credentials() { @@ -8661,7 +9037,7 @@ network_policies: .await .unwrap(); workload.shutdown().await.unwrap(); - let handler = tokio::spawn(handle_mediated_connection( + let mut handler = tokio::spawn(handle_mediated_connection( tokio::io::BufReader::new(stream), supplied_identity, socket_addrs, @@ -8683,10 +9059,11 @@ network_policies: None, None, )); - let request = tokio::time::timeout(std::time::Duration::from_secs(5), request_rx) - .await - .expect("upstream never received the workload's request") - .unwrap(); + let request = tokio::select! { + result = tokio::time::timeout(std::time::Duration::from_secs(5), request_rx) => + result.expect("upstream never received the workload's request").unwrap(), + result = &mut handler => panic!("proxy ended before forwarding the workload request: {result:?}"), + }; assert!( request.starts_with(b"GET /ping HTTP/1.1\r\n"), "upstream received {:?} instead of the workload's request", @@ -9666,6 +10043,122 @@ network_policies: {} } } + #[tokio::test] + async fn plaintext_forward_allow_retains_path_and_process_in_primary_record() { + use tracing::instrument::WithSubscriber as _; + use tracing_subscriber::layer::SubscriberExt as _; + + let Some(upstream_ip) = non_loopback_test_ipv4() else { + eprintln!("skipping: no routable non-loopback IPv4 test address"); + return; + }; + let listener = TcpListener::bind((upstream_ip, 0)).await.unwrap(); + let upstream_port = listener.local_addr().unwrap().port(); + let engine = Arc::new( + native_identity_engine( + include_str!("../data/sandbox-policy.rego"), + &format!( + r#" +network_policies: + plain_api: + name: plain_api + endpoints: + - host: "{upstream_ip}" + port: {upstream_port} + allowed_ips: ["{upstream_ip}/32"] + binaries: + - path: /usr/bin/python3 +"# + ), + ) + .unwrap(), + ); + let binary = executable_path("/usr/bin/python3"); + let command = executable_path("/workspace/investigate.py"); + let identity = Ok(ContractBinaryIdentity { + executable: ContractExecutableIdentity { + path: binary.clone(), + digest: Some("00".repeat(32).parse().unwrap()), + }, + ancestors: Vec::new(), + cmdline_paths: vec![command.clone()], + }); + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let (mut app, proxy) = tokio::io::duplex(8192); + let mut proxy: ProxyClient = tokio::io::BufReader::new(Box::new(proxy)); + let target = format!("http://{upstream_ip}:{upstream_port}/v1/incidents?token=not-logged"); + let request = format!( + "GET {target} HTTP/1.1\r\nHost: {upstream_ip}:{upstream_port}\r\nx-openshell-request-id: app-owned-in\r\nConnection: close\r\n\r\n" + ); + let run = async { + tokio::join!( + async { + let result = Box::pin(handle_forward_proxy( + "GET", &target, request.as_bytes(), request.len(), &mut proxy, + Some(&identity), + Some(("192.0.2.20:42123".parse().unwrap(), "192.0.2.10:3128".parse().unwrap())), + engine, Arc::new(BinaryIdentityCache::new()), Arc::new(AtomicU32::new(0)), + None, AgentProposals::default(), Arc::new(None), Arc::new(None), + None, None, None, None, None, + )).await; + drop(proxy); + result + }, + async { + let (mut upstream, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + request.push(upstream.read_u8().await.unwrap()); + } + assert!(request.starts_with(b"GET /v1/incidents?token=not-logged HTTP/1.1\r\n")); + assert!(String::from_utf8_lossy(&request).contains("x-openshell-request-id: app-owned-in\r\n")); + upstream.write_all(b"HTTP/1.1 200 OK\r\nx-openshell-request-id: app-owned-out\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok").await.unwrap(); + }, + async { + let mut response = Vec::new(); + app.read_to_end(&mut response).await.unwrap(); + response + }, + ) + }.with_subscriber(subscriber); + let (result, (), response) = + tokio::time::timeout(std::time::Duration::from_secs(5), Box::pin(run)) + .await + .unwrap(); + result.unwrap(); + assert!(response.starts_with(b"HTTP/1.1 200 OK\r\n")); + assert!( + String::from_utf8_lossy(&response) + .contains("x-openshell-request-id: app-owned-out\r\n") + ); + let records: Vec<_> = capture + .json_events() + .into_iter() + .filter(|event| event["class_uid"] == 4002) + .collect(); + assert_eq!(records.len(), 1, "one primary FORWARD record: {records:?}"); + let record = &records[0]; + assert_eq!(record["action"], "Allowed"); + assert_eq!(record["status"], "Success"); + assert_eq!(record["http_request"]["url"]["path"], "/v1/incidents"); + assert_eq!( + record["actor"]["process"]["name"], + binary.to_string_lossy().as_ref() + ); + assert_eq!( + record["actor"]["process"]["cmd_line"], + command.to_string_lossy().as_ref() + ); + assert!( + record["message"] + .as_str() + .unwrap() + .starts_with("FORWARD allowed") + ); + assert!(!record.to_string().contains("not-logged")); + } + #[tokio::test] async fn plaintext_mcp_forwarding_preserves_initialization_and_selected_revision() { if !cfg!(target_os = "linux") { @@ -10548,7 +11041,7 @@ network_policies: #[test] fn middleware_failure_response_uses_platform_text_without_policy_guidance() { - let response = build_middleware_failure_response("api-policy", None); + let response = build_middleware_failure_response("api-policy", None, None); let response = String::from_utf8(response).expect("UTF-8 error response"); let (_, body) = response.split_once("\r\n\r\n").expect("HTTP response"); let body: serde_json::Value = serde_json::from_str(body).expect("JSON response"); @@ -10566,7 +11059,7 @@ network_policies: #[test] fn middleware_unavailable_response_is_complete_and_has_no_retry_hint() { - let response = build_middleware_unavailable_response("api-policy"); + let response = build_middleware_unavailable_response("api-policy", None); let response = String::from_utf8(response).expect("UTF-8 error response"); assert!(response.starts_with("HTTP/1.1 503 Service Unavailable\r\n")); assert!(!response.to_ascii_lowercase().contains("retry-after")); @@ -11324,6 +11817,7 @@ network_policies: engine: &tunnel_engine, request_info: &request_info, }), + correlation: None, }; let (_app, mut client) = tokio::io::duplex(8192); @@ -11427,6 +11921,7 @@ network_policies: scheme: "http", exchange: &exchange, l7_reevaluation: None, + correlation: None, }; let (_app, mut client) = tokio::io::duplex(8192); let revoke = async { @@ -11868,6 +12363,7 @@ network_policies: None, FORWARD_REQUEST_ID, None, + None, ) .await } @@ -14336,6 +14832,7 @@ network_policies: scheme: "http", exchange: &exchange, l7_reevaluation: None, + correlation: None, }; let (_app, mut client) = tokio::io::duplex(8192); @@ -15154,6 +15651,7 @@ network_policies: config_name: "prototype-content-guard".into(), reason_code: Some("content_match".into()), }, + None, ); let resp_str = String::from_utf8(resp).unwrap(); let body_start = resp_str.find("\r\n\r\n").unwrap() + 4; diff --git a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs index 42f1c6f8d9..455f177a5f 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs @@ -37,6 +37,7 @@ fn assert_json_response( expected_status: &str, expected_error: &str, expected_detail: &str, + expected_request_id: Option<&str>, ) { let (headers, body) = response .windows(4) @@ -48,12 +49,19 @@ fn assert_json_response( assert!(headers.contains("Content-Type: application/json\r\n")); assert!(headers.contains(&format!("Content-Length: {}\r\n", body.len()))); assert!(headers.contains("Connection: close\r\n")); + if let Some(request_id) = expected_request_id { + assert!(headers.contains(&format!("x-openshell-request-id: {request_id}\r\n"))); + } + let mut expected = serde_json::json!({ + "error": expected_error, + "detail": expected_detail, + }); + if let Some(request_id) = expected_request_id { + expected["request_id"] = serde_json::json!(request_id); + } assert_eq!( serde_json::from_slice::(body).unwrap(), - serde_json::json!({ - "error": expected_error, - "detail": expected_detail, - }) + expected ); } @@ -111,9 +119,26 @@ async fn destination_denials_preserve_adapter_specific_wire_contracts() { "HTTP/1.1 403 Forbidden\r\n", "ssrf_denied", &format!("CONNECT target.example:8443 blocked: {detail}"), + None, ); let (mut app, mut proxy) = tcp_pair().await; + let request = crate::l7::provider::L7Request { + action: "POST".to_string(), + target: "/v1/items".to_string(), + query_params: std::collections::HashMap::new(), + raw_header: b"POST /v1/items HTTP/1.1\r\nHost: target.example:8080\r\n\r\n".to_vec(), + body_length: crate::l7::provider::BodyLength::None, + }; + let correlation = crate::l7::correlation::RequestCorrelation::with_request_id( + &request, + &crate::l7::relay::L7EvalContext { + host: "target.example".to_string(), + port: 8080, + ..Default::default() + }, + "test-request-id".to_string(), + ); deny_forward_destination( &mut proxy, &tracing::Span::none(), @@ -134,6 +159,7 @@ async fn destination_denials_preserve_adapter_specific_wire_contracts() { )), None, None, + Some(&correlation), ) .await .unwrap(); @@ -145,6 +171,7 @@ async fn destination_denials_preserve_adapter_specific_wire_contracts() { "HTTP/1.1 403 Forbidden\r\n", "ssrf_denied", &format!("POST target.example:8080 blocked: {detail}"), + Some("test-request-id"), ); } } diff --git a/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs b/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs new file mode 100644 index 0000000000..db572fcedb --- /dev/null +++ b/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs @@ -0,0 +1,453 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use crate::test_support::request_tracing::RequestTracing; +use opentelemetry_sdk::trace::Sampler; +use tracing::instrument::WithSubscriber as _; +use tracing_subscriber::layer::SubscriberExt as _; + +const WORKLOAD_TRACE_ID: &str = "4bf92f3577b34da6a3ce929d0e0e4736"; +const WORKLOAD_SPAN_ID: &str = "00f067aa0ba902b7"; + +#[derive(Clone, Copy)] +enum Outcome { + Allowed, + PolicyDenied, + AuditDenied, + UpstreamClosed, +} + +async fn forward_case(ip: Ipv4Addr, outcome: Outcome, sampler: Option, flags: &str) { + let listener = TcpListener::bind((ip, 0)).await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let mode = if matches!(outcome, Outcome::AuditDenied) { + "audit" + } else { + "enforce" + }; + let path = if matches!(outcome, Outcome::PolicyDenied | Outcome::AuditDenied) { + "/blocked" + } else { + "/allowed" + }; + let engine = Arc::new( + native_identity_engine( + include_str!("../../../data/sandbox-policy.rego"), + &format!( + r#" +network_policies: + traced_api: + name: traced_api + endpoints: + - host: "{ip}" + port: {port} + allowed_ips: ["{ip}/32"] + protocol: rest + enforcement: {mode} + rules: + - allow: {{ method: GET, path: /allowed }} + binaries: + - path: /usr/bin/python3 +"# + ), + ) + .unwrap(), + ); + let identity = Ok(ContractBinaryIdentity { + executable: ContractExecutableIdentity { + path: executable_path("/usr/bin/python3"), + digest: Some("00".repeat(32).parse().unwrap()), + }, + ancestors: Vec::new(), + cmdline_paths: vec![executable_path("/workspace/investigate.py")], + }); + let expect_export = matches!(sampler, Some(Sampler::AlwaysOn)); + let capture = RequestTracing::new(sampler); + let subscriber = capture + .subscriber() + .with(openshell_otel::supervisor_span_filter("info")); + let (mut app, proxy) = tokio::io::duplex(8192); + let mut proxy: ProxyClient = tokio::io::BufReader::new(Box::new(proxy)); + let target = format!("http://{ip}:{port}{path}?token=SENTINEL-QUERY"); + let request = format!( + "GET {target} HTTP/1.1\r\nHost: {ip}:{port}\r\ntraceparent: 00-{WORKLOAD_TRACE_ID}-{WORKLOAD_SPAN_ID}-{flags}\r\ntracestate: vendor=SENTINEL-VENDOR\r\nx-openshell-request-id: forged-receipt\r\nConnection: close\r\n\r\n" + ); + let exchange = async { + tokio::join!( + async { + let result = Box::pin(handle_forward_proxy( + "GET", + &target, + request.as_bytes(), + request.len(), + &mut proxy, + Some(&identity), + Some(( + "192.0.2.20:42123".parse().unwrap(), + "192.0.2.10:3128".parse().unwrap(), + )), + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(0)), + None, + AgentProposals::default(), + Arc::new(None), + Arc::new(None), + None, + None, + None, + None, + None, + )) + .await; + drop(proxy); + result + }, + async { + if matches!(outcome, Outcome::PolicyDenied) { + assert!( + tokio::time::timeout( + std::time::Duration::from_millis(50), + listener.accept() + ) + .await + .is_err(), + "policy denial reached upstream" + ); + } else { + let (mut upstream, _) = listener.accept().await.unwrap(); + let mut bytes = Vec::new(); + while !bytes.ends_with(b"\r\n\r\n") { + bytes.push(upstream.read_u8().await.unwrap()); + } + assert!( + String::from_utf8_lossy(&bytes) + .starts_with(&format!("GET {path}?token=SENTINEL-QUERY HTTP/1.1\r\n")) + ); + assert!(!String::from_utf8_lossy(&bytes).contains("forged-receipt")); + if !matches!(outcome, Outcome::UpstreamClosed) { + upstream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok").await.unwrap(); + } + } + }, + async { + let mut response = Vec::new(); + app.read_to_end(&mut response).await.unwrap(); + response + }, + ) + }; + let (result, (), response) = + async { tokio::time::timeout(std::time::Duration::from_secs(5), Box::pin(exchange)).await } + .with_subscriber(subscriber) + .await + .unwrap(); + result.unwrap(); + let spans = capture.spans(); + let expected = (!matches!(outcome, Outcome::UpstreamClosed)).then_some( + if matches!(outcome, Outcome::PolicyDenied) { + 403 + } else { + 200 + }, + ); + let response = String::from_utf8(response).unwrap(); + let records = capture + .events + .json_events() + .into_iter() + .filter(|event| event["class_uid"] == 4002) + .collect::>(); + assert_eq!(records.len(), 1, "{records:?}"); + let receipt = records[0]["metadata"]["correlation_uid"].as_str().unwrap(); + assert!(uuid::Uuid::parse_str(receipt).is_ok()); + if let Some(expected) = expected { + assert!( + response.starts_with(&format!("HTTP/1.1 {expected} ")), + "{response}" + ); + let wire_receipt = response + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("x-openshell-request-id") + .then(|| value.trim()) + }) + .expect("inspected response receipt"); + assert_eq!(wire_receipt, receipt); + assert_eq!(records[0]["http_response"]["code"], expected); + } else { + assert!( + response.is_empty(), + "upstream EOF must not synthesize a response" + ); + assert!(records[0].get("http_response").is_none()); + assert_eq!(records[0]["status_id"], 2); + } + assert_eq!( + records[0]["action_id"], + if matches!(outcome, Outcome::PolicyDenied) { + 2 + } else { + 1 + } + ); + let created = capture.created_span_names(); + assert!( + !created.contains(&"supervisor.egress.deny"), + "inspected paths must never create a deny-only span" + ); + assert_eq!( + created + .iter() + .filter(|name| **name == "openshell.http.request") + .count(), + 1 + ); + let roots = spans + .iter() + .filter(|span| span.name == "openshell.http.request") + .collect::>(); + assert_eq!( + roots.len(), + usize::from(expect_export), + "local sampler/export configuration controls request span export" + ); + if roots.is_empty() { + assert!(spans.is_empty()); + assert!(records[0].get("trace").is_none()); + } else { + assert_eq!(roots.len(), 1); + let root = roots[0]; + assert_eq!(root.parent_span_id, opentelemetry::trace::SpanId::INVALID); + assert_ne!(root.span_context.trace_id().to_string(), WORKLOAD_TRACE_ID); + assert_eq!(root.links.links.len(), 1); + assert_eq!( + root.links.links[0].span_context.trace_id().to_string(), + WORKLOAD_TRACE_ID + ); + assert_eq!( + root.links.links[0].span_context.span_id().to_string(), + WORKLOAD_SPAN_ID + ); + assert!( + root.links.links[0] + .span_context + .trace_state() + .header() + .is_empty() + ); + assert_eq!( + records[0]["trace"]["uid"], + root.span_context.trace_id().to_string() + ); + assert!(root.attributes.iter().any(|attribute| { + attribute.key.as_str() == "openshell.request.id" && attribute.value.as_str() == receipt + })); + let stages = spans + .iter() + .filter(|span| span.parent_span_id == root.span_context.span_id()) + .collect::>(); + assert!( + stages + .iter() + .any(|span| span.name == "openshell.policy.evaluate") + ); + assert_eq!( + stages + .iter() + .any(|span| span.name == "openshell.upstream.connect"), + !matches!(outcome, Outcome::PolicyDenied) + ); + assert_eq!( + stages + .iter() + .any(|span| span.name == "openshell.upstream.resolve"), + !matches!(outcome, Outcome::PolicyDenied) + ); + assert_eq!( + matches!(root.status, opentelemetry::trace::Status::Error { .. }), + matches!(outcome, Outcome::UpstreamClosed), + "a policy refusal is an outcome, not an operational tracing failure" + ); + assert!( + stages + .iter() + .all(|span| span.span_context.trace_id() == root.span_context.trace_id()) + ); + assert!(!format!("{spans:?}").contains("SENTINEL")); + } + capture.shutdown(); +} + +#[tokio::test] +async fn inspected_forward_traces_all_outcomes_without_sampling_or_export_dependencies() { + let _lock = openshell_otel_test_support::tracing_test_lock().await; + let ip = non_loopback_test_ipv4() + .expect("FORWARD trace qualification requires a non-loopback IPv4 address"); + for outcome in [ + Outcome::Allowed, + Outcome::PolicyDenied, + Outcome::AuditDenied, + Outcome::UpstreamClosed, + ] { + for flags in ["00", "01"] { + for sampler in [Some(Sampler::AlwaysOn), Some(Sampler::AlwaysOff), None] { + forward_case(ip, outcome, sampler, flags).await; + } + } + } +} + +#[tokio::test] +async fn inspected_forward_http_trace_ends_before_websocket_session() { + let _lock = openshell_otel_test_support::tracing_test_lock().await; + let ip = non_loopback_test_ipv4() + .expect("FORWARD trace qualification requires a non-loopback IPv4 address"); + let listener = TcpListener::bind((ip, 0)).await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let engine = Arc::new( + native_identity_engine( + include_str!("../../../data/sandbox-policy.rego"), + &format!( + r#" +network_policies: + traced_websocket: + name: traced_websocket + endpoints: + - host: "{ip}" + port: {port} + allowed_ips: ["{ip}/32"] + protocol: websocket + enforcement: enforce + rules: + - allow: {{ method: GET, path: /ws }} + - allow: {{ method: WEBSOCKET_TEXT, path: /ws }} + binaries: + - path: /usr/bin/python3 +"# + ), + ) + .unwrap(), + ); + let capture = RequestTracing::new(Some(Sampler::AlwaysOn)); + let subscriber = capture + .subscriber() + .with(openshell_otel::supervisor_span_filter("info")); + let (mut app, proxy) = tokio::io::duplex(8192); + let target = format!("http://{ip}:{port}/ws"); + let request = format!( + "GET {target} HTTP/1.1\r\nHost: {ip}:{port}\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\ntraceparent: 00-{WORKLOAD_TRACE_ID}-{WORKLOAD_SPAN_ID}-01\r\n\r\n" + ); + let handler = tokio::spawn( + async move { + let identity = Ok(ContractBinaryIdentity { + executable: ContractExecutableIdentity { + path: executable_path("/usr/bin/python3"), + digest: Some("00".repeat(32).parse().unwrap()), + }, + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + }); + let mut proxy: ProxyClient = tokio::io::BufReader::new(Box::new(proxy)); + Box::pin(handle_forward_proxy( + "GET", + &target, + request.as_bytes(), + request.len(), + &mut proxy, + Some(&identity), + Some(( + "192.0.2.20:42123".parse().unwrap(), + "192.0.2.10:3128".parse().unwrap(), + )), + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(0)), + None, + AgentProposals::default(), + Arc::new(None), + Arc::new(None), + None, + None, + None, + None, + None, + )) + .await + } + .with_subscriber(subscriber), + ); + let (release, released) = tokio::sync::oneshot::channel(); + let (activity, observed_activity) = tokio::sync::oneshot::channel(); + let (handshake, ready) = tokio::sync::oneshot::channel(); + let upstream = tokio::spawn(async move { + let (mut upstream, _) = listener.accept().await.unwrap(); + let request = read_http_headers(&mut upstream).await; + assert!(String::from_utf8_lossy(&request).starts_with("GET /ws HTTP/1.1\r\n")); + upstream + .write_all(b"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=\r\n\r\n") + .await + .unwrap(); + handshake.send(()).unwrap(); + let frame = crate::l7::websocket::read_frame_for_test(&mut upstream).await; + assert_eq!(frame, masked_text_frame(b"after-http-root")); + activity.send(std::time::SystemTime::now()).unwrap(); + upstream.write_all(b"\x81\x02ok").await.unwrap(); + released.await.unwrap(); + }); + let scenario = async { + ready.await.unwrap(); + let response = read_http_headers(&mut app).await; + assert!(String::from_utf8_lossy(&response).starts_with("HTTP/1.1 101 ")); + let root = tokio::time::timeout(std::time::Duration::from_secs(2), async { + loop { + if let Some(root) = capture + .spans() + .into_iter() + .find(|span| span.name == "openshell.http.request") + { + break root; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("HTTP root must finish while the upgraded connection is still open"); + assert!(!handler.is_finished()); + assert!(!upstream.is_finished()); + assert!(root.attributes.iter().any(|attribute| { + attribute.key.as_str() == "http.response.status_code" + && attribute.value == opentelemetry::Value::I64(101) + })); + assert!(root.attributes.iter().any(|attribute| { + attribute.key.as_str() == "openshell.request.outcome" + && attribute.value.as_str() == "upgraded" + })); + app.write_all(&masked_text_frame(b"after-http-root")) + .await + .unwrap(); + assert!(root.end_time <= observed_activity.await.unwrap()); + let mut frame = [0; 4]; + app.read_exact(&mut frame).await.unwrap(); + assert_eq!(&frame, b"\x81\x02ok"); + assert!(!handler.is_finished()); + release.send(()).unwrap(); + drop(app); + upstream.await.unwrap(); + handler.await.unwrap().unwrap(); + let records = capture.events.json_events(); + assert_eq!( + records + .iter() + .filter(|event| event["class_uid"] == 4002 && event["http_response"]["code"] == 101) + .count(), + 1 + ); + }; + tokio::time::timeout(std::time::Duration::from_secs(5), scenario) + .await + .unwrap(); + capture.shutdown(); +} diff --git a/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs b/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs index 0d26319749..b1dbeb00d4 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs @@ -80,6 +80,7 @@ async fn forward_inspection_selects_only_the_owner_admitting_the_request() { }), "5c4b3a29-1807-4f6e-8d5c-4b3a29180716", None, + None, ) .await .expect("prepare admitted forward request"); @@ -130,6 +131,7 @@ async fn forward_inspection_retains_installation_through_guarded_write() { }), "5c4b3a29-1807-4f6e-8d5c-4b3a29180716", None, + None, ) .await .expect("prepare authenticated request"); diff --git a/crates/openshell-supervisor-network/src/proxy/trace_tests.rs b/crates/openshell-supervisor-network/src/proxy/trace_tests.rs index 1602913885..d6871ca941 100644 --- a/crates/openshell-supervisor-network/src/proxy/trace_tests.rs +++ b/crates/openshell-supervisor-network/src/proxy/trace_tests.rs @@ -16,6 +16,7 @@ async fn paired_credential_denial_shares_one_deny_span() { 443, "credential-policy", &tracing::Span::none(), + None, ); }) .await; @@ -38,6 +39,7 @@ async fn repeated_credential_denials_keep_decision_spans_separate() { 443, "credential-policy", &connect, + None, ); } let allowed = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -133,6 +135,7 @@ async fn forward_denials_use_the_saved_connect_parent() { &decision, None, None, + None, )) .unwrap(); assert!(output.starts_with(b"HTTP/1.1 403 Forbidden")); @@ -194,7 +197,7 @@ async fn forward_evaluation_error_keeps_the_http_decision_policy_context() { "evaluation unavailable", ); // Use the same grouping as the forward-proxy production path. - emit_forward_l7_decision(decision, Some(error), &connect); + emit_forward_l7_decision(decision, Some(error), &connect, None); }) .await; diff --git a/crates/openshell-supervisor-network/src/telemetry.rs b/crates/openshell-supervisor-network/src/telemetry.rs index 73e573b985..103c49b5d5 100644 --- a/crates/openshell-supervisor-network/src/telemetry.rs +++ b/crates/openshell-supervisor-network/src/telemetry.rs @@ -1,13 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Keep a deny span active during OCSF emission. +//! Emit egress audit records in their owning trace context. use std::collections::HashMap; use std::sync::{LazyLock, Mutex}; use std::time::{Duration, Instant}; use openshell_ocsf::{ActionId, HttpActivityEvent, OcsfEvent}; +use tracing_opentelemetry::OpenTelemetrySpanExt; /// Emit one egress event, entering an INFO deny span if it is denied HTTP Activity. macro_rules! egress_emit { @@ -63,6 +64,27 @@ pub fn emit_decision(events: impl IntoIterator, parent: Option } } +/// Inspected requests own an all-outcome span, so their records never create a +/// denial-only span, including when that request span is disabled or unsampled. +#[allow(clippy::disallowed_macros)] +pub fn emit_inspected_decision( + events: impl IntoIterator, + parent: Option<&tracing::Span>, +) { + let emit = || { + for event in events { + openshell_ocsf::ocsf_emit!(event); + } + }; + let context = parent.map_or_else(opentelemetry::Context::new, OpenTelemetrySpanExt::context); + let _context = context.attach(); + if let Some(parent) = parent { + parent.in_scope(emit); + } else { + emit(); + } +} + /// Returns `None` when deny spans are disabled or the destination has used up /// its deny spans. Disabled spans spend no tokens. fn deny_span(event: &HttpActivityEvent, parent: Option<&tracing::Span>) -> Option { diff --git a/crates/openshell-supervisor-network/src/telemetry/tests.rs b/crates/openshell-supervisor-network/src/telemetry/tests.rs index 7a3388ad0d..0d0d2111be 100644 --- a/crates/openshell-supervisor-network/src/telemetry/tests.rs +++ b/crates/openshell-supervisor-network/src/telemetry/tests.rs @@ -81,6 +81,10 @@ static REGISTER: LazyLock<()> = LazyLock::new(|| { )); }); +pub fn init_trace_extractor() { + LazyLock::force(®ISTER); +} + fn capture( log_level: &str, provider: &opentelemetry_sdk::trace::SdkTracerProvider, @@ -600,3 +604,82 @@ async fn disabled_deny_spans_spend_no_tokens() { assert!(super::LIMITER.lock().unwrap().buckets.is_empty()); assert!(super::LIMITER.lock().unwrap().global.is_none()); } + +#[tokio::test] +async fn inspected_audit_records_do_not_depend_on_sampling_or_create_deny_spans() { + let _lock = tracing_test_lock().await; + LazyLock::force(®ISTER); + for sampled in [false, true] { + super::reset_deny_span_limiter(); + let exporter = opentelemetry_sdk::trace::InMemorySpanExporter::default(); + let sampler = if sampled { + opentelemetry_sdk::trace::Sampler::AlwaysOn + } else { + opentelemetry_sdk::trace::Sampler::AlwaysOff + }; + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_sampler(sampler) + .with_simple_exporter(exporter.clone()) + .build(); + let request_id = uuid::Uuid::new_v4().to_string(); + let events = capture("warn", &provider, || { + let request = tracing::info_span!(parent: None, "openshell.http.request"); + let mut denied = http_deny("blocked.example", "middleware_blocked"); + denied.set_correlation_uid(request_id.clone()); + super::emit_inspected_decision([denied], Some(&request)); + }); + provider.force_flush().unwrap(); + let spans = exporter.get_finished_spans().unwrap(); + assert_eq!(spans.len(), usize::from(sampled)); + assert!( + spans + .iter() + .all(|span| span.name == "openshell.http.request") + ); + assert_eq!(events.len(), 1); + assert_eq!(events[0]["metadata"]["correlation_uid"], request_id); + assert_eq!(events[0]["action_id"], 2); + if sampled { + assert_eq!( + events[0]["trace"]["uid"], + spans[0].span_context.trace_id().to_string() + ); + } else { + assert!(events[0].get("trace").is_none()); + } + assert!(super::LIMITER.lock().unwrap().buckets.is_empty()); + assert!(super::LIMITER.lock().unwrap().global.is_none()); + provider.shutdown().unwrap(); + } +} + +#[tokio::test] +async fn inspected_emission_without_a_request_span_never_falls_back_to_deny_tracing() { + let (events, received) = exported("warn", || { + use opentelemetry::trace::TraceContextExt as _; + let ambient = opentelemetry::trace::SpanContext::new( + opentelemetry::trace::TraceId::from(1), + opentelemetry::trace::SpanId::from(2), + opentelemetry::trace::TraceFlags::SAMPLED, + true, + opentelemetry::trace::TraceState::default(), + ); + let _ambient = opentelemetry::Context::new() + .with_remote_span_context(ambient.clone()) + .attach(); + super::emit_inspected_decision([http_deny("blocked.example", "no_parent")], None); + super::emit_inspected_decision( + [http_deny("blocked.example", "disabled_parent")], + Some(&tracing::Span::none()), + ); + assert_eq!( + opentelemetry::Context::current().span().span_context(), + &ambient + ); + }) + .await; + assert_eq!(events.len(), 2); + assert!(received.spans.is_empty()); + assert!(events.iter().all(|event| event["action_id"] == 2)); + assert_no_deny(&events, &received); +} diff --git a/crates/openshell-supervisor-network/src/test_support.rs b/crates/openshell-supervisor-network/src/test_support.rs index 5c302b91de..4326ade7c3 100644 --- a/crates/openshell-supervisor-network/src/test_support.rs +++ b/crates/openshell-supervisor-network/src/test_support.rs @@ -1,7 +1,44 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Credential, executable identity, and socket-owner subprocess fixtures. +//! Credential, identity, socket-owner and request-tracing fixtures. + +pub mod request_tracing; + +pub fn response_request_id(response: &[u8]) -> &str { + let end = response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .expect("complete response headers"); + std::str::from_utf8(&response[..end]) + .expect("ASCII response headers") + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("x-openshell-request-id") + .then(|| value.trim()) + }) + .expect("response carries a request receipt") +} + +pub fn assert_http_request_records( + events: &[serde_json::Value], + expected_invocations: usize, +) -> (&serde_json::Value, Vec<&serde_json::Value>) { + let (primary, invocations): (Vec<_>, Vec<_>) = events + .iter() + .filter(|event| event["class_uid"] == 4002) + .partition(|event| event["unmapped"]["http_event_kind"] == "request"); + assert_eq!(primary.len(), 1, "{events:?}"); + assert_eq!(invocations.len(), expected_invocations, "{events:?}"); + assert!( + invocations + .iter() + .all(|event| event["unmapped"]["http_event_kind"] == "middleware_invocation"), + "unexpected HTTP Activity record: {events:?}" + ); + (primary[0], invocations) +} pub fn header_placement_metadata( keys: &[&str], diff --git a/crates/openshell-supervisor-network/src/test_support/request_tracing.rs b/crates/openshell-supervisor-network/src/test_support/request_tracing.rs new file mode 100644 index 0000000000..2cee89d2ab --- /dev/null +++ b/crates/openshell-supervisor-network/src/test_support/request_tracing.rs @@ -0,0 +1,86 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::{Arc, Mutex}; + +use openshell_ocsf::test_support::OcsfCapture; +use opentelemetry_sdk::trace::{InMemorySpanExporter, Sampler, SdkTracerProvider, SpanData}; +use tracing_subscriber::{layer::SubscriberExt as _, registry::LookupSpan}; + +pub fn span_attribute<'a>(span: &'a SpanData, name: &str) -> Option<&'a opentelemetry::Value> { + span.attributes + .iter() + .rev() + .find(|attribute| attribute.key.as_str() == name) + .map(|attribute| &attribute.value) +} + +#[derive(Clone, Default)] +struct CreatedSpans(Arc>>); + +impl tracing_subscriber::Layer for CreatedSpans { + fn on_new_span( + &self, + attributes: &tracing::span::Attributes<'_>, + _id: &tracing::Id, + _context: tracing_subscriber::layer::Context<'_, S>, + ) { + self.0.lock().unwrap().push(attributes.metadata().name()); + } +} + +pub struct RequestTracing { + pub events: OcsfCapture, + exporter: InMemorySpanExporter, + provider: Option, + created: CreatedSpans, +} + +impl RequestTracing { + pub fn new(sampler: Option) -> Self { + crate::telemetry::tests::init_trace_extractor(); + let exporter = InMemorySpanExporter::default(); + let provider = sampler.map(|sampler| { + SdkTracerProvider::builder() + .with_sampler(sampler) + .with_simple_exporter(exporter.clone()) + .build() + }); + Self { + events: OcsfCapture::default(), + exporter, + provider, + created: CreatedSpans::default(), + } + } + + pub fn subscriber( + &self, + ) -> impl tracing::Subscriber + for<'span> LookupSpan<'span> + Send + Sync + 'static { + let layer = self + .provider + .as_ref() + .map(|provider| openshell_otel::layer(provider, "request-test")); + tracing_subscriber::registry() + .with(self.events.clone()) + .with(self.created.clone()) + .with(layer) + } + + pub fn spans(&self) -> Vec { + if let Some(provider) = &self.provider { + provider.force_flush().unwrap(); + } + self.exporter.get_finished_spans().unwrap() + } + + pub fn created_span_names(&self) -> Vec<&'static str> { + self.created.0.lock().unwrap().clone() + } + + pub fn shutdown(&self) { + if let Some(provider) = &self.provider { + provider.shutdown().unwrap(); + } + } +} diff --git a/docs/observability/ocsf-json-export.mdx b/docs/observability/ocsf-json-export.mdx index 2f4b52c10f..6ada756106 100644 --- a/docs/observability/ocsf-json-export.mdx +++ b/docs/observability/ocsf-json-export.mdx @@ -311,12 +311,100 @@ HTTP Activity and API Activity records emitted inside a sampled OpenTelemetry sp OTLP log export links every OCSF record, whatever its class, to the active sampled span through the log record's own trace and span ID fields. Use those fields for log-to-trace links in your backend. -With OTLP trace export enabled on a dedicated supervisor, denied HTTP requests can create an INFO `supervisor.egress.deny` span around the audit records for that decision. The span records the policy, reason, and destination when available. At the default log level, every deny span is a root span. At debug level, FORWARD policy, destination, credential-binding, and request middleware denials can join the available connect trace and its authorization, resolve, and dial spans. Other deny spans, including L7 denials inside CONNECT tunnels and response middleware denials, remain root spans. Shared supervisor hosts do not initialize OTLP exporters, so their records carry no trace correlation. Workload `traceparent` headers never supply supervisor correlation. +With OTLP trace export enabled on a dedicated supervisor, each inspected HTTP request has an INFO `openshell.http.request` span for its allowed, denied or failed outcome. Children measure the policy, middleware, credential, upstream and response stages that actually run. HTTP upgrade tracing ends before the upgraded WebSocket session. -To bound telemetry volume when a workload retries blocked destinations, the supervisor creates at most 10 deny spans per destination at once and one more per second after that. A process-wide budget also allows at most 100 spans at once and refills at 100 per second across destinations. Rate-limited decisions create no additional deny span, while all audit records are still emitted. Records can retain correlation to an already active sampled span, including a connect span at debug level. +The request span ends after its owned stages, not when response headers are logged. `openshell.upstream.send` measures request writes and flush, not the full upstream round trip. Response handling has a separate span. + +The supervisor owns the request trace. A valid workload `traceparent`, sampled or unsampled, becomes an explicitly untrusted link, never its parent. Invalid or duplicate context is ignored; workload `tracestate` is not adopted. Your backend may show the agent and supervisor as separate linked traces rather than one nested tree. + +Inspected requests do not also create a denial-only `supervisor.egress.deny` span. Other denied HTTP decisions retain that legacy span: a root at the default log level, or a child of an available connect span at debug level. Shared supervisor hosts do not initialize OTLP exporters, so their records carry no trace correlation. + +The legacy deny-span limiter allows at most 10 spans per destination at once and one more per second after that. A process-wide budget also allows at most 100 spans at once and refills at 100 per second across destinations. Rate-limited decisions create no additional deny span, while all audit records are still emitted. These denial-specific budgets do not select inspected request spans, which cover every outcome rather than denials alone. Gateway records and their OTLP trace context can inherit the caller's incoming `traceparent`, including before authentication succeeds. The caller also decides whether they are correlated at all, because an unsampled `traceparent` suppresses `trace.uid`. Treat gateway trace IDs as correlation hints, not evidence of the caller's identity, and do not treat a missing `trace.uid` as meaningful. +## Join Inspected HTTP Requests + +The supervisor assigns each inspected HTTP request a UUID shared by middleware +`RequestContext.request_id`, OCSF `metadata.correlation_uid`, and the +`x-openshell-request-id` response header. It also exports the ID as the OTLP log +attribute `ocsf.metadata.correlation_uid`. + +This header is reserved for OpenShell: workload and upstream values are removed, +and middleware cannot set or remove it. Use a different header for +application-owned correlation. The receipt is a join key, not proof of identity, +authorization or downstream execution. L4 and uninspected traffic do not provide +this receipt guarantee. + +One primary HTTP Activity record is emitted when the final response head is +written, with `unmapped.http_event_kind: "request"`. If the request ends earlier, +a guard emits the available outcome. The response header does not prove that +the body was delivered. + +Per-middleware HTTP Activity records remain available with their existing +attribution, transformation flags and response-stage fields. They carry the same +request ID, `unmapped.http_event_kind: "middleware_invocation"`, and +`unmapped.middleware_phase: "request"` or `"response"`. Response observations can +arrive after the primary record. Their status describes that invocation, not a +second request decision. Security findings and late failure records remain +separate. Collection and the existing diagnostic limits remain best-effort. + +The request ID join and enforcement work without tracing, when spans are not +sampled, and when the collector is unavailable. Sampled OCSF trace fields identify +the supervisor trace; the request ID remains the join key when those fields are +absent. An instrumented client can record the receipt on its own trace and use it +to find OpenShell's security records; it does not need to receive those records +inside the sandbox. + +### Update Existing Queries + +Existing middleware queries can continue using their original fields and +messages. To count inspected request decisions, select primary records rather +than counting every HTTP Activity or every `action: "Denied"` record: + +```shell +jq 'select(.class_uid == 4002 and .unmapped.http_event_kind == "request")' supervisor.jsonl +``` + +Add `.action == "Denied"` to count refused inspected requests. To inspect the +middleware work for one receipt, select `metadata.correlation_uid` with that +UUID and `unmapped.http_event_kind == "middleware_invocation"`. Do not count +companion observations as extra requests, or discard them when auditing what +middleware changed. These filters apply to inspected HTTP, not L4 decisions. + +For inspected-request traces, query span name `openshell.http.request` instead +of `supervisor.egress.deny`. `openshell.request.outcome == "denied"` selects +classified denials, but is not a complete replacement for the old denial query: +a fail-closed platform error or admission refusal can have outcome `"failed"`. +For all refusals, select primary OCSF records with `action: "Denied"`, then join +spans through `openshell.request.id`. Policy name (`firewall_rule.name`) and +reason (`status_detail`) remain on the OCSF primary; those legacy policy +attributes are not copied to the new root. Keep the legacy span query separately +for non-inspected egress paths. + +### Size Supervisor Trace Collection + +The old deny-span budgets do not bound inspected request spans. With supervisor +trace export enabled, allowed traffic also creates spans at INFO level. At +`R` inspected requests per second and an average of `S` descendant spans per +request, expect approximately `R × (1 + S)` spans per second before exporter or +collector loss. Include middleware RPCs and repeated stages in `S`; background +and lifecycle spans add separate volume. This is not a fixed per-request count. + +The exporter queue is bounded, but queue bounds do not cap span generation or +preserve the previous collector load. Collector-side sampling can reduce stored +traces; it does not remove the supervisor's span-generation cost. The gateway's +`OTEL_TRACES_SAMPLER` setting does not configure supervisor sampling. Operator +control of supervisor sampling is tracked in +[#5362](https://github.com/NVIDIA/OpenShell/issues/5362). + +Before enabling supervisor trace export, update dashboards and size the collector +for the inspected request rate. If the old denial budget is required for your +current deployment, leave supervisor trace export disabled until a suitable +sampling or capacity policy is available. OCSF records and request receipts +continue to work without it. This change does not itself add a sampling setting +or restore denial-only spans. + ## OCSF Event Classes in JSON The `class_uid` field identifies the event type: diff --git a/docs/observability/traces.mdx b/docs/observability/traces.mdx index 3cbdba70a3..abe075e27b 100644 --- a/docs/observability/traces.mdx +++ b/docs/observability/traces.mdx @@ -28,11 +28,11 @@ In the Helm chart, set the table under `gatewayConfig`. The gateway passes the e | Gateway requests | One server span for each inbound gRPC or HTTP request. Request spans carry `method`, `path`, and the `request_id` that also appears in gateway logs. | | Gateway background work | Reconciliation, driver watches, and credential refresh. These start their own traces because no request supplies a parent. Credential refresh creates a span only when a credential is due or needs cleanup. | | Compute drivers | Child spans for each compute-driver operation the gateway performs, from both the gateway's side and the driver's side. | -| Sandbox supervisors | Supervisor startup, such as loading policy. | +| Sandbox supervisors | Supervisor startup, such as loading policy, plus inspected HTTP requests and the stages they execute. | These are OpenShell's own spans. Spans that the agent produces travel separately, through the [agent trace relay](/observability/overview#agent-traces). -Spans for routine or high-volume operations use DEBUG level, and the default INFO level does not export them. These include store operations, health checks, the RPCs that supervisors call on a timer (`GetSandboxConfig` and `ReportProviderReadiness`), and how supervisors handle egress connections. Set the gateway or sandbox log level to `debug` to include them. +Store operations, health checks, periodic supervisor RPCs (`GetSandboxConfig` and `ReportProviderReadiness`), and connection-level egress diagnostics use DEBUG level. Set the gateway or sandbox log level to `debug` to include them. Inspected HTTP request spans and their executed stage children use INFO level, including allowed requests. Their volume is not limited by the legacy denial-span budgets. See [query and capacity migration](/observability/ocsf-json-export#update-existing-queries) before enabling supervisor trace export. ## Span Names @@ -43,7 +43,7 @@ Each component names its spans after the operation it performs: | Gateway requests | The gRPC service and method, or the HTTP method for other requests | `openshell.v1.OpenShell/CreateSandbox`, `GET` | | Gateway background work | The task | `reconcile.sandboxes`, `refresh.provider_credentials` | | Compute drivers | The compute-driver RPC, with child spans prefixed by the driver name | `openshell.compute.v1.ComputeDriver/CreateSandbox`, `docker.provision` | -| Sandbox supervisors | Prefixed with `supervisor.` | `supervisor.startup`, `supervisor.policy.load` | +| Sandbox supervisors | Lifecycle spans use `supervisor.`; inspected HTTP uses request and stage operation names | `supervisor.startup`, `supervisor.policy.load`, `openshell.http.request`, `openshell.policy.evaluate` | The gateway's client span and the driver's server span for a compute-driver call share the same name, which also appears in `rpc.method`. Their service names and span kinds tell them apart. A driver watch stream records a terminal status when it observes one. If the gateway stops consuming the stream first, the span status stays unset. From d0bdbe9ca68009f55e571b536cda84689a9a7067 Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Sun, 15 Nov 2026 18:53:31 -0700 Subject: [PATCH 2/7] fix(observability): narrow receipt validation and simplify tests Signed-off-by: Alex Fournier --- crates/openshell-supervisor-network/README.md | 3 +- .../l7/relay/token_grant_ownership_tests.rs | 38 +++-------- .../src/l7/rest/http_response.rs | 66 +++++++------------ docs/observability/ocsf-json-export.mdx | 4 ++ 4 files changed, 39 insertions(+), 72 deletions(-) diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index da8cfc19b2..8f03beb5a8 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -84,4 +84,5 @@ legacy bounded deny-span behavior. End HTTP tracing before an upgraded session. The request span ends after its owned stages, not when response headers are logged. The `openshell.upstream.send` stage measures request writes and flush; response handling is measured separately. Receipt filtering preserves raw HTTP bytes -and borrows unchanged header blocks. +and borrows unchanged header blocks. It retains the existing status-line parsing +but rejects ambiguous header line endings and folded fields. diff --git a/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs index 1c1d7f2977..c6ece78731 100644 --- a/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs +++ b/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs @@ -105,7 +105,6 @@ struct Route<'a> { method: &'a str, binary: &'a str, enforcement: &'a str, - deny_path: Option<&'a str>, body_rewrite: bool, protocol: &'a str, } @@ -117,7 +116,6 @@ fn route<'a>(name: &'a str, path: &'a str) -> Route<'a> { method: "GET", binary: "/usr/bin/curl", enforcement: "enforce", - deny_path: None, body_rewrite: false, protocol: "rest", } @@ -187,10 +185,6 @@ impl Fixture { if route.protocol == "mcp" { endpoint["mcp"] = serde_json::json!({"versions": ["2025-11-25"]}); } - // An explicit empty deny_rules list is invalid authored policy. - if let Some(path) = route.deny_path { - endpoint["deny_rules"] = serde_json::json!([{"method": "GET", "path": path}]); - } if route.body_rewrite { endpoint["request_body_credential_rewrite"] = true.into(); } @@ -247,11 +241,7 @@ impl Fixture { } fn grant_key(&self, path: &str, owner: &str) -> String { - let snapshot = self.state.snapshot(); - format!( - "{HOST}\t{PORT}\t{path}\trev:{}\tinstallation:{}\t{owner}:access_token", - snapshot.revision, snapshot.installation_id - ) + self.state.snapshot().scoped_key(&key(path, owner)) } fn reload_policy(&mut self) { @@ -1041,20 +1031,12 @@ async fn token_grant_refusal_never_creates_a_denial_only_span() { .all(|span| span.name != "openshell.upstream.send") ); } - let records: Vec<_> = evidence - .events - .iter() - .filter(|event| event["class_uid"] == 4002) - .collect(); - assert_eq!(records.len(), 1, "{:?}", evidence.events); - assert_eq!(records[0]["action"], "Denied"); - assert_eq!(records[0]["http_response"]["code"], 502); - let receipt = records[0]["metadata"]["correlation_uid"].as_str().unwrap(); - assert!( - evidence - .output - .to_ascii_lowercase() - .contains(&format!("x-openshell-request-id: {receipt}\r\n")) + let (record, _) = crate::test_support::assert_http_request_records(&evidence.events, 0); + assert_eq!(record["action"], "Denied"); + assert_eq!(record["http_response"]["code"], 502); + assert_eq!( + record["metadata"]["correlation_uid"], + crate::test_support::response_request_id(evidence.output.as_bytes()) ); } } @@ -1488,11 +1470,7 @@ async fn transformed_jsonrpc_body_selects_final_operation_owner() { std::str::from_utf8(&injected.raw_header).expect("injected HTTP request"), "aud-transformed", ); - let snapshot = state.snapshot(); - let expected_key = format!( - "{HOST}\t{PORT}\t/api/**\trev:{}\tinstallation:{}\ttransformed:access_token", - snapshot.revision, snapshot.installation_id - ); + let expected_key = state.snapshot().scoped_key(&key("/api/**", "transformed")); resolver.assert_calls(&[(&expected_key, "aud-transformed")]); } diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs index 11ecc60ee5..86bb4f3d61 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs @@ -379,11 +379,8 @@ fn validate_http_response_header_block(headers: &[u8]) -> Result<()> { let header_lines = &headers[..headers.len() - 2]; let mut lines = header_lines .split(|byte| *byte == b'\n') - .map(|line| line.strip_suffix(b"\r").unwrap_or(line)); - let status_line = lines - .next() - .ok_or_else(|| miette!("HTTP response status line is missing"))?; - validate_http_response_status_line(status_line)?; + .map(|line| line.strip_suffix(b"\r").unwrap_or(line)) + .skip(1); while let Some(line) = lines.next() { if line.is_empty() { @@ -420,26 +417,6 @@ fn validate_http_response_header_block(headers: &[u8]) -> Result<()> { Ok(()) } -fn validate_http_response_status_line(status_line: &[u8]) -> Result<()> { - let remainder = if let Some(remainder) = status_line.strip_prefix(b"HTTP/1.0 ") { - remainder - } else if let Some(remainder) = status_line.strip_prefix(b"HTTP/1.1 ") { - remainder - } else { - return Err(miette!( - "HTTP response status line must use HTTP/1.0 or HTTP/1.1" - )); - }; - if remainder.len() < 3 || (remainder.len() > 3 && remainder[3] != b' ') { - return Err(miette!("HTTP response status line is malformed")); - } - http::StatusCode::from_bytes(&remainder[..3]) - .map_err(|_| miette!("HTTP response status code is invalid"))?; - http::HeaderValue::from_bytes(remainder.get(4..).unwrap_or_default()) - .map_err(|_| miette!("HTTP response reason phrase is malformed"))?; - Ok(()) -} - fn final_response_head<'a>( headers: &'a [u8], correlation: Option<&RequestCorrelation>, @@ -3028,16 +3005,28 @@ mod tests { } #[tokio::test] - async fn correlated_response_without_reason_phrase_retains_legacy_tolerance() { - let correlation = test_correlation("empty-reason"); - let (outcome, client) = relay_bytes( - b"HTTP/1.1 200\r\nContent-Length: 0\r\n\r\n", - Some(&correlation), - ) - .await; - outcome.expect("tolerated status line"); - assert!(client.starts_with(b"HTTP/1.1 200\r\n")); - assert_eq!(request_id_header_count(&client), 1); + async fn correlated_response_preserves_legacy_status_line_tolerance() { + for status in [ + "HTTP/1.1 200", + "HTTP/1.1 200 OK", + "HTTP/1.1\t200 OK", + "HTTP/1.2 200 OK", + ] { + let response = format!("{status}\r\nContent-Length: 0\r\n\r\n"); + let (outcome, client) = relay_bytes(response.as_bytes(), None).await; + outcome.expect("legacy status line"); + assert_eq!(client, response.as_bytes()); + + let correlation = test_correlation("legacy-status"); + let (outcome, client) = relay_bytes(response.as_bytes(), Some(&correlation)).await; + outcome.unwrap_or_else(|error| panic!("{status:?}: {error}")); + assert!(client.starts_with(format!("{status}\r\n").as_bytes())); + assert_eq!(request_id_header_count(&client), 1); + assert_eq!( + crate::test_support::response_request_id(&client), + "legacy-status" + ); + } } #[tokio::test] @@ -3242,7 +3231,7 @@ mod tests { } #[test] - fn response_head_validation_rejects_malformed_syntax() { + fn response_head_validation_rejects_ambiguous_header_blocks() { let cases: &[(&str, &[u8])] = &[ ("incomplete block", b"HTTP/1.1 200 OK\r\nX-Test: yes\r\n"), ("bare LF", b"HTTP/1.1 200 OK\r\nX-Test: yes\n\r\n"), @@ -3251,10 +3240,6 @@ mod tests { "obsolete fold", b"HTTP/1.1 200 OK\r\nX-Test: yes\r\n continued\r\n\r\n", ), - ("unsupported protocol", b"HTTP/2 200 OK\r\n\r\n"), - ("invalid status range", b"HTTP/1.1 099 Nope\r\n\r\n"), - ("non-numeric status", b"HTTP/1.1 2O0 Nope\r\n\r\n"), - ("four-digit status", b"HTTP/1.1 2000 Nope\r\n\r\n"), ("malformed field", b"HTTP/1.1 200 OK\r\nX-Test\r\n\r\n"), ("empty field name", b"HTTP/1.1 200 OK\r\n: value\r\n\r\n"), ( @@ -3273,7 +3258,6 @@ mod tests { "field delete byte", b"HTTP/1.1 200 OK\r\nX-Test: value\x7f\r\n\r\n", ), - ("reason control byte", b"HTTP/1.1 200 O\x00K\r\n\r\n"), ( "early empty line", b"HTTP/1.1 200 OK\r\n\r\nX-Smuggled: yes\r\n\r\n", diff --git a/docs/observability/ocsf-json-export.mdx b/docs/observability/ocsf-json-export.mdx index 6ada756106..a0d03af65c 100644 --- a/docs/observability/ocsf-json-export.mdx +++ b/docs/observability/ocsf-json-export.mdx @@ -336,6 +336,10 @@ application-owned correlation. The receipt is a join key, not proof of identity, authorization or downstream execution. L4 and uninspected traffic do not provide this receipt guarantee. +Receipt insertion retains existing status-line tolerance. Ambiguous header line +endings and folded fields are rejected so upstream headers cannot hide a forged +receipt. + One primary HTTP Activity record is emitted when the final response head is written, with `unmapped.http_event_kind: "request"`. If the request ends earlier, a guard emits the available outcome. The response header does not prove that From 7778e71c3ad1e342316337fcd0b509e9aa4b11a1 Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Mon, 16 Nov 2026 11:52:54 -0700 Subject: [PATCH 3/7] refactor(observability): preserve per-control HTTP audit records Correlate original control records at emission with a request-scoped ID and sequence. Remove primary-record aggregation and broad response-head validation while retaining protected receipts and request tracing. Signed-off-by: Alex Fournier --- AGENTS.md | 2 +- crates/openshell-ocsf/src/lib.rs | 2 +- .../src/tracing_layers/event_bridge.rs | 120 ++- .../openshell-ocsf/src/tracing_layers/mod.rs | 2 +- crates/openshell-supervisor-network/README.md | 25 +- .../src/l7/correlation.rs | 656 ++++----------- .../src/l7/correlation/tests.rs | 750 ++++-------------- .../src/l7/relay.rs | 358 +++++---- .../l7/relay/correlation_dispatch_tests.rs | 126 ++- .../l7/relay/token_grant_ownership_tests.rs | 32 +- .../src/l7/rest.rs | 29 +- .../src/l7/rest/http_response.rs | 375 ++++----- .../src/l7/rest/http_response/http_hooks.rs | 23 +- .../src/l7/token_grant_injection.rs | 2 + .../openshell-supervisor-network/src/proxy.rs | 196 ++--- .../src/proxy/tests/request_trace.rs | 181 ++++- .../src/proxy/tests/token_grants.rs | 1 + .../src/telemetry.rs | 9 + .../src/test_support.rs | 19 - docs/observability/ocsf-json-export.mdx | 87 +- 20 files changed, 1145 insertions(+), 1850 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b12dd0ede7..0ed5902932 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -153,7 +153,7 @@ ocsf_emit!(event); - `crate::ocsf_ctx()` returns the process-wide `EventContext`. It is always available (falls back to defaults in tests). - `ocsf_emit!()` is non-blocking and cannot panic. It stores the event in a thread-local and emits via `tracing::info!()`. -- In `openshell-supervisor-network`, route inspected request records through `RequestCorrelation`, which uses their existing request span without creating a denial-only span. Other egress decisions use `crate::telemetry::egress_emit!` or `crate::telemetry::emit_decision`. Clippy rejects direct `ocsf_emit!` calls there. +- In `openshell-supervisor-network`, the inspected request scope automatically correlates all OCSF records at emission, including findings, and uses the request span without creating a denial-only span. Emit through `crate::telemetry::egress_emit!` or `crate::telemetry::emit_decision`; Clippy rejects direct `ocsf_emit!` calls there. - The shorthand layer and JSONL layer extract the event from the thread-local. The shorthand format is derived automatically from the builder fields. - For security findings, **dual-emit**: one domain event (e.g., `SshActivityBuilder`) AND one `DetectionFindingBuilder` for the same incident. - Never log secrets, credentials, or query parameters in OCSF messages. The OCSF JSONL file may be shipped to external systems. diff --git a/crates/openshell-ocsf/src/lib.rs b/crates/openshell-ocsf/src/lib.rs index 4b169e165c..35d949b160 100644 --- a/crates/openshell-ocsf/src/lib.rs +++ b/crates/openshell-ocsf/src/lib.rs @@ -74,5 +74,5 @@ pub use builders::{ pub use tracing_layers::{ OCSF_TARGET, OcsfJsonSettings, OcsfJsonlLayer, OcsfShorthandLayer, bind_thread_json_settings, clear_current_event, clone_current_event, emit_ocsf_event, emit_ocsf_event_routed, - set_current_event, + register_event_enricher, set_current_event, }; diff --git a/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs b/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs index ca5a5e9cdd..98a0031ce7 100644 --- a/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs +++ b/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs @@ -8,7 +8,11 @@ //! layers intercept this target, clone the event, and format it. //! After dispatch, `emit_ocsf_event` clears the thread-local. -use crate::events::OcsfEvent; +use std::sync::OnceLock; + +use crate::events::{BaseEventData, OcsfEvent}; + +static EVENT_ENRICHER: OnceLock = OnceLock::new(); std::thread_local! { // Thread-local storage for the current OCSF event being emitted. @@ -19,6 +23,27 @@ std::thread_local! { /// Target string used to identify OCSF tracing events. pub const OCSF_TARGET: &str = "ocsf"; +/// Register optional process-wide event enrichment at startup. +/// +/// The callback runs once per emission, before any output layer receives the +/// record, independently of trace sampling or export. It receives the common +/// base fields for every event class and should add only contextual fields, +/// preserving the record's content, classification, and timestamp. It must not +/// block, panic, or emit another OCSF event. +/// +/// Returns `false` if an enricher is already registered. Without an enricher, +/// emission does not add application-specific context. +pub fn register_event_enricher(enricher: fn(&mut BaseEventData)) -> bool { + EVENT_ENRICHER.set(enricher).is_ok() +} + +fn enrich(event: &mut OcsfEvent) { + crate::trace::enrich(event); + if let Some(enrich) = EVENT_ENRICHER.get() { + enrich(event.base_mut()); + } +} + /// Clone the current thread-local OCSF event, if any. /// /// Multiple layers can call this for the same event — each receives @@ -35,7 +60,7 @@ pub fn clone_current_event() -> Option { /// /// Both layers receive the event — `clone_current_event()` is non-consuming. pub fn emit_ocsf_event(mut event: OcsfEvent) { - crate::trace::enrich(&mut event); + enrich(&mut event); // Store the event in thread-local so layers can access it set_current_event(event); @@ -83,7 +108,7 @@ pub fn clear_current_event() { /// per-event `sandbox_id` field here instead. One tracing event feeds both the /// JSONL audit file and the routing bus. pub fn emit_ocsf_event_routed(sandbox_id: &str, mut event: OcsfEvent) { - crate::trace::enrich(&mut event); + enrich(&mut event); let message = event.format_shorthand(); set_current_event(event); tracing::info!(target: "ocsf", sandbox_id = %sandbox_id, message = %message); @@ -111,6 +136,22 @@ mod tests { use crate::events::{BaseEvent, OcsfEvent}; use crate::objects::{Metadata, Product}; + std::thread_local! { + // Registration is process-wide, but this test's callback must be inert + // for unrelated tests running in parallel or reusing this thread later. + static TEST_ENRICHER_SEQUENCE: std::cell::Cell> = const { std::cell::Cell::new(None) }; + } + + fn test_enricher(event: &mut BaseEventData) { + TEST_ENRICHER_SEQUENCE.with(|sequence| { + if let Some(next) = sequence.get() { + sequence.set(Some(next + 1)); + event.metadata.correlation_uid = Some("emit-time-request".into()); + event.add_unmapped("request_sequence", next); + } + }); + } + fn test_event() -> OcsfEvent { OcsfEvent::Base(BaseEvent { base: BaseEventData::new( @@ -238,4 +279,77 @@ mod tests { "routed emit must match the bare path JSON" ); } + + #[test] + fn event_enrichment_runs_once_for_every_class_and_output_path() { + use crate::test_support::OcsfCapture; + use crate::tracing_layers::OcsfJsonlLayer; + use tracing_subscriber::prelude::*; + + assert!(register_event_enricher(test_enricher)); + assert!(!register_event_enricher(test_enricher)); + let capture = OcsfCapture::default(); + let bytes = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let subscriber = tracing_subscriber::registry() + .with(capture.clone()) + .with(OcsfJsonlLayer::new(SharedWriter(bytes.clone()))); + let context = crate::builders::test_sandbox_context(); + let mut expected = Vec::new(); + + tracing::subscriber::with_default(subscriber, || { + struct ResetEnricher; + impl Drop for ResetEnricher { + fn drop(&mut self) { + TEST_ENRICHER_SEQUENCE.with(|sequence| sequence.set(None)); + } + } + let plain = test_event(); + expected.push(plain.clone()); + emit_ocsf_event(plain); + + { + let _reset = ResetEnricher; + TEST_ENRICHER_SEQUENCE.with(|sequence| sequence.set(Some(1))); + let mut sequence = 1; + for routed in [false, true] { + for (_, mut event) in crate::builders::sample_events(&context) { + event.base_mut().add_unmapped("source_detail", "keep"); + event + .base_mut() + .add_unmapped("http_event_kind", "original_kind"); + let mut enriched = event.clone(); + enriched.set_correlation_uid("emit-time-request"); + enriched + .base_mut() + .add_unmapped("request_sequence", sequence); + expected.push(enriched); + if routed { + emit_ocsf_event_routed("test-sandbox", event); + } else { + emit_ocsf_event(event); + } + assert!(clone_current_event().is_none()); + sequence += 1; + } + } + TEST_ENRICHER_SEQUENCE.with(|next| assert_eq!(next.get(), Some(sequence))); + } + + let plain = test_event(); + expected.push(plain.clone()); + emit_ocsf_event_routed("test-sandbox", plain); + }); + + assert_eq!(capture.events(), expected); + let expected_json: Vec<_> = expected + .iter() + .map(|event| event.to_json().unwrap()) + .collect(); + let output = String::from_utf8(bytes.lock().unwrap().clone()).unwrap(); + let json: Vec = output + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + assert_eq!(json, expected_json); + } } diff --git a/crates/openshell-ocsf/src/tracing_layers/mod.rs b/crates/openshell-ocsf/src/tracing_layers/mod.rs index 808051b061..23e55ab305 100644 --- a/crates/openshell-ocsf/src/tracing_layers/mod.rs +++ b/crates/openshell-ocsf/src/tracing_layers/mod.rs @@ -13,7 +13,7 @@ mod shorthand_layer; pub use event_bridge::{ OCSF_TARGET, clear_current_event, clone_current_event, emit_ocsf_event, emit_ocsf_event_routed, - set_current_event, + register_event_enricher, set_current_event, }; pub use jsonl_layer::{OcsfJsonSettings, OcsfJsonlLayer, bind_thread_json_settings}; pub use shorthand_layer::OcsfShorthandLayer; diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index 8f03beb5a8..d87ded103e 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -64,16 +64,16 @@ header. Application correlation must use another header. The receipt is only a join key: it does not establish identity, authorization or complete delivery. L4 and passthrough traffic do not provide this header guarantee. -The audit state commits one primary HTTP Activity record when the final response -head is written, with an early-termination fallback. It retains the original -request and process context. Middleware invocation records remain separate, -request-correlated observations, including after the primary is committed. They -retain attribution and transformation details without changing the final -response status or consuming the late-failure budget. Security findings remain -separate. A process crash or indefinitely hung request cannot be finalized by -the fallback. - -This includes HTTP-hook preflight and completion invocation records. +The emission hook adds request correlation to every OCSF class in the current +request scope, including findings. Policy, middleware, token-grant and finding +records retain their original content, event IDs, timestamps and classification; +they are not merged or delayed. `unmapped.http_event_kind` identifies `decision`, +`middleware_invocation` or `token_grant`, and middleware records retain +`unmapped.middleware_phase`, including HTTP-hook preflight and completion. +`unmapped.request_sequence` starts at 1 per request and describes local emission +order, not downstream delivery order. Count policy outcomes by selecting the +actual policy/firewall control record within a receipt group, not every +`decision` record; companion findings also use `decision`. Request IDs and audit outcomes do not depend on OpenTelemetry, sampling or collector availability. Inspected requests have a supervisor-owned request span @@ -84,5 +84,6 @@ legacy bounded deny-span behavior. End HTTP tracing before an upgraded session. The request span ends after its owned stages, not when response headers are logged. The `openshell.upstream.send` stage measures request writes and flush; response handling is measured separately. Receipt filtering preserves raw HTTP bytes -and borrows unchanged header blocks. It retains the existing status-line parsing -but rejects ambiguous header line endings and folded fields. +and borrows unchanged header blocks. It removes only protected receipt fields +and their `Connection` or `Trailer` nominations, including folded continuations, +without adding general response-header validation. diff --git a/crates/openshell-supervisor-network/src/l7/correlation.rs b/crates/openshell-supervisor-network/src/l7/correlation.rs index d36bbabcd2..8a9508415a 100644 --- a/crates/openshell-supervisor-network/src/l7/correlation.rs +++ b/crates/openshell-supervisor-network/src/l7/correlation.rs @@ -1,25 +1,67 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Per-request receipts and consolidated HTTP audit records. +//! Request receipts and emission-time correlation without changing audit records. -use std::sync::Mutex; -use std::{ - pin::Pin, - task::{Context, Poll}, -}; +use std::cell::RefCell; +use std::future::Future; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; -use openshell_ocsf::{ - ActionId, DispositionId, Endpoint, HttpActivityBuilder, HttpActivityEvent, HttpRequest, - HttpResponse, OcsfEvent, SeverityId, StatusId, -}; -use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use openshell_ocsf::OcsfEvent; use crate::l7::provider::L7Request; use crate::l7::relay::L7EvalContext; pub use crate::l7::request_trace::RequestStage; use crate::l7::request_trace::RequestTrace; +tokio::task_local! { + static CURRENT_REQUEST: RefCell>>; +} + +/// Give each connection its own request slot, including across task migration. +pub fn scope_connection(future: F) -> impl Future { + CURRENT_REQUEST.scope(RefCell::new(None), future) +} + +pub fn is_inspected_request() -> bool { + CURRENT_REQUEST + .try_with(|current| { + current + .borrow() + .as_ref() + .is_some_and(|scope| scope.inspected) + }) + .unwrap_or(false) +} + +pub fn current_request_span() -> Option { + CURRENT_REQUEST + .try_with(|current| current.borrow().clone()) + .ok() + .flatten() + .filter(|scope| scope.inspected) + .map(|scope| scope.span()) +} + +fn enrich_request(event: &mut openshell_ocsf::BaseEventData) { + let _ = CURRENT_REQUEST.try_with(|current| { + let current = current.borrow(); + let Some(scope) = current.as_ref().filter(|scope| scope.inspected) else { + return; + }; + event.metadata.correlation_uid = Some(scope.request_id.clone()); + let sequence = scope.sequence.fetch_add(1, Ordering::Relaxed); + let fields = event.unmapped.get_or_insert_with(|| serde_json::json!({})); + if let Some(fields) = fields.as_object_mut() { + fields + .entry("http_event_kind") + .or_insert_with(|| "decision".into()); + fields.insert("request_sequence".into(), sequence.into()); + } + }); +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum HttpOutcome { Allowed, @@ -29,78 +71,46 @@ pub enum HttpOutcome { Upgraded, } -impl HttpOutcome { - fn as_str(self) -> &'static str { - match self { - Self::Allowed => "allowed", - Self::Denied => "denied", - Self::PlatformError => "platform_error", - Self::UpstreamError => "upstream_error", - Self::Upgraded => "upgraded", - } - } -} - #[derive(Debug)] -pub struct RequestCorrelation { +struct RequestScope { request_id: String, inspected: bool, - // Keep the audit payload off the async state machine. - state: Box>, + sequence: AtomicU64, + trace: Mutex>, } -#[derive(Debug, Default)] -struct RequestState { - trace: Option, - primary: Option, - seeded: bool, - status: Option, - upstream_status: Option, - outcome: Option, - blocked: bool, - failure_reason: Option<&'static str>, - late_failure_emitted: bool, -} - -impl RequestState { - fn record_outcome(&mut self, outcome: HttpOutcome) { - self.outcome = Some(outcome); - if let Some(trace) = &mut self.trace { - trace.record_outcome(outcome); - } - } - - fn record_status(&mut self, status: u16) { - self.status = Some(status); - if let Some(trace) = &self.trace { - trace.record_status(status); - } +impl RequestScope { + fn span(&self) -> tracing::Span { + self.trace + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .map_or_else(tracing::Span::none, RequestTrace::span) } +} - fn record_failure_reason(&mut self, reason: &'static str) { - self.failure_reason = Some(reason); - if let Some(trace) = &self.trace { - trace.record_failure_reason(reason); - } - } +#[derive(Debug)] +pub struct RequestCorrelation { + scope: Arc, + // Request loops are sequential; nested scopes restore in LIFO order. + previous: Option>, } impl RequestCorrelation { - #[must_use] pub(crate) fn new(request: &L7Request, ctx: &L7EvalContext) -> Self { Self::build(request, ctx, uuid::Uuid::new_v4().to_string()) } pub(crate) fn uninspected() -> Self { - Self { + Self::install(RequestScope { request_id: uuid::Uuid::new_v4().to_string(), inspected: false, - state: Box::default(), - } + sequence: AtomicU64::new(1), + trace: Mutex::new(None), + }) } #[cfg(test)] - #[must_use] pub(crate) fn with_request_id( request: &L7Request, ctx: &L7EvalContext, @@ -110,15 +120,6 @@ impl RequestCorrelation { } fn build(request: &L7Request, ctx: &L7EvalContext, request_id: String) -> Self { - let target = request.target.split(['?', '#']).next().unwrap_or("/"); - let target = if target.starts_with('/') { target } else { "/" }; - let path = openshell_core::secrets::redact_target_for_policy(target) - .unwrap_or_else(|_| "/".to_string()); - let fallback = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .http_request(HttpRequest::new(&request.action, ctx.ocsf_url(&path))) - .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) - .message("HTTP request ended before a response was selected") - .build(); let trace = RequestTrace::new( &request_id, &request.action, @@ -126,91 +127,72 @@ impl RequestCorrelation { ctx.port, &request.raw_header, ); - Self { + Self::install(RequestScope { request_id, inspected: true, - state: Box::new(Mutex::new(RequestState { - trace: Some(trace), - primary: Some(fallback), - ..RequestState::default() - })), - } + sequence: AtomicU64::new(1), + trace: Mutex::new(Some(trace)), + }) + } + + fn install(scope: RequestScope) -> Self { + openshell_ocsf::register_event_enricher(enrich_request); + let scope = Arc::new(scope); + let previous = CURRENT_REQUEST + .try_with(|current| current.replace(Some(Arc::clone(&scope)))) + .ok() + .flatten(); + Self { scope, previous } } - #[must_use] pub(crate) fn request_id(&self) -> &str { - &self.request_id + &self.scope.request_id } pub(crate) fn stage(&self, stage: RequestStage) -> tracing::Span { - self.state - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) + self.scope .trace - .as_ref() - .map_or_else(tracing::Span::none, |trace| trace.stage(stage)) - } - - #[cfg(test)] - pub(crate) fn trace_span(&self) -> tracing::Span { - self.state .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .trace .as_ref() - .map_or_else(tracing::Span::none, RequestTrace::span) + .map_or_else(tracing::Span::none, |trace| trace.stage(stage)) } pub(crate) fn finish_http_trace(&self) { - let (mut trace, status) = { - let mut state = self - .state - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - (state.trace.take(), state.status) - }; - if status == Some(101) - && let Some(trace) = &mut trace - { - trace.record_outcome(HttpOutcome::Upgraded); - } - drop(trace); - } - - pub(crate) fn observe_response<'a, C>(&'a self, client: &'a mut C) -> AuditedClient<'a, C> { - AuditedClient { - inner: client, - correlation: self, - head: ResponseHead::default(), - } - } - - pub(crate) fn record_enforcement_block(&self) { - self.state + let trace = self + .scope + .trace .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .blocked = true; + .take(); + drop(trace); } pub(crate) fn record_outcome(&self, outcome: HttpOutcome) { - self.state + if let Some(trace) = self + .scope + .trace .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .record_outcome(outcome); + .as_mut() + { + trace.record_outcome(outcome); + } } pub(crate) fn record_response_status(&self, status: u16) { - self.state - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .record_status(status); - } - - pub(crate) fn record_upstream_status(&self, status: u16) { - self.state + if let Some(trace) = self + .scope + .trace .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .upstream_status = Some(status); + .as_mut() + { + trace.record_status(status); + if status == 101 { + trace.record_outcome(HttpOutcome::Upgraded); + } + } } pub(crate) fn record_rejection( @@ -219,201 +201,44 @@ impl RequestCorrelation { outcome: HttpOutcome, reason: Option<&'static str>, ) { - let mut state = self - .state - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - state.blocked = true; - state.record_outcome(outcome); - state.record_status(status); + self.record_outcome(outcome); + self.record_response_status(status); if let Some(reason) = reason { - state.record_failure_reason(reason); + self.record_failure(outcome, reason); } } pub(crate) fn record_failure(&self, outcome: HttpOutcome, reason: &'static str) { - let mut state = self - .state + if let Some(trace) = self + .scope + .trace .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - state.record_outcome(outcome); - state.record_failure_reason(reason); + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_mut() + { + trace.record_outcome(outcome); + trace.record_failure_reason(reason); + } } pub(crate) fn emit(&self, event: OcsfEvent) { - if !self.inspected { + if !self.scope.inspected { crate::telemetry::emit_decision([event], None); return; } - if let OcsfEvent::HttpActivity(mut incoming) = event { - let mut state = self - .state + if let OcsfEvent::HttpActivity(http) = &event + && let Some(trace) = self + .scope + .trace .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - if is_middleware_invocation(&incoming) { - if incoming.action == Some(ActionId::Denied) - && let Some(OcsfEvent::HttpActivity(primary)) = &mut state.primary - { - merge_middleware_denial(primary, &incoming); - } - drop(state); - self.emit_correlated(OcsfEvent::HttpActivity(incoming)); - return; - } - if !state.seeded && state.primary.is_some() { - state.primary = Some(OcsfEvent::HttpActivity(incoming)); - state.seeded = true; - return; - } - if let Some(OcsfEvent::HttpActivity(primary)) = &mut state.primary { - merge_http_outcome(primary, incoming); - return; - } - let fail_open = incoming.action == Some(ActionId::Other) - && (incoming.disposition == Some(DispositionId::Allowed) - || incoming - .base - .unmapped - .as_ref() - .and_then(|fields| fields.get("response_middleware_outcome")) - .and_then(serde_json::Value::as_str) - == Some("failopen")); - if !is_failure(&incoming) || fail_open || state.late_failure_emitted { - return; - } - state.late_failure_emitted = true; - if let Some(code) = state.status { - incoming.http_response = Some(HttpResponse { code }); - } - drop(state); - self.emit_correlated(OcsfEvent::HttpActivity(incoming)); - } else { - if let OcsfEvent::NetworkActivity(network) = &event { - let mut state = self - .state - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - if !state.seeded - && (network.action == Some(ActionId::Denied) - || network.base.status == Some(StatusId::Failure)) - && let Some(OcsfEvent::HttpActivity(primary)) = &mut state.primary - { - primary.action = network.action; - primary.disposition = network.disposition; - primary.base.status = Some(StatusId::Failure); - primary - .base - .status_detail - .clone_from(&network.base.status_detail); - primary.base.severity = network.base.severity; - primary.base.message = - Some("HTTP request rejected before policy evaluation".into()); - primary.src_endpoint.clone_from(&network.src_endpoint); - primary.actor.clone_from(&network.actor); - } - } - self.emit_correlated(event); - } - } - - pub(crate) fn response_selected(&self, status: u16) { - self.record_response_status(status); - self.emit_primary(false); - } - - fn emit_primary(&self, incomplete: bool) { - if !self.inspected { - return; - } - let (mut event, status, upstream_status, outcome, blocked, failure_reason) = { - let mut state = self - .state - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let Some(primary) = state.primary.take() else { - return; - }; - ( - primary, - state.status, - state.upstream_status, - state.outcome, - state.blocked, - state.failure_reason, - ) - }; - if let OcsfEvent::HttpActivity(http) = &mut event { - let has_failure_reason = is_failure(http) && http.base.status_detail.is_some(); - if let Some(code) = status { - http.http_response = Some(HttpResponse { code }); - } - if let serde_json::Value::Object(unmapped) = http - .base - .unmapped - .get_or_insert_with(|| serde_json::json!({})) - { - unmapped.insert("http_event_kind".into(), "request".into()); - if let Some(upstream) = upstream_status.filter(|upstream| Some(*upstream) != status) - { - unmapped.insert("upstream_response_code".into(), serde_json::json!(upstream)); - } - } - if blocked || outcome == Some(HttpOutcome::Denied) { - if http.action != Some(ActionId::Denied) { - http.base.message = Some("HTTP request blocked by supervisor".into()); - } - http.action = Some(ActionId::Denied); - http.disposition = Some(DispositionId::Blocked); - http.base.status = Some(StatusId::Failure); - raise_severity(http, SeverityId::Medium); - if !has_failure_reason { - http.base.status_detail = Some( - failure_reason - .or_else(|| outcome.map(HttpOutcome::as_str)) - .unwrap_or("supervisor_blocked") - .into(), - ); - } - } else if matches!( - outcome, - Some(HttpOutcome::PlatformError | HttpOutcome::UpstreamError) - ) { - http.base.status = Some(StatusId::Failure); - raise_severity(http, SeverityId::Low); - if !has_failure_reason { - http.base.status_detail = failure_reason - .or_else(|| outcome.map(HttpOutcome::as_str)) - .map(str::to_string); - } - } - if incomplete && outcome.is_none() { - http.base.status = Some(StatusId::Failure); - if !has_failure_reason { - http.base.status_detail = Some("request_incomplete".into()); - } - raise_severity(http, SeverityId::Low); - } + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_mut() + { + trace.observe_http(http); } - self.emit_correlated(event); - } - - fn emit_correlated(&self, mut event: OcsfEvent) { - event.set_correlation_uid(self.request_id.clone()); - let parent = { - let mut state = self - .state - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - state.trace.as_mut().map(|trace| { - if let OcsfEvent::HttpActivity(http) = &event - && !is_middleware_invocation(http) - { - trace.observe_http(http); - } - trace.span() - }) - }; - crate::telemetry::emit_inspected_decision([event], parent.as_ref()); + CURRENT_REQUEST.sync_scope(RefCell::new(Some(Arc::clone(&self.scope))), || { + crate::telemetry::emit_inspected_decision([event], Some(&self.scope.span())); + }); } } @@ -430,209 +255,18 @@ pub fn emit_request_events( } } -fn is_failure(event: &HttpActivityEvent) -> bool { - event.action == Some(ActionId::Denied) || event.base.status == Some(StatusId::Failure) -} - -fn is_middleware_invocation(event: &HttpActivityEvent) -> bool { - event - .base - .unmapped - .as_ref() - .and_then(|fields| fields.get("http_event_kind")) - .and_then(serde_json::Value::as_str) - == Some("middleware_invocation") -} - -fn merge_middleware_denial(primary: &mut HttpActivityEvent, invocation: &HttpActivityEvent) { - primary.action = invocation.action; - primary.disposition = invocation.disposition; - primary.base.status = Some(StatusId::Failure); - raise_severity(primary, invocation.base.severity); - if invocation.base.message.is_some() { - primary.base.message.clone_from(&invocation.base.message); - } - if invocation.base.status_detail.is_some() { - primary - .base - .status_detail - .clone_from(&invocation.base.status_detail); - } -} - -fn raise_severity(event: &mut HttpActivityEvent, severity: SeverityId) { - if event.base.severity.as_u8() < severity.as_u8() { - event.base.severity = severity; - } -} - -fn merge_http_outcome(primary: &mut HttpActivityEvent, incoming: HttpActivityEvent) { - let failed = is_failure(&incoming); - if primary.actor.is_none() { - primary.actor = incoming.actor; - } - if primary.src_endpoint.is_none() { - primary.src_endpoint = incoming.src_endpoint; - } - if primary.dst_endpoint.is_none() { - primary.dst_endpoint = incoming.dst_endpoint; - } - if primary.firewall_rule.is_none() { - primary.firewall_rule = incoming.firewall_rule; - } - if primary - .http_request - .as_ref() - .and_then(|request| request.url.as_ref()) - .is_none() - && incoming - .http_request - .as_ref() - .and_then(|request| request.url.as_ref()) - .is_some() - { - primary.http_request = incoming.http_request; - } - if !failed { - if incoming - .base - .unmapped - .as_ref() - .and_then(|fields| fields.get("transformed_body_policy")) - .and_then(serde_json::Value::as_str) - == Some("audit") - { - if let serde_json::Value::Object(fields) = primary - .base - .unmapped - .get_or_insert_with(|| serde_json::json!({})) - { - fields.insert("transformed_body_policy".into(), "audit".into()); - } - if !is_failure(primary) && incoming.base.status_detail.is_some() { - primary - .base - .status_detail - .clone_from(&incoming.base.status_detail); - } - } - if primary.action.is_none() { - primary.action = incoming.action; - primary.disposition = incoming.disposition; - primary.base.status = incoming.base.status; - primary.base.message = incoming.base.message; - primary.base.status_detail = incoming.base.status_detail; - } - return; - } - if incoming.action == Some(ActionId::Denied) { - primary.action = incoming.action; - primary.disposition = incoming.disposition; - } - primary.base.status = Some(StatusId::Failure); - raise_severity(primary, incoming.base.severity); - if incoming.base.message.is_some() { - primary.base.message = incoming.base.message; - } - if incoming.base.status_detail.is_some() { - primary.base.status_detail = incoming.base.status_detail; - } - if incoming.http_response.is_some() { - primary.http_response = incoming.http_response; - } - if let Some(serde_json::Value::Object(fields)) = incoming.base.unmapped - && let serde_json::Value::Object(existing) = primary - .base - .unmapped - .get_or_insert_with(|| serde_json::json!({})) - { - existing.extend(fields); - } -} - -/// Observe accepted response bytes without buffering or changing delivery. -pub struct AuditedClient<'a, C> { - inner: &'a mut C, - correlation: &'a RequestCorrelation, - head: ResponseHead, -} - -#[derive(Default)] -struct ResponseHead { - prefix: [u8; 12], - prefix_len: usize, - delimiter: u32, - complete: bool, -} - -impl ResponseHead { - fn accept(&mut self, bytes: &[u8], correlation: &RequestCorrelation) { - for &byte in bytes { - if self.complete { - break; - } - if self.prefix_len < self.prefix.len() { - self.prefix[self.prefix_len] = byte; - self.prefix_len += 1; - } - self.delimiter = (self.delimiter << 8) | u32::from(byte); - if self.delimiter != u32::from_be_bytes(*b"\r\n\r\n") { - continue; - } - let status = std::str::from_utf8(&self.prefix[9..12]) - .ok() - .and_then(|code| code.parse::().ok()); - if status.is_some_and(|code| (100..200).contains(&code) && code != 101) { - *self = Self::default(); - } else { - self.complete = true; - if let Some(status) = status { - correlation.response_selected(status); - } - } - } - } -} - -impl AsyncRead for AuditedClient<'_, C> { - fn poll_read( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { - Pin::new(&mut *self.get_mut().inner).poll_read(cx, buf) - } -} - -impl AsyncWrite for AuditedClient<'_, C> { - fn poll_write( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - bytes: &[u8], - ) -> Poll> { - let this = self.get_mut(); - let result = Pin::new(&mut *this.inner).poll_write(cx, bytes); - if let Poll::Ready(Ok(accepted)) = result - && this.correlation.inspected - { - this.head.accept(&bytes[..accepted], this.correlation); - } - result - } - - fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut *self.get_mut().inner).poll_flush(cx) - } - - fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut *self.get_mut().inner).poll_shutdown(cx) - } -} - impl Drop for RequestCorrelation { fn drop(&mut self) { - self.emit_primary(true); self.finish_http_trace(); + let _ = CURRENT_REQUEST.try_with(|current| { + let mut current = current.borrow_mut(); + if current + .as_ref() + .is_some_and(|scope| Arc::ptr_eq(scope, &self.scope)) + { + *current = self.previous.take(); + } + }); } } diff --git a/crates/openshell-supervisor-network/src/l7/correlation/tests.rs b/crates/openshell-supervisor-network/src/l7/correlation/tests.rs index c6b00b2b18..eae7bd4ab9 100644 --- a/crates/openshell-supervisor-network/src/l7/correlation/tests.rs +++ b/crates/openshell-supervisor-network/src/l7/correlation/tests.rs @@ -2,9 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 use super::*; -use crate::test_support::request_tracing::{RequestTracing, span_attribute}; -use openshell_ocsf::{ActivityId, NetworkActivityBuilder, Process}; -use tokio::io::AsyncWriteExt as _; +use crate::test_support::request_tracing::RequestTracing; +use openshell_ocsf::test_support::OcsfCapture; +use openshell_ocsf::{ + ActionId, ActivityId, DetectionFindingBuilder, DispositionId, HttpActivityBuilder, HttpRequest, + HttpResponse, NetworkActivityBuilder, SeverityId, StatusId, +}; fn request() -> L7Request { L7Request { @@ -20,7 +23,6 @@ fn context() -> L7EvalContext { L7EvalContext { host: "example.test".into(), port: 443, - policy_name: "protected-api".into(), ..Default::default() } } @@ -28,657 +30,205 @@ fn context() -> L7EvalContext { fn allowed() -> OcsfEvent { HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) .http_request(HttpRequest::new("GET", context().ocsf_url("/data"))) - .actor_process(Process::new("curl", 42).with_cmd_line("curl /data")) .firewall_rule("protected-api", "l7") .action(ActionId::Allowed) .disposition(DispositionId::Allowed) .status(StatusId::Success) .message("HTTP request admitted by policy") - .status_detail("matched_rule") .build() } -fn denied() -> OcsfEvent { +fn deny() -> OcsfEvent { HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .http_request(HttpRequest { - http_method: "GET".parse().unwrap(), - url: None, - }) + .http_request(HttpRequest::new("GET", context().ocsf_url("/data"))) .http_response(HttpResponse { code: 403 }) - .firewall_rule("protected-api", "credential-binding") + .firewall_rule("content-guard", "supervisor-middleware") .action(ActionId::Denied) .disposition(DispositionId::Blocked) .status(StatusId::Failure) - .severity(SeverityId::High) - .message("Credential use denied") - .status_detail("credential_endpoint_mismatch") - .build() -} - -fn http_events(capture: &openshell_ocsf::test_support::OcsfCapture) -> Vec { - capture - .json_events() - .into_iter() - .filter(|event| event["class_uid"] == 4002) - .collect() -} - -#[derive(Clone, Copy)] -enum InvocationOutcome { - Pass, - Transform, - FailOpen, - Deny, -} - -fn invocation(name: &str, phase: &str, outcome: InvocationOutcome) -> OcsfEvent { - let (action, disposition, status) = match outcome { - InvocationOutcome::Pass | InvocationOutcome::Transform => { - (ActionId::Allowed, DispositionId::Allowed, StatusId::Success) - } - InvocationOutcome::FailOpen => (ActionId::Other, DispositionId::Error, StatusId::Failure), - InvocationOutcome::Deny => (ActionId::Denied, DispositionId::Blocked, StatusId::Failure), - }; - HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .http_request(HttpRequest::new("POST", context().ocsf_url("/observed"))) - .http_response(HttpResponse { code: 200 }) - .firewall_rule("middleware-policy", "supervisor-middleware") - .action(action) - .disposition(disposition) - .status(status) .severity(SeverityId::Medium) - .status_detail(name) - .message(format!("middleware invocation {name}")) + .message("middleware blocked request") .unmapped("http_event_kind", "middleware_invocation") - .unmapped("middleware_phase", phase) - .unmapped("middleware_config", name) - .unmapped( - "transformed", - matches!(outcome, InvocationOutcome::Transform), - ) - .unmapped("failed", matches!(outcome, InvocationOutcome::FailOpen)) + .unmapped("middleware_phase", "response") + .unmapped("transformed", true) .build() } #[test] -fn middleware_invocations_are_retained_without_seeding_or_poisoning_the_request() { - use opentelemetry_sdk::trace::Sampler; - - for sampler in [Some(Sampler::AlwaysOn), Some(Sampler::AlwaysOff), None] { - let sampled = matches!(sampler, Some(Sampler::AlwaysOn)); - for phase in ["request", "response"] { - let capture = RequestTracing::new(sampler.clone()); - let mut expected = Vec::new(); - tracing::subscriber::with_default(capture.subscriber(), || { - let correlation = RequestCorrelation::new(&request(), &context()); - for (name, outcome) in [ - ("pass", InvocationOutcome::Pass), - ("redact", InvocationOutcome::Transform), - ("optional", InvocationOutcome::FailOpen), - ] { - let event = invocation(name, phase, outcome); - expected.push(event.to_json().unwrap()); - correlation.emit(event); - } - correlation.emit(allowed()); - correlation.response_selected(200); - }); - let events = http_events(&capture.events); - assert_eq!(events.len(), 4); - for (event, expected) in events[..3].iter().zip(&expected) { - for field in [ - "action", - "disposition", - "status", - "message", - "status_detail", - "http_request", - "http_response", - "firewall_rule", - "unmapped", - ] { - assert_eq!(event[field], expected[field], "{field}"); - } - assert_eq!(event["trace"]["uid"].is_string(), sampled); - } - let primary = &events[3]; - assert_eq!(primary["unmapped"]["http_event_kind"], "request"); - assert_eq!(primary["action"], "Allowed"); - assert_eq!(primary["status"], "Success"); - assert_eq!(primary["http_request"]["url"]["path"], "/data"); - assert_eq!(primary["actor"]["process"]["pid"], 42); - assert_eq!(primary["firewall_rule"]["type"], "l7"); - assert!(events.iter().all(|event| { - event["metadata"]["correlation_uid"] == primary["metadata"]["correlation_uid"] - })); - assert!( - !capture - .created_span_names() - .contains(&"supervisor.egress.deny") - ); - let spans = capture.spans(); - assert_eq!(spans.len(), usize::from(sampled)); - if sampled { - assert_eq!( - span_attribute(&spans[0], "openshell.request.outcome"), - Some(&"allowed".into()) - ); - assert_eq!( - span_attribute(&spans[0], "http.response.status_code"), - Some(&200_i64.into()) - ); - } - capture.shutdown(); - } - } -} - -#[test] -fn denied_middleware_invocation_only_overlays_the_preheader_denial_summary() { - for seeded in [false, true] { - let capture = openshell_ocsf::test_support::OcsfCapture::default(); - let _subscriber = capture.install(); - let correlation = RequestCorrelation::new(&request(), &context()); - if seeded { - correlation.emit(allowed()); - } - correlation.record_upstream_status(200); - correlation.emit(invocation("guard", "response", InvocationOutcome::Deny)); - correlation.response_selected(403); - drop(correlation); - let events = http_events(&capture); - assert_eq!(events.len(), 2); - let observation = &events[0]; - let primary = &events[1]; - assert_eq!( - observation["unmapped"]["http_event_kind"], - "middleware_invocation" - ); - assert_eq!(observation["http_response"]["code"], 200); - assert_eq!(observation["http_request"]["url"]["path"], "/observed"); - assert_eq!(primary["unmapped"]["http_event_kind"], "request"); - assert_eq!(primary["http_response"]["code"], 403); - assert_eq!(primary["unmapped"]["upstream_response_code"], 200); - assert_eq!(primary["http_request"]["url"]["path"], "/data"); - assert_eq!(primary["action"], "Denied"); - assert_eq!(primary["status"], "Failure"); - assert_eq!(primary["message"], "middleware invocation guard"); - assert_eq!(primary["status_detail"], "guard"); - if seeded { - assert_eq!(primary["firewall_rule"]["type"], "l7"); - assert_eq!(primary["actor"]["process"]["pid"], 42); - } else { - assert!(primary.get("firewall_rule").is_none()); - assert!(primary.get("actor").is_none()); +fn records_emit_immediately_and_keep_their_original_meaning() { + let capture = OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = RequestCorrelation::new(&request(), &context()); + for (index, source) in [allowed(), deny(), deny()].into_iter().enumerate() { + let mut expected = source.to_json().unwrap(); + expected["metadata"]["correlation_uid"] = correlation.request_id().into(); + expected["unmapped"]["request_sequence"] = (index + 1).into(); + if index == 0 { + expected["unmapped"]["http_event_kind"] = "decision".into(); } + correlation.emit(source); + assert_eq!(capture.json_events()[index], expected); + correlation.record_response_status(502); } + drop(correlation); + // No replacement, fallback, or late-failure suppression. + assert_eq!(capture.json_events().len(), 3); } #[test] -fn late_middleware_invocations_do_not_consume_the_late_failure_budget() { - use opentelemetry_sdk::trace::Sampler; - - for sampler in [Some(Sampler::AlwaysOn), Some(Sampler::AlwaysOff), None] { - let sampled = matches!(sampler, Some(Sampler::AlwaysOn)); - for late_failure in [false, true] { - let capture = RequestTracing::new(sampler.clone()); - tracing::subscriber::with_default(capture.subscriber(), || { - let correlation = RequestCorrelation::new(&request(), &context()); - correlation.emit(allowed()); - correlation.response_selected(200); - for (name, outcome) in [ - ("optional", InvocationOutcome::FailOpen), - ("guard", InvocationOutcome::Deny), - ("pass", InvocationOutcome::Pass), - ] { - correlation.emit(invocation(name, "response", outcome)); - } - if late_failure { - correlation.emit(denied()); - correlation.emit(denied()); - } - }); - let events = http_events(&capture.events); - assert_eq!(events.len(), 4 + usize::from(late_failure)); - assert_eq!(events[0]["unmapped"]["http_event_kind"], "request"); - assert_eq!(events[0]["action"], "Allowed"); - for event in &events[1..4] { - assert_eq!( - event["unmapped"]["http_event_kind"], - "middleware_invocation" - ); - assert_eq!(event["http_response"]["code"], 200); - } - if late_failure { - assert!(events[4]["unmapped"].get("http_event_kind").is_none()); - assert_eq!(events[4]["status_detail"], "credential_endpoint_mismatch"); - assert_eq!(events[4]["http_response"]["code"], 200); - } - assert!( - events - .iter() - .all(|event| event["metadata"]["correlation_uid"] - == events[0]["metadata"]["correlation_uid"]) - ); - assert!( - !capture - .created_span_names() - .contains(&"supervisor.egress.deny") - ); - let spans = capture.spans(); - assert_eq!(spans.len(), usize::from(sampled)); - if sampled { - assert_eq!( - span_attribute(&spans[0], "openshell.request.outcome"), - Some(&if late_failure { "denied" } else { "allowed" }.into()) - ); - assert_eq!( - span_attribute(&spans[0], "http.response.status_code"), - Some(&200_i64.into()) - ); - } - capture.shutdown(); - } - } +fn unused_request_scope_does_not_create_an_audit_record() { + let capture = OcsfCapture::default(); + let _subscriber = capture.install(); + drop(RequestCorrelation::new(&request(), &context())); + assert!(capture.json_events().is_empty()); } #[test] -fn audit_receipt_and_denial_are_independent_of_sampling_or_exporter() { +fn request_join_does_not_depend_on_export_and_never_creates_denial_spans() { use opentelemetry_sdk::trace::Sampler; - for sampler in [Some(Sampler::AlwaysOn), Some(Sampler::AlwaysOff), None] { let sampled = matches!(sampler, Some(Sampler::AlwaysOn)); let capture = RequestTracing::new(sampler); tracing::subscriber::with_default(capture.subscriber(), || { - let correlation = RequestCorrelation::with_request_id( - &request(), - &context(), - "00000000-0000-4000-8000-000000000001".into(), - ); + let correlation = RequestCorrelation::new(&request(), &context()); correlation.emit(allowed()); - correlation.emit(denied()); - correlation.record_outcome(HttpOutcome::Denied); - correlation.response_selected(403); + correlation.emit(deny()); }); - let events = http_events(&capture.events); - assert_eq!(events.len(), 1); - let event = &events[0]; + let records = capture.events.json_events(); + assert_eq!(records.len(), 2); + assert_eq!(records[0]["action"], "Allowed"); + assert_eq!(records[1]["action"], "Denied"); assert_eq!( - event["metadata"]["correlation_uid"], - "00000000-0000-4000-8000-000000000001" + records[0]["metadata"]["correlation_uid"], + records[1]["metadata"]["correlation_uid"] + ); + assert!(records[0]["metadata"]["correlation_uid"].is_string()); + assert_eq!(records[0]["trace"]["uid"].is_string(), sampled); + assert!( + !capture + .created_span_names() + .contains(&"supervisor.egress.deny") ); - assert_eq!(event["action"], "Denied"); - assert_eq!(event["status"], "Failure"); - assert_eq!(event["http_response"]["code"], 403); - assert_eq!(event["http_request"]["url"]["path"], "/data"); - assert_eq!(event["trace"]["uid"].is_string(), sampled); - let spans = capture.spans(); - assert_eq!(spans.len(), usize::from(sampled)); - if sampled { - assert_eq!(spans[0].name, "openshell.http.request"); - assert_eq!( - event["trace"]["uid"], - spans[0].span_context.trace_id().to_string() - ); - } capture.shutdown(); } } -#[test] -fn finished_http_trace_is_not_retained_by_an_upgraded_session() { - use opentelemetry_sdk::trace::Sampler; - - let capture = RequestTracing::new(Some(Sampler::AlwaysOn)); - tracing::subscriber::with_default(capture.subscriber(), || { +#[tokio::test] +async fn ambient_scope_stamps_companions_without_call_site_wiring() { + let capture = OcsfCapture::default(); + let _subscriber = capture.install(); + scope_connection(async { let correlation = RequestCorrelation::new(&request(), &context()); - correlation.emit(allowed()); - correlation.response_selected(101); - correlation.finish_http_trace(); - assert_eq!(capture.spans().len(), 1); - assert!(correlation.trace_span().is_disabled()); - assert!(correlation.stage(RequestStage::UpstreamSend).is_disabled()); - correlation.finish_http_trace(); + crate::telemetry::emit_decision( + [ + NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Open) + .action(ActionId::Denied) + .message("credential refused") + .dst_endpoint(openshell_ocsf::Endpoint::from_domain("example.test", 443)) + .build(), + DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .finding_info(openshell_ocsf::FindingInfo::new( + "credential", + "Credential refused", + )) + .build(), + ], + None, + ); + let records = capture.json_events(); + assert_eq!(records.len(), 2); + for (index, record) in records.iter().enumerate() { + assert_eq!( + record["metadata"]["correlation_uid"], + correlation.request_id() + ); + assert_eq!(record["unmapped"]["request_sequence"], index + 1); + } drop(correlation); - assert_eq!(capture.spans().len(), 1); - }); - capture.shutdown(); -} - -#[test] -fn a_provider_without_an_exporter_still_emits_the_request_id_join() { - use opentelemetry::trace::TracerProvider as _; - use opentelemetry_sdk::trace::{Sampler, SdkTracerProvider}; - use tracing_subscriber::layer::SubscriberExt as _; - - crate::telemetry::tests::init_trace_extractor(); - let provider = SdkTracerProvider::builder() - .with_sampler(Sampler::AlwaysOn) - .build(); - let capture = openshell_ocsf::test_support::OcsfCapture::default(); - let subscriber = tracing_subscriber::registry() - .with(capture.clone()) - .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("no-exporter-test"))); - let mut id = None; - tracing::subscriber::with_default(subscriber, || { - let correlation = RequestCorrelation::new(&request(), &context()); - id = Some(correlation.request_id().to_owned()); - correlation - .stage(RequestStage::PolicyEvaluate) - .in_scope(|| correlation.emit(allowed())); - correlation.record_outcome(HttpOutcome::Allowed); - correlation.response_selected(200); - }); - let events = http_events(&capture); - assert_eq!(events.len(), 1); - assert_eq!(events[0]["metadata"]["correlation_uid"], id.unwrap()); - assert_eq!(events[0]["http_response"]["code"], 200); - assert_eq!(events[0]["action"], "Allowed"); - provider.shutdown().unwrap(); + crate::telemetry::emit_decision([allowed()], None); + assert!(capture.json_events()[2]["metadata"]["correlation_uid"].is_null()); + }) + .await; } #[tokio::test] -async fn primary_record_waits_for_final_response_headers_and_is_emitted_once_without_otel() { - let capture = openshell_ocsf::test_support::OcsfCapture::default(); +async fn concurrent_connections_and_nested_requests_restore_their_own_scope() { + let capture = OcsfCapture::default(); let _subscriber = capture.install(); - let correlation = RequestCorrelation::new(&request(), &context()); - let id = correlation.request_id().to_string(); - correlation.emit(allowed()); - let mut output = Vec::new(); - { - let mut client = correlation.observe_response(&mut output); - client - .write_all(b"HTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 4\r\n") - .await - .unwrap(); - assert!(capture.events().is_empty()); - client.write_all(b"\r\nbody").await.unwrap(); - } - drop(correlation); - let events = http_events(&capture); - assert_eq!(events.len(), 1); - assert_eq!(events[0]["http_response"]["code"], 200); - assert_eq!(events[0]["http_request"]["url"]["path"], "/data"); - assert_eq!(events[0]["action"], "Allowed"); - assert_eq!(events[0]["status"], "Success"); - assert_eq!(events[0]["actor"]["process"]["pid"], 42); - assert_eq!(events[0]["metadata"]["correlation_uid"], id); - assert!(events[0].get("trace").is_none()); - assert!(output.ends_with(b"\r\n\r\nbody")); -} - -#[test] -fn later_credential_denial_preserves_policy_request_context() { - let capture = openshell_ocsf::test_support::OcsfCapture::default(); - let _subscriber = capture.install(); - let correlation = RequestCorrelation::new(&request(), &context()); - let id = correlation.request_id().to_string(); - correlation.emit(allowed()); - correlation.emit(denied()); - // A routine later middleware result cannot undo the denial. - correlation.emit(allowed()); - correlation.response_selected(403); - drop(correlation); - let events = http_events(&capture); - assert_eq!(events.len(), 1); - let event = &events[0]; - assert_eq!(event["action"], "Denied"); - assert_eq!(event["disposition"], "Blocked"); - assert_eq!(event["status"], "Failure"); - assert_eq!(event["severity"], "High"); - assert_eq!(event["status_detail"], "credential_endpoint_mismatch"); - assert_eq!(event["http_response"]["code"], 403); - assert_eq!(event["http_request"]["url"]["path"], "/data"); - assert_eq!(event["firewall_rule"]["type"], "l7"); - assert_eq!(event["actor"]["process"]["pid"], 42); - assert_eq!(event["metadata"]["correlation_uid"], id); -} - -#[test] -fn local_refusals_and_upstream_failures_are_distinct() { - for (outcome, blocked, code, expected_action, expected_disposition) in [ - (HttpOutcome::PlatformError, true, 503, "Denied", "Blocked"), - (HttpOutcome::UpstreamError, false, 502, "Allowed", "Allowed"), - (HttpOutcome::PlatformError, false, 502, "Allowed", "Allowed"), - (HttpOutcome::Allowed, false, 500, "Allowed", "Allowed"), - ] { - let capture = openshell_ocsf::test_support::OcsfCapture::default(); - let _subscriber = capture.install(); - let correlation = RequestCorrelation::new(&request(), &context()); - correlation.emit(allowed()); - if blocked { - correlation.record_enforcement_block(); - } - correlation.record_outcome(outcome); - correlation.response_selected(code); - drop(correlation); - let events = http_events(&capture); - assert_eq!(events.len(), 1); - assert_eq!(events[0]["action"], expected_action); - assert_eq!(events[0]["disposition"], expected_disposition); - assert_eq!( - events[0]["status"], - if outcome == HttpOutcome::Allowed { - "Success" - } else { - "Failure" + let run = |name: &'static str| { + scope_connection(async move { + let outer = RequestCorrelation::with_request_id(&request(), &context(), name.into()); + crate::telemetry::emit_decision([allowed()], None); + tokio::task::yield_now().await; + { + let _nested = RequestCorrelation::with_request_id( + &request(), + &context(), + format!("{name}-nested"), + ); + crate::telemetry::emit_decision([allowed()], None); } + crate::telemetry::emit_decision([allowed()], None); + drop(outer); + let _uninspected = RequestCorrelation::uninspected(); + crate::telemetry::emit_decision([allowed()], None); + }) + }; + tokio::join!(run("first"), run("second")); + let records = capture.json_events(); + assert_eq!(records.len(), 8); + for name in ["first", "second"] { + let outer: Vec<_> = records + .iter() + .filter(|record| record["metadata"]["correlation_uid"] == name) + .collect(); + assert_eq!(outer.len(), 2); + assert_eq!(outer[0]["unmapped"]["request_sequence"], 1); + assert_eq!(outer[1]["unmapped"]["request_sequence"], 2); + assert_eq!( + records + .iter() + .filter(|record| record["metadata"]["correlation_uid"] == format!("{name}-nested")) + .count(), + 1 ); - assert_eq!(events[0]["http_response"]["code"], code); } -} - -#[test] -fn pre_policy_rejection_retains_reason_without_claiming_policy_admission() { - let capture = openshell_ocsf::test_support::OcsfCapture::default(); - let _subscriber = capture.install(); - let correlation = RequestCorrelation::new(&request(), &context()); - let id = correlation.request_id().to_string(); - correlation.emit( - NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .dst_endpoint(Endpoint::from_domain("example.test", 443)) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .status(StatusId::Failure) - .severity(SeverityId::Medium) - .status_detail("invalid_jsonrpc_request") - .actor_process(Process::new("curl", 42)) - .build(), - ); - correlation.record_enforcement_block(); - correlation.record_outcome(HttpOutcome::Denied); - correlation.response_selected(400); - drop(correlation); - assert_eq!(capture.events().len(), 2); - let events = http_events(&capture); - assert_eq!(events.len(), 1); - assert_eq!(events[0]["http_request"]["url"]["path"], "/data"); - assert_eq!(events[0]["status_detail"], "invalid_jsonrpc_request"); assert_eq!( - events[0]["message"], - "HTTP request rejected before policy evaluation" - ); - assert_eq!(events[0]["action"], "Denied"); - assert_eq!(events[0]["actor"]["process"]["pid"], 42); - assert!(events[0].get("firewall_rule").is_none()); - assert!( - capture - .json_events() - .iter() - .all(|event| event["metadata"]["correlation_uid"] == id) - ); -} - -#[test] -fn late_failure_is_preserved_once_but_routine_late_records_are_not_duplicated() { - let capture = openshell_ocsf::test_support::OcsfCapture::default(); - let _subscriber = capture.install(); - let correlation = RequestCorrelation::new(&request(), &context()); - let id = correlation.request_id().to_string(); - correlation.emit(allowed()); - correlation.response_selected(200); - correlation.emit(allowed()); - correlation.emit(denied()); - correlation.emit(denied()); - drop(correlation); - let events = http_events(&capture); - assert_eq!(events.len(), 2); - assert_eq!(events[0]["action"], "Allowed"); - assert_eq!(events[0]["http_response"]["code"], 200); - assert_eq!(events[1]["action"], "Denied"); - assert_eq!(events[1]["status_detail"], "credential_endpoint_mismatch"); - assert_eq!(events[1]["http_response"]["code"], 200); - assert!( - events + records .iter() - .all(|event| event["metadata"]["correlation_uid"] == id) - ); -} - -#[test] -fn unfinished_requests_preserve_admission_context_and_omit_uncommitted_response() { - for admitted in [false, true] { - let capture = openshell_ocsf::test_support::OcsfCapture::default(); - let _subscriber = capture.install(); - let mut request = request(); - request.target = "/data?secret=SENTINEL-QUERY".into(); - let correlation = RequestCorrelation::new(&request, &context()); - let id = correlation.request_id().to_string(); - if admitted { - correlation.emit(allowed()); - } else { - let mut head = ResponseHead::default(); - head.accept(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n", &correlation); - } - assert!(capture.events().is_empty()); - drop(correlation); - let events = http_events(&capture); - assert_eq!(events.len(), 1); - assert_eq!(events[0]["metadata"]["correlation_uid"], id); - assert_eq!(events[0]["status"], "Failure"); - assert_eq!(events[0]["status_detail"], "request_incomplete"); - assert_eq!(events[0]["http_request"]["url"]["path"], "/data"); - assert!(events[0].get("http_response").is_none()); - assert!(!events[0].to_string().contains("SENTINEL-QUERY")); - if admitted { - assert_eq!(events[0]["action"], "Allowed"); - assert_eq!(events[0]["actor"]["process"]["pid"], 42); - } else { - assert!(events[0].get("firewall_rule").is_none()); - } - } -} - -#[test] -fn workload_traceparent_does_not_supply_audit_trace_identity() { - let capture = openshell_ocsf::test_support::OcsfCapture::default(); - let _subscriber = capture.install(); - let mut request = request(); - request.raw_header = b"GET /data HTTP/1.1\r\nHost: example.test\r\ntraceparent: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\r\n\r\n".to_vec(); - let correlation = RequestCorrelation::new(&request, &context()); - correlation.emit(allowed()); - correlation.response_selected(200); - drop(correlation); - let events = http_events(&capture); - assert_eq!(events.len(), 1); - assert!(events[0].get("trace").is_none()); - assert!( - !events[0] - .to_string() - .contains("4bf92f3577b34da6a3ce929d0e0e4736") + .filter(|record| record["metadata"]["correlation_uid"].is_null()) + .count(), + 2 ); + assert!(!is_inspected_request()); } #[tokio::test] -async fn uninspected_response_and_events_preserve_legacy_behavior() { - let capture = openshell_ocsf::test_support::OcsfCapture::default(); +async fn cancelling_a_request_does_not_leak_its_context() { + let capture = OcsfCapture::default(); let _subscriber = capture.install(); - let correlation = RequestCorrelation::uninspected(); - assert!(uuid::Uuid::parse_str(correlation.request_id()).is_ok()); - let mut output = Vec::new(); - { - let mut client = correlation.observe_response(&mut output); - client - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") - .await - .unwrap(); + let (ready, started) = tokio::sync::oneshot::channel(); + let mut pending = Box::pin(scope_connection(async { + let _correlation = RequestCorrelation::new(&request(), &context()); + ready.send(()).unwrap(); + std::future::pending::<()>().await; + })); + tokio::select! { + () = &mut pending => unreachable!(), + _ = started => {}, } - assert!(capture.events().is_empty()); - let expected = allowed(); - correlation.emit(expected.clone()); - drop(correlation); - assert_eq!(capture.events(), vec![expected]); - assert_eq!(output, b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"); -} - -#[test] -fn policy_classification_fills_an_existing_forward_request_context() { - let capture = openshell_ocsf::test_support::OcsfCapture::default(); - let _subscriber = capture.install(); - let correlation = RequestCorrelation::new(&request(), &context()); - let seed = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .http_request(HttpRequest::new("GET", context().ocsf_url("/forward"))) - .actor_process(Process::new("python3", 77)) - .message("HTTP request awaiting inspection") - .build(); - correlation.emit(seed); - correlation.emit(allowed()); - correlation.response_selected(200); - drop(correlation); - let events = http_events(&capture); - assert_eq!(events.len(), 1); - assert_eq!(events[0]["http_request"]["url"]["path"], "/forward"); - assert_eq!(events[0]["actor"]["process"]["pid"], 77); - assert_eq!(events[0]["action"], "Allowed"); - assert_eq!(events[0]["status"], "Success"); - assert_eq!(events[0]["firewall_rule"]["type"], "l7"); - assert_eq!(events[0]["message"], "HTTP request admitted by policy"); -} - -#[test] -fn failure_reason_replaces_only_a_prior_success_reason() { - let capture = openshell_ocsf::test_support::OcsfCapture::default(); - let _subscriber = capture.install(); - let correlation = RequestCorrelation::new(&request(), &context()); - correlation.emit(allowed()); - correlation.record_rejection( - 503, - HttpOutcome::PlatformError, - Some("middleware_admission"), - ); - correlation.response_selected(503); - drop(correlation); - let events = http_events(&capture); - assert_eq!(events.len(), 1); - assert_eq!(events[0]["status_detail"], "middleware_admission"); - assert_eq!(events[0]["action"], "Denied"); - assert_eq!(events[0]["status"], "Failure"); -} - -#[test] -fn late_fail_open_does_not_hide_a_later_blocking_failure() { - let capture = openshell_ocsf::test_support::OcsfCapture::default(); - let _subscriber = capture.install(); - let correlation = RequestCorrelation::new(&request(), &context()); - let id = correlation.request_id().to_string(); - correlation.emit(allowed()); - correlation.response_selected(200); - correlation.emit( - HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .http_request(HttpRequest::new("GET", context().ocsf_url("/data"))) - .action(ActionId::Other) - .disposition(DispositionId::Allowed) - .status(StatusId::Failure) - .status_detail("middleware_fail_open") - .build(), - ); - correlation.emit(denied()); - drop(correlation); - let events = http_events(&capture); - assert_eq!(events.len(), 2); - assert_eq!(events[1]["action"], "Denied"); - assert_eq!(events[1]["status_detail"], "credential_endpoint_mismatch"); - assert_eq!(events[1]["metadata"]["correlation_uid"], id); - assert_eq!(events[1]["http_response"]["code"], 200); + assert!(!is_inspected_request()); + // Drop while pending, then start a new connection on the same task. + drop(pending); + scope_connection(async { + let correlation = RequestCorrelation::new(&request(), &context()); + crate::telemetry::emit_decision([allowed()], None); + assert_eq!( + capture.json_events()[0]["metadata"]["correlation_uid"], + correlation.request_id() + ); + assert_eq!(capture.json_events()[0]["unmapped"]["request_sequence"], 1); + }) + .await; } diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 1b14041d85..dd2b9e1bb9 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -293,12 +293,8 @@ async fn reject_request_authority_mismatch( where W: AsyncWrite + Unpin, { - let activity = build_request_authority_mismatch_event(ctx, method); - let finding = build_request_authority_mismatch_finding(ctx); if let Some(correlation) = correlation { correlation.record_rejection(403, HttpOutcome::Denied, None); - correlation.emit(activity); - correlation.emit(finding); crate::l7::rest::send_json_response_with_request_id( &ctx.policy_name, serde_json::json!({ @@ -311,7 +307,6 @@ where ) .await?; } else { - crate::telemetry::emit_decision([activity, finding], None); crate::l7::rest::send_json_response( &ctx.policy_name, serde_json::json!({ @@ -323,6 +318,13 @@ where ) .await?; } + crate::l7::correlation::emit_request_events( + correlation, + [ + build_request_authority_mismatch_event(ctx, method), + build_request_authority_mismatch_finding(ctx), + ], + ); Ok(()) } @@ -539,32 +541,8 @@ where /// Reinspect the buffered outgoing MCP request after request transformations. /// The forwarding adapter must call this before any upstream request write. -#[cfg(test)] -pub(crate) async fn enforce_final_mcp_protocol_version( - config: &L7EndpointConfig, - request: &crate::l7::provider::L7Request, - client: &mut W, - ctx: &L7EvalContext, - redacted_target: &str, - observer: Option<&EndpointObserver>, -) -> Result -where - W: AsyncWrite + Unpin, -{ - enforce_final_mcp_protocol_version_correlated( - config, - request, - client, - ctx, - redacted_target, - observer, - None, - ) - .await -} - #[allow(clippy::too_many_arguments)] -pub(crate) async fn enforce_final_mcp_protocol_version_correlated( +pub(crate) async fn enforce_final_mcp_protocol_version( config: &L7EndpointConfig, request: &crate::l7::provider::L7Request, client: &mut W, @@ -722,6 +700,24 @@ where "message": "Credential placeholder could not be resolved", }) }; + if let Some(correlation) = correlation { + let (response_code, outcome) = if endpoint_mismatch || placement_mismatch { + (403, HttpOutcome::Denied) + } else { + (500, HttpOutcome::PlatformError) + }; + correlation.record_rejection(response_code, outcome, None); + crate::l7::rest::send_json_response_with_request_id( + &ctx.policy_name, + body, + client, + status, + correlation.request_id(), + ) + .await?; + } else { + crate::l7::rest::send_json_response(&ctx.policy_name, body, client, status).await?; + } let mut events = if placement_mismatch { vec![ HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -753,28 +749,7 @@ where if endpoint_mismatch { events.push(build_credential_endpoint_mismatch_finding(ctx)); } - if let Some(correlation) = correlation { - let (response_code, outcome) = if endpoint_mismatch || placement_mismatch { - (403, HttpOutcome::Denied) - } else { - (500, HttpOutcome::PlatformError) - }; - correlation.record_rejection(response_code, outcome, None); - for event in events { - correlation.emit(event); - } - crate::l7::rest::send_json_response_with_request_id( - &ctx.policy_name, - body, - client, - status, - correlation.request_id(), - ) - .await?; - } else { - crate::telemetry::emit_decision(events, None); - crate::l7::rest::send_json_response(&ctx.policy_name, body, client, status).await?; - } + crate::l7::correlation::emit_request_events(correlation, events); Ok(()) } @@ -1076,13 +1051,7 @@ where observer.observe_credential_failure(false); } if let Some(correlation) = correlation { - correlation.record_enforcement_block(); correlation.record_outcome(HttpOutcome::PlatformError); - let mut event = build_credential_resolution_event(ctx, &request.action, false); - if let openshell_ocsf::OcsfEvent::HttpActivity(http) = &mut event { - http.http_response = None; - } - correlation.emit(event); } Err(report) } else { @@ -1532,6 +1501,23 @@ pub async fn relay_with_route_selection( upstream: &mut U, ctx: &L7EvalContext, ) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send, + U: AsyncRead + AsyncWrite + Unpin + Send, +{ + crate::l7::correlation::scope_connection(relay_with_route_selection_inner( + configs, engine, client, upstream, ctx, + )) + .await +} + +async fn relay_with_route_selection_inner( + configs: &[L7EndpointConfig], + engine: TunnelPolicyEngine, + client: &mut C, + upstream: &mut U, + ctx: &L7EvalContext, +) -> Result<()> where C: AsyncRead + AsyncWrite + Unpin + Send, U: AsyncRead + AsyncWrite + Unpin + Send, @@ -1582,8 +1568,6 @@ where } }; let correlation = RequestCorrelation::new(&req, ctx); - let mut audited_client = correlation.observe_response(client); - let client = &mut audited_client; if !request_authority_matches_endpoint(&req, ctx) { reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)).await?; return Ok(()); @@ -1703,7 +1687,6 @@ where let detail = parse_rejection_detail(&e.to_string(), ParseRejectionMode::L7Endpoint); { - correlation.record_enforcement_block(); correlation.record_outcome(HttpOutcome::Denied); } emit_parse_rejection_correlated( @@ -1748,7 +1731,6 @@ where ParseRejectionMode::L7Endpoint, ); { - correlation.record_enforcement_block(); correlation.record_outcome(HttpOutcome::Denied); } emit_parse_rejection_correlated( @@ -1810,16 +1792,6 @@ where if config.protocol == L7Protocol::Websocket && !websocket_request { let reason = "websocket endpoint requires a valid WebSocket upgrade request"; correlation.record_rejection(403, HttpOutcome::Denied, None); - emit_l7_request_log_correlated( - ctx, - &request_info.action, - &redacted_target, - "deny", - "l7-websocket", - reason, - None, - Some(&correlation), - ); let deny_context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx) .with_request_id(correlation.request_id()); crate::l7::rest::RestProvider::default() @@ -1855,11 +1827,6 @@ where (false, EnforcementMode::Audit) => "audit", (false, EnforcementMode::Enforce) => "deny", }; - { - if !allowed && (config.enforcement != EnforcementMode::Audit || force_deny) { - correlation.record_enforcement_block(); - } - } let engine_type = engine_type_for_protocol(config.protocol); let protocol_summary = l7_protocol_log_summary(graphql_info.as_ref(), jsonrpc_info.as_ref()); @@ -2555,6 +2522,23 @@ async fn relay_rest( upstream: &mut U, ctx: &L7EvalContext, ) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send, + U: AsyncRead + AsyncWrite + Unpin + Send, +{ + crate::l7::correlation::scope_connection(relay_rest_inner( + config, engine, client, upstream, ctx, + )) + .await +} + +async fn relay_rest_inner( + config: &L7EndpointConfig, + engine: &TunnelPolicyEngine, + client: &mut C, + upstream: &mut U, + ctx: &L7EvalContext, +) -> Result<()> where C: AsyncRead + AsyncWrite + Unpin + Send, U: AsyncRead + AsyncWrite + Unpin + Send, @@ -2598,8 +2582,6 @@ where } }; let correlation = RequestCorrelation::new(&req, ctx); - let mut audited_client = correlation.observe_response(client); - let client = &mut audited_client; if !request_authority_matches_endpoint(&req, ctx) { reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)).await?; return Ok(()); @@ -2643,16 +2625,6 @@ where if config.protocol == L7Protocol::Websocket && !websocket_request { let reason = "websocket endpoint requires a valid WebSocket upgrade request"; correlation.record_rejection(403, HttpOutcome::Denied, None); - emit_l7_request_log_correlated( - ctx, - &request_info.action, - &redacted_target, - "deny", - "l7-websocket", - reason, - None, - Some(&correlation), - ); provider .deny_with_redacted_target( &req, @@ -2697,9 +2669,6 @@ where (false, EnforcementMode::Audit, _) => "audit", (false, EnforcementMode::Enforce, _) => "deny", }; - if !allowed && config.enforcement != EnforcementMode::Audit { - correlation.record_enforcement_block(); - } // Log every L7 decision as an OCSF HTTP Activity event. // Uses redacted_target (path only, no query params) to avoid logging secrets. @@ -3070,6 +3039,23 @@ async fn relay_jsonrpc( upstream: &mut U, ctx: &L7EvalContext, ) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send, + U: AsyncRead + AsyncWrite + Unpin + Send, +{ + crate::l7::correlation::scope_connection(relay_jsonrpc_inner( + config, engine, client, upstream, ctx, + )) + .await +} + +async fn relay_jsonrpc_inner( + config: &L7EndpointConfig, + engine: &TunnelPolicyEngine, + client: &mut C, + upstream: &mut U, + ctx: &L7EvalContext, +) -> Result<()> where C: AsyncRead + AsyncWrite + Unpin + Send, U: AsyncRead + AsyncWrite + Unpin + Send, @@ -3118,8 +3104,6 @@ where let req = parsed.request; let jsonrpc_info = parsed.info; let correlation = RequestCorrelation::new(&req, ctx); - let mut audited_client = correlation.observe_response(client); - let client = &mut audited_client; let observer = EndpointObserver::begin_captured( ctx.endpoint_observation_tx.as_ref(), config, @@ -3208,9 +3192,6 @@ where (false, EnforcementMode::Audit) => "audit", (false, EnforcementMode::Enforce) => "deny", }; - if !allowed && (config.enforcement != EnforcementMode::Audit || force_deny) { - correlation.record_enforcement_block(); - } { let (action_id, disposition_id, severity) = match decision_str { @@ -3407,6 +3388,23 @@ async fn relay_graphql( upstream: &mut U, ctx: &L7EvalContext, ) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send, + U: AsyncRead + AsyncWrite + Unpin + Send, +{ + crate::l7::correlation::scope_connection(relay_graphql_inner( + config, engine, client, upstream, ctx, + )) + .await +} + +async fn relay_graphql_inner( + config: &L7EndpointConfig, + engine: &TunnelPolicyEngine, + client: &mut C, + upstream: &mut U, + ctx: &L7EvalContext, +) -> Result<()> where C: AsyncRead + AsyncWrite + Unpin + Send, U: AsyncRead + AsyncWrite + Unpin + Send, @@ -3450,8 +3448,6 @@ where } }; let correlation = RequestCorrelation::new(&req, ctx); - let mut audited_client = correlation.observe_response(client); - let client = &mut audited_client; if !request_authority_matches_endpoint(&req, ctx) { reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)).await?; @@ -3489,7 +3485,6 @@ where } else { let detail = parse_rejection_detail(&e.to_string(), ParseRejectionMode::L7Endpoint); - correlation.record_enforcement_block(); correlation.record_outcome(HttpOutcome::Denied); emit_parse_rejection_correlated(ctx, &detail, "l7-graphql", Some(&correlation)); } @@ -3550,9 +3545,6 @@ where (false, EnforcementMode::Audit) => "audit", (false, EnforcementMode::Enforce) => "deny", }; - if !allowed && (config.enforcement != EnforcementMode::Audit || force_deny) { - correlation.record_enforcement_block(); - } { let (action_id, disposition_id, severity) = match decision_str { @@ -4103,7 +4095,7 @@ fn emit_transformed_body_decision( SeverityId::Informational, ), }; - let mut event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Other) .action(action_id) .disposition(disposition_id) @@ -4114,17 +4106,11 @@ fn emit_transformed_body_decision( )) .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) .firewall_rule(&ctx.policy_name, engine_type) - .status_detail(reason) .message(format!( "L7_REQUEST_TRANSFORMED {decision_str} {} {}:{}{} reason={}", request_info.action, ctx.host, ctx.port, request_info.target, reason )) .build(); - if decision_str == "audit" - && let openshell_ocsf::OcsfEvent::HttpActivity(http) = &mut event - { - http.base.unmapped = Some(serde_json::json!({"transformed_body_policy": "audit"})); - } crate::l7::correlation::emit_request_events(correlation, [event]); } @@ -4285,6 +4271,27 @@ pub async fn relay_passthrough_with_credentials( generation_guard: &PolicyGenerationGuard, middleware_engine: Option<&crate::opa::OpaEngine>, ) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send, + U: AsyncRead + AsyncWrite + Unpin + Send, +{ + crate::l7::correlation::scope_connection(relay_passthrough_with_credentials_inner( + client, + upstream, + ctx, + generation_guard, + middleware_engine, + )) + .await +} + +async fn relay_passthrough_with_credentials_inner( + client: &mut C, + upstream: &mut U, + ctx: &L7EvalContext, + generation_guard: &PolicyGenerationGuard, + middleware_engine: Option<&crate::opa::OpaEngine>, +) -> Result<()> where C: AsyncRead + AsyncWrite + Unpin + Send, U: AsyncRead + AsyncWrite + Unpin + Send, @@ -4338,8 +4345,6 @@ where RequestCorrelation::uninspected() }; let receipt = inspected.then_some(&correlation); - let mut audited_client = correlation.observe_response(client); - let client = &mut audited_client; if !request_authority_matches_endpoint(&req, ctx) { reject_request_authority_mismatch(client, ctx, &req.action, receipt).await?; return Ok(()); @@ -4658,7 +4663,7 @@ mod trace_tests { mod tests { use super::*; use crate::opa::{NetworkInput, OpaEngine}; - use crate::test_support::{assert_http_request_records, response_request_id}; + use crate::test_support::response_request_id; use openshell_core::proto::{StaticCredentialBinding, StaticCredentialEndpointBinding}; use openshell_core::provider_credentials::ProviderCredentialState; use openshell_core::token_grant_failure::{ @@ -4669,6 +4674,20 @@ mod tests { use std::path::PathBuf; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; + fn assert_http_control_records( + events: &[serde_json::Value], + expected_controls: usize, + expected_invocations: usize, + ) -> (Vec<&serde_json::Value>, Vec<&serde_json::Value>) { + let (invocations, controls): (Vec<_>, Vec<_>) = events + .iter() + .filter(|event| event["class_uid"] == 4002) + .partition(|event| event["unmapped"]["http_event_kind"] == "middleware_invocation"); + assert_eq!(controls.len(), expected_controls, "{events:?}"); + assert_eq!(invocations.len(), expected_invocations, "{events:?}"); + (controls, invocations) + } + fn relay_grant_failure() -> TokenGrantFailure { TokenGrantFailure::new( TokenGrantFailureKind::ProviderAuthorizationRequired, @@ -8530,7 +8549,7 @@ network_policies: } #[tokio::test] - async fn transformed_jsonrpc_policy_outcome_has_one_primary_and_shared_receipt() { + async fn transformed_jsonrpc_policy_records_keep_their_decisions_and_shared_receipt() { use tracing::instrument::WithSubscriber as _; use tracing_subscriber::layer::SubscriberExt as _; @@ -8545,23 +8564,48 @@ network_policies: .await; let request_id = response_request_id(response.as_bytes()); let events = capture.json_events(); - let (primary, companions) = assert_http_request_records(&events, 1); + let (controls, invocations) = assert_http_control_records(&events, 2, 1); + let initial = controls[0]; + let transformed = controls[1]; + let companions = invocations; assert_eq!(companions[0]["unmapped"]["middleware_phase"], "request"); assert_eq!(companions[0]["unmapped"]["transformed"], true); assert_eq!(companions[0]["metadata"]["correlation_uid"], request_id); - assert_eq!(primary["metadata"]["correlation_uid"], request_id); - assert_eq!(primary["http_request"]["url"]["path"], "/rpc"); - assert_eq!(primary["firewall_rule"]["type"], "l7-jsonrpc"); + assert_eq!(initial["action"], "Allowed"); + assert!( + initial["message"] + .as_str() + .unwrap() + .starts_with("JSONRPC_L7_REQUEST decision=allow ") + ); + for control in &controls { + assert_eq!(control["metadata"]["correlation_uid"], request_id); + assert_eq!(control["http_request"]["url"]["path"], "/rpc"); + assert_eq!(control["firewall_rule"]["type"], "l7-jsonrpc"); + assert!(control.get("http_response").is_none()); + } + assert!( + transformed["message"] + .as_str() + .unwrap() + .contains("middleware transformation denied") + ); + let sequences: Vec<_> = events + .iter() + .filter(|event| event["class_uid"] == 4002) + .filter_map(|event| event["unmapped"]["request_sequence"].as_u64()) + .collect(); + assert_eq!(sequences.len(), 3, "{events:?}"); + assert!(sequences.windows(2).all(|pair| pair[0] < pair[1])); if enforcement == "enforce" { assert_middleware_failure_response(&response, "jsonrpc_api"); assert!(upstream_seen.is_none(), "upstream must not see the request"); - assert_eq!(primary["action"], "Denied"); - assert_eq!(primary["http_response"]["code"], 403); + assert_eq!(transformed["action"], "Denied"); assert!( - primary["status_detail"] + transformed["message"] .as_str() .unwrap() - .contains("middleware transformation denied") + .starts_with("L7_REQUEST_TRANSFORMED deny ") ); } else { assert!(response.contains("204 No Content"), "{response}"); @@ -8570,9 +8614,13 @@ network_policies: .expect("audited request reaches upstream") .contains("admin.delete") ); - assert_eq!(primary["action"], "Allowed"); - assert_eq!(primary["http_response"]["code"], 204); - assert_eq!(primary["unmapped"]["transformed_body_policy"], "audit"); + assert_eq!(transformed["action"], "Allowed"); + assert!( + transformed["message"] + .as_str() + .unwrap() + .starts_with("L7_REQUEST_TRANSFORMED audit ") + ); } } } @@ -9463,7 +9511,7 @@ network_policies: } #[tokio::test] - async fn passthrough_receipts_require_selected_middleware_and_keep_one_primary() { + async fn passthrough_receipts_require_selected_middleware_and_preserve_control_records() { use tracing::instrument::WithSubscriber as _; use tracing_subscriber::layer::SubscriberExt as _; @@ -9561,38 +9609,26 @@ network_policies: "{forwarded}" ); let records = capture.json_events(); - let http: Vec<_> = records - .iter() - .filter(|event| event["class_uid"] == 4002) - .collect(); - let primary: Vec<_> = records - .iter() - .filter(|event| { - event["class_uid"] == 4002 - && if selected { - event["unmapped"]["http_event_kind"] == "request" - } else { - event["unmapped"].get("http_event_kind").is_none() - } - }) - .collect(); - assert_eq!(primary.len(), 1, "selected={selected}: {records:?}"); - let companions: Vec<_> = http - .iter() - .filter(|event| event["unmapped"]["http_event_kind"] == "middleware_invocation") - .collect(); - assert_eq!(companions.len(), usize::from(selected)); - assert_eq!(http.len(), primary.len() + companions.len()); + let (controls, companions) = + assert_http_control_records(&records, 1, usize::from(selected)); + let control = controls[0]; if selected { assert_eq!(companions[0]["unmapped"]["middleware_phase"], "request"); assert_eq!(companions[0]["unmapped"]["failed"], false); assert_eq!( companions[0]["metadata"]["correlation_uid"], - primary[0]["metadata"]["correlation_uid"] + control["metadata"]["correlation_uid"] ); } - assert_eq!(primary[0]["action"], "Allowed"); - assert_eq!(primary[0]["http_request"]["url"]["path"], "/v1/messages"); + assert_eq!(control["action"], "Allowed"); + assert_eq!(control["http_request"]["url"]["path"], "/v1/messages"); + assert!( + control["message"] + .as_str() + .unwrap() + .starts_with("HTTP_REQUEST ") + ); + assert!(control.get("http_response").is_none()); assert!( !serde_json::to_string(&records) .unwrap() @@ -9603,8 +9639,7 @@ network_policies: assert!(!response.contains("application-out")); let receipt = response_request_id(response.as_bytes()); assert!(uuid::Uuid::parse_str(receipt).is_ok()); - assert_eq!(primary[0]["metadata"]["correlation_uid"], receipt); - assert_eq!(primary[0]["http_response"]["code"], 204); + assert_eq!(control["metadata"]["correlation_uid"], receipt); } else { assert!( forwarded.contains("X-OpenShell-Request-ID: application-in"), @@ -9614,7 +9649,7 @@ network_policies: response.contains("X-OpenShell-Request-ID: application-out"), "{response}" ); - assert!(primary[0]["metadata"].get("correlation_uid").is_none()); + assert!(control["metadata"].get("correlation_uid").is_none()); } } } @@ -13994,14 +14029,16 @@ network_policies: request_id ); let events = capture.events.json_events(); - let (primary, companions) = assert_http_request_records(&events, 1); + let (controls, companions) = assert_http_control_records(&events, 1, 1); assert_eq!(companions[0]["unmapped"]["middleware_phase"], "request"); assert_eq!(companions[0]["metadata"]["correlation_uid"], request_id); assert_eq!( companions[0]["action"], if reject { "Denied" } else { "Allowed" } ); - assert_eq!(primary["metadata"]["correlation_uid"], request_id); + assert_eq!(controls[0]["metadata"]["correlation_uid"], request_id); + assert_eq!(controls[0]["action"], "Allowed"); + assert!(controls[0].get("http_response").is_none()); let spans = capture.spans(); let stage = |stage_name: &str| { spans @@ -15007,6 +15044,7 @@ network_policies: &ctx, "/mcp", None, + None, ) .await .unwrap(); @@ -15039,7 +15077,8 @@ network_policies: &mut response, &ctx, "/mcp", - None + None, + None, ) .await .unwrap() @@ -15490,6 +15529,7 @@ network_policies: &ctx, "/mcp", Some(&observer), + None, ) .await; if disconnect_client { @@ -15602,6 +15642,7 @@ network_policies: &ctx, "/mcp", Some(&observer), + None, ) .await .expect("initialize version exemption") @@ -15626,6 +15667,7 @@ network_policies: &ctx, "/mcp", Some(&observer), + None, ) .await; if disconnect_client { @@ -15734,7 +15776,7 @@ network_policies: let correlation = RequestCorrelation::with_request_id(&request, &ctx, REQUEST_ID.to_string()); - let allowed = enforce_final_mcp_protocol_version_correlated( + let allowed = enforce_final_mcp_protocol_version( &config, &request, &mut relay_client, diff --git a/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs index c2c4a9e6df..3dcfaf9f97 100644 --- a/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs +++ b/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs @@ -7,7 +7,7 @@ use super::L7EvalContext; use crate::l7::L7EndpointConfig; use crate::opa::{NetworkInput, OpaEngine, TunnelPolicyEngine}; use crate::test_support::request_tracing::RequestTracing; -use crate::test_support::{assert_http_request_records, response_request_id as request_id}; +use crate::test_support::response_request_id as request_id; use opentelemetry_sdk::trace::{Sampler, SpanData}; use std::future::Future; use std::path::PathBuf; @@ -309,33 +309,61 @@ async fn run_exchange( } } -fn assert_primary_record<'a>( +fn assert_control_records<'a>( exchange: &'a TracedOperation, - engine_type: &str, - status: u16, - action: &str, + expected: &[(&str, &str)], + finding_uids: &[&str], path: &str, -) -> &'a serde_json::Value { +) -> Vec<&'a serde_json::Value> { let receipt = request_id(&exchange.output.response); assert!( uuid::Uuid::parse_str(receipt).is_ok(), "canonical receipt: {receipt}" ); - let primary: Vec<_> = exchange - .events - .iter() - .filter(|event| event["class_uid"] == 4002) - .collect(); - assert_eq!(primary.len(), 1, "one HTTP Activity: {:?}", exchange.events); - let event = primary[0]; - assert_eq!(event["metadata"]["correlation_uid"], receipt); - assert_eq!(event["firewall_rule"]["type"], engine_type); - assert_eq!(event["http_response"]["code"], status); - assert_eq!(event["action"], action); - assert_eq!(event["http_request"]["url"]["path"], path); + assert_eq!( + exchange.events.len(), + expected.len() + finding_uids.len(), + "{:?}", + exchange.events + ); + for (sequence, event) in exchange.events.iter().enumerate() { + assert_eq!(event["metadata"]["correlation_uid"], receipt); + assert_eq!(event["unmapped"]["request_sequence"], sequence + 1); + } + for (sequence, (event, (engine_type, action))) in + exchange.events.iter().zip(expected).enumerate() + { + assert_eq!(event["class_uid"], 4002); + assert_eq!(event["unmapped"]["request_sequence"], sequence + 1); + assert_eq!(event["firewall_rule"]["type"], *engine_type); + assert_eq!(event["action"], *action); + assert_eq!(event["http_request"]["url"]["path"], path); + assert!( + event.get("http_response").is_none(), + "delivery status must not be overlaid on a control decision: {event}" + ); + } + for (event, uid) in exchange.events[expected.len()..].iter().zip(finding_uids) { + assert_eq!(event["class_uid"], 2004); + assert_eq!(event["finding_info"]["uid"], *uid); + assert_eq!(event["action"], "Denied"); + assert!(event.get("trace").is_none()); + } + exchange.events[..expected.len()].iter().collect() +} + +fn assert_policy_record<'a>( + exchange: &'a TracedOperation, + engine_type: &str, + action: &str, + path: &str, +) -> &'a serde_json::Value { + let event = assert_control_records(exchange, &[(engine_type, action)], &[], path)[0]; + assert_eq!(event["unmapped"]["http_event_kind"], "decision"); + assert!(event.get("status").is_none()); assert!( event.get("trace").is_none(), - "receipts must work without tracing" + "receipts work without tracing" ); event } @@ -487,12 +515,14 @@ async fn inspected_allow_creates_only_the_stages_that_run() { assert_eq!(names.contains(&name), expected, "{name}: {names:?}"); } } - let (record, _) = assert_http_request_records(&evidence.events, 0); - assert_eq!( - record["metadata"]["correlation_uid"], - request_id(&evidence.output.response) + let records = assert_control_records(&evidence, &[("l7", "Allowed")], &[], "/item"); + assert!(records[0].get("status").is_none()); + assert!( + records[0]["message"] + .as_str() + .unwrap() + .starts_with("L7_REQUEST allow ") ); - assert_eq!(record["action"], "Allowed"); } } @@ -532,16 +562,32 @@ async fn request_middleware_block_never_creates_a_denial_only_span() { .all(|span| span.name != "openshell.upstream.send") ); } - let (record, companions) = assert_http_request_records(&evidence.events, 1); - assert_eq!(companions[0]["unmapped"]["middleware_phase"], "request"); - assert_eq!(companions[0]["action"], "Denied"); - assert!(companions[0].get("http_response").is_none()); + let records = assert_control_records( + &evidence, + &[("l7", "Allowed"), ("middleware", "Denied")], + &["openshell.middleware.failure"], + "/item", + ); + assert_eq!(records[0]["unmapped"]["http_event_kind"], "decision"); + assert!(records[0].get("status").is_none()); + assert!( + records[0]["message"] + .as_str() + .unwrap() + .starts_with("L7_REQUEST allow ") + ); assert_eq!( - record["metadata"]["correlation_uid"], - request_id(&evidence.output.response) + records[1]["unmapped"]["http_event_kind"], + "middleware_invocation" + ); + assert_eq!(records[1]["unmapped"]["middleware_phase"], "request"); + assert_eq!(records[1]["status"], "Failure"); + assert!( + records[1]["message"] + .as_str() + .unwrap() + .starts_with("MIDDLEWARE request test/block-request decision=Deny") ); - assert_eq!(record["action"], "Denied"); - assert_eq!(record["http_response"]["code"], 403); } } @@ -609,13 +655,17 @@ async fn inspected_keep_alive_requests_have_distinct_receipts_without_tracing() assert_eq!( records.len(), 2, - "one primary per inspected request: {records:?}" + "one policy decision per request: {records:?}" ); for (record, receipt) in records.iter().zip(receipts) { assert!(uuid::Uuid::parse_str(receipt).is_ok()); assert_eq!(record["metadata"]["correlation_uid"], receipt); assert_eq!(record["action"], "Allowed"); - assert_eq!(record["http_response"]["code"], 200); + assert!(record.get("http_response").is_none()); + assert!(record.get("status").is_none()); + assert_eq!(record["firewall_rule"]["type"], "l7"); + assert_eq!(record["unmapped"]["http_event_kind"], "decision"); + assert_eq!(record["unmapped"]["request_sequence"], 1); assert_eq!(record["http_request"]["url"]["path"], "/item"); assert!(record.get("trace").is_none()); assert!( @@ -643,7 +693,7 @@ async fn production_single_mcp_allow_correlates_record_and_response_receipt() { .starts_with(b"POST /mcp HTTP/1.1\r\n") ); assert!(exchange.output.response.starts_with(b"HTTP/1.1 200 OK\r\n")); - assert_primary_record(&exchange, "l7-mcp", 200, "Allowed", "/mcp"); + assert_policy_record(&exchange, "l7-mcp", "Allowed", "/mcp"); } #[tokio::test] @@ -671,7 +721,7 @@ async fn production_single_mcp_invalid_version_correlates_denial_and_receipt() { String::from_utf8_lossy(&exchange.output.response) .contains("unsupported_mcp_protocol_version") ); - assert_primary_record(&exchange, "l7-mcp", 400, "Denied", "/mcp"); + assert_policy_record(&exchange, "l7-mcp", "Denied", "/mcp"); } #[tokio::test] @@ -695,7 +745,7 @@ async fn production_single_mcp_policy_deny_correlates_record_and_receipt() { .response .starts_with(b"HTTP/1.1 403 Forbidden\r\n") ); - let event = assert_primary_record(&exchange, "l7-mcp", 403, "Denied", "/mcp"); + let event = assert_policy_record(&exchange, "l7-mcp", "Denied", "/mcp"); assert!( event["message"] .as_str() @@ -741,5 +791,5 @@ network_policies: .starts_with(b"POST /graphql HTTP/1.1\r\n") ); assert!(exchange.output.response.starts_with(b"HTTP/1.1 200 OK\r\n")); - assert_primary_record(&exchange, "l7-graphql", 200, "Allowed", "/graphql"); + assert_policy_record(&exchange, "l7-graphql", "Allowed", "/graphql"); } diff --git a/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs index c6ece78731..d53b1e0abd 100644 --- a/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs +++ b/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs @@ -1031,13 +1031,33 @@ async fn token_grant_refusal_never_creates_a_denial_only_span() { .all(|span| span.name != "openshell.upstream.send") ); } - let (record, _) = crate::test_support::assert_http_request_records(&evidence.events, 0); - assert_eq!(record["action"], "Denied"); - assert_eq!(record["http_response"]["code"], 502); - assert_eq!( - record["metadata"]["correlation_uid"], - crate::test_support::response_request_id(evidence.output.as_bytes()) + let records: Vec<_> = evidence + .events + .iter() + .filter(|event| event["class_uid"] == 4002) + .collect(); + assert_eq!(records.len(), 2, "{:?}", evidence.events); + assert_eq!(records[0]["action"], "Allowed"); + assert!( + records[0]["message"] + .as_str() + .unwrap() + .starts_with("L7_REQUEST allow ") ); + assert_eq!(records[1]["action"], "Denied"); + assert!( + records[1]["message"] + .as_str() + .unwrap() + .starts_with("Token grant failed ") + ); + for record in records { + assert!(record.get("http_response").is_none()); + assert_eq!( + record["metadata"]["correlation_uid"], + crate::test_support::response_request_id(evidence.output.as_bytes()) + ); + } } } diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 3e5636193e..5cf92e8b79 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -940,7 +940,7 @@ where U: AsyncRead + AsyncWrite + Unpin, { relay_http_request_with_response_middleware_guarded_observed( - req, client, upstream, options, None, None, + req, client, upstream, options, None, None, None, ) .await } @@ -952,30 +952,6 @@ pub(crate) async fn relay_http_request_with_response_middleware_guarded_observed options: RelayRequestOptions<'_>, response_middleware: Option>, observer: Option<&EndpointObserver>, -) -> Result -where - C: AsyncRead + AsyncWrite + Unpin, - U: AsyncRead + AsyncWrite + Unpin, -{ - relay_http_request_with_response_middleware_guarded_observed_correlated( - req, - client, - upstream, - options, - response_middleware, - observer, - None, - ) - .await -} - -pub(crate) async fn relay_http_request_with_response_middleware_guarded_observed_correlated( - req: &L7Request, - client: &mut C, - upstream: &mut U, - options: RelayRequestOptions<'_>, - response_middleware: Option>, - observer: Option<&EndpointObserver>, correlation: Option<&RequestCorrelation>, ) -> Result where @@ -1129,7 +1105,7 @@ where raw_header, body_length: req.body_length, }; - if !crate::l7::relay::enforce_final_mcp_protocol_version_correlated( + if !crate::l7::relay::enforce_final_mcp_protocol_version( validation.config, &outgoing, client, @@ -6851,6 +6827,7 @@ mod tests { }, None, Some(&observer), + None, ), ) .await diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs index 86bb4f3d61..e7af4ec0e3 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs @@ -103,15 +103,6 @@ where } if let Some(header_end) = header_end { - if options.correlation.is_some() - && let Err(error) = validate_http_response_header_block(&buf[..header_end]) - { - if let Some(observer) = options.observer.as_ref() { - observer.observe(EndpointResult::TransportFailed); - } - record_upstream_response_error(options.correlation, "upstream_response_headers"); - return Err(error); - } let header_str = String::from_utf8_lossy(&buf[..header_end]); if matches!( parse_observed_http_status_code(&header_str), @@ -119,7 +110,7 @@ where ) { informational_header_bytes += header_end; let sanitized = if options.correlation.is_some() { - strip_platform_request_id_from_head(&buf[..header_end]) + strip_platform_request_id_from_response_head(&buf[..header_end]) } else { Cow::Borrowed(&buf[..header_end]) }; @@ -146,7 +137,8 @@ where }; if options.correlation.is_some() - && let Cow::Owned(sanitized) = strip_platform_request_id_from_head(&buf[..header_end]) + && let Cow::Owned(sanitized) = + strip_platform_request_id_from_response_head(&buf[..header_end]) { let overflow = buf.split_off(header_end); buf = sanitized; @@ -157,9 +149,6 @@ where // Parse response framing let header_str = String::from_utf8_lossy(&buf[..header_end]); let observed_status = parse_observed_http_status_code(&header_str); - if let (Some(correlation), Some(status_code)) = (options.correlation, observed_status) { - correlation.record_upstream_status(status_code); - } if let Some(observer) = options.observer.as_ref() { match observed_status { Some(status_code) if status_code < 400 => { @@ -359,64 +348,6 @@ fn record_committed_response_error( } } -fn validate_http_response_header_block(headers: &[u8]) -> Result<()> { - if !headers.ends_with(b"\r\n\r\n") { - return Err(miette!("HTTP response header block is incomplete")); - } - - for (index, byte) in headers.iter().copied().enumerate() { - match byte { - b'\r' if headers.get(index + 1) != Some(&b'\n') => { - return Err(miette!("HTTP response headers contain a bare CR")); - } - b'\n' if index == 0 || headers.get(index - 1) != Some(&b'\r') => { - return Err(miette!("HTTP response headers contain a bare LF")); - } - _ => {} - } - } - - let header_lines = &headers[..headers.len() - 2]; - let mut lines = header_lines - .split(|byte| *byte == b'\n') - .map(|line| line.strip_suffix(b"\r").unwrap_or(line)) - .skip(1); - - while let Some(line) = lines.next() { - if line.is_empty() { - if lines.next().is_some() { - return Err(miette!( - "HTTP response header block contains an unexpected empty line" - )); - } - break; - } - if line - .first() - .is_some_and(|byte| matches!(byte, b' ' | b'\t')) - { - return Err(miette!( - "HTTP response header continuation lines are not supported" - )); - } - let colon = line - .iter() - .position(|byte| *byte == b':') - .ok_or_else(|| miette!("HTTP response header field is malformed"))?; - let name = &line[..colon]; - if name.last().is_some_and(|byte| matches!(byte, b' ' | b'\t')) { - return Err(miette!( - "HTTP response header field contains whitespace before the colon" - )); - } - http::HeaderName::from_bytes(name) - .map_err(|_| miette!("HTTP response field name is malformed"))?; - http::HeaderValue::from_bytes(&line[colon + 1..]) - .map_err(|_| miette!("HTTP response field value is malformed"))?; - } - Ok(()) -} - fn final_response_head<'a>( headers: &'a [u8], correlation: Option<&RequestCorrelation>, @@ -424,8 +355,10 @@ fn final_response_head<'a>( let Some(correlation) = correlation else { return Cow::Borrowed(headers); }; - let stripped = - strip_platform_request_id_from_connection(strip_platform_request_id_from_head(headers)); + if let Some(status_code) = parse_status_code(&String::from_utf8_lossy(headers)) { + correlation.record_response_status(status_code); + } + let stripped = strip_platform_request_id_from_response_head(headers); let Some(header_end) = stripped.windows(4).position(|window| window == b"\r\n\r\n") else { return stripped; }; @@ -441,7 +374,11 @@ fn final_response_head<'a>( Cow::Owned(output) } -fn strip_platform_request_id_from_connection(headers: Cow<'_, [u8]>) -> Cow<'_, [u8]> { +/// Remove only receipt fields and their hop-by-hop/trailer nominations. +/// Folded fields are grouped so removing a receipt never leaves its +/// continuation attached to an unrelated field. Recognize physical LF lines +/// without normalizing unrelated bytes or tightening response parsing. +fn strip_platform_request_id_from_response_head(headers: &[u8]) -> Cow<'_, [u8]> { let header_end = headers .windows(4) .position(|window| window == b"\r\n\r\n") @@ -453,62 +390,91 @@ fn strip_platform_request_id_from_connection(headers: Cow<'_, [u8]>) -> Cow<'_, while position < header_block.len() { let line_end = header_block[position..] - .windows(2) - .position(|window| window == b"\r\n") + .iter() + .position(|byte| *byte == b'\n') .map_or(header_block.len(), |end| position + end); - let line = &header_block[position..line_end]; + let raw_line = &header_block[position..line_end]; + let line = raw_line.strip_suffix(b"\r").unwrap_or(raw_line); if line.is_empty() { - if position + 2 < header_block.len() { - output.get_or_insert_with(|| header_block[..position].to_vec()); - } if let Some(output) = &mut output { - output.extend_from_slice(b"\r\n"); + output.extend_from_slice(&header_block[position..]); } break; } - if let Some(colon) = line.iter().position(|byte| *byte == b':') - && trim_ascii_ows(&line[..colon]).eq_ignore_ascii_case(b"connection") - && line[colon + 1..] - .split(|byte| *byte == b',') - .any(|value| trim_ascii_ows(value).eq_ignore_ascii_case(protected)) + let mut field_end = (line_end + 1).min(header_block.len()); + while header_block + .get(field_end) + .is_some_and(|byte| matches!(byte, b' ' | b'\t')) { - let output = output.get_or_insert_with(|| { - let mut output = Vec::with_capacity(headers.len()); - output.extend_from_slice(&header_block[..position]); - output + field_end = header_block[field_end..] + .iter() + .position(|byte| *byte == b'\n') + .map_or(header_block.len(), |end| field_end + end + 1); + } + let field = &header_block[position..field_end]; + if let Some(colon) = line.iter().position(|byte| *byte == b':') { + let name = trim_ascii_ows(&line[..colon]); + let receipt = name.eq_ignore_ascii_case(protected); + let nomination = (name.eq_ignore_ascii_case(b"connection") + || name.eq_ignore_ascii_case(b"trailer")) + && field + .windows(protected.len()) + .any(|window| window.eq_ignore_ascii_case(protected)); + let unfolded = nomination.then(|| { + let mut value = Vec::with_capacity(field.len() - colon); + for segment in field[colon + 1..].split(|byte| *byte == b'\n') { + let segment = trim_ascii_ows(segment.strip_suffix(b"\r").unwrap_or(segment)); + if !value.is_empty() && !segment.is_empty() { + value.push(b' '); + } + value.extend_from_slice(segment); + } + value + }); + let protected_nomination = unfolded.as_ref().is_some_and(|value| { + value + .split(|byte| *byte == b',') + .any(|value| trim_ascii_ows(value).eq_ignore_ascii_case(protected)) }); - let mut retained = line[colon + 1..] - .split(|byte| *byte == b',') - .map(trim_ascii_ows) - .filter(|token| !token.is_empty() && !token.eq_ignore_ascii_case(protected)); - if let Some(first) = retained.next() { - output.extend_from_slice(&line[..=colon]); - output.push(b' '); - output.extend_from_slice(first); - for token in retained { - output.extend_from_slice(b", "); - output.extend_from_slice(token); + if receipt || protected_nomination { + let output = output.get_or_insert_with(|| { + let mut output = Vec::with_capacity(headers.len()); + output.extend_from_slice(&header_block[..position]); + output + }); + if let Some(unfolded) = unfolded { + let mut retained = unfolded + .split(|byte| *byte == b',') + .map(trim_ascii_ows) + .filter(|token| { + !token.is_empty() && !token.eq_ignore_ascii_case(protected) + }); + if let Some(first) = retained.next() { + output.extend_from_slice(&line[..=colon]); + output.push(b' '); + output.extend_from_slice(first); + for token in retained { + output.extend_from_slice(b", "); + output.extend_from_slice(token); + } + output.extend_from_slice(b"\r\n"); + } } - output.extend_from_slice(b"\r\n"); + position = field_end; + continue; } - position = line_end.saturating_add(2); - continue; - } - if line_end + 2 > header_block.len() { - output.get_or_insert_with(|| header_block[..position].to_vec()); } if let Some(output) = &mut output { - output.extend_from_slice(line); - output.extend_from_slice(b"\r\n"); + output.extend_from_slice(field); } - position = line_end.saturating_add(2); + position = field_end; } if let Some(output) = &mut output { output.extend_from_slice(buffered_suffix); } - output.map_or(headers, Cow::Owned) + output.map_or(Cow::Borrowed(headers), Cow::Owned) } /// Body framing determines when delivery finishes, not whether another @@ -2603,7 +2569,7 @@ pub(super) fn response_is_event_stream(headers: &str) -> bool { #[cfg(test)] mod tests { - use super::{final_response_head, strip_platform_request_id_from_connection}; + use super::{final_response_head, strip_platform_request_id_from_response_head}; use crate::l7::correlation::HttpOutcome; use crate::l7::rest::strip_platform_request_id_from_head; use openshell_supervisor_middleware::{ @@ -2763,7 +2729,7 @@ mod tests { .status(openshell_ocsf::StatusId::Success) .build(), ); - correlation.response_selected(200); + correlation.record_response_status(200); for (outcome, failed) in [(Outcome::FailOpen, true), (Outcome::BlockDelivery, false)] { for event in super::http_response_middleware_invocation_events( "response-policy", @@ -2781,7 +2747,7 @@ mod tests { .filter(|event| event["class_uid"] == 4002) .collect(); assert_eq!(events.len(), 3); - assert_eq!(events[0]["unmapped"]["http_event_kind"], "request"); + assert_eq!(events[0]["unmapped"]["http_event_kind"], "decision"); assert_eq!(events[0]["action"], "Allowed"); assert_eq!(events[1]["action"], "Other"); assert_eq!( @@ -2811,7 +2777,7 @@ mod tests { use super::{ RelayResponseOptions, ResponseFraming, parse_connection_keep_alive, parse_response_head_for_middleware, relay_response, response_is_http_10, - serialize_response_head, validate_http_response_header_block, + serialize_response_head, }; use crate::l7::provider::{BodyLength, L7Request, RelayOutcome}; use crate::l7::relay::L7EvalContext; @@ -2875,7 +2841,7 @@ mod tests { let stripped = strip_platform_request_id_from_head(head); assert!(matches!(stripped, Cow::Borrowed(_))); - let stripped = strip_platform_request_id_from_connection(stripped); + let stripped = strip_platform_request_id_from_response_head(&stripped); assert!(matches!(stripped, Cow::Borrowed(_))); assert_eq!(stripped.as_ref(), head); } @@ -2936,7 +2902,7 @@ mod tests { } #[tokio::test] - async fn delivery_failure_receipt_distinguishes_upstream_error_from_local_block() { + async fn delivery_failure_receipt_does_not_synthesize_audit_records() { for outcome in [HttpOutcome::UpstreamError, HttpOutcome::PlatformError] { let capture = openshell_ocsf::test_support::OcsfCapture::default(); let _subscriber = capture.install(); @@ -2950,36 +2916,27 @@ mod tests { query: String::new(), }; let mut client = Vec::new(); - { - let mut client = correlation.observe_response(&mut client); - super::send_response_delivery_failure( - &mut client, - "GET", - "policy", - &target, - None, - outcome, - "response_body", - Some(&correlation), - ) - .await - .unwrap(); - } + super::send_response_delivery_failure( + &mut client, + "GET", + "policy", + &target, + None, + outcome, + "response_body", + Some(&correlation), + ) + .await + .unwrap(); drop(correlation); assert_eq!(request_id_header_count(&client), 1); - let events = capture.json_events(); - assert_eq!(events.len(), 1); - assert_eq!(events[0]["http_response"]["code"], 502); - assert_eq!(events[0]["metadata"]["correlation_uid"], "delivery-failure"); - assert_eq!( - events[0]["action"] == "Denied", - outcome == HttpOutcome::PlatformError - ); + assert!(client.starts_with(b"HTTP/1.1 502 Bad Gateway\r\n")); + assert!(capture.json_events().is_empty()); } } #[tokio::test] - async fn correlated_upstream_eof_preserves_transport_close_and_audits_failure() { + async fn correlated_upstream_eof_preserves_transport_close_without_synthetic_records() { let capture = openshell_ocsf::test_support::OcsfCapture::default(); let _subscriber = capture.install(); let correlation = test_correlation("eof-request"); @@ -2993,15 +2950,7 @@ mod tests { "EOF must not become a synthetic HTTP response" ); drop(correlation); - let records = capture.json_events(); - let primary: Vec<_> = records - .iter() - .filter(|event| event["class_uid"] == 4002) - .collect(); - assert_eq!(primary.len(), 1, "{records:?}"); - assert_eq!(primary[0]["status"], "Failure"); - assert_eq!(primary[0]["metadata"]["correlation_uid"], "eof-request"); - assert!(primary[0].get("http_response").is_none()); + assert!(capture.json_events().is_empty()); } #[tokio::test] @@ -3039,7 +2988,7 @@ mod tests { } #[tokio::test] - async fn response_middleware_denial_has_one_correlated_primary_without_tracing() { + async fn response_middleware_denial_preserves_correlated_control_records_without_tracing() { let capture = openshell_ocsf::test_support::OcsfCapture::default(); let _subscriber = capture.install(); exercise_response_middleware_block().await; @@ -3112,20 +3061,23 @@ mod tests { companions[0]["metadata"]["correlation_uid"], "response-denied" ); - let primary: Vec<_> = records + let policy: Vec<_> = records .iter() .filter(|event| { - event["class_uid"] == 4002 && event["unmapped"]["http_event_kind"] == "request" + event["class_uid"] == 4002 && event["firewall_rule"]["type"] == "l7-rest" }) .collect(); - assert_eq!(primary.len(), 1, "{records:?}"); - assert_eq!(primary[0]["action"], "Denied"); - assert_eq!(primary[0]["status"], "Failure"); - assert_eq!(primary[0]["http_response"]["code"], 403); - assert_eq!(primary[0]["http_request"]["url"]["path"], "/mutate"); - assert_eq!(primary[0]["firewall_rule"]["type"], "l7-rest"); - assert_eq!(primary[0]["metadata"]["correlation_uid"], "response-denied"); - assert_eq!(primary[0].get("trace").is_some(), traced); + assert_eq!(policy.len(), 1, "{records:?}"); + assert_eq!(policy[0]["action"], "Allowed"); + assert_eq!(policy[0]["status"], "Success"); + assert!(policy[0].get("http_response").is_none()); + assert_eq!(policy[0]["http_request"]["url"]["path"], "/mutate"); + for record in records { + assert_eq!(record["metadata"]["correlation_uid"], "response-denied"); + if record["class_uid"] == 4002 { + assert_eq!(record.get("trace").is_some(), traced); + } + } } #[tokio::test] @@ -3204,84 +3156,29 @@ mod tests { } #[tokio::test] - async fn ordinary_response_rejects_folded_receipt_connection_nomination() { + async fn ordinary_response_sanitizes_only_folded_receipt_fields_and_nominations() { let correlation = test_correlation("trusted"); - let response = &b"HTTP/1.1 200 OK\r\nConnection: keep-alive,\r\n X-OpenShell-Request-ID\r\nContent-Length: 0\r\n\r\n"[..]; + let response = &b"HTTP/1.1 200 OK\r\nX-Unrelated: opaque\r\n continuation\r\nX-OpenShell-Request-ID: forged\r\n more-forgery\r\nConnection: keep-alive,\r\n X-OpenShell-Request-ID\r\nTrailer: X-Checksum,\r\n\tx-openshell-request-id\r\nContent-Length: 0\r\n\r\n"[..]; let (outcome, client) = relay_bytes(response, Some(&correlation)).await; - let error = outcome.expect_err("folded response fields must fail closed"); - - assert!( - error - .to_string() - .contains("response header continuation lines are not supported"), - "{error}" + outcome.expect("protected folded fields are sanitized"); + assert_eq!( + client, + b"HTTP/1.1 200 OK\r\nX-Unrelated: opaque\r\n continuation\r\nConnection: keep-alive\r\nTrailer: X-Checksum\r\nContent-Length: 0\r\nx-openshell-request-id: trusted\r\n\r\n" ); - assert!(client.is_empty()); } #[tokio::test] - async fn ordinary_response_rejects_bare_lf_receipt_smuggling() { + async fn ordinary_response_strips_bare_lf_receipt_without_normalizing_unrelated_fields() { let correlation = test_correlation("trusted"); - let response = &b"HTTP/1.1 200 OK\r\nX-Safe: yes\nX-OpenShell-Request-ID: forged\r\nContent-Length: 0\r\n\r\n"[..]; + let response = b"HTTP/1.1 200 OK\r\nX-Safe: yes\nX-OpenShell-Request-ID: forged\r\nX-Unrelated: opaque\n retained\r\nContent-Length: 0\r\n\r\n"; let (outcome, client) = relay_bytes(response, Some(&correlation)).await; - let error = outcome.expect_err("bare LF response fields must fail closed"); + outcome.expect("protected physical LF fields are sanitized"); - assert!(error.to_string().contains("bare LF"), "{error}"); - assert!(client.is_empty()); - } - - #[test] - fn response_head_validation_rejects_ambiguous_header_blocks() { - let cases: &[(&str, &[u8])] = &[ - ("incomplete block", b"HTTP/1.1 200 OK\r\nX-Test: yes\r\n"), - ("bare LF", b"HTTP/1.1 200 OK\r\nX-Test: yes\n\r\n"), - ("bare CR", b"HTTP/1.1 200 OK\r\nX-Test: yes\rX: no\r\n\r\n"), - ( - "obsolete fold", - b"HTTP/1.1 200 OK\r\nX-Test: yes\r\n continued\r\n\r\n", - ), - ("malformed field", b"HTTP/1.1 200 OK\r\nX-Test\r\n\r\n"), - ("empty field name", b"HTTP/1.1 200 OK\r\n: value\r\n\r\n"), - ( - "non-token field name", - b"HTTP/1.1 200 OK\r\nBad Name: value\r\n\r\n", - ), - ( - "whitespace before colon", - b"HTTP/1.1 200 OK\r\nX-Test : value\r\n\r\n", - ), - ( - "field control byte", - b"HTTP/1.1 200 OK\r\nX-Test: value\x0bmore\r\n\r\n", - ), - ( - "field delete byte", - b"HTTP/1.1 200 OK\r\nX-Test: value\x7f\r\n\r\n", - ), - ( - "early empty line", - b"HTTP/1.1 200 OK\r\n\r\nX-Smuggled: yes\r\n\r\n", - ), - ]; - - for (name, input) in cases { - assert!( - validate_http_response_header_block(input).is_err(), - "{name} unexpectedly passed validation" - ); - } - } - - #[test] - fn response_head_validation_accepts_supported_informational_and_upgrade_responses() { - for response in [ - &b"HTTP/1.0 200 OK\r\nX-Test: yes\r\n\r\n"[..], - &b"HTTP/1.1 200\r\nContent-Length: 0\r\n\r\n"[..], - &b"HTTP/1.1 103 Early Hints\r\nLink: ; rel=preload\r\n\r\n"[..], - &b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n"[..], - ] { - validate_http_response_header_block(response).expect("valid HTTP/1.x response head"); - } + assert_eq!( + client, + b"HTTP/1.1 200 OK\r\nX-Safe: yes\nX-Unrelated: opaque\n retained\r\nContent-Length: 0\r\nx-openshell-request-id: trusted\r\n\r\n" + ); + assert_eq!(request_id_header_count(&client), 1); } #[tokio::test] @@ -3382,19 +3279,19 @@ mod tests { } #[tokio::test] - async fn switching_protocols_rejects_folded_receipt_connection_nomination() { + async fn switching_protocols_sanitizes_folded_receipt_connection_nomination() { let correlation = test_correlation("trusted-upgrade"); let response = &b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade,\r\n\tX-OpenShell-Request-ID\r\nUpgrade: websocket\r\n\r\n"[..]; let (outcome, client) = relay_bytes(response, Some(&correlation)).await; - let error = outcome.expect_err("folded upgrade fields must fail closed"); - + assert!(matches!( + outcome.expect("sanitized upgrade"), + RelayOutcome::Upgraded { .. } + )); assert!( - error - .to_string() - .contains("response header continuation lines are not supported"), - "{error}" + String::from_utf8_lossy(&client).contains("Connection: Upgrade\r\n"), + "{client:?}" ); - assert!(client.is_empty()); + assert_eq!(request_id_header_count(&client), 1); } #[test] diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs index 7d15476208..5489c89474 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs @@ -441,7 +441,6 @@ where } Ok(failure) if committed => { if let Some(correlation) = response_correlation { - correlation.record_enforcement_block(); correlation.record_failure( if failure.denial.is_some() { HttpOutcome::Denied @@ -1007,7 +1006,7 @@ mod tests { use tracing::instrument::WithSubscriber as _; #[test] - fn response_hook_invocations_survive_request_consolidation_without_tracing() { + fn response_hook_invocations_keep_individual_correlated_records_without_tracing() { let capture = RequestTracing::new(None); tracing::subscriber::with_default(capture.subscriber(), || { let request = L7Request { @@ -1062,16 +1061,18 @@ mod tests { }, ); if outcome == HttpStageOutcome::WholeBody { - correlation.response_selected(200); + correlation.record_response_status(200); } } }); let records = capture.events.json_events(); - let (primary, invocations) = crate::test_support::assert_http_request_records(&records, 2); - assert_eq!(primary["firewall_rule"]["type"], "l7-rest"); - assert_eq!(primary["action"], "Allowed"); - assert_eq!(primary["http_response"]["code"], 200); + assert_eq!(records.len(), 3); + let policy = &records[0]; + assert_eq!(policy["firewall_rule"]["type"], "l7-rest"); + assert_eq!(policy["action"], "Allowed"); + assert!(policy.get("http_response").is_none()); + let invocations = &records[1..]; for (event, (outcome, input_bytes)) in invocations.iter().zip([("wholebody", 0), ("unchanged", 5)]) { @@ -1125,9 +1126,8 @@ mod tests { }; let (mut relay, mut app) = tokio::io::duplex(4096); { - let mut audited = correlation.observe_response(&mut relay); let mut writer = ResponseWriter { - client: &mut audited, + client: &mut relay, status_line: "HTTP/1.1 200 OK", headers: &parsed.headers, parsed: &parsed, @@ -1171,10 +1171,7 @@ mod tests { ); assert_eq!(receipts.next(), None); assert!(!response.contains("upstream-forgery")); - let events = capture.events.json_events(); - assert_eq!(events.len(), 1); - assert_eq!(events[0]["metadata"]["correlation_uid"], "pipeline-receipt"); - assert_eq!(events[0]["http_response"]["code"], 200); + assert!(capture.events.json_events().is_empty()); } .with_subscriber(capture.subscriber()) .await; diff --git a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs index 057dfd7104..d24a77ec34 100644 --- a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs +++ b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs @@ -430,6 +430,7 @@ fn report_refused_grant( ctx.host, ctx.port )) .unmapped("request_id", request_id) + .unmapped("http_event_kind", "token_grant") .build()], ); } @@ -494,6 +495,7 @@ async fn inject_selected( ctx.host, ctx.port )) + .unmapped("http_event_kind", "token_grant") .build()], ); } diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index e75053e520..b94ecfbd9e 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -2087,24 +2087,6 @@ fn build_forward_allow_ocsf_event( .build() } -fn build_forward_inspection_rejection_ocsf_event( - method: &str, - reason: &'static str, -) -> openshell_ocsf::OcsfEvent { - HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .http_request(HttpRequest { - http_method: method.parse().expect("HTTP method parsing is infallible"), - url: None, - }) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .status_detail(reason) - .message("FORWARD request rejected before policy evaluation") - .build() -} - fn build_forward_rejection_event( host: &str, port: u16, @@ -5439,7 +5421,6 @@ struct ForwardResponseMiddleware<'a> { exchange: &'a crate::l7::middleware::HttpMiddlewareExchange, } -#[cfg(test)] async fn relay_rewritten_forward_request( method: &str, path: &str, @@ -5447,24 +5428,6 @@ async fn relay_rewritten_forward_request( client: &mut C, upstream: &mut U, options: ForwardRelayOptions<'_>, -) -> Result -where - C: TokioAsyncRead + TokioAsyncWrite + Unpin, - U: TokioAsyncRead + TokioAsyncWrite + Unpin, -{ - relay_rewritten_forward_request_correlated( - method, path, rewritten, client, upstream, options, None, - ) - .await -} - -async fn relay_rewritten_forward_request_correlated( - method: &str, - path: &str, - rewritten: Vec, - client: &mut C, - upstream: &mut U, - options: ForwardRelayOptions<'_>, correlation: Option<&crate::l7::correlation::RequestCorrelation>, ) -> Result where @@ -5492,7 +5455,7 @@ where .response_relay(&req, middleware.ctx, middleware.scheme) }); - crate::l7::rest::relay_http_request_with_response_middleware_guarded_observed_correlated( + crate::l7::rest::relay_http_request_with_response_middleware_guarded_observed( &req, client, upstream, @@ -5611,6 +5574,54 @@ async fn handle_forward_proxy( denial_tx: Option<&mpsc::UnboundedSender>, activity_tx: Option<&ActivitySender>, endpoint_observation_tx: Option, +) -> Result<()> { + Box::pin(crate::l7::correlation::scope_connection( + handle_forward_proxy_inner( + method, + target_uri, + buf, + used, + client, + supplied_identity, + socket_addrs, + opa_engine, + identity_cache, + entrypoint_pid, + policy_local_ctx, + agent_proposals, + backend_host_gateway, + trusted_host_gateway, + provider_credentials, + secret_resolver, + denial_tx, + activity_tx, + endpoint_observation_tx, + ), + )) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn handle_forward_proxy_inner( + method: &str, + target_uri: &str, + buf: &[u8], + used: usize, + client: &mut ProxyClient, + supplied_identity: Option<&Result>, + socket_addrs: Option<(SocketAddr, SocketAddr)>, + opa_engine: Arc, + identity_cache: Arc, + entrypoint_pid: Arc, + policy_local_ctx: Option>, + agent_proposals: openshell_core::proposals::AgentProposals, + backend_host_gateway: Arc>, + trusted_host_gateway: Arc>, + provider_credentials: Option, + secret_resolver: Option>, + denial_tx: Option<&mpsc::UnboundedSender>, + activity_tx: Option<&ActivitySender>, + endpoint_observation_tx: Option, ) -> Result<()> { let endpoint_observation_context = endpoint_observation_tx .as_ref() @@ -5978,30 +5989,6 @@ async fn handle_forward_proxy( let request_correlation = inspected.then_some(&correlation); let receipt_id = request_correlation.map(crate::l7::correlation::RequestCorrelation::request_id); - let mut audited_client = correlation.observe_response(client); - let client = &mut audited_client; - if inspected { - let mut primary = build_forward_allow_ocsf_event( - workload_addr, - method, - &host_lc, - port, - &telemetry_path, - &binary_str, - &pid_str, - &ancestors_str, - &cmdline_str, - policy_str, - ); - if l7_inspected && let openshell_ocsf::OcsfEvent::HttpActivity(event) = &mut primary { - event.action = None; - event.disposition = None; - event.base.status = None; - event.firewall_rule = None; - event.base.message = Some("FORWARD request awaiting inspection".into()); - } - correlation.emit(primary); - } forward_request_bytes = correlation_request.raw_header; if let Some(route) = decision .endpoint @@ -6091,33 +6078,7 @@ async fn handle_forward_proxy( return Ok(()); }; let Some(l7_config) = select_l7_config_for_path(&route.configs, &redacted_path) else { - let reason = "no L7 endpoint path matched request"; correlation.record_rejection(403, HttpOutcome::Denied, None); - correlation.emit( - HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::for_http_method(method)) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("http", &host_lc, &telemetry_path, port), - )) - .http_response(HttpResponse { code: 403 }) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint(Endpoint::from_ip(workload_addr.ip(), workload_addr.port())) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule(policy_str, "l7") - .message(format!( - "FORWARD_L7 denied {method} {host_lc}:{port}{telemetry_path}" - )) - .status_detail(reason) - .build(), - ); emit_activity_simple(activity_tx, true, "l7_policy"); respond( client, @@ -6152,10 +6113,6 @@ async fn handle_forward_proxy( && crate::l7::path::canonical_path_has_encoded_slash(&path) { correlation.record_rejection(400, HttpOutcome::Denied, None); - correlation.emit(build_forward_inspection_rejection_ocsf_event( - method, - "invalid_request_target", - )); correlation.emit(build_forward_l7_parse_rejection_ocsf_event( workload_addr, method, @@ -6275,10 +6232,6 @@ async fn handle_forward_proxy( Ok(info) => info, Err(e) => { correlation.record_rejection(400, HttpOutcome::Denied, None); - correlation.emit(build_forward_inspection_rejection_ocsf_event( - method, - "invalid_graphql_request", - )); let event = build_forward_rejection_event( &host_lc, port, @@ -6333,10 +6286,6 @@ async fn handle_forward_proxy( Ok(body) => body, Err(e) => { correlation.record_rejection(400, HttpOutcome::Denied, None); - correlation.emit(build_forward_inspection_rejection_ocsf_event( - method, - "invalid_jsonrpc_request", - )); let event = build_forward_rejection_event( &host_lc, port, @@ -6424,11 +6373,6 @@ async fn handle_forward_proxy( (false, crate::l7::EnforcementMode::Audit) => "audit", (false, crate::l7::EnforcementMode::Enforce) => "deny", }; - if !allowed - && (l7_config.config.enforcement != crate::l7::EnforcementMode::Audit || force_deny) - { - correlation.record_enforcement_block(); - } { let (action_id, disposition_id, severity) = match decision_str { @@ -6940,7 +6884,6 @@ async fn handle_forward_proxy( }; if let Some(guard) = credential_generation { guard.ensure_current().inspect_err(|_| { - correlation.record_enforcement_block(); correlation.record_outcome(HttpOutcome::PlatformError); })?; } @@ -7048,7 +6991,7 @@ async fn handle_forward_proxy( &upstream_target, rewritten, )?; - if !crate::l7::relay::enforce_final_mcp_protocol_version_correlated( + if !crate::l7::relay::enforce_final_mcp_protocol_version( config, &request, client, @@ -7199,7 +7142,7 @@ async fn handle_forward_proxy( let signing_region = forward_upgrade_config .as_ref() .map_or("", |config| config.signing_region.as_str()); - let outcome_result = relay_rewritten_forward_request_correlated( + let outcome_result = relay_rewritten_forward_request( method, &upstream_target, rewritten, @@ -7278,20 +7221,21 @@ async fn handle_forward_proxy( ) .await?; - if !inspected { - correlation.emit(build_forward_allow_ocsf_event( - workload_addr, - method, - &host_lc, - port, - &telemetry_path, - &binary_str, - &pid_str, - &ancestors_str, - &cmdline_str, - policy_str, - )); - } + // The request has now survived middleware, token grant, credential + // rewriting, generation checks, and the HTTP relay. Only now record the + // final allowed outcome. + correlation.emit(build_forward_allow_ocsf_event( + workload_addr, + method, + &host_lc, + port, + &telemetry_path, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + policy_str, + )); emit_forward_success_activity(activity_tx, l7_activity_pending); match outcome { @@ -12116,6 +12060,7 @@ network_policies: }), endpoint_observer: None, }, + None, ) .await .expect("plaintext forward relay"); @@ -12205,6 +12150,7 @@ network_policies: }), endpoint_observer: None, }, + None, ) .await .expect("plaintext forward response denial"); @@ -12323,6 +12269,7 @@ network_policies: response_middleware: None, endpoint_observer: None, }, + None, ) .await?; @@ -12627,6 +12574,7 @@ network_policies: response_middleware: None, endpoint_observer: None, }, + None, ) .await?; if let crate::l7::provider::RelayOutcome::Upgraded { @@ -15210,6 +15158,7 @@ network_policies: response_middleware: None, endpoint_observer: None, }, + None, ) .await .expect_err("unresolved body placeholder should fail closed"); @@ -15294,6 +15243,7 @@ network_policies: response_middleware: None, endpoint_observer: None, }, + None, ) .await .expect_err("SigV4 endpoint mismatch should fail closed"); @@ -15385,6 +15335,7 @@ network_policies: response_middleware: None, endpoint_observer: None, }, + None, ) .await; assert!( @@ -15438,6 +15389,7 @@ network_policies: response_middleware: None, endpoint_observer: None, }, + None, ) .await; assert!(result.is_err(), "forward relay must reject CL/TE ambiguity"); diff --git a/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs b/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs index db572fcedb..f615587de6 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs @@ -3,6 +3,7 @@ use super::*; use crate::test_support::request_tracing::RequestTracing; +use crate::test_support::response_request_id; use opentelemetry_sdk::trace::Sampler; use tracing::instrument::WithSubscriber as _; use tracing_subscriber::layer::SubscriberExt as _; @@ -18,7 +19,25 @@ enum Outcome { UpstreamClosed, } -async fn forward_case(ip: Ipv4Addr, outcome: Outcome, sampler: Option, flags: &str) { +fn assert_request_record_sequence(records: &[serde_json::Value], receipt: &str) { + for (index, record) in records.iter().enumerate() { + assert_eq!( + record["metadata"]["correlation_uid"], receipt, + "{records:?}" + ); + assert_eq!( + record["unmapped"]["request_sequence"], + index + 1, + "{records:?}" + ); + assert_eq!( + record["unmapped"]["http_event_kind"], "decision", + "{records:?}" + ); + } +} + +async fn forward_case(ip: Ipv4Addr, outcome: Outcome, sampler: Option) { let listener = TcpListener::bind((ip, 0)).await.unwrap(); let port = listener.local_addr().unwrap().port(); let mode = if matches!(outcome, Outcome::AuditDenied) { @@ -71,7 +90,7 @@ network_policies: let mut proxy: ProxyClient = tokio::io::BufReader::new(Box::new(proxy)); let target = format!("http://{ip}:{port}{path}?token=SENTINEL-QUERY"); let request = format!( - "GET {target} HTTP/1.1\r\nHost: {ip}:{port}\r\ntraceparent: 00-{WORKLOAD_TRACE_ID}-{WORKLOAD_SPAN_ID}-{flags}\r\ntracestate: vendor=SENTINEL-VENDOR\r\nx-openshell-request-id: forged-receipt\r\nConnection: close\r\n\r\n" + "GET {target} HTTP/1.1\r\nHost: {ip}:{port}\r\ntraceparent: 00-{WORKLOAD_TRACE_ID}-{WORKLOAD_SPAN_ID}-01\r\ntracestate: vendor=SENTINEL-VENDOR\r\nx-openshell-request-id: forged-receipt\r\nConnection: close\r\n\r\n" ); let exchange = async { tokio::join!( @@ -153,40 +172,64 @@ network_policies: }, ); let response = String::from_utf8(response).unwrap(); - let records = capture - .events - .json_events() - .into_iter() - .filter(|event| event["class_uid"] == 4002) - .collect::>(); - assert_eq!(records.len(), 1, "{records:?}"); - let receipt = records[0]["metadata"]["correlation_uid"].as_str().unwrap(); + let records = capture.events.json_events(); + let expected_records = if matches!(outcome, Outcome::PolicyDenied) { + 1 + } else { + 2 + }; + assert_eq!(records.len(), expected_records, "{records:?}"); + assert!(records.iter().all(|record| record["class_uid"] == 4002)); + let record = &records[0]; + let decision = match outcome { + Outcome::PolicyDenied => "deny", + Outcome::AuditDenied => "audit", + Outcome::Allowed | Outcome::UpstreamClosed => "allow", + }; + assert!( + record["message"] + .as_str() + .unwrap() + .starts_with(&format!("FORWARD_L7 {decision} GET ")) + ); + assert_eq!(record["firewall_rule"]["type"], "l7"); + assert!(record.get("status_id").is_none()); + assert!(record.get("status_detail").is_none()); + for control in &records { + assert_eq!(control["http_request"]["url"]["path"], path); + assert!(control.get("http_response").is_none()); + } + if !matches!(outcome, Outcome::PolicyDenied) { + let forwarded = &records[1]; + assert!( + forwarded["message"] + .as_str() + .unwrap() + .starts_with("FORWARD allowed GET ") + ); + assert_eq!(forwarded["firewall_rule"]["type"], "opa"); + assert_eq!(forwarded["action_id"], 1); + assert_eq!(forwarded["status_id"], 1); + } + let receipt = record["metadata"]["correlation_uid"].as_str().unwrap(); assert!(uuid::Uuid::parse_str(receipt).is_ok()); + assert_request_record_sequence(&records, receipt); + assert!(!format!("{records:?}").contains("SENTINEL")); + assert!(!format!("{records:?}").contains("forged-receipt")); if let Some(expected) = expected { assert!( response.starts_with(&format!("HTTP/1.1 {expected} ")), "{response}" ); - let wire_receipt = response - .lines() - .find_map(|line| { - let (name, value) = line.split_once(':')?; - name.eq_ignore_ascii_case("x-openshell-request-id") - .then(|| value.trim()) - }) - .expect("inspected response receipt"); - assert_eq!(wire_receipt, receipt); - assert_eq!(records[0]["http_response"]["code"], expected); + assert_eq!(response_request_id(response.as_bytes()), receipt); } else { assert!( response.is_empty(), "upstream EOF must not synthesize a response" ); - assert!(records[0].get("http_response").is_none()); - assert_eq!(records[0]["status_id"], 2); } assert_eq!( - records[0]["action_id"], + record["action_id"], if matches!(outcome, Outcome::PolicyDenied) { 2 } else { @@ -216,7 +259,7 @@ network_policies: ); if roots.is_empty() { assert!(spans.is_empty()); - assert!(records[0].get("trace").is_none()); + assert!(records.iter().all(|record| record.get("trace").is_none())); } else { assert_eq!(roots.len(), 1); let root = roots[0]; @@ -238,10 +281,24 @@ network_policies: .header() .is_empty() ); - assert_eq!( - records[0]["trace"]["uid"], - root.span_context.trace_id().to_string() - ); + for record in &records { + assert_eq!( + record["trace"]["uid"], + root.span_context.trace_id().to_string() + ); + } + if let Some(expected) = expected { + assert!(root.attributes.iter().any(|attribute| { + attribute.key.as_str() == "http.response.status_code" + && attribute.value == opentelemetry::Value::I64(expected) + })); + } else { + assert!( + root.attributes + .iter() + .all(|attribute| attribute.key.as_str() != "http.response.status_code") + ); + } assert!(root.attributes.iter().any(|attribute| { attribute.key.as_str() == "openshell.request.id" && attribute.value.as_str() == receipt })); @@ -292,10 +349,8 @@ async fn inspected_forward_traces_all_outcomes_without_sampling_or_export_depend Outcome::AuditDenied, Outcome::UpstreamClosed, ] { - for flags in ["00", "01"] { - for sampler in [Some(Sampler::AlwaysOn), Some(Sampler::AlwaysOff), None] { - forward_case(ip, outcome, sampler, flags).await; - } + for sampler in [Some(Sampler::AlwaysOn), Some(Sampler::AlwaysOff), None] { + forward_case(ip, outcome, sampler).await; } } } @@ -401,14 +456,17 @@ network_policies: ready.await.unwrap(); let response = read_http_headers(&mut app).await; assert!(String::from_utf8_lossy(&response).starts_with("HTTP/1.1 101 ")); + let receipt = response_request_id(&response); let root = tokio::time::timeout(std::time::Duration::from_secs(2), async { loop { - if let Some(root) = capture + let mut roots: Vec<_> = capture .spans() .into_iter() - .find(|span| span.name == "openshell.http.request") - { - break root; + .filter(|span| span.name == "openshell.http.request") + .collect(); + if !roots.is_empty() { + assert_eq!(roots.len(), 1); + break roots.pop().unwrap(); } tokio::task::yield_now().await; } @@ -438,13 +496,52 @@ network_policies: upstream.await.unwrap(); handler.await.unwrap().unwrap(); let records = capture.events.json_events(); - assert_eq!( - records - .iter() - .filter(|event| event["class_uid"] == 4002 && event["http_response"]["code"] == 101) - .count(), - 1 + assert_eq!(records.len(), 4, "{records:?}"); + assert_request_record_sequence(&records, receipt); + for record in &records[..2] { + assert_eq!(record["class_uid"], 4002); + assert_eq!(record["action_id"], 1); + assert_eq!(record["http_request"]["url"]["path"], "/ws"); + assert!(record.get("http_response").is_none()); + assert_eq!( + record["trace"]["uid"], + root.span_context.trace_id().to_string() + ); + } + assert!( + records[0]["message"] + .as_str() + .unwrap() + .starts_with("FORWARD_L7 allow GET ") + ); + assert_eq!(records[0]["firewall_rule"]["type"], "l7"); + assert!(records[0].get("status_id").is_none()); + assert!( + records[1]["message"] + .as_str() + .unwrap() + .starts_with("FORWARD allowed GET ") + ); + assert_eq!(records[1]["firewall_rule"]["type"], "opa"); + assert_eq!(records[1]["status_id"], 1); + assert_eq!(records[2]["class_uid"], 4001); + assert_eq!(records[2]["activity_name"], "Upgrade"); + assert!( + records[2]["message"] + .as_str() + .unwrap() + .starts_with("101 Switching Protocols — websocket parsed relay ") + ); + assert!(records[2].get("http_response").is_none()); + assert_eq!(records[3]["class_uid"], 4001); + assert!( + records[3]["message"] + .as_str() + .unwrap() + .starts_with("WEBSOCKET_L7_REQUEST allow WEBSOCKET_TEXT ") ); + assert_eq!(records[3]["firewall_rule"]["type"], "l7-websocket"); + assert!(!format!("{records:?}").contains("after-http-root")); }; tokio::time::timeout(std::time::Duration::from_secs(5), scenario) .await diff --git a/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs b/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs index b1dbeb00d4..8c85b1e380 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs @@ -180,6 +180,7 @@ async fn forward_inspection_retains_installation_through_guarded_write() { response_middleware: None, endpoint_observer: None, }, + None, ); let result = tokio::time::timeout(std::time::Duration::from_secs(5), relay) .await diff --git a/crates/openshell-supervisor-network/src/telemetry.rs b/crates/openshell-supervisor-network/src/telemetry.rs index 103c49b5d5..8de4530dac 100644 --- a/crates/openshell-supervisor-network/src/telemetry.rs +++ b/crates/openshell-supervisor-network/src/telemetry.rs @@ -39,8 +39,17 @@ fn as_http(event: &OcsfEvent) -> Option<&HttpActivityEvent> { /// a span. The span describes the denied HTTP record, or the first HTTP record /// when only a companion finding is denied. A saved `parent` keeps the span in /// the connect trace after authorization exits. +/// +/// An inspected request already owns an all-outcome span. Its ambient request +/// scope suppresses denial-only spans even for legacy emitters and when tracing +/// is disabled, filtered, or unsampled. #[allow(clippy::disallowed_macros)] pub fn emit_decision(events: impl IntoIterator, parent: Option<&tracing::Span>) { + if crate::l7::correlation::is_inspected_request() { + let request_span = crate::l7::correlation::current_request_span(); + emit_inspected_decision(events, request_span.as_ref()); + return; + } let events: Vec<_> = events.into_iter().collect(); let span = if events.iter().any(is_denied) { events diff --git a/crates/openshell-supervisor-network/src/test_support.rs b/crates/openshell-supervisor-network/src/test_support.rs index 4326ade7c3..4ad244dae5 100644 --- a/crates/openshell-supervisor-network/src/test_support.rs +++ b/crates/openshell-supervisor-network/src/test_support.rs @@ -21,25 +21,6 @@ pub fn response_request_id(response: &[u8]) -> &str { .expect("response carries a request receipt") } -pub fn assert_http_request_records( - events: &[serde_json::Value], - expected_invocations: usize, -) -> (&serde_json::Value, Vec<&serde_json::Value>) { - let (primary, invocations): (Vec<_>, Vec<_>) = events - .iter() - .filter(|event| event["class_uid"] == 4002) - .partition(|event| event["unmapped"]["http_event_kind"] == "request"); - assert_eq!(primary.len(), 1, "{events:?}"); - assert_eq!(invocations.len(), expected_invocations, "{events:?}"); - assert!( - invocations - .iter() - .all(|event| event["unmapped"]["http_event_kind"] == "middleware_invocation"), - "unexpected HTTP Activity record: {events:?}" - ); - (primary[0], invocations) -} - pub fn header_placement_metadata( keys: &[&str], ) -> std::collections::HashMap { diff --git a/docs/observability/ocsf-json-export.mdx b/docs/observability/ocsf-json-export.mdx index a0d03af65c..31ae42e3e0 100644 --- a/docs/observability/ocsf-json-export.mdx +++ b/docs/observability/ocsf-json-export.mdx @@ -336,22 +336,20 @@ application-owned correlation. The receipt is a join key, not proof of identity, authorization or downstream execution. L4 and uninspected traffic do not provide this receipt guarantee. -Receipt insertion retains existing status-line tolerance. Ambiguous header line -endings and folded fields are rejected so upstream headers cannot hide a forged -receipt. - -One primary HTTP Activity record is emitted when the final response head is -written, with `unmapped.http_event_kind: "request"`. If the request ends earlier, -a guard emits the available outcome. The response header does not prove that -the body was delivered. - -Per-middleware HTTP Activity records remain available with their existing -attribution, transformation flags and response-stage fields. They carry the same -request ID, `unmapped.http_event_kind: "middleware_invocation"`, and -`unmapped.middleware_phase: "request"` or `"response"`. Response observations can -arrive after the primary record. Their status describes that invocation, not a -second request decision. Security findings and late failure records remain -separate. Collection and the existing diagnostic limits remain best-effort. +All OCSF event classes emitted in the inspected request scope carry the request +ID, including Detection Findings. Policy, middleware, token-grant and finding +records remain separate, with their original event IDs, timestamps, +classification and decision fields. The additive `unmapped.http_event_kind` +field identifies `decision`, `middleware_invocation` or `token_grant` records. +Middleware records retain `unmapped.middleware_phase` and their existing +attribution and transformation details. + +`unmapped.request_sequence` starts at 1 for each request and records local +emission order. It does not guarantee downstream delivery order or complete +collection. To count policy outcomes, group by `metadata.correlation_uid` and +select the actual policy or firewall control record. Do not count every HTTP +Activity or `decision` record as another request; companion findings also use +`decision`. The request ID join and enforcement work without tracing, when spans are not sampled, and when the collector is unavailable. Sampled OCSF trace fields identify @@ -360,54 +358,27 @@ absent. An instrumented client can record the receipt on its own trace and use i to find OpenShell's security records; it does not need to receive those records inside the sandbox. -### Update Existing Queries - -Existing middleware queries can continue using their original fields and -messages. To count inspected request decisions, select primary records rather -than counting every HTTP Activity or every `action: "Denied"` record: - -```shell -jq 'select(.class_uid == 4002 and .unmapped.http_event_kind == "request")' supervisor.jsonl -``` - -Add `.action == "Denied"` to count refused inspected requests. To inspect the -middleware work for one receipt, select `metadata.correlation_uid` with that -UUID and `unmapped.http_event_kind == "middleware_invocation"`. Do not count -companion observations as extra requests, or discard them when auditing what -middleware changed. These filters apply to inspected HTTP, not L4 decisions. - For inspected-request traces, query span name `openshell.http.request` instead -of `supervisor.egress.deny`. `openshell.request.outcome == "denied"` selects -classified denials, but is not a complete replacement for the old denial query: -a fail-closed platform error or admission refusal can have outcome `"failed"`. -For all refusals, select primary OCSF records with `action: "Denied"`, then join -spans through `openshell.request.id`. Policy name (`firewall_rule.name`) and -reason (`status_detail`) remain on the OCSF primary; those legacy policy -attributes are not copied to the new root. Keep the legacy span query separately -for non-inspected egress paths. +of `supervisor.egress.deny` and join through `openshell.request.id`. A refusal +can have outcome `"denied"` or `"failed"`; use the original OCSF control record +for its policy and reason. Keep the legacy span query for non-inspected egress. ### Size Supervisor Trace Collection -The old deny-span budgets do not bound inspected request spans. With supervisor -trace export enabled, allowed traffic also creates spans at INFO level. At -`R` inspected requests per second and an average of `S` descendant spans per -request, expect approximately `R × (1 + S)` spans per second before exporter or -collector loss. Include middleware RPCs and repeated stages in `S`; background -and lifecycle spans add separate volume. This is not a fixed per-request count. - -The exporter queue is bounded, but queue bounds do not cap span generation or -preserve the previous collector load. Collector-side sampling can reduce stored -traces; it does not remove the supervisor's span-generation cost. The gateway's -`OTEL_TRACES_SAMPLER` setting does not configure supervisor sampling. Operator -control of supervisor sampling is tracked in +The old deny-span budgets do not bound inspected request spans, which include +allowed traffic at INFO level. At `R` inspected requests per second and an +average of `S` descendant spans per request, expect approximately +`R × (1 + S)` spans per second before exporter or collector loss, plus background +and lifecycle spans. Size the collector for this traffic before enabling export. + +The bounded exporter queue does not cap span generation. Collector-side sampling +reduces stored traces, not the supervisor's span-generation cost. The gateway's +`OTEL_TRACES_SAMPLER` setting does not configure supervisor sampling; operator +control is tracked in [#5362](https://github.com/NVIDIA/OpenShell/issues/5362). -Before enabling supervisor trace export, update dashboards and size the collector -for the inspected request rate. If the old denial budget is required for your -current deployment, leave supervisor trace export disabled until a suitable -sampling or capacity policy is available. OCSF records and request receipts -continue to work without it. This change does not itself add a sampling setting -or restore denial-only spans. +Leave supervisor trace export disabled if your deployment requires the old +denial-only volume bound. OCSF records and request receipts still work without it. ## OCSF Event Classes in JSON From 54a11a15d52ac942acadd9efbec3316db224c793 Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Mon, 16 Nov 2026 13:22:53 -0700 Subject: [PATCH 4/7] fix(observability): preserve HTTP correlation through relay failures Avoid parsing queries solely for correlation. Carry request attribution through terminal errors and policy reloads, and clear HTTP scope at WebSocket upgrade. Preserve per-control audit records while reporting bounded stage outcomes and filtering receipt trailers consistently. Remove JSON receipt additions and the unused correlation setter. Signed-off-by: Alex Fournier --- crates/openshell-ocsf/src/events/mod.rs | 7 +- .../src/tracing_layers/event_bridge.rs | 3 +- crates/openshell-otel/src/ocsf.rs | 11 +- crates/openshell-supervisor-network/README.md | 13 +- .../src/l7/correlation.rs | 291 +++++++++-- .../src/l7/correlation/tests.rs | 359 ++++++++++++- .../src/l7/middleware.rs | 236 ++++++--- .../src/l7/relay.rs | 471 +++++++++++++----- .../l7/relay/correlation_dispatch_tests.rs | 253 +++++++++- .../src/l7/request_trace.rs | 16 +- .../src/l7/request_trace/tests.rs | 32 ++ .../src/l7/rest.rs | 101 ++-- .../src/l7/rest/http_response.rs | 61 ++- .../src/l7/rest/http_response/http_hooks.rs | 37 ++ .../src/l7/rest/request_body.rs | 20 +- .../openshell-supervisor-network/src/proxy.rs | 119 +++-- .../src/proxy/relay.rs | 37 +- .../src/proxy/tests/compatibility.rs | 5 +- .../src/proxy/tests/request_trace.rs | 135 +++-- .../openshell-supervisor-network/src/run.rs | 1 + .../src/telemetry/tests.rs | 5 +- docs/observability/ocsf-json-export.mdx | 12 +- 22 files changed, 1797 insertions(+), 428 deletions(-) diff --git a/crates/openshell-ocsf/src/events/mod.rs b/crates/openshell-ocsf/src/events/mod.rs index 22ecbb6dfc..ee3cf27b69 100644 --- a/crates/openshell-ocsf/src/events/mod.rs +++ b/crates/openshell-ocsf/src/events/mod.rs @@ -169,11 +169,6 @@ impl OcsfEvent { } } - /// Associate this event with other events produced for the same operation. - pub fn set_correlation_uid(&mut self, uid: impl Into) { - self.base_mut().metadata.correlation_uid = Some(uid.into()); - } - /// Returns mutable base data common to all event classes. pub(crate) fn base_mut(&mut self) -> &mut BaseEventData { match self { @@ -297,7 +292,7 @@ mod tests { .status(StatusId::Success) .build(); - event.set_correlation_uid("request-123"); + event.base_mut().metadata.correlation_uid = Some("request-123".into()); assert_eq!( event.base().metadata.correlation_uid.as_deref(), diff --git a/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs b/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs index 98a0031ce7..f5a8d41419 100644 --- a/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs +++ b/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs @@ -318,7 +318,8 @@ mod tests { .base_mut() .add_unmapped("http_event_kind", "original_kind"); let mut enriched = event.clone(); - enriched.set_correlation_uid("emit-time-request"); + enriched.base_mut().metadata.correlation_uid = + Some("emit-time-request".into()); enriched .base_mut() .add_unmapped("request_sequence", sequence); diff --git a/crates/openshell-otel/src/ocsf.rs b/crates/openshell-otel/src/ocsf.rs index 86ff38de6a..cc4f6c44f7 100644 --- a/crates/openshell-otel/src/ocsf.rs +++ b/crates/openshell-otel/src/ocsf.rs @@ -636,7 +636,10 @@ mod tests { #[test] fn correlation_uid_is_a_stable_otlp_attribute() { let mut event = event("sandbox-1", SeverityId::Informational); - event.set_correlation_uid("request-123"); + let OcsfEvent::DeviceConfigStateChange(config) = &mut event else { + unreachable!("event fixture returns a configuration event"); + }; + config.base.metadata.correlation_uid = Some("request-123".into()); assert_eq!( attribute(&record(&event), CORRELATION_UID_ATTRIBUTE), Some(AnyValue::String("request-123".into())) @@ -803,7 +806,11 @@ mod tests { let span_context = tracing::subscriber::with_default(subscriber, || { let span = tracing::info_span!("decision"); let mut event = event("sandbox-1", SeverityId::Medium); - event.set_correlation_uid("request-with-or-without-sampling"); + let OcsfEvent::DeviceConfigStateChange(config) = &mut event else { + unreachable!("event fixture returns a configuration event"); + }; + config.base.metadata.correlation_uid = + Some("request-with-or-without-sampling".into()); span.in_scope(|| ocsf_emit!(event)); span.context().span().span_context().clone() }); diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index d87ded103e..d9fa595a56 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -67,20 +67,25 @@ L4 and passthrough traffic do not provide this header guarantee. The emission hook adds request correlation to every OCSF class in the current request scope, including findings. Policy, middleware, token-grant and finding records retain their original content, event IDs, timestamps and classification; -they are not merged or delayed. `unmapped.http_event_kind` identifies `decision`, -`middleware_invocation` or `token_grant`, and middleware records retain +they are not merged or delayed. `unmapped.http_event_kind` distinguishes decisions, +findings, connection events and middleware invocations. Explicit control kinds, +such as `token_grant`, remain unchanged. Middleware records retain `unmapped.middleware_phase`, including HTTP-hook preflight and completion. `unmapped.request_sequence` starts at 1 per request and describes local emission order, not downstream delivery order. Count policy outcomes by selecting the actual policy/firewall control record within a receipt group, not every -`decision` record; companion findings also use `decision`. +`decision` record. Request IDs and audit outcomes do not depend on OpenTelemetry, sampling or collector availability. Inspected requests have a supervisor-owned request span and children around stages that actually run, with valid workload W3C context only as an untrusted link. Their records never create an extra denial-only span, even if the request span is disabled or unsampled. Other egress paths retain the -legacy bounded deny-span behavior. End HTTP tracing before an upgraded session. +legacy bounded deny-span behavior. Both HTTP tracing and the request's ambient +audit scope end before an upgraded session; later WebSocket frames do not reuse +the handshake receipt. Terminal errors retain their request context explicitly, +and policy-reload cancellation records the active request before dropping it. +Idle connection closures have no HTTP receipt. The request span ends after its owned stages, not when response headers are logged. The `openshell.upstream.send` stage measures request writes and flush; response handling is measured separately. Receipt filtering preserves raw HTTP bytes diff --git a/crates/openshell-supervisor-network/src/l7/correlation.rs b/crates/openshell-supervisor-network/src/l7/correlation.rs index 8a9508415a..5951ee4235 100644 --- a/crates/openshell-supervisor-network/src/l7/correlation.rs +++ b/crates/openshell-supervisor-network/src/l7/correlation.rs @@ -3,12 +3,13 @@ //! Request receipts and emission-time correlation without changing audit records. -use std::cell::RefCell; +use std::fmt; use std::future::Future; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, OnceLock}; use openshell_ocsf::OcsfEvent; +use tracing::Instrument as _; use crate::l7::provider::L7Request; use crate::l7::relay::L7EvalContext; @@ -16,50 +17,98 @@ pub use crate::l7::request_trace::RequestStage; use crate::l7::request_trace::RequestTrace; tokio::task_local! { - static CURRENT_REQUEST: RefCell>>; + static CURRENT_REQUEST: Arc>>>; } /// Give each connection its own request slot, including across task migration. pub fn scope_connection(future: F) -> impl Future { - CURRENT_REQUEST.scope(RefCell::new(None), future) + #[cfg(test)] + init_request_correlation().expect("request correlation enricher registration"); + let slot = CURRENT_REQUEST + .try_with(Arc::clone) + .unwrap_or_else(|_| Arc::new(Mutex::new(None))); + CURRENT_REQUEST.scope(slot, future) +} + +/// Install the network supervisor's process-wide audit enrichment at startup. +/// +/// A different enricher is a configuration conflict, not a reason to silently +/// emit inspected records without their request receipt. +pub fn init_request_correlation() -> miette::Result<()> { + static REGISTRATION: OnceLock> = OnceLock::new(); + match REGISTRATION.get_or_init(|| { + openshell_ocsf::register_event_enricher(enrich_request) + .then_some(()) + .ok_or("cannot initialize HTTP request correlation: an OCSF event enricher is already registered") + }) { + Ok(()) => Ok(()), + Err(message) => Err(miette::miette!("{message}")), + } } pub fn is_inspected_request() -> bool { - CURRENT_REQUEST - .try_with(|current| { - current - .borrow() - .as_ref() - .is_some_and(|scope| scope.inspected) - }) - .unwrap_or(false) + current_request().is_some_and(|scope| scope.inspected) } pub fn current_request_span() -> Option { + current_request() + .filter(|scope| scope.inspected) + .map(|scope| scope.span()) +} + +fn current_request() -> Option> { CURRENT_REQUEST - .try_with(|current| current.borrow().clone()) + .try_with(|current| { + current + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + }) .ok() .flatten() - .filter(|scope| scope.inspected) - .map(|scope| scope.span()) +} + +fn in_request_scope(scope: &Arc, operation: impl FnOnce() -> T) -> T { + if current_request().is_some_and(|current| Arc::ptr_eq(¤t, scope)) { + operation() + } else { + let slot = Arc::new(Mutex::new(Some(Arc::clone(scope)))); + CURRENT_REQUEST.sync_scope(slot, operation) + } +} + +pub fn record_current_failure(outcome: HttpOutcome, reason: &'static str) { + if let Some(scope) = current_request().filter(|scope| scope.inspected) { + scope.record_failure(outcome, reason); + } } fn enrich_request(event: &mut openshell_ocsf::BaseEventData) { - let _ = CURRENT_REQUEST.try_with(|current| { - let current = current.borrow(); - let Some(scope) = current.as_ref().filter(|scope| scope.inspected) else { - return; - }; - event.metadata.correlation_uid = Some(scope.request_id.clone()); - let sequence = scope.sequence.fetch_add(1, Ordering::Relaxed); - let fields = event.unmapped.get_or_insert_with(|| serde_json::json!({})); - if let Some(fields) = fields.as_object_mut() { - fields - .entry("http_event_kind") - .or_insert_with(|| "decision".into()); - fields.insert("request_sequence".into(), sequence.into()); - } - }); + let Some(scope) = current_request().filter(|scope| scope.inspected) else { + return; + }; + event.metadata.correlation_uid = Some(scope.request_id.clone()); + let sequence = scope.sequence.fetch_add(1, Ordering::Relaxed); + let event_kind = default_event_kind(event.class_uid); + let fields = event.unmapped.get_or_insert_with(|| serde_json::json!({})); + if let Some(fields) = fields.as_object_mut() { + fields + .entry("http_event_kind") + .or_insert_with(|| event_kind.into()); + fields.insert("request_sequence".into(), sequence.into()); + } +} + +fn default_event_kind(class_uid: u32) -> &'static str { + match class_uid { + 4002 => "decision", + 2004 => "finding", + 4001 | 4007 => "connection", + 1007 | 6002 => "lifecycle", + 5019 => "configuration", + 3002 => "authentication", + _ => "event", + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -87,6 +136,45 @@ impl RequestScope { .as_ref() .map_or_else(tracing::Span::none, RequestTrace::span) } + + fn record_failure(&self, outcome: HttpOutcome, reason: &'static str) { + if let Some(trace) = self + .trace + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_mut() + { + trace.record_outcome(outcome); + trace.record_failure_reason(reason); + } + } +} + +/// Carry request attribution only with an actual terminal error, never as a +/// connection-wide "last request" that could misattribute unrelated events. +#[derive(Debug)] +struct CapturedRequestError { + scope: Arc, + span: tracing::Span, + detail: String, +} + +impl fmt::Display for CapturedRequestError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.detail) + } +} + +/// Emit the existing terminal record in the request that returned the error, +/// even after its request guard has restored the connection's ambient scope. +pub fn emit_error_event(event: OcsfEvent, error: &miette::Report) { + if let Some(captured) = error.downcast_ref::() { + in_request_scope(&captured.scope, || { + crate::telemetry::emit_inspected_decision([event], Some(&captured.span)); + }); + } else { + crate::telemetry::emit_decision([event], None); + } } #[derive(Debug)] @@ -96,9 +184,41 @@ pub struct RequestCorrelation { previous: Option>, } +/// An interrupted stage is an error even if its future never returns a result. +struct StageResult { + span: tracing::Span, + completed: bool, +} + +impl StageResult { + fn record(&mut self, result: &Result, reason: &'static str) { + self.completed = true; + if result.is_err() { + self.span.record("otel.status_code", "ERROR"); + self.span.record("openshell.request.failure_reason", reason); + } else { + self.span.record("otel.status_code", "OK"); + } + } +} + +impl Drop for StageResult { + fn drop(&mut self) { + if !self.completed { + self.span.record("otel.status_code", "ERROR"); + self.span + .record("openshell.request.failure_reason", "cancelled"); + } + } +} + impl RequestCorrelation { pub(crate) fn new(request: &L7Request, ctx: &L7EvalContext) -> Self { - Self::build(request, ctx, uuid::Uuid::new_v4().to_string()) + Self::new_http(&request.action, &request.raw_header, ctx) + } + + pub(crate) fn new_http(method: &str, raw_header: &[u8], ctx: &L7EvalContext) -> Self { + Self::build(method, raw_header, ctx, uuid::Uuid::new_v4().to_string()) } pub(crate) fn uninspected() -> Self { @@ -116,17 +236,11 @@ impl RequestCorrelation { ctx: &L7EvalContext, request_id: String, ) -> Self { - Self::build(request, ctx, request_id) + Self::build(&request.action, &request.raw_header, ctx, request_id) } - fn build(request: &L7Request, ctx: &L7EvalContext, request_id: String) -> Self { - let trace = RequestTrace::new( - &request_id, - &request.action, - &ctx.host, - ctx.port, - &request.raw_header, - ); + fn build(method: &str, raw_header: &[u8], ctx: &L7EvalContext, request_id: String) -> Self { + let trace = RequestTrace::new(&request_id, method, &ctx.host, ctx.port, raw_header); Self::install(RequestScope { request_id, inspected: true, @@ -136,10 +250,14 @@ impl RequestCorrelation { } fn install(scope: RequestScope) -> Self { - openshell_ocsf::register_event_enricher(enrich_request); let scope = Arc::new(scope); let previous = CURRENT_REQUEST - .try_with(|current| current.replace(Some(Arc::clone(&scope)))) + .try_with(|current| { + current + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .replace(Arc::clone(&scope)) + }) .ok() .flatten(); Self { scope, previous } @@ -149,6 +267,27 @@ impl RequestCorrelation { &self.scope.request_id } + pub(crate) fn capture_error(&self, error: miette::Report) -> miette::Report { + if !self.scope.inspected || error.downcast_ref::().is_some() { + return error; + } + if let Some(trace) = self + .scope + .trace + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_mut() + { + trace.record_unclassified_failure(); + } + let captured = CapturedRequestError { + scope: Arc::clone(&self.scope), + span: self.scope.span(), + detail: error.to_string(), + }; + error.wrap_err(captured) + } + pub(crate) fn stage(&self, stage: RequestStage) -> tracing::Span { self.scope .trace @@ -158,7 +297,56 @@ impl RequestCorrelation { .map_or_else(tracing::Span::none, |trace| trace.stage(stage)) } - pub(crate) fn finish_http_trace(&self) { + /// Record a stage's actual result without exporting its potentially + /// sensitive error value or changing cancellation behavior. + pub(crate) fn run_stage( + &self, + stage: RequestStage, + failure_outcome: HttpOutcome, + reason: &'static str, + future: impl Future>, + ) -> impl Future> { + let mut stage = StageResult { + span: self.stage(stage), + completed: false, + }; + let future = future.instrument(stage.span.clone()); + async move { + let result = future.await; + stage.record(&result, reason); + if result.is_err() { + self.record_failure(failure_outcome, reason); + } + result + } + } + + pub(crate) fn run_stage_sync( + &self, + stage: RequestStage, + failure_outcome: HttpOutcome, + reason: &'static str, + operation: impl FnOnce() -> Result, + ) -> Result { + let mut stage = StageResult { + span: self.stage(stage), + completed: false, + }; + let result = stage.span.in_scope(operation); + stage.record(&result, reason); + if result.is_err() { + self.record_failure(failure_outcome, reason); + } + result + } + + /// End HTTP tracing and restore the previous ambient scope before handing + /// an upgraded connection to its longer-lived protocol relay. + pub(crate) fn finish_http_request(self) { + drop(self); + } + + fn finish_http_trace(&self) { let trace = self .scope .trace @@ -209,16 +397,7 @@ impl RequestCorrelation { } pub(crate) fn record_failure(&self, outcome: HttpOutcome, reason: &'static str) { - if let Some(trace) = self - .scope - .trace - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .as_mut() - { - trace.record_outcome(outcome); - trace.record_failure_reason(reason); - } + self.scope.record_failure(outcome, reason); } pub(crate) fn emit(&self, event: OcsfEvent) { @@ -236,7 +415,7 @@ impl RequestCorrelation { { trace.observe_http(http); } - CURRENT_REQUEST.sync_scope(RefCell::new(Some(Arc::clone(&self.scope))), || { + in_request_scope(&self.scope, || { crate::telemetry::emit_inspected_decision([event], Some(&self.scope.span())); }); } @@ -259,7 +438,9 @@ impl Drop for RequestCorrelation { fn drop(&mut self) { self.finish_http_trace(); let _ = CURRENT_REQUEST.try_with(|current| { - let mut current = current.borrow_mut(); + let mut current = current + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); if current .as_ref() .is_some_and(|scope| Arc::ptr_eq(scope, &self.scope)) diff --git a/crates/openshell-supervisor-network/src/l7/correlation/tests.rs b/crates/openshell-supervisor-network/src/l7/correlation/tests.rs index eae7bd4ab9..6f0bcd8de4 100644 --- a/crates/openshell-supervisor-network/src/l7/correlation/tests.rs +++ b/crates/openshell-supervisor-network/src/l7/correlation/tests.rs @@ -2,14 +2,16 @@ // SPDX-License-Identifier: Apache-2.0 use super::*; -use crate::test_support::request_tracing::RequestTracing; +use crate::test_support::request_tracing::{RequestTracing, span_attribute}; use openshell_ocsf::test_support::OcsfCapture; use openshell_ocsf::{ ActionId, ActivityId, DetectionFindingBuilder, DispositionId, HttpActivityBuilder, HttpRequest, HttpResponse, NetworkActivityBuilder, SeverityId, StatusId, }; +use tracing::instrument::WithSubscriber as _; fn request() -> L7Request { + init_request_correlation().unwrap(); L7Request { action: "GET".into(), target: "/data".into(), @@ -138,6 +140,8 @@ async fn ambient_scope_stamps_companions_without_call_site_wiring() { ); let records = capture.json_events(); assert_eq!(records.len(), 2); + assert_eq!(records[0]["unmapped"]["http_event_kind"], "connection"); + assert_eq!(records[1]["unmapped"]["http_event_kind"], "finding"); for (index, record) in records.iter().enumerate() { assert_eq!( record["metadata"]["correlation_uid"], @@ -232,3 +236,356 @@ async fn cancelling_a_request_does_not_leak_its_context() { }) .await; } + +#[test] +fn event_kinds_distinguish_decisions_from_companions() { + for (class_uid, expected) in [ + (4002, "decision"), + (2004, "finding"), + (4001, "connection"), + (4007, "connection"), + (1007, "lifecycle"), + (6002, "lifecycle"), + (5019, "configuration"), + (3002, "authentication"), + (6003, "event"), + (0, "event"), + ] { + assert_eq!(default_event_kind(class_uid), expected); + } +} + +#[tokio::test] +async fn finishing_http_restores_the_previous_scope_before_protocol_relay() { + let capture = OcsfCapture::default(); + let _subscriber = capture.install(); + scope_connection(async { + let outer = RequestCorrelation::with_request_id(&request(), &context(), "outer".into()); + let nested = RequestCorrelation::with_request_id(&request(), &context(), "nested".into()); + crate::telemetry::emit_decision([allowed()], None); + + nested.finish_http_request(); + crate::telemetry::emit_decision([allowed()], None); + assert!(is_inspected_request()); + + outer.finish_http_request(); + assert!(!is_inspected_request()); + assert!(current_request_span().is_none()); + tokio::task::yield_now().await; + crate::telemetry::emit_decision([allowed()], None); + }) + .await; + let records = capture.json_events(); + assert_eq!(records[0]["metadata"]["correlation_uid"], "nested"); + assert_eq!(records[1]["metadata"]["correlation_uid"], "outer"); + assert_eq!(records[1]["unmapped"]["request_sequence"], 1); + assert!(records[2]["metadata"]["correlation_uid"].is_null()); + assert!(records[2]["unmapped"]["request_sequence"].is_null()); +} + +#[tokio::test] +async fn stage_results_record_bounded_errors_and_preserve_a_failed_root() { + use opentelemetry::trace::Status; + use opentelemetry_sdk::trace::Sampler; + + let capture = RequestTracing::new(Some(Sampler::AlwaysOn)); + scope_connection(async { + let correlation = RequestCorrelation::new_http( + "POST", + b"POST / HTTP/1.1\r\nX-Opaque: \xff\r\n\r\n", + &context(), + ); + let success = correlation.run_stage_sync( + RequestStage::PolicyEvaluate, + HttpOutcome::PlatformError, + "policy_evaluation_failed", + || Ok::<_, &str>(42), + ); + assert_eq!(success, Ok(42)); + let failure = correlation + .run_stage( + RequestStage::UpstreamDial, + HttpOutcome::UpstreamError, + "upstream_connect_failed", + async { Err::<(), _>("DO-NOT-RECORD-ERROR") }, + ) + .await; + assert_eq!(failure, Err("DO-NOT-RECORD-ERROR")); + correlation.record_failure(HttpOutcome::PlatformError, "request_failed"); + correlation.record_outcome(HttpOutcome::Allowed); + }) + .with_subscriber(capture.subscriber()) + .await; + let spans = capture.spans(); + let policy = spans + .iter() + .find(|span| span.name == "openshell.policy.evaluate") + .unwrap(); + assert_eq!(policy.status, Status::Ok); + let dial = spans + .iter() + .find(|span| span.name == "openshell.upstream.connect") + .unwrap(); + assert!(matches!(dial.status, Status::Error { .. })); + assert_eq!( + span_attribute(dial, "openshell.request.failure_reason") + .unwrap() + .as_str(), + "upstream_connect_failed" + ); + let root = spans + .iter() + .find(|span| span.name == "openshell.http.request") + .unwrap(); + assert!(matches!(root.status, Status::Error { .. })); + assert_eq!( + span_attribute(root, "openshell.request.outcome") + .unwrap() + .as_str(), + "failed" + ); + assert_eq!( + span_attribute(root, "openshell.request.failure_reason") + .unwrap() + .as_str(), + "upstream_connect_failed" + ); + assert_eq!( + span_attribute(root, "http.request.method") + .unwrap() + .as_str(), + "POST" + ); + assert!(!format!("{spans:?}").contains("DO-NOT-RECORD-ERROR")); + capture.shutdown(); +} + +#[tokio::test] +async fn a_cancelled_stage_drops_its_future_and_closes_with_a_bounded_reason() { + use opentelemetry::trace::Status; + use opentelemetry_sdk::trace::Sampler; + use std::sync::atomic::AtomicBool; + + struct Dropped(Arc); + impl Drop for Dropped { + fn drop(&mut self) { + self.0.store(true, Ordering::Relaxed); + } + } + + let dropped = Arc::new(AtomicBool::new(false)); + let capture = RequestTracing::new(Some(Sampler::AlwaysOn)); + scope_connection(async { + let correlation = RequestCorrelation::new(&request(), &context()); + let mut pending = Box::pin(correlation.run_stage( + RequestStage::Response, + HttpOutcome::UpstreamError, + "response_failed", + async { + let _drop = Dropped(Arc::clone(&dropped)); + std::future::pending::>().await + }, + )); + assert!(futures::poll!(&mut pending).is_pending()); + drop(pending); + assert!(dropped.load(Ordering::Relaxed)); + }) + .with_subscriber(capture.subscriber()) + .await; + assert!(!is_inspected_request()); + let spans = capture.spans(); + let response = spans + .iter() + .find(|span| span.name == "openshell.http.response") + .unwrap(); + assert!(matches!(response.status, Status::Error { .. })); + assert_eq!( + span_attribute(response, "openshell.request.failure_reason") + .unwrap() + .as_str(), + "cancelled" + ); + let root = spans + .iter() + .find(|span| span.name == "openshell.http.request") + .unwrap(); + assert!(matches!(root.status, Status::Error { .. })); + assert_eq!( + span_attribute(root, "openshell.request.outcome") + .unwrap() + .as_str(), + "incomplete" + ); + capture.shutdown(); +} + +#[tokio::test] +async fn captured_terminal_errors_keep_request_attribution_after_the_guard_drops() { + use opentelemetry::trace::Status; + use opentelemetry_sdk::trace::Sampler; + + for sampler in [Some(Sampler::AlwaysOn), Some(Sampler::AlwaysOff), None] { + let sampled = matches!(sampler, Some(Sampler::AlwaysOn)); + let capture = RequestTracing::new(sampler); + scope_connection(async { + let correlation = RequestCorrelation::with_request_id( + &request(), + &context(), + "failed-request".into(), + ); + correlation.emit(allowed()); + correlation.record_failure(HttpOutcome::UpstreamError, "upstream_send_failed"); + let original = miette::miette!("original transfer error").wrap_err("transfer failed"); + let original_chain: Vec<_> = original.chain().map(ToString::to_string).collect(); + let captured = correlation.capture_error(original); + assert_eq!(captured.to_string(), "transfer failed"); + assert_eq!( + captured + .chain() + .skip(1) + .map(ToString::to_string) + .collect::>(), + original_chain + ); + let captured = correlation.capture_error(captured); + assert_eq!(captured.chain().count(), original_chain.len() + 1); + drop(correlation); + assert!(!is_inspected_request()); + + let next = + RequestCorrelation::with_request_id(&request(), &context(), "next-request".into()); + emit_error_event(deny(), &captured); + emit_error_event( + DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .finding_info(openshell_ocsf::FindingInfo::new( + "transfer-failed", + "Transfer failed", + )) + .build(), + &captured, + ); + crate::telemetry::emit_decision([allowed()], None); + drop(next); + }) + .with_subscriber(capture.subscriber()) + .await; + let events = capture.events.json_events(); + assert_eq!(events.len(), 4); + for (index, event) in events[..3].iter().enumerate() { + assert_eq!(event["metadata"]["correlation_uid"], "failed-request"); + assert_eq!(event["unmapped"]["request_sequence"], index + 1); + } + assert_eq!(events[1]["trace"]["uid"].is_string(), sampled); + assert_eq!(events[2]["unmapped"]["http_event_kind"], "finding"); + assert_eq!(events[3]["metadata"]["correlation_uid"], "next-request"); + assert_eq!(events[3]["unmapped"]["request_sequence"], 1); + assert!( + !capture + .created_span_names() + .contains(&"supervisor.egress.deny") + ); + if sampled { + let spans = capture.spans(); + let failed = spans + .iter() + .find(|span| { + span_attribute(span, "openshell.request.id") + .is_some_and(|id| id.as_str() == "failed-request") + }) + .unwrap(); + assert!(matches!(failed.status, Status::Error { .. })); + assert_eq!( + span_attribute(failed, "openshell.request.failure_reason") + .unwrap() + .as_str(), + "upstream_send_failed" + ); + } + capture.shutdown(); + } +} + +#[test] +fn failed_denial_delivery_marks_the_root_failed_without_replacing_the_reason() { + let capture = RequestTracing::new(Some(opentelemetry_sdk::trace::Sampler::AlwaysOn)); + tracing::subscriber::with_default(capture.subscriber(), || { + let correlation = RequestCorrelation::new(&request(), &context()); + correlation.record_failure(HttpOutcome::Denied, "request_middleware_denied"); + let error = correlation.capture_error(miette::miette!("response write failed")); + drop(correlation); + drop(error); + }); + let spans = capture.spans(); + let root = spans + .iter() + .find(|span| span.name == "openshell.http.request") + .unwrap(); + assert_eq!( + span_attribute(root, "openshell.request.outcome") + .unwrap() + .as_str(), + "failed" + ); + assert_eq!( + span_attribute(root, "openshell.request.failure_reason") + .unwrap() + .as_str(), + "request_middleware_denied" + ); + assert!(matches!( + root.status, + opentelemetry::trace::Status::Error { .. } + )); + capture.shutdown(); +} + +#[test] +fn an_unscoped_terminal_error_does_not_borrow_a_finished_request() { + let capture = OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = RequestCorrelation::new(&request(), &context()); + let error = correlation.capture_error(miette::miette!("request error")); + drop(correlation); + emit_error_event(allowed(), &error); + emit_error_event(allowed(), &miette::miette!("unrelated connection error")); + let records = capture.json_events(); + assert!(records[0]["metadata"]["correlation_uid"].is_string()); + assert!(records[1]["metadata"]["correlation_uid"].is_null()); + assert!(records[1]["unmapped"]["request_sequence"].is_null()); +} + +#[tokio::test] +async fn nested_connection_scopes_share_the_active_request_with_a_sibling_watcher() { + let capture = OcsfCapture::default(); + let _subscriber = capture.install(); + scope_connection(async { + let (ready, started) = tokio::sync::oneshot::channel(); + let relay = scope_connection(async { + let _request = RequestCorrelation::with_request_id( + &request(), + &context(), + "active-request".into(), + ); + ready.send(()).unwrap(); + std::future::pending::<()>().await; + }); + let watcher = async { + started.await.unwrap(); + assert!(is_inspected_request()); + record_current_failure(HttpOutcome::Denied, "policy_reloaded"); + crate::telemetry::emit_decision([deny()], None); + }; + tokio::select! { + () = relay => unreachable!(), + () = watcher => {}, + } + assert!(!is_inspected_request()); + crate::telemetry::emit_decision([allowed()], None); + }) + .await; + let records = capture.json_events(); + assert_eq!(records.len(), 2); + assert_eq!(records[0]["metadata"]["correlation_uid"], "active-request"); + assert_eq!(records[0]["unmapped"]["request_sequence"], 1); + assert!(records[1]["metadata"]["correlation_uid"].is_null()); +} diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index e5d5b8641a..b2754879d7 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -426,7 +426,12 @@ pub fn emit_websocket_preflight_events_correlated( outcome: &openshell_supervisor_middleware::WebSocketPreflightResult, correlation: Option<&super::correlation::RequestCorrelation>, ) { - emit_websocket_invocations_correlated(ctx, &outcome.invocations, correlation); + emit_websocket_invocations_correlated( + ctx, + &outcome.invocations, + WebSocketMiddlewarePhase::Preflight, + correlation, + ); emit_request_events( correlation, websocket_coverage_events(ctx, &outcome.coverage), @@ -508,14 +513,18 @@ pub(super) fn emit_websocket_session_start_events( ctx: &L7EvalContext, outcome: &openshell_supervisor_middleware::WebSocketSessionStartOutcome, ) { - emit_websocket_invocations(ctx, &outcome.invocations); + emit_websocket_invocations( + ctx, + &outcome.invocations, + WebSocketMiddlewarePhase::SessionStart, + ); } pub(super) fn emit_websocket_message_events( ctx: &L7EvalContext, outcome: &openshell_supervisor_middleware::WebSocketMessageOutcome, ) { - emit_websocket_invocations(ctx, &outcome.invocations); + emit_websocket_invocations(ctx, &outcome.invocations, WebSocketMiddlewarePhase::Message); if outcome.saturated { emit_websocket_saturation(ctx); } @@ -568,16 +577,35 @@ pub(super) fn middleware_finding_events( .collect() } +#[derive(Clone, Copy)] +enum WebSocketMiddlewarePhase { + Preflight, + SessionStart, + Message, +} + +impl WebSocketMiddlewarePhase { + const fn as_str(self) -> &'static str { + match self { + Self::Preflight => "websocket_preflight", + Self::SessionStart => "websocket_session_start", + Self::Message => "websocket_message", + } + } +} + fn emit_websocket_invocations( ctx: &L7EvalContext, invocations: &[openshell_supervisor_middleware::WebSocketInvocation], + phase: WebSocketMiddlewarePhase, ) { - emit_websocket_invocations_correlated(ctx, invocations, None); + emit_websocket_invocations_correlated(ctx, invocations, phase, None); } fn emit_websocket_invocations_correlated( ctx: &L7EvalContext, invocations: &[openshell_supervisor_middleware::WebSocketInvocation], + phase: WebSocketMiddlewarePhase, correlation: Option<&super::correlation::RequestCorrelation>, ) { for invocation in invocations { @@ -642,6 +670,8 @@ fn emit_websocket_invocations_correlated( .status(status) .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) .firewall_rule(&ctx.policy_name, "supervisor-middleware") + .unmapped("http_event_kind", "middleware_invocation") + .unmapped("middleware_phase", phase.as_str()) .message(format!( "WEBSOCKET_MIDDLEWARE {outcome_name} config={} implementation={} sequence={sequence} input_bytes={} replacement_bytes={replacement_size} transformed={} reason_code={reason_code}", invocation.config_name, @@ -1023,68 +1053,80 @@ async fn apply_middleware_chain_for_scheme_with_request_id_and_delivery_correlat return Ok(MiddlewareApplyResult::Allowed(req)); } let chain = runner.describe_chain(&chain).await?; - let span = if chain.is_empty() { - tracing::Span::none() - } else { - correlation.map_or_else(tracing::Span::none, |correlation| { - correlation.stage(RequestStage::RequestMiddleware) - }) - }; - match openshell_supervisor_middleware::chain_http_hook_version(&chain) { - ChainHttpHookVersion::Legacy => { - Box::pin(apply_protocol1_middleware_chain( - req, - client, - ctx, - scheme, - chain, - runner, - generation_guard, - transformed_body_policy, - request_id, - correlation, - span.clone(), - )) - .await - } - ChainHttpHookVersion::Mixed => { - emit_mixed_protocol_denial(ctx, &req, &chain, correlation); - Ok(MiddlewareApplyResult::Denied { - denial: None, - failure: Some(MiddlewareFailureDetail { - config_name: None, - code: HttpFailureCode::HookVersionsMixed, - }), - }) + let has_stages = !chain.is_empty(); + let apply = async { + let span = if has_stages && correlation.is_some() { + tracing::Span::current() + } else { + tracing::Span::none() + }; + match openshell_supervisor_middleware::chain_http_hook_version(&chain) { + ChainHttpHookVersion::Legacy => { + Box::pin(apply_protocol1_middleware_chain( + req, + client, + ctx, + scheme, + chain, + runner, + generation_guard, + transformed_body_policy, + request_id, + correlation, + )) + .await + } + ChainHttpHookVersion::Mixed => { + emit_mixed_protocol_denial(ctx, &req, &chain, correlation); + Ok(MiddlewareApplyResult::Denied { + denial: None, + failure: Some(MiddlewareFailureDetail { + config_name: None, + code: HttpFailureCode::HookVersionsMixed, + }), + }) + } + ChainHttpHookVersion::Current => { + // A body-aware policy re-checks every replaced body, so the + // upstream must not see any of it before the last stage's output + // is checked. + let delivery = match transformed_body_policy { + openshell_supervisor_middleware::TransformedBodyPolicy::NotPolicyRelevant => { + delivery + } + openshell_supervisor_middleware::TransformedBodyPolicy::Reevaluate(_) => { + delivery.or_withhold() + } + }; + Box::pin(apply_pipeline_middleware_chain( + req, + client, + ctx, + scheme, + chain, + runner, + generation_guard, + transformed_body_policy, + request_id, + delivery, + correlation, + span.clone(), + )) + .await + } } - ChainHttpHookVersion::Current => { - // A body-aware policy re-checks every replaced body, so the - // upstream must not see any of it before the last stage's output - // is checked. - let delivery = match transformed_body_policy { - openshell_supervisor_middleware::TransformedBodyPolicy::NotPolicyRelevant => { - delivery - } - openshell_supervisor_middleware::TransformedBodyPolicy::Reevaluate(_) => { - delivery.or_withhold() - } - }; - Box::pin(apply_pipeline_middleware_chain( - req, - client, - ctx, - scheme, - chain, - runner, - generation_guard, - transformed_body_policy, - request_id, - delivery, - correlation, - span.clone(), - )) + }; + if has_stages && let Some(correlation) = correlation { + correlation + .run_stage( + RequestStage::RequestMiddleware, + super::correlation::HttpOutcome::PlatformError, + "request_middleware_failed", + apply, + ) .await - } + } else { + apply.await } } @@ -1101,15 +1143,11 @@ async fn apply_protocol1_middleware_chain, request_id: &str, correlation: Option<&super::correlation::RequestCorrelation>, - span: tracing::Span, ) -> Result { let admission = if chain.is_empty() { None } else { - let outcome = runner - .reserve_middleware_work() - .instrument(span.clone()) - .await?; + let outcome = runner.reserve_middleware_work().await?; match outcome { openshell_supervisor_middleware::MiddlewareWorkAdmissionOutcome::Admitted( admission, @@ -1143,7 +1181,6 @@ async fn apply_protocol1_middleware_chain buffered, @@ -1202,7 +1238,6 @@ async fn apply_protocol1_middleware_chain prepared, Err(error) => { if let Some(correlation) = correlation { - correlation.record_rejection(502, HttpOutcome::PlatformError, None); + correlation.record_rejection( + 502, + HttpOutcome::PlatformError, + Some("credential_prepare_failed"), + ); } if let Some(refusal) = error.downcast_ref::() @@ -1051,7 +1059,8 @@ where observer.observe_credential_failure(false); } if let Some(correlation) = correlation { - correlation.record_outcome(HttpOutcome::PlatformError); + correlation + .record_failure(HttpOutcome::PlatformError, "credential_unavailable"); } Err(report) } else { @@ -1096,7 +1105,7 @@ async fn reject_live_request_body( } else { HttpOutcome::PlatformError }, - None, + denial.is_none().then_some("request_middleware_failed"), ); } if let Some(observer) = observer { @@ -1568,8 +1577,11 @@ where } }; let correlation = RequestCorrelation::new(&req, ctx); + let capture_error = |error| correlation.capture_error(error); if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)) + .await + .map_err(&capture_error)?; return Ok(()); } @@ -1577,7 +1589,8 @@ where Ok(target) => target, Err(error) => { reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } }; @@ -1606,7 +1619,8 @@ where .with_request_id(correlation.request_id()), ), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); }; let observer = EndpointObserver::begin_captured( @@ -1650,7 +1664,8 @@ where None, Some(deny_context), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } if deny_unsupported_upgrade_if_requested( @@ -1661,7 +1676,8 @@ where client, Some(&correlation), ) - .await? + .await + .map_err(&capture_error)? { return Ok(()); } @@ -1759,7 +1775,8 @@ where observer.observe_credential_failure(error.is_endpoint_mismatch()); } reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } }; @@ -1775,7 +1792,8 @@ where observer.as_ref(), Some(&correlation), ) - .await? + .await + .map_err(&capture_error)? else { return Ok(()); }; @@ -1803,7 +1821,8 @@ where Some(&redacted_target), Some(deny_context), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } @@ -1813,8 +1832,13 @@ where (false, reason) } else { correlation - .stage(RequestStage::PolicyEvaluate) - .in_scope(|| evaluate_l7_request(&engine, ctx, &request_info))? + .run_stage_sync( + RequestStage::PolicyEvaluate, + HttpOutcome::PlatformError, + "policy_evaluation_failed", + || evaluate_l7_request(&engine, ctx, &request_info), + ) + .map_err(&capture_error)? }; if close_if_stale(engine.generation_guard(), ctx) { @@ -1842,7 +1866,9 @@ where ); if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { - let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; + let chain = engine + .query_middleware_chain(&middleware_network_input(ctx)) + .map_err(&capture_error)?; let response_chain = chain.clone(); let request_id = correlation.request_id(); let websocket_chain = websocket_request.then(|| chain.clone()); @@ -1873,7 +1899,7 @@ where Some(&correlation), ) .await; - let middleware_result = middleware_result?; + let middleware_result = middleware_result.map_err(&capture_error)?; let (req, mut live_body) = match middleware_result { MiddlewareApplyResult::Allowed(request) => (request, None), MiddlewareApplyResult::Streamed { request, body } => (request, Some(body)), @@ -1885,7 +1911,8 @@ where &redacted_target, Some(&correlation), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::Denied { denial, failure } => { @@ -1899,7 +1926,7 @@ where } else { HttpOutcome::PlatformError }, - None, + denial.is_none().then_some("request_middleware_failed"), ); let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), @@ -1917,11 +1944,16 @@ where &redacted_target, Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::AdmissionExhausted => { - correlation.record_rejection(503, HttpOutcome::PlatformError, None); + correlation.record_rejection( + 503, + HttpOutcome::PlatformError, + Some("middleware_admission_exhausted"), + ); let unavailable_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -1936,31 +1968,43 @@ where &redacted_target, Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } }; let mut middleware_session = if let Some(chain) = websocket_chain.as_deref() { - let preflight_span = if chain.is_empty() { - tracing::Span::none() - } else { - correlation.stage(RequestStage::WebsocketPreflight) - }; let preflight = websocket_middleware_preflight( &req, chain, engine.middleware_runner(), ctx, ctx.websocket_scheme(), - ) - .instrument(preflight_span) - .await; + ); + let preflight = if chain.is_empty() { + preflight.await + } else { + correlation + .run_stage( + RequestStage::WebsocketPreflight, + HttpOutcome::PlatformError, + "websocket_preflight_failed", + preflight, + ) + .await + }; let preflight = match preflight { Ok(preflight) => preflight, Err(error) => { warn!(error = %error, "WebSocket middleware preflight failed"); - correlation.record_rejection(502, HttpOutcome::PlatformError, None); - write_bad_gateway_response(client, Some(correlation.request_id())).await?; + correlation.record_rejection( + 502, + HttpOutcome::PlatformError, + Some("websocket_preflight_failed"), + ); + write_bad_gateway_response(client, Some(correlation.request_id())) + .await + .map_err(&capture_error)?; return Ok(()); } }; @@ -1977,7 +2021,10 @@ where } else { HttpOutcome::PlatformError }, - None, + preflight + .denial + .is_none() + .then_some("websocket_preflight_failed"), ); crate::l7::middleware::send_middleware_rejection_response( &req, @@ -1988,7 +2035,8 @@ where &redacted_target, Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } preflight.session @@ -2036,7 +2084,10 @@ where &ctx.policy_name, outcome_result, ) - .await?; + .await + .map_err(&capture_error)?; + drop(live_body); + drop(validate); match outcome { RelayOutcome::Reusable => { if let Some(session) = middleware_session.take() { @@ -2090,7 +2141,7 @@ where options.websocket.permessage_deflate = websocket_permessage_deflate; options.middleware_session = middleware_session.take(); options.selected_subprotocol = websocket_subprotocol; - correlation.finish_http_trace(); + correlation.finish_http_request(); return handle_upgrade( client, upstream, overflow, &ctx.host, ctx.port, options, ) @@ -2116,7 +2167,8 @@ where Some(&redacted_target), Some(deny_context), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } } @@ -2582,8 +2634,11 @@ where } }; let correlation = RequestCorrelation::new(&req, ctx); + let capture_error = |error| correlation.capture_error(error); if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)) + .await + .map_err(&capture_error)?; return Ok(()); } if deny_unsupported_upgrade_if_requested( @@ -2594,7 +2649,8 @@ where client, Some(&correlation), ) - .await? + .await + .map_err(&capture_error)? { return Ok(()); } @@ -2609,7 +2665,8 @@ where Ok(target) => target, Err(error) => { reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } }; @@ -2637,14 +2694,20 @@ where .with_request_id(correlation.request_id()), ), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } // Evaluate L7 policy via Rego (using redacted target) let (allowed, reason) = correlation - .stage(RequestStage::PolicyEvaluate) - .in_scope(|| evaluate_l7_request(engine, ctx, &request_info))?; + .run_stage_sync( + RequestStage::PolicyEvaluate, + HttpOutcome::PlatformError, + "policy_evaluation_failed", + || evaluate_l7_request(engine, ctx, &request_info), + ) + .map_err(&capture_error)?; if close_if_stale(engine.generation_guard(), ctx) { return Ok(()); @@ -2706,7 +2769,9 @@ where } if allowed || config.enforcement == EnforcementMode::Audit { - let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; + let chain = engine + .query_middleware_chain(&middleware_network_input(ctx)) + .map_err(&capture_error)?; let response_chain = chain.clone(); let request_id = correlation.request_id(); let websocket_chain = websocket_request.then(|| chain.clone()); @@ -2726,7 +2791,7 @@ where Some(&correlation), ) .await; - let middleware_result = middleware_result?; + let middleware_result = middleware_result.map_err(&capture_error)?; let (req, mut live_body) = match middleware_result { MiddlewareApplyResult::Allowed(request) => (request, None), MiddlewareApplyResult::Streamed { request, body } => (request, Some(body)), @@ -2738,7 +2803,8 @@ where &redacted_target, Some(&correlation), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::Denied { denial, failure } => { @@ -2749,7 +2815,7 @@ where } else { HttpOutcome::PlatformError }, - None, + denial.is_none().then_some("request_middleware_failed"), ); let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), @@ -2767,11 +2833,16 @@ where &redacted_target, Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::AdmissionExhausted => { - correlation.record_rejection(503, HttpOutcome::PlatformError, None); + correlation.record_rejection( + 503, + HttpOutcome::PlatformError, + Some("middleware_admission_exhausted"), + ); let unavailable_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -2786,31 +2857,43 @@ where &redacted_target, Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } }; let mut middleware_session = if let Some(chain) = websocket_chain.as_deref() { - let preflight_span = if chain.is_empty() { - tracing::Span::none() - } else { - correlation.stage(RequestStage::WebsocketPreflight) - }; let preflight = websocket_middleware_preflight( &req, chain, engine.middleware_runner(), ctx, ctx.websocket_scheme(), - ) - .instrument(preflight_span) - .await; + ); + let preflight = if chain.is_empty() { + preflight.await + } else { + correlation + .run_stage( + RequestStage::WebsocketPreflight, + HttpOutcome::PlatformError, + "websocket_preflight_failed", + preflight, + ) + .await + }; let preflight = match preflight { Ok(preflight) => preflight, Err(error) => { warn!(error = %error, "WebSocket middleware preflight failed"); - correlation.record_rejection(502, HttpOutcome::PlatformError, None); - write_bad_gateway_response(client, Some(correlation.request_id())).await?; + correlation.record_rejection( + 502, + HttpOutcome::PlatformError, + Some("websocket_preflight_failed"), + ); + write_bad_gateway_response(client, Some(correlation.request_id())) + .await + .map_err(&capture_error)?; return Ok(()); } }; @@ -2827,7 +2910,10 @@ where } else { HttpOutcome::PlatformError }, - None, + preflight + .denial + .is_none() + .then_some("websocket_preflight_failed"), ); crate::l7::middleware::send_middleware_rejection_response( &req, @@ -2838,7 +2924,8 @@ where &redacted_target, Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } preflight.session @@ -2886,7 +2973,8 @@ where &ctx.policy_name, outcome_result, ) - .await?; + .await + .map_err(&capture_error)?; match outcome { RelayOutcome::Reusable => { if let Some(session) = middleware_session.take() { @@ -2930,7 +3018,7 @@ where options.websocket.permessage_deflate = websocket_permessage_deflate; options.middleware_session = middleware_session.take(); options.selected_subprotocol = websocket_subprotocol; - correlation.finish_http_trace(); + correlation.finish_http_request(); return handle_upgrade( client, upstream, overflow, &ctx.host, ctx.port, options, ) @@ -2953,7 +3041,8 @@ where .with_request_id(correlation.request_id()), ), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } } @@ -2974,6 +3063,7 @@ pub(crate) fn emit_policy_reload( port: u16, policy_name: &str, ) { + crate::l7::correlation::record_current_failure(HttpOutcome::PlatformError, "policy_reloaded"); egress_emit!( NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Open) @@ -3104,6 +3194,7 @@ where let req = parsed.request; let jsonrpc_info = parsed.info; let correlation = RequestCorrelation::new(&req, ctx); + let capture_error = |error| correlation.capture_error(error); let observer = EndpointObserver::begin_captured( ctx.endpoint_observation_tx.as_ref(), config, @@ -3115,7 +3206,9 @@ where if let Some(observer) = observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } - reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)) + .await + .map_err(&capture_error)?; return Ok(()); } if deny_unsupported_upgrade_if_requested( @@ -3126,7 +3219,8 @@ where client, Some(&correlation), ) - .await? + .await + .map_err(&capture_error)? { return Ok(()); } @@ -3141,7 +3235,8 @@ where observer.observe_credential_failure(error.is_endpoint_mismatch()); } reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } }; @@ -3156,7 +3251,8 @@ where observer.as_ref(), Some(&correlation), ) - .await? + .await + .map_err(&capture_error)? else { return Ok(()); }; @@ -3175,10 +3271,20 @@ where (false, reason, jsonrpc_info.clone()) } else { let evaluation = correlation - .stage(RequestStage::PolicyEvaluate) - .in_scope(|| { - evaluate_jsonrpc_l7_request_for_log(engine, ctx, &request_info, &jsonrpc_info) - })?; + .run_stage_sync( + RequestStage::PolicyEvaluate, + HttpOutcome::PlatformError, + "policy_evaluation_failed", + || { + evaluate_jsonrpc_l7_request_for_log( + engine, + ctx, + &request_info, + &jsonrpc_info, + ) + }, + ) + .map_err(&capture_error)?; (evaluation.allowed, evaluation.reason, evaluation.log_info) }; @@ -3228,7 +3334,9 @@ where } if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { - let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; + let chain = engine + .query_middleware_chain(&middleware_network_input(ctx)) + .map_err(&capture_error)?; let response_chain = chain.clone(); let request_id = correlation.request_id(); // Policy admitted the original body above; re-check the body @@ -3248,13 +3356,14 @@ where request_id, Some(&correlation), ) - .await?; + .await + .map_err(&capture_error)?; let req = match middleware_result { MiddlewareApplyResult::Allowed(request) => request, MiddlewareApplyResult::Streamed { .. } => { - return Err(miette!( + return Err(correlation.capture_error(miette!( "body-aware request middleware must collect the request body" - )); + ))); } // The body was read before middleware ran, so the client // cannot stall it; answer like the REST path if it happens. @@ -3266,7 +3375,8 @@ where &redacted_target, Some(&correlation), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::Denied { denial, failure } => { @@ -3280,7 +3390,7 @@ where } else { HttpOutcome::PlatformError }, - None, + denial.is_none().then_some("request_middleware_failed"), ); let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), @@ -3298,11 +3408,16 @@ where &redacted_target, Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::AdmissionExhausted => { - correlation.record_rejection(503, HttpOutcome::PlatformError, None); + correlation.record_rejection( + 503, + HttpOutcome::PlatformError, + Some("middleware_admission_exhausted"), + ); let unavailable_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -3317,7 +3432,8 @@ where &redacted_target, Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } }; @@ -3339,7 +3455,8 @@ where live_body: None, }, ) - .await? + .await + .map_err(&capture_error)? else { return Ok(()); }; @@ -3375,7 +3492,8 @@ where .with_request_id(correlation.request_id()), ), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } } @@ -3448,9 +3566,12 @@ where } }; let correlation = RequestCorrelation::new(&req, ctx); + let capture_error = |error| correlation.capture_error(error); if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)) + .await + .map_err(&capture_error)?; return Ok(()); } if deny_unsupported_upgrade_if_requested( @@ -3461,7 +3582,8 @@ where client, Some(&correlation), ) - .await? + .await + .map_err(&capture_error)? { return Ok(()); } @@ -3496,7 +3618,9 @@ where // without Host must still be denied if that body contains reserved // credential markers, so repeat the authority check on the full request. if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)) + .await + .map_err(&capture_error)?; return Ok(()); } @@ -3508,7 +3632,8 @@ where Ok(target) => target, Err(error) => { reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } }; @@ -3531,8 +3656,13 @@ where (false, reason) } else { correlation - .stage(RequestStage::PolicyEvaluate) - .in_scope(|| evaluate_l7_request(engine, ctx, &request_info))? + .run_stage_sync( + RequestStage::PolicyEvaluate, + HttpOutcome::PlatformError, + "policy_evaluation_failed", + || evaluate_l7_request(engine, ctx, &request_info), + ) + .map_err(&capture_error)? }; if close_if_stale(engine.generation_guard(), ctx) { @@ -3581,7 +3711,9 @@ where } if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { - let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; + let chain = engine + .query_middleware_chain(&middleware_network_input(ctx)) + .map_err(&capture_error)?; let response_chain = chain.clone(); let request_id = correlation.request_id(); // Policy admitted the original body above; re-check the body @@ -3601,13 +3733,14 @@ where request_id, Some(&correlation), ) - .await?; + .await + .map_err(&capture_error)?; let req = match middleware_result { MiddlewareApplyResult::Allowed(request) => request, MiddlewareApplyResult::Streamed { .. } => { - return Err(miette!( + return Err(correlation.capture_error(miette!( "body-aware request middleware must collect the request body" - )); + ))); } // The body was read before middleware ran, so the client // cannot stall it; answer like the REST path if it happens. @@ -3619,7 +3752,8 @@ where &redacted_target, Some(&correlation), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::Denied { denial, failure } => { @@ -3630,7 +3764,7 @@ where } else { HttpOutcome::PlatformError }, - None, + denial.is_none().then_some("request_middleware_failed"), ); let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), @@ -3648,11 +3782,16 @@ where &redacted_target, Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::AdmissionExhausted => { - correlation.record_rejection(503, HttpOutcome::PlatformError, None); + correlation.record_rejection( + 503, + HttpOutcome::PlatformError, + Some("middleware_admission_exhausted"), + ); let unavailable_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -3667,7 +3806,8 @@ where &redacted_target, Some(correlation.request_id()), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } }; @@ -3689,7 +3829,8 @@ where live_body: None, }, ) - .await? + .await + .map_err(&capture_error)? else { return Ok(()); }; @@ -3734,7 +3875,8 @@ where .with_request_id(correlation.request_id()), ), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } } @@ -4037,11 +4179,15 @@ fn reevaluate_transformed_body( return Ok(Some(reason)); } - let (allowed, reason) = correlation - .map_or_else(tracing::Span::none, |correlation| { - correlation.stage(RequestStage::PolicyEvaluate) - }) - .in_scope(|| evaluate_l7_request(engine, ctx, &transformed_info))?; + let evaluate = || evaluate_l7_request(engine, ctx, &transformed_info); + let (allowed, reason) = correlation.map_or_else(evaluate, |correlation| { + correlation.run_stage_sync( + RequestStage::PolicyEvaluate, + HttpOutcome::PlatformError, + "policy_evaluation_failed", + evaluate, + ) + })?; if allowed { return Ok(None); } @@ -4344,9 +4490,12 @@ where } else { RequestCorrelation::uninspected() }; + let capture_error = |error| correlation.capture_error(error); let receipt = inspected.then_some(&correlation); if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action, receipt).await?; + reject_request_authority_mismatch(client, ctx, &req.action, receipt) + .await + .map_err(&capture_error)?; return Ok(()); } if close_if_stale(generation_guard, ctx) { @@ -4359,7 +4508,9 @@ where let redacted_target = match secrets::redact_target_for_policy(&req.target) { Ok(target) => target, Err(error) => { - reject_credential_resolution(client, ctx, &req.action, &error, receipt).await?; + reject_credential_resolution(client, ctx, &req.action, &error, receipt) + .await + .map_err(&capture_error)?; return Ok(()); } }; @@ -4390,11 +4541,11 @@ where let mut response_selection = None; let mut live_body = None; let req = if let Some((engine, selection)) = middleware_engine.zip(middleware_selection) { - let (chain, generation) = selection?; + let (chain, generation) = selection.map_err(&capture_error)?; if generation != generation_guard.captured_generation() { return Ok(()); } - let runner = engine.middleware_runner()?; + let runner = engine.middleware_runner().map_err(&capture_error)?; let exchange = crate::l7::middleware::HttpMiddlewareExchange::new( request_id.clone(), chain, @@ -4412,7 +4563,8 @@ where openshell_supervisor_middleware::TransformedBodyPolicy::NotPolicyRelevant, receipt, ) - .await?; + .await + .map_err(&capture_error)?; let request = match result { MiddlewareApplyResult::Allowed(request) => request, MiddlewareApplyResult::Streamed { request, body } => { @@ -4427,7 +4579,8 @@ where &redacted_target, receipt, ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::Denied { denial, failure } => { @@ -4438,7 +4591,7 @@ where } else { HttpOutcome::PlatformError }, - None, + denial.is_none().then_some("request_middleware_failed"), ); let denied_request = crate::l7::provider::L7Request { action: "HTTP".into(), @@ -4456,11 +4609,16 @@ where &redacted_target, receipt.map(RequestCorrelation::request_id), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } MiddlewareApplyResult::AdmissionExhausted => { - correlation.record_rejection(503, HttpOutcome::PlatformError, None); + correlation.record_rejection( + 503, + HttpOutcome::PlatformError, + Some("middleware_admission_exhausted"), + ); let unavailable_request = crate::l7::provider::L7Request { action: "HTTP".into(), target: redacted_target.clone(), @@ -4475,7 +4633,8 @@ where &redacted_target, receipt.map(RequestCorrelation::request_id), ) - .await?; + .await + .map_err(&capture_error)?; return Ok(()); } }; @@ -4504,10 +4663,16 @@ where )), }; let Ok((req_with_auth, injected)) = injected else { - correlation.record_rejection(502, HttpOutcome::PlatformError, None); + correlation.record_rejection( + 502, + HttpOutcome::PlatformError, + Some("credential_prepare_failed"), + ); // The refusal event names the credential and request ID. // The sandbox-level grant record does not require an MCP endpoint identity. - write_bad_gateway_response(client, Some(&request_id)).await?; + write_bad_gateway_response(client, Some(&request_id)) + .await + .map_err(&capture_error)?; return Ok(()); }; let ctx = &grant_ctx; @@ -4542,7 +4707,8 @@ where correlation: receipt, }, ) - .await? + .await + .map_err(&capture_error)? else { return Ok(()); }; @@ -4551,7 +4717,7 @@ where RelayOutcome::Reusable => {} // continue loop RelayOutcome::Consumed => break, RelayOutcome::Upgraded { overflow, .. } => { - correlation.finish_http_trace(); + correlation.finish_http_request(); return handle_upgrade( client, upstream, @@ -4759,7 +4925,7 @@ mod tests { "credential_placeholder_in_request_body" ); assert_eq!(body["error"]["reason"], "classification_unavailable"); - assert_eq!(body["request_id"], REQUEST_ID); + assert!(body.get("request_id").is_none()); let mut sent = String::new(); server.read_to_string(&mut sent).await.unwrap(); assert!(!sent.contains("openshell:resolve:")); @@ -5222,7 +5388,7 @@ mod tests { ); let body: serde_json::Value = serde_json::from_str(response.split_once("\r\n\r\n").unwrap().1).unwrap(); - assert_eq!(body["request_id"], REQUEST_ID); + assert!(body.get("request_id").is_none()); let mut forwarded = Vec::new(); upstream_peer.read_to_end(&mut forwarded).await.unwrap(); assert!( @@ -5812,6 +5978,65 @@ mod tests { assert!(!sent.contains("synthetic-signing-secret")); } + #[tokio::test] + async fn header_endpoint_mismatch_traces_refusal_without_failing_request_root() { + use crate::test_support::request_tracing::{RequestTracing, span_attribute}; + use opentelemetry_sdk::trace::Sampler; + use tracing::instrument::WithSubscriber as _; + + let _lock = openshell_otel_test_support::tracing_test_lock().await; + let capture = RequestTracing::new(Some(Sampler::AlwaysOn)); + let (_state, resolver) = endpoint_mismatch_resolver(TestHashMap::from([( + "API_TOKEN".to_string(), + "secret".to_string(), + )])); + let request = crate::l7::provider::L7Request { + action: "GET".into(), + target: "/outside".into(), + query_params: TestHashMap::new(), + raw_header: b"GET /outside HTTP/1.1\r\nHost: denied.example.test\r\nAuthorization: Bearer openshell:resolve:env:v1_API_TOKEN\r\nContent-Length: 0\r\n\r\n".to_vec(), + body_length: crate::l7::provider::BodyLength::ContentLength(0), + }; + Box::pin(assert_credential_relay_rejected( + request, + resolver.as_ref(), + crate::l7::rest::RelayRequestOptions::default(), + )) + .with_subscriber(capture.subscriber()) + .await; + let spans = capture.spans(); + capture.shutdown(); + let root = spans + .iter() + .find(|span| span.name == "openshell.http.request") + .unwrap(); + assert_eq!( + span_attribute(root, "openshell.request.outcome") + .unwrap() + .as_str(), + "denied" + ); + assert!(span_attribute(root, "openshell.request.failure_reason").is_none()); + assert!(!matches!( + root.status, + opentelemetry::trace::Status::Error { .. } + )); + let prepare = spans + .iter() + .find(|span| span.name == "openshell.credential.prepare") + .unwrap(); + assert_eq!(prepare.parent_span_id, root.span_context.span_id()); + assert!(matches!( + prepare.status, + opentelemetry::trace::Status::Error { .. } + )); + assert!( + spans + .iter() + .all(|span| span.name != "supervisor.egress.deny") + ); + } + #[tokio::test] async fn body_only_endpoint_mismatch_returns_typed_403() { let (_state, resolver) = endpoint_mismatch_resolver(TestHashMap::from([( @@ -15808,7 +16033,7 @@ network_policies: ); let response_body: serde_json::Value = serde_json::from_str(response.split_once("\r\n\r\n").unwrap().1).unwrap(); - assert_eq!(response_body["request_id"], REQUEST_ID); + assert!(response_body.get("request_id").is_none()); } #[tokio::test] diff --git a/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs index 3dcfaf9f97..bf045b3883 100644 --- a/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs +++ b/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs @@ -6,7 +6,7 @@ use super::L7EvalContext; use crate::l7::L7EndpointConfig; use crate::opa::{NetworkInput, OpaEngine, TunnelPolicyEngine}; -use crate::test_support::request_tracing::RequestTracing; +use crate::test_support::request_tracing::{RequestTracing, span_attribute}; use crate::test_support::response_request_id as request_id; use opentelemetry_sdk::trace::{Sampler, SpanData}; use std::future::Future; @@ -676,6 +676,257 @@ async fn inspected_keep_alive_requests_have_distinct_receipts_without_tracing() } } +#[derive(Clone, Copy, Debug)] +enum ReloadAt { + BeforeRequest, + DuringResponse, + BetweenRequests, +} + +async fn reload_inspected_relay(at: ReloadAt) -> Option { + let Fixture { + config, + tunnel_engine, + middleware_engine, + ctx, + } = rest_fixture("GET", false); + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let mut relay = Box::pin(crate::proxy::relay_inspected_http_stream_for_test( + &mut relay_client, + &mut relay_upstream, + vec![config], + tunnel_engine, + &middleware_engine, + &ctx, + )); + let mut receipt = None; + if !matches!(at, ReloadAt::BeforeRequest) { + app.write_all(b"GET /item HTTP/1.1\r\nHost: rest.example.test:8000\r\n\r\n") + .await + .unwrap(); + tokio::select! { + result = relay.as_mut() => panic!("relay stopped before forwarding: {result:?}"), + forwarded = read_http_message(&mut upstream) => { + assert!(forwarded.starts_with(b"GET /item HTTP/1.1\r\n")); + } + } + if matches!(at, ReloadAt::BetweenRequests) { + upstream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .await + .unwrap(); + let response = tokio::select! { + result = relay.as_mut() => panic!("persistent relay stopped: {result:?}"), + response = read_http_message(&mut app) => response, + }; + receipt = Some(request_id(&response).to_string()); + } + } + // Drive the real relay into the pending response read or next request + // read before reloading, without a scheduling delay or a test-only hook. + assert!(futures::poll!(relay.as_mut()).is_pending(), "{at:?}"); + middleware_engine + .reload(POLICY, "network_policies: {}\n") + .expect("reload closes the generation-pinned relay"); + relay.await.expect("stale relay closes without a new error"); + receipt +} + +fn assert_request_outcome( + evidence: &TracedOperation, + receipt: &str, + outcome: &str, + reason: Option<&str>, +) { + evidence.assert_no_deny_span(true); + let root = evidence + .spans + .iter() + .find(|span| span.name == "openshell.http.request") + .unwrap(); + assert_eq!( + span_attribute(root, "openshell.request.id") + .unwrap() + .as_str(), + receipt + ); + assert_eq!( + span_attribute(root, "openshell.request.outcome") + .unwrap() + .as_str(), + outcome, + ); + assert_eq!( + span_attribute(root, "openshell.request.failure_reason").map(|value| value.as_str()), + reason.map(std::borrow::Cow::Borrowed), + ); + assert_eq!( + matches!(root.status, opentelemetry::trace::Status::Error { .. }), + outcome == "failed", + ); +} + +#[tokio::test] +async fn production_policy_reload_correlates_only_the_active_request() { + for at in [ + ReloadAt::BeforeRequest, + ReloadAt::DuringResponse, + ReloadAt::BetweenRequests, + ] { + let evidence = capture_request_spans( + true, + tokio::time::timeout( + std::time::Duration::from_secs(5), + Box::pin(reload_inspected_relay(at)), + ), + ) + .await; + let completed_receipt = evidence.output.as_ref().expect("policy reload completes"); + let started = !matches!(at, ReloadAt::BeforeRequest); + let events: Vec<_> = evidence + .events + .iter() + .filter(|event| matches!(event["class_uid"].as_u64(), Some(4001 | 4002))) + .collect(); + assert_eq!(events.len(), 1 + usize::from(started), "{at:?}"); + let close = events.last().unwrap(); + assert_eq!(close["class_uid"], 4001); + assert_eq!(close["action"], "Denied"); + assert_eq!(close["status"], "Failure"); + assert!( + close["message"] + .as_str() + .unwrap() + .starts_with("L7 tunnel closed before inspection because policy changed:") + ); + if matches!(at, ReloadAt::DuringResponse) { + let policy = events[0]; + let receipt = policy["metadata"]["correlation_uid"].as_str().unwrap(); + assert!(uuid::Uuid::parse_str(receipt).is_ok()); + assert_eq!(policy["action"], "Allowed"); + assert_eq!(close["metadata"]["correlation_uid"], receipt); + assert_eq!(close["unmapped"]["request_sequence"], 2); + assert_eq!(close["unmapped"]["http_event_kind"], "connection"); + assert_request_outcome(&evidence, receipt, "failed", Some("policy_reloaded")); + } else { + assert!( + close["metadata"].get("correlation_uid").is_none(), + "{at:?}: {close}" + ); + assert!( + close["unmapped"].get("request_sequence").is_none(), + "{at:?}: {close}" + ); + if let Some(receipt) = completed_receipt { + assert_eq!(events[0]["metadata"]["correlation_uid"], *receipt); + assert_request_outcome(&evidence, receipt, "allowed", None); + } else { + assert!(evidence.spans.is_empty()); + assert!(!evidence.created.contains(&"openshell.http.request")); + } + } + } +} + +#[tokio::test] +async fn oversized_response_header_keeps_request_attribution_after_dispatch_returns_error() { + let operation = async { + let Fixture { + config, + tunnel_engine, + middleware_engine, + ctx, + } = rest_fixture("GET", false); + let (mut app, mut relay_client) = tokio::io::duplex(32 * 1024); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(32 * 1024); + let mut relay = Box::pin(crate::proxy::relay_inspected_http_stream_for_test( + &mut relay_client, + &mut relay_upstream, + vec![config], + tunnel_engine, + &middleware_engine, + &ctx, + )); + app.write_all(b"GET /item HTTP/1.1\r\nHost: rest.example.test:8000\r\n\r\n") + .await + .unwrap(); + tokio::select! { + result = relay.as_mut() => panic!("relay stopped before forwarding: {result:?}"), + forwarded = read_http_message(&mut upstream) => { + assert!(forwarded.starts_with(b"GET /item HTTP/1.1\r\n")); + } + } + let oversized = format!( + "HTTP/1.1 200 OK\r\nX-Fill: {}\r\n\r\n", + "x".repeat(16 * 1024) + ); + upstream.write_all(oversized.as_bytes()).await.unwrap(); + let error = relay + .await + .expect_err("oversized upstream response headers fail closed"); + assert!( + error + .to_string() + .contains("HTTP response headers exceed limit") + ); + + // This is the existing outer plaintext relay-error record, emitted + // only after the production dispatch has dropped its request guard. + let message = format!("L7 relay error: {error}"); + let terminal = openshell_ocsf::NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(openshell_ocsf::ActivityId::Fail) + .severity(openshell_ocsf::SeverityId::Low) + .status(openshell_ocsf::StatusId::Failure) + .dst_endpoint(openshell_ocsf::Endpoint::from_domain(&ctx.host, ctx.port)) + .message(&message) + .build(); + let event_uid = terminal.to_json().unwrap()["metadata"]["uid"].clone(); + crate::l7::correlation::emit_error_event(terminal, &error); + (message, event_uid) + }; + let evidence = capture_request_spans( + true, + tokio::time::timeout(std::time::Duration::from_secs(5), Box::pin(operation)), + ) + .await; + let (message, event_uid) = evidence + .output + .as_ref() + .expect("oversized response header fails without waiting for EOF"); + let events: Vec<_> = evidence + .events + .iter() + .filter(|event| matches!(event["class_uid"].as_u64(), Some(4001 | 4002))) + .collect(); + assert_eq!(events.len(), 2, "{:?}", evidence.events); + let policy = events[0]; + let terminal = events[1]; + let receipt = policy["metadata"]["correlation_uid"].as_str().unwrap(); + assert!(uuid::Uuid::parse_str(receipt).is_ok()); + assert_eq!(policy["class_uid"], 4002); + assert_eq!(policy["action"], "Allowed"); + assert_eq!(terminal["class_uid"], 4001); + assert_eq!(terminal["metadata"]["correlation_uid"], receipt); + assert_eq!(terminal["metadata"]["uid"], *event_uid); + assert_eq!(terminal["unmapped"]["request_sequence"], 2); + assert_eq!(terminal["unmapped"]["http_event_kind"], "connection"); + assert_eq!(terminal["message"], *message); + assert_eq!(terminal["status"], "Failure"); + assert!(terminal.get("action").is_none()); + assert!(terminal.get("disposition").is_none()); + assert_request_outcome(&evidence, receipt, "failed", Some("response_failed")); + let response = evidence + .spans + .iter() + .find(|span| span.name == "openshell.http.response") + .unwrap(); + assert!(matches!( + response.status, + opentelemetry::trace::Status::Error { .. } + )); +} + #[tokio::test] async fn production_single_mcp_allow_correlates_record_and_response_receipt() { let body = r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"test","version":"1"}}}"#; diff --git a/crates/openshell-supervisor-network/src/l7/request_trace.rs b/crates/openshell-supervisor-network/src/l7/request_trace.rs index 5e5065e553..e3abf00852 100644 --- a/crates/openshell-supervisor-network/src/l7/request_trace.rs +++ b/crates/openshell-supervisor-network/src/l7/request_trace.rs @@ -19,6 +19,7 @@ const MAX_TRACEPARENT_BYTES: usize = 256; pub(super) struct RequestTrace { span: Span, outcome: &'static str, + failure_reason: Option<&'static str>, } #[derive(Debug, Clone, Copy)] @@ -83,6 +84,7 @@ impl RequestTrace { Self { span, outcome: "incomplete", + failure_reason: None, } } @@ -99,6 +101,7 @@ impl RequestTrace { "openshell.http.stage", otel.name = stage.name(), otel.status_code = tracing::field::Empty, + openshell.request.failure_reason = tracing::field::Empty, ) } @@ -109,6 +112,7 @@ impl RequestTrace { pub(super) fn record_outcome(&mut self, outcome: HttpOutcome) { let outcome = match outcome { + _ if self.outcome == "failed" => "failed", HttpOutcome::Allowed if self.outcome == "denied" => "denied", HttpOutcome::Allowed => "allowed", HttpOutcome::Denied => "denied", @@ -119,8 +123,16 @@ impl RequestTrace { self.span.record("openshell.request.outcome", outcome); } - pub(super) fn record_failure_reason(&self, reason: &'static str) { - self.span.record("openshell.request.failure_reason", reason); + pub(super) fn record_failure_reason(&mut self, reason: &'static str) { + if self.failure_reason.is_none() { + self.failure_reason = Some(reason); + self.span.record("openshell.request.failure_reason", reason); + } + } + + pub(super) fn record_unclassified_failure(&mut self) { + self.record_outcome(HttpOutcome::PlatformError); + self.record_failure_reason("request_failed"); } pub(super) fn observe_http(&mut self, event: &HttpActivityEvent) { diff --git a/crates/openshell-supervisor-network/src/l7/request_trace/tests.rs b/crates/openshell-supervisor-network/src/l7/request_trace/tests.rs index b0818f4f3e..c4ef2e6d04 100644 --- a/crates/openshell-supervisor-network/src/l7/request_trace/tests.rs +++ b/crates/openshell-supervisor-network/src/l7/request_trace/tests.rs @@ -95,6 +95,38 @@ fn explicit_request_root_does_not_adopt_ambient_otel_context() { assert_eq!(root.links.links.len(), 1); } +#[test] +fn failures_cannot_be_overwritten_by_later_success_or_decisions() { + for failure in [HttpOutcome::PlatformError, HttpOutcome::UpstreamError] { + let spans = capture(Sampler::AlwaysOn, || { + let mut trace = RequestTrace::new("local", "GET", "example.test", 443, &request("01")); + trace.record_outcome(failure); + trace.record_failure_reason("upstream_connect_failed"); + trace.record_outcome(HttpOutcome::Allowed); + trace.record_outcome(HttpOutcome::Denied); + trace.record_outcome(HttpOutcome::Upgraded); + trace.record_failure_reason("request_failed"); + }); + let root = root(&spans); + assert!(matches!( + root.status, + opentelemetry::trace::Status::Error { .. } + )); + assert_eq!( + span_attribute(root, "openshell.request.outcome") + .unwrap() + .as_str(), + "failed" + ); + assert_eq!( + span_attribute(root, "openshell.request.failure_reason") + .unwrap() + .as_str(), + "upstream_connect_failed" + ); + } +} + #[test] fn request_completion_encloses_stages_after_an_early_root_exit() { let spans = capture(Sampler::AlwaysOn, || { diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 5cf92e8b79..0f6015da62 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -27,7 +27,7 @@ pub(crate) use request_body::{ }; use crate::l7::EndpointObserver; -use crate::l7::correlation::{RequestCorrelation, RequestStage}; +use crate::l7::correlation::{HttpOutcome, RequestCorrelation, RequestStage}; use crate::l7::http::{parse_chunk_size_line, validate_chunk_trailer_line}; use crate::l7::provider::{BodyLength, L7Provider, L7Request, RelayOutcome}; use crate::opa::PolicyGenerationGuard; @@ -49,7 +49,7 @@ use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::fmt::{self, Write as _}; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; -use tracing::{Instrument as _, debug}; +use tracing::debug; const MAX_HEADER_BYTES: usize = 16384; // 16 KiB for HTTP headers pub(crate) const MAX_REWRITE_BODY_BYTES: usize = 256 * 1024; @@ -1087,10 +1087,24 @@ where } else { tracing::Span::none() }; - let rewrite_result = rewrite_span - .in_scope(|| rewrite_http_header_block(&header_bytes, options.resolver)) - .map_err(miette::Report::new)?; + let rewrite_result = + rewrite_span.in_scope(|| rewrite_http_header_block(&header_bytes, options.resolver)); + rewrite_span.record( + "otel.status_code", + if rewrite_result.is_ok() { + "OK" + } else { + "ERROR" + }, + ); + if rewrite_result.is_err() { + rewrite_span.record( + "openshell.request.failure_reason", + "credential_resolution_refused", + ); + } drop(rewrite_span); + let rewrite_result = rewrite_result.map_err(miette::Report::new)?; if let Some(validation) = options.mcp_request_validation { // Header credential resolution and hop-by-hop cleanup must finish @@ -1434,10 +1448,7 @@ where upstream.flush().await.into_diagnostic()?; upstream.request_span.take(); - let response_span = correlation.map_or_else(tracing::Span::none, |correlation| { - correlation.stage(RequestStage::Response) - }); - let outcome = relay_response( + let response = relay_response( &req.action, upstream, client, @@ -1449,9 +1460,18 @@ where correlation, }, response_middleware, - ) - .instrument(response_span) - .await?; + ); + let outcome = if let Some(correlation) = correlation { + Box::pin(correlation.run_stage( + RequestStage::Response, + HttpOutcome::UpstreamError, + "response_failed", + response, + )) + .await? + } else { + response.await? + }; Ok(outcome) } @@ -1480,6 +1500,18 @@ struct ObservedUpstream<'a, U> { track_request_span: bool, } +impl ObservedUpstream<'_, U> { + fn record_write_failure(&self) { + if let Some(span) = &self.request_span { + span.record("otel.status_code", "ERROR"); + span.record("openshell.request.failure_reason", "upstream_send_failed"); + } + if let Some(correlation) = self.correlation { + correlation.record_failure(HttpOutcome::UpstreamError, "upstream_send_failed"); + } + } +} + impl AsyncRead for ObservedUpstream<'_, U> { fn poll_read( self: std::pin::Pin<&mut Self>, @@ -1515,10 +1547,11 @@ impl AsyncWrite for ObservedUpstream<'_, U> { } else { std::pin::Pin::new(&mut *this.upstream).poll_write(cx, buf) }; - if matches!(result, std::task::Poll::Ready(Err(_))) - && let Some(observer) = this.observer - { - observer.observe(EndpointResult::TransportFailed); + if matches!(result, std::task::Poll::Ready(Err(_))) { + this.record_write_failure(); + if let Some(observer) = this.observer { + observer.observe(EndpointResult::TransportFailed); + } } result } @@ -1533,10 +1566,15 @@ impl AsyncWrite for ObservedUpstream<'_, U> { } else { std::pin::Pin::new(&mut *this.upstream).poll_flush(cx) }; - if matches!(result, std::task::Poll::Ready(Err(_))) - && let Some(observer) = this.observer + if matches!(result, std::task::Poll::Ready(Err(_))) { + this.record_write_failure(); + if let Some(observer) = this.observer { + observer.observe(EndpointResult::TransportFailed); + } + } else if matches!(result, std::task::Poll::Ready(Ok(()))) + && let Some(span) = &this.request_span { - observer.observe(EndpointResult::TransportFailed); + span.record("otel.status_code", "OK"); } result } @@ -3161,7 +3199,7 @@ pub(crate) async fn send_middleware_deny_response( let body = middleware_deny_response_body(req, policy_name, denial, redacted_target, context); send_json_response_internal( policy_name, - body_with_request_id(body, context.and_then(|context| context.request_id)), + body, client, "403 Forbidden", None, @@ -3190,7 +3228,7 @@ pub(crate) async fn send_middleware_failure_response( } send_json_response_internal( policy_name, - body_with_request_id(body, context.and_then(|context| context.request_id)), + body, client, "403 Forbidden", None, @@ -3236,7 +3274,7 @@ pub(crate) async fn send_middleware_unavailable_response( ); send_json_response_internal( policy_name, - body_with_request_id(body, context.and_then(|context| context.request_id)), + body, client, "503 Service Unavailable", None, @@ -3245,16 +3283,6 @@ pub(crate) async fn send_middleware_unavailable_response( .await } -fn body_with_request_id( - mut body: serde_json::Value, - request_id: Option<&str>, -) -> serde_json::Value { - if let (Some(request_id), serde_json::Value::Object(fields)) = (request_id, &mut body) { - fields.insert("request_id".to_string(), serde_json::json!(request_id)); - } - body -} - /// Send a platform-owned 408 when the sandbox stops sending a request body /// that request middleware is processing. pub(crate) async fn send_request_timeout_response( @@ -3274,7 +3302,7 @@ pub(crate) async fn send_request_timeout_response( } send_json_response_internal( policy_name, - body_with_request_id(body, context.and_then(|context| context.request_id)), + body, client, "408 Request Timeout", None, @@ -3315,7 +3343,7 @@ pub(crate) async fn send_json_response_with_allow_and_request_id Result<()> { send_json_response_internal( policy_name, - body_with_request_id(body, Some(request_id)), + body, client, status, allowed_methods, @@ -3498,9 +3526,6 @@ fn deny_response_body( if let Some(binary) = binary { body.insert("binary".to_string(), serde_json::json!(binary)); } - if let Some(request_id) = context.request_id { - body.insert("request_id".to_string(), serde_json::json!(request_id)); - } body.insert( "rule_missing".to_string(), serde_json::Value::Object(rule_missing), @@ -5353,7 +5378,7 @@ mod tests { assert_eq!(body["error"], "policy_denied"); assert_eq!(body["method"], "POST"); assert_eq!(body["path"], "/user/repos"); - assert_eq!(body["request_id"], "request-123"); + assert!(body.get("request_id").is_none()); assert_eq!(body["rule_missing"]["host"], "api.github.com"); assert_eq!(body["next_steps"][2]["action"], "inspect_recent_denials"); assert!(body["agent_guidance"].as_str().unwrap().contains("retry")); diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs index e7af4ec0e3..3ceb430676 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs @@ -532,10 +532,7 @@ where } Err(error) => Err(error), }; - let span = response_correlation.map_or_else(tracing::Span::none, |correlation| { - correlation.stage(RequestStage::ResponseMiddleware) - }); - relay_described_response_through_middleware( + let response = relay_described_response_through_middleware( request_method, upstream, client, @@ -548,9 +545,18 @@ where event_stream, response_correlation, parsed, - ) - .instrument(span) - .await + ); + if let Some(correlation) = response_correlation { + Box::pin(correlation.run_stage( + RequestStage::ResponseMiddleware, + HttpOutcome::PlatformError, + "response_middleware_failed", + response, + )) + .await + } else { + response.await + } } #[allow(clippy::too_many_arguments)] @@ -2364,12 +2370,17 @@ where C: AsyncWrite + Unpin, { let mut trailers = Vec::new(); + let mut field_count = 0; loop { let line = read_response_line_with_deadline(reader, session, client, framing).await?; validate_chunk_trailer_line(&line)?; if line.is_empty() { return Ok(trailers); } + field_count += 1; + if field_count > openshell_supervisor_middleware::MAX_MIDDLEWARE_HEADERS { + return Err(miette!("HTTP response trailer count exceeds limit")); + } let line = std::str::from_utf8(&line) .map_err(|_| miette!("HTTP response trailer contains invalid UTF-8"))?; let (name, value) = line @@ -2378,6 +2389,11 @@ where validate_http_field_name(name)?; validate_http_field_value(value.trim())?; let name = name.to_ascii_lowercase(); + // The upstream cannot supply the platform receipt. Strip it just as + // the ordinary response relay does, before exposing trailers to stages. + if name.eq_ignore_ascii_case(openshell_core::token_grant_failure::REQUEST_ID_HEADER) { + continue; + } if is_protected_response_field(&name) || connection_nominated_headers.contains(&name) { return Err(miette!("HTTP response trailer uses a protected field name")); } @@ -2385,9 +2401,6 @@ where name, value: value.trim().to_string(), }); - if trailers.len() > openshell_supervisor_middleware::MAX_MIDDLEWARE_HEADERS { - return Err(miette!("HTTP response trailer count exceeds limit")); - } } } @@ -2483,7 +2496,6 @@ async fn send_response_delivery_failure( if let Some(failure) = failure { insert_middleware_failure(&mut body, failure); } - let body = body_with_request_id(body, correlation.map(RequestCorrelation::request_id)); let body = serde_json::to_vec(&body).into_diagnostic()?; let head = format!( "HTTP/1.1 502 Bad Gateway\r\nContent-Type: application/json\r\nContent-Length: {}\r\nX-OpenShell-Policy: {policy_name}\r\nConnection: close\r\n\r\n", @@ -3182,16 +3194,24 @@ mod tests { } #[tokio::test] - async fn response_middleware_rejects_forged_request_id_trailer() { + async fn response_middleware_strips_upstream_receipt_trailers_without_aborting_streams() { let correlation = test_correlation("trusted"); let (runner, chain) = response_middleware_fixture(ResponseRelayScript::Stream); let mut middleware = response_middleware_context(&runner, &chain, "GET"); middleware.request_context.request_id = "middleware-request".into(); middleware.request_head = None; - let mut upstream = &b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\nX-OpenShell-Request-ID: forged\r\n\r\n"[..]; + let response = b"HTTP/1.1 200 OK\r\n\ + Transfer-Encoding: chunked\r\n\ + Trailer: X-OpenShell-Request-ID, X-Checksum\r\n\r\n\ + 4\r\nbody\r\n0\r\n\ + x-OPENshell-request-ID: forged\r\n\ + X-Checksum: ok\r\n\r\n"; + let (ordinary_outcome, ordinary) = relay_bytes(response, Some(&correlation)).await; + assert!(matches!(ordinary_outcome.unwrap(), RelayOutcome::Reusable)); + let mut upstream = response.as_slice(); let mut client = Vec::new(); - let error = relay_response( + let outcome = relay_response( "GET", &mut upstream, &mut client, @@ -3202,19 +3222,22 @@ mod tests { Some(middleware), ) .await - .expect_err("platform correlation trailers must fail closed"); + .expect("upstream receipt trailers are stripped without aborting delivery"); - assert!( - error.to_string().contains("protected field name"), - "{error}" - ); + assert!(matches!(outcome, RelayOutcome::Reusable)); let delivered = String::from_utf8_lossy(&client); assert!(!delivered.contains("forged"), "{delivered}"); + assert!(delivered.contains("4\r\nBODY\r\n"), "{delivered}"); + assert!( + delivered.ends_with("0\r\nx-checksum: ok\r\n\r\n"), + "{delivered}" + ); assert!( delivered.contains("x-openshell-request-id: trusted\r\n"), "{delivered}" ); assert_eq!(request_id_header_count(&client), 1); + assert_eq!(request_id_header_count(&ordinary), 1); } #[test] diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs index 5489c89474..2c6efa78fb 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs @@ -631,8 +631,15 @@ async fn read_trailer_fields( connection_nominated: &[String], ) -> Result> { let mut trailers = Vec::new(); + let mut field_count = 0; loop { let line = reader.read_line().await?; + if !line.is_empty() { + field_count += 1; + if field_count > openshell_supervisor_middleware::MAX_MIDDLEWARE_HEADERS { + return Err(miette!("HTTP response trailer count exceeds limit")); + } + } if !push_trailer(&line, connection_nominated, &mut trailers)? { return Ok(trailers); } @@ -950,6 +957,11 @@ fn push_trailer( validate_http_field_name(name)?; validate_http_field_value(value.trim())?; let name = name.to_ascii_lowercase(); + // Only the platform owns receipt fields. Do not abort a valid upstream + // stream or expose its forged receipt to a response middleware stage. + if name.eq_ignore_ascii_case(openshell_core::token_grant_failure::REQUEST_ID_HEADER) { + return Ok(true); + } if is_protected_response_field(&name) || connection_nominated.contains(&name) { return Err(miette!("HTTP response trailer uses a protected field name")); } @@ -1005,6 +1017,31 @@ mod tests { use tokio::io::AsyncReadExt as _; use tracing::instrument::WithSubscriber as _; + #[test] + fn upstream_receipt_trailers_are_hidden_from_http_hooks() { + let mut trailers = Vec::new(); + let nominated = vec!["x-openshell-request-id".into(), "x-private".into()]; + assert!( + push_trailer(b"x-OPENshell-request-ID: forged", &nominated, &mut trailers).unwrap() + ); + assert!(trailers.is_empty()); + assert!(push_trailer(b"X-Checksum: ok", &nominated, &mut trailers).unwrap()); + assert_eq!(trailers.len(), 1); + assert_eq!(trailers[0].name, "x-checksum"); + assert_eq!(trailers[0].value, "ok"); + assert!(!push_trailer(b"", &nominated, &mut trailers).unwrap()); + + for line in [ + b"Content-Length: 7".as_slice(), + b"Set-Cookie: secret", + b"X-Private: opaque", + b"X-OpenShell-Request-ID: bad\0value", + ] { + push_trailer(line, &nominated, &mut Vec::new()) + .expect_err("other protected or malformed trailers still fail closed"); + } + } + #[test] fn response_hook_invocations_keep_individual_correlated_records_without_tracing() { let capture = RequestTracing::new(None); diff --git a/crates/openshell-supervisor-network/src/l7/rest/request_body.rs b/crates/openshell-supervisor-network/src/l7/rest/request_body.rs index 780d7d7384..eb0069532c 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/request_body.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/request_body.rs @@ -693,18 +693,24 @@ where correlation, )); let mut response = Box::pin(async { - let span = correlation.map_or_else(tracing::Span::none, |correlation| { - correlation.stage(RequestStage::Response) - }); - relay_response( + let response = relay_response( &req.action, &mut upstream_reader, &mut client_writer, response_options, response_middleware, - ) - .instrument(span) - .await + ); + if let Some(correlation) = correlation { + Box::pin(correlation.run_stage( + RequestStage::Response, + HttpOutcome::UpstreamError, + "response_failed", + response, + )) + .await + } else { + response.await + } }); tokio::select! { diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index b94ecfbd9e..6ea77f0f4c 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -3356,7 +3356,7 @@ async fn handle_mediated_connection( .dst_endpoint(Endpoint::from_domain(&host_lc, port)) .message(format!("TLS relay error: {e}")) .build(); - egress_emit!(event); + crate::l7::correlation::emit_error_event(event, &e); } } } else { @@ -3431,7 +3431,7 @@ async fn handle_mediated_connection( .dst_endpoint(Endpoint::from_domain(&host_lc, port)) .message(message) .build(); - egress_emit!(event); + crate::l7::correlation::emit_error_event(event, &e); } } } else { @@ -5975,21 +5975,18 @@ async fn handle_forward_proxy_inner( || middleware_chain .as_ref() .is_ok_and(|(chain, _)| !chain.is_empty()); - let correlation_request = crate::l7::rest::request_from_buffered_http_unprotected( - method, - path.clone(), - &upstream_target, - forward_request_bytes, - )?; let correlation = if inspected { - crate::l7::correlation::RequestCorrelation::new(&correlation_request, &l7_ctx) + crate::l7::correlation::RequestCorrelation::new_http( + method, + &forward_request_bytes, + &l7_ctx, + ) } else { crate::l7::correlation::RequestCorrelation::uninspected() }; let request_correlation = inspected.then_some(&correlation); let receipt_id = request_correlation.map(crate::l7::correlation::RequestCorrelation::request_id); - forward_request_bytes = correlation_request.raw_header; if let Some(route) = decision .endpoint .l7_route @@ -6347,13 +6344,18 @@ async fn handle_forward_proxy_inner( let mut evaluation_error = None; let (allowed, reason) = hard_deny_reason.map_or_else( || { - let stage = correlation.stage(RequestStage::PolicyEvaluate); - let evaluation = stage.in_scope(|| { - crate::l7::relay::evaluate_l7_request(&tunnel_engine, &l7_ctx, &request_info) - }); - if evaluation.is_err() { - egress::mark_error(&stage); - } + let evaluation = correlation.run_stage_sync( + RequestStage::PolicyEvaluate, + HttpOutcome::PlatformError, + "policy_evaluation_failed", + || { + crate::l7::relay::evaluate_l7_request( + &tunnel_engine, + &l7_ctx, + &request_info, + ) + }, + ); evaluation.unwrap_or_else(|e| { evaluation_error = Some(build_forward_l7_evaluation_error( &host_lc, @@ -6528,8 +6530,14 @@ async fn handle_forward_proxy_inner( }) .instrument(resolve_span.clone()) .await; - if inspected && resolved.is_err() { - egress::mark_error(&resolve_span); + if inspected { + if resolved.is_err() { + egress::mark_error(&resolve_span); + resolve_span.record("openshell.request.failure_reason", "destination_denied"); + correlation.record_failure(HttpOutcome::Denied, "destination_denied"); + } else { + resolve_span.record("otel.status_code", "OK"); + } } drop(resolve_span); let connector = match resolved { @@ -6682,7 +6690,7 @@ async fn handle_forward_proxy_inner( } else { HttpOutcome::PlatformError }, - None, + Some("request_middleware_denied"), ); let response = denial.as_ref().map_or_else( || { @@ -6702,7 +6710,11 @@ async fn handle_forward_proxy_inner( } crate::l7::middleware::MiddlewareApplyResult::AdmissionExhausted => { emit_activity_simple(activity_tx, true, "middleware"); - correlation.record_rejection(503, HttpOutcome::PlatformError, None); + correlation.record_rejection( + 503, + HttpOutcome::PlatformError, + Some("middleware_admission_exhausted"), + ); let response = build_middleware_unavailable_response(&l7_ctx.policy_name, receipt_id); respond(client, &response).await?; @@ -6720,29 +6732,34 @@ async fn handle_forward_proxy_inner( forward_request_bytes.clone(), )?; let middleware_runner = opa_engine.middleware_runner()?; - let stage = if chain.is_empty() { - tracing::Span::none() - } else { - correlation.stage(RequestStage::WebsocketPreflight) - }; let preflight = crate::l7::relay::websocket_middleware_preflight( &request, chain, &middleware_runner, &l7_ctx, "ws", - ) - .instrument(stage.clone()) - .await; - if preflight.is_err() { - egress::mark_error(&stage); - } - drop(stage); + ); + let preflight = if chain.is_empty() { + preflight.await + } else { + correlation + .run_stage( + RequestStage::WebsocketPreflight, + HttpOutcome::PlatformError, + "websocket_preflight_failed", + preflight, + ) + .await + }; let preflight = match preflight { Ok(preflight) => preflight, Err(error) => { warn!(error = %error, "Plaintext WebSocket middleware preflight failed"); - correlation.record_rejection(502, HttpOutcome::PlatformError, None); + correlation.record_rejection( + 502, + HttpOutcome::PlatformError, + Some("websocket_preflight_failed"), + ); respond( client, &build_json_error_response_with_request_id( @@ -6809,7 +6826,11 @@ async fn handle_forward_proxy_inner( Err(e) => { let refusal = e.downcast_ref::(); - correlation.record_rejection(502, HttpOutcome::PlatformError, None); + correlation.record_rejection( + 502, + HttpOutcome::PlatformError, + Some("token_grant_failed"), + ); if refusal.is_none() { warn!( dst_host = %host_lc, @@ -6884,7 +6905,7 @@ async fn handle_forward_proxy_inner( }; if let Some(guard) = credential_generation { guard.ensure_current().inspect_err(|_| { - correlation.record_outcome(HttpOutcome::PlatformError); + correlation.record_failure(HttpOutcome::PlatformError, "credential_generation_stale"); })?; } @@ -7055,9 +7076,21 @@ async fn handle_forward_proxy_inner( }; let dial_result = connector.connect().instrument(dial_span.clone()).await; let mut upstream = match dial_result { - Ok(s) => s, + Ok(s) => { + if inspected { + dial_span.record("otel.status_code", "OK"); + } + s + } Err(e) => { egress::mark_error(&dial_span); + if inspected { + dial_span.record( + "openshell.request.failure_reason", + "upstream_connect_failed", + ); + correlation.record_failure(HttpOutcome::UpstreamError, "upstream_connect_failed"); + } let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Fail) .severity(SeverityId::Low) @@ -7219,7 +7252,8 @@ async fn handle_forward_proxy_inner( policy_str, outcome_result, ) - .await?; + .await + .map_err(|error| correlation.capture_error(error))?; // The request has now survived middleware, token grant, credential // rewriting, generation checks, and the HTTP relay. Only now record the @@ -7252,7 +7286,7 @@ async fn handle_forward_proxy_inner( websocket_permessage_deflate, websocket_subprotocol, } => { - correlation.finish_http_trace(); + correlation.finish_http_request(); // Protocols whose rules apply to individual HTTP requests never // upgrade (see `upgrade_refusal_for_protocol`). No current path // forwards upgrade headers for them: the request-side refusal @@ -7398,7 +7432,6 @@ fn build_correlated_json_error_response( let body = serde_json::json!({ "error": error, "detail": detail, - "request_id": request_id, }) .to_string(); format!( @@ -7486,9 +7519,6 @@ fn build_middleware_deny_response( if let Some(reason_code) = &denial.reason_code { body.insert("reason_code".to_string(), serde_json::json!(reason_code)); } - if let Some(request_id) = request_id { - body.insert("request_id".to_string(), serde_json::json!(request_id)); - } let body_str = serde_json::Value::Object(body).to_string(); let request_id_header = request_id.map_or_else(String::new, |request_id| { format!( @@ -7541,9 +7571,6 @@ fn build_middleware_platform_response( "detail": "Request could not be processed by configured middleware", "policy": policy_name, }); - if let (Some(request_id), serde_json::Value::Object(fields)) = (request_id, &mut body) { - fields.insert("request_id".to_string(), serde_json::json!(request_id)); - } if let Some(failure) = failure { crate::l7::rest::insert_middleware_failure(&mut body, failure); } diff --git a/crates/openshell-supervisor-network/src/proxy/relay.rs b/crates/openshell-supervisor-network/src/proxy/relay.rs index 495c22241b..e53e546d81 100644 --- a/crates/openshell-supervisor-network/src/proxy/relay.rs +++ b/crates/openshell-supervisor-network/src/proxy/relay.rs @@ -240,6 +240,19 @@ pub(super) async fn relay_http_stream( upstream: &mut U, context: RelayContext<'_>, ) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send, + U: AsyncRead + AsyncWrite + Unpin + Send, +{ + crate::l7::correlation::scope_connection(relay_http_stream_inner(client, upstream, context)) + .await +} + +async fn relay_http_stream_inner( + client: &mut C, + upstream: &mut U, + context: RelayContext<'_>, +) -> Result<()> where C: AsyncRead + AsyncWrite + Unpin + Send, U: AsyncRead + AsyncWrite + Unpin + Send, @@ -255,10 +268,7 @@ where upstream, context.request, ) => result, - () = generation_guard.wait_until_stale() => { - emit_stale_relay_close(context.request, &generation_guard); - Ok(()) - } + () = close_stale_http_relay(context.request, &generation_guard) => Ok(()) } } PreparedHttpPolicy::Inspect { configs, evaluator } => { @@ -271,10 +281,7 @@ where upstream, context.request, ) => result, - () = generation_guard.wait_until_stale() => { - emit_stale_relay_close(context.request, &generation_guard); - Ok(()) - } + () = close_stale_http_relay(context.request, &generation_guard) => Ok(()) } } PreparedHttpPolicy::Passthrough { generation_guard } => { @@ -286,15 +293,21 @@ where &generation_guard, Some(context.middleware_engine), ) => result, - () = generation_guard.wait_until_stale() => { - emit_stale_relay_close(context.request, &generation_guard); - Ok(()) - } + () = close_stale_http_relay(context.request, &generation_guard) => Ok(()) } } } } +async fn close_stale_http_relay(request: &L7EvalContext, guard: &PolicyGenerationGuard) { + guard.wait_until_stale().await; + crate::l7::correlation::record_current_failure( + crate::l7::correlation::HttpOutcome::PlatformError, + "policy_reloaded", + ); + emit_stale_relay_close(request, guard); +} + // Exercise the production single/multiple-route dispatch from L7 fixtures. #[cfg(test)] pub async fn relay_inspected_http_stream_for_test( diff --git a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs index 455f177a5f..1644f96058 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs @@ -52,13 +52,10 @@ fn assert_json_response( if let Some(request_id) = expected_request_id { assert!(headers.contains(&format!("x-openshell-request-id: {request_id}\r\n"))); } - let mut expected = serde_json::json!({ + let expected = serde_json::json!({ "error": expected_error, "detail": expected_detail, }); - if let Some(request_id) = expected_request_id { - expected["request_id"] = serde_json::json!(request_id); - } assert_eq!( serde_json::from_slice::(body).unwrap(), expected diff --git a/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs b/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs index f615587de6..05cda6da9f 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs @@ -15,6 +15,7 @@ const WORKLOAD_SPAN_ID: &str = "00f067aa0ba902b7"; enum Outcome { Allowed, PolicyDenied, + MalformedQueryPolicyDenied, AuditDenied, UpstreamClosed, } @@ -40,12 +41,16 @@ fn assert_request_record_sequence(records: &[serde_json::Value], receipt: &str) async fn forward_case(ip: Ipv4Addr, outcome: Outcome, sampler: Option) { let listener = TcpListener::bind((ip, 0)).await.unwrap(); let port = listener.local_addr().unwrap().port(); + let policy_denied = matches!( + outcome, + Outcome::PolicyDenied | Outcome::MalformedQueryPolicyDenied + ); let mode = if matches!(outcome, Outcome::AuditDenied) { "audit" } else { "enforce" }; - let path = if matches!(outcome, Outcome::PolicyDenied | Outcome::AuditDenied) { + let path = if policy_denied || matches!(outcome, Outcome::AuditDenied) { "/blocked" } else { "/allowed" @@ -88,7 +93,12 @@ network_policies: .with(openshell_otel::supervisor_span_filter("info")); let (mut app, proxy) = tokio::io::duplex(8192); let mut proxy: ProxyClient = tokio::io::BufReader::new(Box::new(proxy)); - let target = format!("http://{ip}:{port}{path}?token=SENTINEL-QUERY"); + let query = if matches!(outcome, Outcome::MalformedQueryPolicyDenied) { + "token=%ff" + } else { + "token=SENTINEL-QUERY" + }; + let target = format!("http://{ip}:{port}{path}?{query}"); let request = format!( "GET {target} HTTP/1.1\r\nHost: {ip}:{port}\r\ntraceparent: 00-{WORKLOAD_TRACE_ID}-{WORKLOAD_SPAN_ID}-01\r\ntracestate: vendor=SENTINEL-VENDOR\r\nx-openshell-request-id: forged-receipt\r\nConnection: close\r\n\r\n" ); @@ -124,7 +134,7 @@ network_policies: result }, async { - if matches!(outcome, Outcome::PolicyDenied) { + if policy_denied { assert!( tokio::time::timeout( std::time::Duration::from_millis(50), @@ -164,25 +174,19 @@ network_policies: .unwrap(); result.unwrap(); let spans = capture.spans(); - let expected = (!matches!(outcome, Outcome::UpstreamClosed)).then_some( - if matches!(outcome, Outcome::PolicyDenied) { - 403 - } else { - 200 - }, - ); + let expected = (!matches!(outcome, Outcome::UpstreamClosed)).then_some(if policy_denied { + 403 + } else { + 200 + }); let response = String::from_utf8(response).unwrap(); let records = capture.events.json_events(); - let expected_records = if matches!(outcome, Outcome::PolicyDenied) { - 1 - } else { - 2 - }; + let expected_records = if policy_denied { 1 } else { 2 }; assert_eq!(records.len(), expected_records, "{records:?}"); assert!(records.iter().all(|record| record["class_uid"] == 4002)); let record = &records[0]; let decision = match outcome { - Outcome::PolicyDenied => "deny", + Outcome::PolicyDenied | Outcome::MalformedQueryPolicyDenied => "deny", Outcome::AuditDenied => "audit", Outcome::Allowed | Outcome::UpstreamClosed => "allow", }; @@ -199,7 +203,7 @@ network_policies: assert_eq!(control["http_request"]["url"]["path"], path); assert!(control.get("http_response").is_none()); } - if !matches!(outcome, Outcome::PolicyDenied) { + if !policy_denied { let forwarded = &records[1]; assert!( forwarded["message"] @@ -216,26 +220,32 @@ network_policies: assert_request_record_sequence(&records, receipt); assert!(!format!("{records:?}").contains("SENTINEL")); assert!(!format!("{records:?}").contains("forged-receipt")); + assert!(!format!("{records:?}").contains("%ff")); if let Some(expected) = expected { assert!( response.starts_with(&format!("HTTP/1.1 {expected} ")), "{response}" ); assert_eq!(response_request_id(response.as_bytes()), receipt); + if policy_denied { + let (_, body) = response.split_once("\r\n\r\n").unwrap(); + let body: serde_json::Value = serde_json::from_str(body).unwrap(); + assert_eq!(body["error"], "policy_denied"); + assert!( + body["detail"] + .as_str() + .unwrap() + .starts_with(&format!("GET {ip}:{port}{path} denied by L7 policy: ")) + ); + assert_eq!(body.as_object().unwrap().len(), 2); + } } else { assert!( response.is_empty(), "upstream EOF must not synthesize a response" ); } - assert_eq!( - record["action_id"], - if matches!(outcome, Outcome::PolicyDenied) { - 2 - } else { - 1 - } - ); + assert_eq!(record["action_id"], if policy_denied { 2 } else { 1 }); let created = capture.created_span_names(); assert!( !created.contains(&"supervisor.egress.deny"), @@ -315,13 +325,13 @@ network_policies: stages .iter() .any(|span| span.name == "openshell.upstream.connect"), - !matches!(outcome, Outcome::PolicyDenied) + !policy_denied ); assert_eq!( stages .iter() .any(|span| span.name == "openshell.upstream.resolve"), - !matches!(outcome, Outcome::PolicyDenied) + !policy_denied ); assert_eq!( matches!(root.status, opentelemetry::trace::Status::Error { .. }), @@ -355,6 +365,68 @@ async fn inspected_forward_traces_all_outcomes_without_sampling_or_export_depend } } +#[tokio::test] +async fn inspected_forward_malformed_query_preserves_correlated_l7_denial() { + let _lock = openshell_otel_test_support::tracing_test_lock().await; + let ip = non_loopback_test_ipv4() + .expect("FORWARD trace qualification requires a non-loopback IPv4 address"); + // This malformed query already reaches the L7 deny path. Creating a + // receipt must not add a stricter parse that aborts before the 403 and its + // existing policy decision record. + for sampler in [Some(Sampler::AlwaysOn), Some(Sampler::AlwaysOff), None] { + forward_case(ip, Outcome::MalformedQueryPolicyDenied, sampler).await; + } +} + +#[test] +fn inspected_forward_error_receipts_preserve_json_bodies() { + let request_id = "server-owned-receipt"; + let denial = openshell_supervisor_middleware::MiddlewareDenial { + config_name: "content-guard".into(), + reason_code: Some("content_match".into()), + }; + for (original, correlated) in [ + ( + build_json_error_response_with_request_id( + 403, + "Forbidden", + "policy_denied", + "denied by L7 policy", + None, + ), + build_json_error_response_with_request_id( + 403, + "Forbidden", + "policy_denied", + "denied by L7 policy", + Some(request_id), + ), + ), + ( + build_middleware_deny_response("api-policy", &denial, None), + build_middleware_deny_response("api-policy", &denial, Some(request_id)), + ), + ( + build_middleware_failure_response("api-policy", None, None), + build_middleware_failure_response("api-policy", None, Some(request_id)), + ), + ( + build_middleware_unavailable_response("api-policy", None), + build_middleware_unavailable_response("api-policy", Some(request_id)), + ), + ] { + let original = String::from_utf8(original).unwrap(); + let correlated = String::from_utf8(correlated).unwrap(); + let (_, original_body) = original.split_once("\r\n\r\n").unwrap(); + let (headers, correlated_body) = correlated.split_once("\r\n\r\n").unwrap(); + assert_eq!(response_request_id(correlated.as_bytes()), request_id); + assert!(headers.contains(&format!("Content-Length: {}\r\n", correlated_body.len()))); + assert_eq!(correlated_body, original_body); + let body: serde_json::Value = serde_json::from_str(correlated_body).unwrap(); + assert!(body.get("request_id").is_none()); + } +} + #[tokio::test] async fn inspected_forward_http_trace_ends_before_websocket_session() { let _lock = openshell_otel_test_support::tracing_test_lock().await; @@ -497,7 +569,7 @@ network_policies: handler.await.unwrap().unwrap(); let records = capture.events.json_events(); assert_eq!(records.len(), 4, "{records:?}"); - assert_request_record_sequence(&records, receipt); + assert_request_record_sequence(&records[..2], receipt); for record in &records[..2] { assert_eq!(record["class_uid"], 4002); assert_eq!(record["action_id"], 1); @@ -533,6 +605,7 @@ network_policies: .starts_with("101 Switching Protocols — websocket parsed relay ") ); assert!(records[2].get("http_response").is_none()); + assert!(records[2].get("http_request").is_none()); assert_eq!(records[3]["class_uid"], 4001); assert!( records[3]["message"] @@ -541,6 +614,12 @@ network_policies: .starts_with("WEBSOCKET_L7_REQUEST allow WEBSOCKET_TEXT ") ); assert_eq!(records[3]["firewall_rule"]["type"], "l7-websocket"); + for record in &records[2..] { + assert!(record["metadata"].get("correlation_uid").is_none()); + assert!(record["unmapped"].get("request_sequence").is_none()); + assert!(record["unmapped"].get("http_event_kind").is_none()); + assert!(record.get("trace").is_none()); + } assert!(!format!("{records:?}").contains("after-http-root")); }; tokio::time::timeout(std::time::Duration::from_secs(5), scenario) diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 0c342c6bab..8a151c4090 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -298,6 +298,7 @@ pub async fn run_networking( proxy_listener: Option, reserved_destination: Option, ) -> Result { + crate::l7::correlation::init_request_correlation()?; // Build the policy-local route context. The orchestrator's policy poll // loop also holds an `Arc` clone (via `Networking::policy_local_ctx`) so // it can publish updated policy snapshots after a successful reload. diff --git a/crates/openshell-supervisor-network/src/telemetry/tests.rs b/crates/openshell-supervisor-network/src/telemetry/tests.rs index 0d0d2111be..59d9f2e408 100644 --- a/crates/openshell-supervisor-network/src/telemetry/tests.rs +++ b/crates/openshell-supervisor-network/src/telemetry/tests.rs @@ -625,7 +625,10 @@ async fn inspected_audit_records_do_not_depend_on_sampling_or_create_deny_spans( let events = capture("warn", &provider, || { let request = tracing::info_span!(parent: None, "openshell.http.request"); let mut denied = http_deny("blocked.example", "middleware_blocked"); - denied.set_correlation_uid(request_id.clone()); + let openshell_ocsf::OcsfEvent::HttpActivity(event) = &mut denied else { + unreachable!("http_deny returns an HTTP activity"); + }; + event.base.metadata.correlation_uid = Some(request_id.clone()); super::emit_inspected_decision([denied], Some(&request)); }); provider.force_flush().unwrap(); diff --git a/docs/observability/ocsf-json-export.mdx b/docs/observability/ocsf-json-export.mdx index 31ae42e3e0..86e8e5977c 100644 --- a/docs/observability/ocsf-json-export.mdx +++ b/docs/observability/ocsf-json-export.mdx @@ -334,13 +334,14 @@ This header is reserved for OpenShell: workload and upstream values are removed, and middleware cannot set or remove it. Use a different header for application-owned correlation. The receipt is a join key, not proof of identity, authorization or downstream execution. L4 and uninspected traffic do not provide -this receipt guarantee. +this receipt guarantee. OpenShell does not add the receipt to JSON error bodies. All OCSF event classes emitted in the inspected request scope carry the request ID, including Detection Findings. Policy, middleware, token-grant and finding records remain separate, with their original event IDs, timestamps, classification and decision fields. The additive `unmapped.http_event_kind` -field identifies `decision`, `middleware_invocation` or `token_grant` records. +field distinguishes decisions, findings, connection events and middleware +invocations. Explicit control kinds such as `token_grant` remain unchanged. Middleware records retain `unmapped.middleware_phase` and their existing attribution and transformation details. @@ -348,8 +349,11 @@ attribution and transformation details. emission order. It does not guarantee downstream delivery order or complete collection. To count policy outcomes, group by `metadata.correlation_uid` and select the actual policy or firewall control record. Do not count every HTTP -Activity or `decision` record as another request; companion findings also use -`decision`. +Activity or `decision` record as another request. + +Errors that end an inspected request retain that request's ID. A policy reload +records the active request before closing it; an idle connection has no request +ID. After a WebSocket upgrade, frame events do not reuse the HTTP handshake ID. The request ID join and enforcement work without tracing, when spans are not sampled, and when the collector is unavailable. Sampled OCSF trace fields identify From c871af57862f895aa75aff642cc0ad0eb069f2d4 Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Mon, 16 Nov 2026 15:50:08 -0700 Subject: [PATCH 5/7] fix(observability): finish stage results and isolate correlation fixtures Signed-off-by: Alex Fournier --- crates/openshell-supervisor-network/README.md | 2 +- .../src/l7/correlation.rs | 11 +- .../src/l7/correlation/tests.rs | 30 +++ .../src/l7/middleware.rs | 111 +++--------- .../src/l7/relay.rs | 81 ++------- .../l7/relay/correlation_dispatch_tests.rs | 76 ++++++++ .../src/l7/request_trace.rs | 18 -- .../src/l7/rest.rs | 171 +++++++++++------- .../openshell-supervisor-network/src/proxy.rs | 36 +++- .../src/proxy/tests/request_trace.rs | 67 ++++++- docs/observability/ocsf-json-export.mdx | 14 +- 11 files changed, 339 insertions(+), 278 deletions(-) diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index d9fa595a56..1dea7f6c0f 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -81,7 +81,7 @@ collector availability. Inspected requests have a supervisor-owned request span and children around stages that actually run, with valid workload W3C context only as an untrusted link. Their records never create an extra denial-only span, even if the request span is disabled or unsampled. Other egress paths retain the -legacy bounded deny-span behavior. Both HTTP tracing and the request's ambient +bounded deny-span behavior. Both HTTP tracing and the request's ambient audit scope end before an upgraded session; later WebSocket frames do not reuse the handshake receipt. Terminal errors retain their request context explicitly, and policy-reload cancellation records the active request before dropping it. diff --git a/crates/openshell-supervisor-network/src/l7/correlation.rs b/crates/openshell-supervisor-network/src/l7/correlation.rs index 5951ee4235..9ad15b3c5f 100644 --- a/crates/openshell-supervisor-network/src/l7/correlation.rs +++ b/crates/openshell-supervisor-network/src/l7/correlation.rs @@ -236,6 +236,7 @@ impl RequestCorrelation { ctx: &L7EvalContext, request_id: String, ) -> Self { + init_request_correlation().expect("request correlation enricher registration"); Self::build(&request.action, &request.raw_header, ctx, request_id) } @@ -405,16 +406,6 @@ impl RequestCorrelation { crate::telemetry::emit_decision([event], None); return; } - if let OcsfEvent::HttpActivity(http) = &event - && let Some(trace) = self - .scope - .trace - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .as_mut() - { - trace.observe_http(http); - } in_request_scope(&self.scope, || { crate::telemetry::emit_inspected_decision([event], Some(&self.scope.span())); }); diff --git a/crates/openshell-supervisor-network/src/l7/correlation/tests.rs b/crates/openshell-supervisor-network/src/l7/correlation/tests.rs index 6f0bcd8de4..d394c69ba6 100644 --- a/crates/openshell-supervisor-network/src/l7/correlation/tests.rs +++ b/crates/openshell-supervisor-network/src/l7/correlation/tests.rs @@ -115,6 +115,36 @@ fn request_join_does_not_depend_on_export_and_never_creates_denial_spans() { } } +#[test] +fn audit_events_do_not_override_the_actual_response_outcome() { + let capture = RequestTracing::new(Some(opentelemetry_sdk::trace::Sampler::AlwaysOn)); + tracing::subscriber::with_default(capture.subscriber(), || { + let correlation = RequestCorrelation::new(&request(), &context()); + correlation.record_response_status(200); + correlation.record_outcome(HttpOutcome::Allowed); + correlation.emit(deny()); + }); + let spans = capture.spans(); + let root = spans + .iter() + .find(|span| span.name == "openshell.http.request") + .unwrap(); + assert_eq!( + span_attribute(root, "openshell.request.outcome") + .unwrap() + .as_str(), + "allowed" + ); + assert_eq!( + span_attribute(root, "http.response.status_code") + .unwrap() + .as_str(), + "200" + ); + assert_eq!(capture.events.json_events()[0]["action"], "Denied"); + capture.shutdown(); +} + #[tokio::test] async fn ambient_scope_stamps_companions_without_call_site_wiring() { let capture = OcsfCapture::default(); diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index b2754879d7..ac7ff4e8fc 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -413,40 +413,22 @@ pub fn emit_middleware_uninspectable(ctx: &L7EvalContext, detail: &str, denied: egress_emit!(event.build()); } -#[cfg(test)] pub fn emit_websocket_preflight_events( ctx: &L7EvalContext, outcome: &openshell_supervisor_middleware::WebSocketPreflightResult, ) { - emit_websocket_preflight_events_correlated(ctx, outcome, None); -} - -pub fn emit_websocket_preflight_events_correlated( - ctx: &L7EvalContext, - outcome: &openshell_supervisor_middleware::WebSocketPreflightResult, - correlation: Option<&super::correlation::RequestCorrelation>, -) { - emit_websocket_invocations_correlated( + emit_websocket_invocations( ctx, &outcome.invocations, WebSocketMiddlewarePhase::Preflight, - correlation, - ); - emit_request_events( - correlation, - websocket_coverage_events(ctx, &outcome.coverage), ); - emit_request_events(correlation, websocket_preflight_finding_events(outcome)); + crate::telemetry::emit_decision(websocket_coverage_events(ctx, &outcome.coverage), None); + crate::telemetry::emit_decision(websocket_preflight_finding_events(outcome), None); if outcome.saturated { - emit_websocket_saturation_correlated(ctx, correlation); + emit_websocket_saturation(ctx); } if outcome.session_capacity_exhausted { - emit_middleware_session_capacity_exhausted( - ctx, - "websocket_message", - !outcome.allowed, - correlation, - ); + emit_middleware_session_capacity_exhausted(ctx, "websocket_message", !outcome.allowed); } } @@ -598,15 +580,6 @@ fn emit_websocket_invocations( ctx: &L7EvalContext, invocations: &[openshell_supervisor_middleware::WebSocketInvocation], phase: WebSocketMiddlewarePhase, -) { - emit_websocket_invocations_correlated(ctx, invocations, phase, None); -} - -fn emit_websocket_invocations_correlated( - ctx: &L7EvalContext, - invocations: &[openshell_supervisor_middleware::WebSocketInvocation], - phase: WebSocketMiddlewarePhase, - correlation: Option<&super::correlation::RequestCorrelation>, ) { for invocation in invocations { use openshell_supervisor_middleware::WebSocketInvocationOutcome as Outcome; @@ -680,7 +653,7 @@ fn emit_websocket_invocations_correlated( invocation.transformed, )) .build(); - emit_request_events(correlation, [event]); + egress_emit!(event); if invocation.failed { let fail_closed = invocation.outcome == Outcome::FailClosed; let mut finding = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()); @@ -706,7 +679,7 @@ fn emit_websocket_invocations_correlated( ]) .message("WebSocket middleware stage failed") .build(); - emit_request_events(correlation, [event]); + egress_emit!(event); } if invocation.stage_disabled { let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) @@ -726,19 +699,12 @@ fn emit_websocket_invocations_correlated( "WebSocket middleware stage stream became unusable and was disabled for this session", ) .build(); - emit_request_events(correlation, [event]); + egress_emit!(event); } } } fn emit_websocket_saturation(ctx: &L7EvalContext) { - emit_websocket_saturation_correlated(ctx, None); -} - -fn emit_websocket_saturation_correlated( - ctx: &L7EvalContext, - correlation: Option<&super::correlation::RequestCorrelation>, -) { let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) .severity(SeverityId::Medium) .finding_info(FindingInfo::new( @@ -752,15 +718,10 @@ fn emit_websocket_saturation_correlated( ]) .message("WebSocket middleware work waited for admission capacity") .build(); - emit_request_events(correlation, [event]); + egress_emit!(event); } -fn emit_middleware_session_capacity_exhausted( - ctx: &L7EvalContext, - operation: &str, - denied: bool, - correlation: Option<&super::correlation::RequestCorrelation>, -) { +fn emit_middleware_session_capacity_exhausted(ctx: &L7EvalContext, operation: &str, denied: bool) { let mut event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) .severity(SeverityId::Medium) .finding_info(FindingInfo::new( @@ -776,7 +737,7 @@ fn emit_middleware_session_capacity_exhausted( if denied { event = event.action(ActionId::Denied); } - emit_request_events(correlation, [event.build()]); + egress_emit!(event.build()); } fn middleware_admission_exhausted_event(ctx: &L7EvalContext) -> openshell_ocsf::OcsfEvent { @@ -1072,12 +1033,11 @@ async fn apply_middleware_chain_for_scheme_with_request_id_and_delivery_correlat generation_guard, transformed_body_policy, request_id, - correlation, )) .await } ChainHttpHookVersion::Mixed => { - emit_mixed_protocol_denial(ctx, &req, &chain, correlation); + emit_mixed_protocol_denial(ctx, &req, &chain); Ok(MiddlewareApplyResult::Denied { denial: None, failure: Some(MiddlewareFailureDetail { @@ -1142,7 +1102,6 @@ async fn apply_protocol1_middleware_chain, request_id: &str, - correlation: Option<&super::correlation::RequestCorrelation>, ) -> Result { let admission = if chain.is_empty() { None @@ -1153,7 +1112,7 @@ async fn apply_protocol1_middleware_chain Some(admission), openshell_supervisor_middleware::MiddlewareWorkAdmissionOutcome::QueueExhausted => { - emit_request_events(correlation, [middleware_admission_exhausted_event(ctx)]); + egress_emit!(middleware_admission_exhausted_event(ctx)); return Ok(MiddlewareApplyResult::AdmissionExhausted); } } @@ -1182,7 +1141,7 @@ async fn apply_protocol1_middleware_chain buffered, crate::l7::rest::BufferResult::OverCapacity { recoverable } => { - return Ok(resolve_unbuffered_body_correlated( - ctx, - req, - &chain, - recoverable, - correlation, - )); + return Ok(resolve_unbuffered_body(ctx, req, &chain, recoverable)); } }; let headers = safe_middleware_headers(&buffered.headers)?; @@ -1239,7 +1192,7 @@ async fn apply_protocol1_middleware_chain, ) { - emit_request_events( - correlation, + crate::telemetry::emit_decision( mixed_protocol_events(ctx, &req.action, &req.target, chain), + None, ); } @@ -2103,42 +2055,27 @@ pub(super) fn raw_query_from_request_headers(headers: &[u8]) -> Result { /// would individually fail with `request_body_over_capacity`; the aggregate is /// a deny unless every attached middleware is `fail_open`, and passing the /// body through is only safe when no bytes were consumed. -#[cfg(test)] pub(super) fn resolve_unbuffered_body( ctx: &L7EvalContext, req: crate::l7::provider::L7Request, chain: &[openshell_supervisor_middleware::DescribedChainEntry], recoverable: bool, -) -> MiddlewareApplyResult { - resolve_unbuffered_body_correlated(ctx, req, chain, recoverable, None) -} - -fn resolve_unbuffered_body_correlated( - ctx: &L7EvalContext, - req: crate::l7::provider::L7Request, - chain: &[openshell_supervisor_middleware::DescribedChainEntry], - recoverable: bool, - correlation: Option<&super::correlation::RequestCorrelation>, ) -> MiddlewareApplyResult { let all_fail_open = chain .iter() .all(|entry| entry.on_error() == openshell_supervisor_middleware::OnError::FailOpen); if recoverable && all_fail_open { - emit_middleware_body_unavailable(ctx, false, correlation); + emit_middleware_body_unavailable(ctx, false); return MiddlewareApplyResult::Allowed(req); } - emit_middleware_body_unavailable(ctx, true, correlation); + emit_middleware_body_unavailable(ctx, true); MiddlewareApplyResult::Denied { denial: None, failure: None, } } -fn emit_middleware_body_unavailable( - ctx: &L7EvalContext, - denied: bool, - correlation: Option<&super::correlation::RequestCorrelation>, -) { +fn emit_middleware_body_unavailable(ctx: &L7EvalContext, denied: bool) { let mut event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) .severity(if denied { SeverityId::High @@ -2162,7 +2099,7 @@ fn emit_middleware_body_unavailable( if denied { event = event.action(ActionId::Denied); } - emit_request_events(correlation, [event.build()]); + egress_emit!(event.build()); } /// Parse the raw header block into middleware-visible headers, preserving @@ -2572,7 +2509,7 @@ mod trace_tests { let (events, received) = exported("warn", || { let ctx = L7EvalContext::default(); emit_middleware_uninspectable(&ctx, "unknown protocol", denied); - emit_middleware_body_unavailable(&ctx, denied, None); + emit_middleware_body_unavailable(&ctx, denied); }) .await; assert_eq!(events.len(), 2); diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 18200a1187..f3f47c7555 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -1322,15 +1322,6 @@ fn parse_rejection_detail(error: &str, mode: ParseRejectionMode) -> String { } fn emit_parse_rejection(ctx: &L7EvalContext, detail: &str, engine_type: &str) { - emit_parse_rejection_correlated(ctx, detail, engine_type, None); -} - -fn emit_parse_rejection_correlated( - ctx: &L7EvalContext, - detail: &str, - engine_type: &str, - correlation: Option<&RequestCorrelation>, -) { let policy_name = if ctx.policy_name.is_empty() { "-" } else { @@ -1350,11 +1341,7 @@ fn emit_parse_rejection_correlated( )) .status_detail(detail) .build(); - if let Some(correlation) = correlation { - correlation.emit(event); - } else { - egress_emit!(event); - } + egress_emit!(event); emit_activity(ctx, true, "l7_parse_rejection"); } @@ -1400,12 +1387,7 @@ where if let Some(correlation) = correlation { correlation.record_rejection(403, HttpOutcome::Denied, None); } - emit_parse_rejection_correlated( - ctx, - detail, - engine_type_for_protocol(config.protocol), - correlation, - ); + emit_parse_rejection(ctx, detail, engine_type_for_protocol(config.protocol)); let body = serde_json::json!({ "error": "unsupported_l7_protocol", "detail": detail, @@ -1597,16 +1579,7 @@ where let Some(config) = select_l7_config_for_path(configs, &route_target) else { let reason = "no L7 endpoint path matched request"; correlation.record_rejection(403, HttpOutcome::Denied, None); - emit_l7_request_log_correlated( - ctx, - &req.action, - &route_target, - "deny", - "l7", - reason, - None, - Some(&correlation), - ); + emit_l7_request_log(ctx, &req.action, &route_target, "deny", "l7", reason, None); crate::l7::rest::RestProvider::default() .deny_with_redacted_target( &req, @@ -1646,12 +1619,7 @@ where if let Some(observer) = observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } - emit_parse_rejection_correlated( - ctx, - detail, - engine_type_for_protocol(config.protocol), - Some(&correlation), - ); + emit_parse_rejection(ctx, detail, engine_type_for_protocol(config.protocol)); correlation.record_rejection(403, HttpOutcome::Denied, None); let deny_context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx) .with_request_id(correlation.request_id()); @@ -1705,12 +1673,7 @@ where { correlation.record_outcome(HttpOutcome::Denied); } - emit_parse_rejection_correlated( - ctx, - &detail, - "l7-graphql", - Some(&correlation), - ); + emit_parse_rejection(ctx, &detail, "l7-graphql"); } return Ok(()); } @@ -1749,12 +1712,7 @@ where { correlation.record_outcome(HttpOutcome::Denied); } - emit_parse_rejection_correlated( - ctx, - &detail, - "l7-jsonrpc", - Some(&correlation), - ); + emit_parse_rejection(ctx, &detail, "l7-jsonrpc"); } return Ok(()); } @@ -1854,7 +1812,7 @@ where let engine_type = engine_type_for_protocol(config.protocol); let protocol_summary = l7_protocol_log_summary(graphql_info.as_ref(), jsonrpc_info.as_ref()); - emit_l7_request_log_correlated( + emit_l7_request_log( ctx, &request_info.action, &redacted_target, @@ -1862,7 +1820,6 @@ where engine_type, &reason, protocol_summary.as_deref(), - Some(&correlation), ); if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { @@ -2008,11 +1965,7 @@ where return Ok(()); } }; - crate::l7::middleware::emit_websocket_preflight_events_correlated( - ctx, - &preflight, - Some(&correlation), - ); + crate::l7::middleware::emit_websocket_preflight_events(ctx, &preflight); if preflight.terminal_reason.is_some() { correlation.record_rejection( 403, @@ -2184,8 +2137,7 @@ fn select_l7_config_for_path<'a>( .max_by_key(|config| config.path_specificity()) } -#[allow(clippy::too_many_arguments)] -fn emit_l7_request_log_correlated( +fn emit_l7_request_log( ctx: &L7EvalContext, action: &str, redacted_target: &str, @@ -2193,7 +2145,6 @@ fn emit_l7_request_log_correlated( engine_type: &str, reason: &str, protocol_summary: Option<&str>, - correlation: Option<&RequestCorrelation>, ) { let event = build_l7_request_event( ctx, @@ -2204,11 +2155,7 @@ fn emit_l7_request_log_correlated( reason, protocol_summary, ); - if let Some(correlation) = correlation { - correlation.emit(event); - } else { - egress_emit!(event); - } + egress_emit!(event); emit_activity(ctx, decision_str == "deny", "l7_policy"); } @@ -2897,11 +2844,7 @@ where return Ok(()); } }; - crate::l7::middleware::emit_websocket_preflight_events_correlated( - ctx, - &preflight, - Some(&correlation), - ); + crate::l7::middleware::emit_websocket_preflight_events(ctx, &preflight); if preflight.terminal_reason.is_some() { correlation.record_rejection( 403, @@ -3608,7 +3551,7 @@ where let detail = parse_rejection_detail(&e.to_string(), ParseRejectionMode::L7Endpoint); correlation.record_outcome(HttpOutcome::Denied); - emit_parse_rejection_correlated(ctx, &detail, "l7-graphql", Some(&correlation)); + emit_parse_rejection(ctx, &detail, "l7-graphql"); } return Ok(()); } diff --git a/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs index bf045b3883..92063327fa 100644 --- a/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs +++ b/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs @@ -676,6 +676,82 @@ async fn inspected_keep_alive_requests_have_distinct_receipts_without_tracing() } } +#[tokio::test] +async fn keep_alive_parse_rejection_does_not_reuse_the_completed_request_scope() { + let Fixture { + config, + tunnel_engine, + middleware_engine, + ctx, + } = rest_fixture("GET", false); + let evidence = capture_request_spans( + true, + tokio::time::timeout(std::time::Duration::from_secs(5), async { + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let mut relay = Box::pin(crate::proxy::relay_inspected_http_stream_for_test( + &mut relay_client, + &mut relay_upstream, + vec![config], + tunnel_engine, + &middleware_engine, + &ctx, + )); + app.write_all(b"GET /item HTTP/1.1\r\nHost: rest.example.test:8000\r\n\r\n") + .await + .unwrap(); + let forwarded = tokio::select! { + result = relay.as_mut() => panic!("relay stopped before forwarding: {result:?}"), + request = read_http_message(&mut upstream) => request, + }; + assert!(forwarded.starts_with(b"GET /item HTTP/1.1\r\n")); + upstream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .await + .unwrap(); + let response = tokio::select! { + result = relay.as_mut() => panic!("persistent relay stopped: {result:?}"), + response = read_http_message(&mut app) => response, + }; + assert!(response.starts_with(b"HTTP/1.1 200 OK\r\n")); + let receipt = request_id(&response).to_string(); + assert!(futures::poll!(relay.as_mut()).is_pending()); + app.write_all(b"GET /item?q=%ff HTTP/1.1\r\nHost: rest.example.test:8000\r\n\r\n") + .await + .unwrap(); + relay.await.expect("parse rejection closes the relay"); + receipt + }), + ) + .await; + let receipt = evidence + .output + .as_ref() + .expect("keep-alive exchange completes"); + assert!(uuid::Uuid::parse_str(receipt).is_ok()); + evidence.assert_no_deny_span(true); + let (rejected, completed): (Vec<_>, Vec<_>) = evidence.events.iter().partition(|event| { + event["message"] + == "HTTP request rejected before policy evaluation for rest.example.test:8000" + }); + assert!(!completed.is_empty()); + for event in completed { + assert_eq!(event["metadata"]["correlation_uid"], receipt.as_str()); + } + assert_eq!(rejected.len(), 1, "actual parse rejection: {rejected:?}"); + let rejection = rejected[0]; + assert_eq!(rejection["class_uid"], 4001); + assert_eq!(rejection["action"], "Denied"); + assert_eq!( + rejection["status_detail"], + "Query component is not valid UTF-8" + ); + assert!(rejection["metadata"]["correlation_uid"].is_null()); + assert!(rejection["unmapped"]["request_sequence"].is_null()); + assert!(rejection["unmapped"]["http_event_kind"].is_null()); + assert!(rejection["trace"]["uid"].is_null()); +} + #[derive(Clone, Copy, Debug)] enum ReloadAt { BeforeRequest, diff --git a/crates/openshell-supervisor-network/src/l7/request_trace.rs b/crates/openshell-supervisor-network/src/l7/request_trace.rs index e3abf00852..1d9c36a509 100644 --- a/crates/openshell-supervisor-network/src/l7/request_trace.rs +++ b/crates/openshell-supervisor-network/src/l7/request_trace.rs @@ -11,7 +11,6 @@ use tracing::Span; use tracing_opentelemetry::OpenTelemetrySpanExt as _; use super::correlation::HttpOutcome; -use openshell_ocsf::{ActionId, HttpActivityEvent, StatusId}; const MAX_TRACEPARENT_BYTES: usize = 256; @@ -134,23 +133,6 @@ impl RequestTrace { self.record_outcome(HttpOutcome::PlatformError); self.record_failure_reason("request_failed"); } - - pub(super) fn observe_http(&mut self, event: &HttpActivityEvent) { - if let Some(response) = &event.http_response { - self.record_status(response.code); - } - if event.action == Some(ActionId::Denied) { - if self.outcome != "failed" { - self.record_outcome(HttpOutcome::Denied); - } - } else if self.outcome == "incomplete" { - self.record_outcome(if event.base.status == Some(StatusId::Failure) { - HttpOutcome::PlatformError - } else { - HttpOutcome::Allowed - }); - } - } } impl Drop for RequestTrace { diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 0f6015da62..caec858eaf 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -1259,11 +1259,7 @@ where guard.ensure_current()?; } - let signing_span = correlation - .map_or_else(tracing::Span::none, |correlation| { - correlation.stage(RequestStage::CredentialPrepare) - }); - let signed = signing_span.in_scope(|| { + let sign = || { crate::sigv4::apply_sigv4_to_request( &full_request, options.host, @@ -1273,23 +1269,26 @@ where secret_key, session_token, ) + }; + let signed = correlation.map_or_else(sign, |correlation| { + correlation.run_stage_sync( + RequestStage::CredentialPrepare, + HttpOutcome::PlatformError, + "credential_signing_failed", + sign, + ) })?; - drop(signing_span); ensure_credential_generation_current(options)?; upstream.write_all(&signed).await.into_diagnostic()?; } else { // Sign headers only, stream body through. - let signable_body = match payload_mode { - SigV4PayloadMode::StreamingUnsignedTrailer => { - SignableBody::StreamingUnsignedPayloadTrailer - } - _ => SignableBody::UnsignedPayload, - }; - let signing_span = correlation - .map_or_else(tracing::Span::none, |correlation| { - correlation.stage(RequestStage::CredentialPrepare) - }); - let signed_headers = signing_span.in_scope(|| { + let sign = || { + let signable_body = match payload_mode { + SigV4PayloadMode::StreamingUnsignedTrailer => { + SignableBody::StreamingUnsignedPayloadTrailer + } + _ => SignableBody::UnsignedPayload, + }; crate::sigv4::apply_sigv4_headers_only_with_body( &rewrite_result.rewritten, options.host, @@ -1300,8 +1299,15 @@ where session_token, signable_body, ) + }; + let signed_headers = correlation.map_or_else(sign, |correlation| { + correlation.run_stage_sync( + RequestStage::CredentialPrepare, + HttpOutcome::PlatformError, + "credential_signing_failed", + sign, + ) })?; - drop(signing_span); ensure_credential_generation_current(options)?; upstream .write_all(&signed_headers) @@ -11881,7 +11887,12 @@ mod tests { } #[tokio::test] - async fn relay_sigv4_body_signing_signs_empty_body_without_content_length() { + async fn relay_sigv4_signing_records_success_and_failure() { + use crate::test_support::request_tracing::{RequestTracing, span_attribute}; + use opentelemetry_sdk::trace::Sampler; + use tracing::instrument::WithSubscriber as _; + + let _lock = openshell_otel_test_support::tracing_test_lock().await; let (_, resolver) = SecretResolver::from_provider_env( [ ("AWS_ACCESS_KEY_ID".to_string(), "AKIATESTKEY".to_string()), @@ -11904,57 +11915,83 @@ mod tests { raw_header, body_length: BodyLength::None, }; - let (mut proxy_to_upstream, mut upstream_side) = tokio::io::duplex(8192); - let (mut _app_side, mut proxy_to_client) = tokio::io::duplex(8192); - - let upstream_task = tokio::spawn(async move { - let mut buf = vec![0u8; 8192]; - let mut total = 0usize; - loop { - let n = upstream_side.read(&mut buf[total..]).await.unwrap(); - if n == 0 { - break; + for signing in [ + crate::l7::CredentialSigning::SigV4Body, + crate::l7::CredentialSigning::SigV4NoBody, + ] { + for host in ["s3.us-east-1.amazonaws.com", "invalid host"] { + let succeeds = host != "invalid host"; + let capture = RequestTracing::new(Some(Sampler::AlwaysOn)); + let (mut proxy_to_upstream, mut upstream_side) = tokio::io::duplex(8192); + let (mut _app_side, mut proxy_to_client) = tokio::io::duplex(8192); + upstream_side + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await + .unwrap(); + let result = tokio::time::timeout(std::time::Duration::from_secs(5), async { + let ctx = crate::l7::relay::L7EvalContext { + host: host.into(), + port: 443, + ..Default::default() + }; + let correlation = RequestCorrelation::new(&req, &ctx); + relay_http_request_with_response_middleware_guarded_observed( + &req, + &mut proxy_to_client, + &mut proxy_to_upstream, + RelayRequestOptions { + resolver: Some(&resolver), + credential_signing: signing, + signing_service: "s3", + signing_region: "us-east-1", + host, + port: 443, + ..Default::default() + }, + None, + None, + Some(&correlation), + ) + .await + }) + .with_subscriber(capture.subscriber()) + .await + .expect("relay must not deadlock"); + assert_eq!(result.is_ok(), succeeds, "{result:?}"); + drop(proxy_to_upstream); + let mut forwarded = String::new(); + upstream_side.read_to_string(&mut forwarded).await.unwrap(); + if succeeds { + assert!(forwarded.contains("authorization:"), "{forwarded}"); + } else { + assert!( + forwarded.is_empty(), + "signing failure must not reach upstream" + ); } - total += n; - if buf[..total].windows(4).any(|w| w == b"\r\n\r\n") { - upstream_side - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") - .await - .unwrap(); - upstream_side.flush().await.unwrap(); - break; + let spans = capture.spans(); + let stages = spans + .iter() + .filter(|span| span.name == "openshell.credential.prepare") + .collect::>(); + assert_eq!(stages.len(), 2, "header resolution and signing"); + assert_eq!(stages[0].status, opentelemetry::trace::Status::Ok); + let signing = stages[1]; + assert_eq!( + matches!(signing.status, opentelemetry::trace::Status::Error { .. }), + !succeeds, + ); + assert_eq!( + span_attribute(signing, "openshell.request.failure_reason") + .map(|value| value.as_str()), + (!succeeds).then_some(Cow::Borrowed("credential_signing_failed")), + ); + if succeeds { + assert_eq!(signing.status, opentelemetry::trace::Status::Ok); } + capture.shutdown(); } - String::from_utf8_lossy(&buf[..total]).to_string() - }); - - let relay_result = tokio::time::timeout( - std::time::Duration::from_secs(5), - relay_http_request_with_options_guarded( - &req, - &mut proxy_to_client, - &mut proxy_to_upstream, - RelayRequestOptions { - resolver: Some(&resolver), - credential_signing: crate::l7::CredentialSigning::SigV4Body, - signing_service: "s3", - signing_region: "us-east-1", - host: "s3.us-east-1.amazonaws.com", - port: 443, - ..Default::default() - }, - ), - ) - .await - .expect("relay must not deadlock"); - drop(proxy_to_upstream); - let forwarded = upstream_task.await.expect("upstream task should complete"); - - relay_result.expect("bodyless GET with SigV4Body should succeed"); - assert!( - forwarded.contains("authorization:"), - "forwarded request should have SigV4 authorization header: {forwarded}" - ); + } } #[tokio::test] diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 6ea77f0f4c..fe5067a77e 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -6059,8 +6059,11 @@ async fn handle_forward_proxy_inner( }; let Ok(redacted_path) = secrets::redact_target_for_policy(&path) else { - correlation.record_outcome(HttpOutcome::PlatformError); - correlation.record_response_status(400); + correlation.record_rejection( + 400, + HttpOutcome::PlatformError, + Some("invalid_credential_placeholder"), + ); respond( client, &build_json_error_response_with_request_id( @@ -6774,11 +6777,7 @@ async fn handle_forward_proxy_inner( return Ok(()); } }; - crate::l7::middleware::emit_websocket_preflight_events_correlated( - &l7_ctx, - &preflight, - request_correlation, - ); + crate::l7::middleware::emit_websocket_preflight_events(&l7_ctx, &preflight); if preflight.terminal_reason.is_some() { correlation.record_rejection( 403, @@ -6787,7 +6786,11 @@ async fn handle_forward_proxy_inner( } else { HttpOutcome::PlatformError }, - None, + Some(if preflight.denial.is_some() { + "websocket_middleware_denied" + } else { + "websocket_preflight_failed" + }), ); let response = preflight.denial.as_ref().map_or_else( || build_middleware_failure_response(&l7_ctx.policy_name, None, receipt_id), @@ -6905,6 +6908,11 @@ async fn handle_forward_proxy_inner( }; if let Some(guard) = credential_generation { guard.ensure_current().inspect_err(|_| { + egress::mark_error(&credential_stage); + credential_stage.record( + "openshell.request.failure_reason", + "credential_generation_stale", + ); correlation.record_failure(HttpOutcome::PlatformError, "credential_generation_stale"); })?; } @@ -6921,6 +6929,12 @@ async fn handle_forward_proxy_inner( }); if rewrite.is_err() { egress::mark_error(&credential_stage); + credential_stage.record( + "openshell.request.failure_reason", + "credential_resolution_refused", + ); + } else { + credential_stage.record("otel.status_code", "OK"); } drop(credential_stage); let rewritten = match rewrite { @@ -6984,7 +6998,11 @@ async fn handle_forward_proxy_inner( }) .await; } else { - correlation.record_rejection(500, HttpOutcome::PlatformError, None); + correlation.record_rejection( + 500, + HttpOutcome::PlatformError, + Some("credential_injection_failed"), + ); respond( client, &build_json_error_response_with_request_id( diff --git a/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs b/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs index 05cda6da9f..e1a6d88820 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use super::*; -use crate::test_support::request_tracing::RequestTracing; +use crate::test_support::request_tracing::{RequestTracing, span_attribute}; use crate::test_support::response_request_id; use opentelemetry_sdk::trace::Sampler; use tracing::instrument::WithSubscriber as _; @@ -17,6 +17,7 @@ enum Outcome { PolicyDenied, MalformedQueryPolicyDenied, AuditDenied, + UnresolvedCredential, UpstreamClosed, } @@ -45,6 +46,15 @@ async fn forward_case(ip: Ipv4Addr, outcome: Outcome, sampler: Option) outcome, Outcome::PolicyDenied | Outcome::MalformedQueryPolicyDenied ); + let credential_failure = matches!(outcome, Outcome::UnresolvedCredential); + let secret_resolver = credential_failure.then(|| { + let (_, resolver) = SecretResolver::from_provider_env( + [("OTHER_KEY".into(), "inert-test-value".into())] + .into_iter() + .collect(), + ); + Arc::new(resolver.expect("test credential resolver")) + }); let mode = if matches!(outcome, Outcome::AuditDenied) { "audit" } else { @@ -99,8 +109,13 @@ network_policies: "token=SENTINEL-QUERY" }; let target = format!("http://{ip}:{port}{path}?{query}"); + let credential_header = if credential_failure { + "Authorization: Bearer openshell:resolve:env:MISSING\r\n" + } else { + "" + }; let request = format!( - "GET {target} HTTP/1.1\r\nHost: {ip}:{port}\r\ntraceparent: 00-{WORKLOAD_TRACE_ID}-{WORKLOAD_SPAN_ID}-01\r\ntracestate: vendor=SENTINEL-VENDOR\r\nx-openshell-request-id: forged-receipt\r\nConnection: close\r\n\r\n" + "GET {target} HTTP/1.1\r\nHost: {ip}:{port}\r\ntraceparent: 00-{WORKLOAD_TRACE_ID}-{WORKLOAD_SPAN_ID}-01\r\ntracestate: vendor=SENTINEL-VENDOR\r\nx-openshell-request-id: forged-receipt\r\n{credential_header}Connection: close\r\n\r\n" ); let exchange = async { tokio::join!( @@ -124,7 +139,7 @@ network_policies: Arc::new(None), Arc::new(None), None, - None, + secret_resolver, None, None, None, @@ -134,7 +149,7 @@ network_policies: result }, async { - if policy_denied { + if policy_denied || credential_failure { assert!( tokio::time::timeout( std::time::Duration::from_millis(50), @@ -142,7 +157,7 @@ network_policies: ) .await .is_err(), - "policy denial reached upstream" + "rejected request reached upstream" ); } else { let (mut upstream, _) = listener.accept().await.unwrap(); @@ -176,19 +191,25 @@ network_policies: let spans = capture.spans(); let expected = (!matches!(outcome, Outcome::UpstreamClosed)).then_some(if policy_denied { 403 + } else if credential_failure { + 500 } else { 200 }); let response = String::from_utf8(response).unwrap(); let records = capture.events.json_events(); - let expected_records = if policy_denied { 1 } else { 2 }; + let expected_records = if policy_denied || credential_failure { + 1 + } else { + 2 + }; assert_eq!(records.len(), expected_records, "{records:?}"); assert!(records.iter().all(|record| record["class_uid"] == 4002)); let record = &records[0]; let decision = match outcome { Outcome::PolicyDenied | Outcome::MalformedQueryPolicyDenied => "deny", Outcome::AuditDenied => "audit", - Outcome::Allowed | Outcome::UpstreamClosed => "allow", + Outcome::Allowed | Outcome::UnresolvedCredential | Outcome::UpstreamClosed => "allow", }; assert!( record["message"] @@ -203,7 +224,7 @@ network_policies: assert_eq!(control["http_request"]["url"]["path"], path); assert!(control.get("http_response").is_none()); } - if !policy_denied { + if !policy_denied && !credential_failure { let forwarded = &records[1]; assert!( forwarded["message"] @@ -325,7 +346,7 @@ network_policies: stages .iter() .any(|span| span.name == "openshell.upstream.connect"), - !policy_denied + !policy_denied && !credential_failure ); assert_eq!( stages @@ -335,9 +356,34 @@ network_policies: ); assert_eq!( matches!(root.status, opentelemetry::trace::Status::Error { .. }), - matches!(outcome, Outcome::UpstreamClosed), + matches!( + outcome, + Outcome::UpstreamClosed | Outcome::UnresolvedCredential + ), "a policy refusal is an outcome, not an operational tracing failure" ); + if credential_failure { + assert_eq!( + span_attribute(root, "openshell.request.failure_reason") + .unwrap() + .as_str(), + "credential_injection_failed", + ); + let preparation = stages + .iter() + .find(|span| span.name == "openshell.credential.prepare") + .expect("credential preparation stage"); + assert!(matches!( + preparation.status, + opentelemetry::trace::Status::Error { .. } + )); + assert_eq!( + span_attribute(preparation, "openshell.request.failure_reason") + .unwrap() + .as_str(), + "credential_resolution_refused", + ); + } assert!( stages .iter() @@ -357,6 +403,7 @@ async fn inspected_forward_traces_all_outcomes_without_sampling_or_export_depend Outcome::Allowed, Outcome::PolicyDenied, Outcome::AuditDenied, + Outcome::UnresolvedCredential, Outcome::UpstreamClosed, ] { for sampler in [Some(Sampler::AlwaysOn), Some(Sampler::AlwaysOff), None] { diff --git a/docs/observability/ocsf-json-export.mdx b/docs/observability/ocsf-json-export.mdx index 86e8e5977c..f0092e98da 100644 --- a/docs/observability/ocsf-json-export.mdx +++ b/docs/observability/ocsf-json-export.mdx @@ -317,9 +317,9 @@ The request span ends after its owned stages, not when response headers are logg The supervisor owns the request trace. A valid workload `traceparent`, sampled or unsampled, becomes an explicitly untrusted link, never its parent. Invalid or duplicate context is ignored; workload `tracestate` is not adopted. Your backend may show the agent and supervisor as separate linked traces rather than one nested tree. -Inspected requests do not also create a denial-only `supervisor.egress.deny` span. Other denied HTTP decisions retain that legacy span: a root at the default log level, or a child of an available connect span at debug level. Shared supervisor hosts do not initialize OTLP exporters, so their records carry no trace correlation. +Inspected requests do not also create a denial-only `supervisor.egress.deny` span. Non-inspected denials use that span: a root at the default log level, or a child of an available connect span at debug level. Shared supervisor hosts do not initialize OTLP exporters, so their records carry no trace correlation. -The legacy deny-span limiter allows at most 10 spans per destination at once and one more per second after that. A process-wide budget also allows at most 100 spans at once and refills at 100 per second across destinations. Rate-limited decisions create no additional deny span, while all audit records are still emitted. These denial-specific budgets do not select inspected request spans, which cover every outcome rather than denials alone. +The non-inspected denial-span limiter allows at most 10 spans per destination at once and one more per second after that. A process-wide budget also allows at most 100 spans at once and refills at 100 per second across destinations. Rate-limited decisions create no additional deny span, while all audit records are still emitted. These denial-specific budgets do not select inspected request spans, which cover every outcome rather than denials alone. Gateway records and their OTLP trace context can inherit the caller's incoming `traceparent`, including before authentication succeeds. The caller also decides whether they are correlated at all, because an unsampled `traceparent` suppresses `trace.uid`. Treat gateway trace IDs as correlation hints, not evidence of the caller's identity, and do not treat a missing `trace.uid` as meaningful. @@ -362,14 +362,14 @@ absent. An instrumented client can record the receipt on its own trace and use i to find OpenShell's security records; it does not need to receive those records inside the sandbox. -For inspected-request traces, query span name `openshell.http.request` instead -of `supervisor.egress.deny` and join through `openshell.request.id`. A refusal +For inspected-request traces, query span name `openshell.http.request` +and join through `openshell.request.id`. A refusal can have outcome `"denied"` or `"failed"`; use the original OCSF control record -for its policy and reason. Keep the legacy span query for non-inspected egress. +for its policy and reason. For non-inspected denials, query `supervisor.egress.deny`. ### Size Supervisor Trace Collection -The old deny-span budgets do not bound inspected request spans, which include +Denial-span budgets do not bound inspected request spans, which include allowed traffic at INFO level. At `R` inspected requests per second and an average of `S` descendant spans per request, expect approximately `R × (1 + S)` spans per second before exporter or collector loss, plus background @@ -381,7 +381,7 @@ reduces stored traces, not the supervisor's span-generation cost. The gateway's control is tracked in [#5362](https://github.com/NVIDIA/OpenShell/issues/5362). -Leave supervisor trace export disabled if your deployment requires the old +Leave supervisor trace export disabled if your deployment requires a denial-only volume bound. OCSF records and request receipts still work without it. ## OCSF Event Classes in JSON From 57ec95ce077aa0a933e8ee753c5a7c0918133af0 Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Mon, 16 Nov 2026 16:43:06 -0700 Subject: [PATCH 6/7] fix(observability): report handled middleware stage failures Signed-off-by: Alex Fournier --- crates/openshell-supervisor-network/README.md | 4 +- .../src/l7/correlation.rs | 23 +- .../src/l7/middleware.rs | 16 +- .../l7/relay/correlation_dispatch_tests.rs | 207 +++++++++++++++--- .../src/l7/rest/http_response.rs | 156 ++++++++++--- .../src/l7/rest/http_response/http_hooks.rs | 32 ++- 6 files changed, 369 insertions(+), 69 deletions(-) diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index 1dea7f6c0f..ca8520f571 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -88,7 +88,9 @@ and policy-reload cancellation records the active request before dropping it. Idle connection closures have no HTTP receipt. The request span ends after its owned stages, not when response headers are logged. The `openshell.upstream.send` stage measures request writes and flush; response -handling is measured separately. Receipt filtering preserves raw HTTP bytes +handling is measured separately. Handled middleware failures remain stage errors +even when the local rejection response is delivered successfully. Receipt +filtering preserves raw HTTP bytes and borrows unchanged header blocks. It removes only protected receipt fields and their `Connection` or `Trailer` nominations, including folded continuations, without adding general response-header validation. diff --git a/crates/openshell-supervisor-network/src/l7/correlation.rs b/crates/openshell-supervisor-network/src/l7/correlation.rs index 9ad15b3c5f..a97f67f966 100644 --- a/crates/openshell-supervisor-network/src/l7/correlation.rs +++ b/crates/openshell-supervisor-network/src/l7/correlation.rs @@ -191,9 +191,9 @@ struct StageResult { } impl StageResult { - fn record(&mut self, result: &Result, reason: &'static str) { + fn record(&mut self, failure_reason: Option<&'static str>) { self.completed = true; - if result.is_err() { + if let Some(reason) = failure_reason { self.span.record("otel.status_code", "ERROR"); self.span.record("openshell.request.failure_reason", reason); } else { @@ -306,6 +306,18 @@ impl RequestCorrelation { failure_outcome: HttpOutcome, reason: &'static str, future: impl Future>, + ) -> impl Future> { + self.run_stage_classified(stage, failure_outcome, reason, future, |_| None) + } + + /// Include handled failures returned as successful transport results. + pub(crate) fn run_stage_classified( + &self, + stage: RequestStage, + failure_outcome: HttpOutcome, + reason: &'static str, + future: impl Future>, + classify_failure: impl FnOnce(&T) -> Option<&'static str>, ) -> impl Future> { let mut stage = StageResult { span: self.stage(stage), @@ -314,8 +326,9 @@ impl RequestCorrelation { let future = future.instrument(stage.span.clone()); async move { let result = future.await; - stage.record(&result, reason); - if result.is_err() { + let failure_reason = result.as_ref().map_or(Some(reason), classify_failure); + stage.record(failure_reason); + if let Some(reason) = failure_reason { self.record_failure(failure_outcome, reason); } result @@ -334,7 +347,7 @@ impl RequestCorrelation { completed: false, }; let result = stage.span.in_scope(operation); - stage.record(&result, reason); + stage.record(result.is_err().then_some(reason)); if result.is_err() { self.record_failure(failure_outcome, reason); } diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index ac7ff4e8fc..52c6dbe967 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -1078,11 +1078,25 @@ async fn apply_middleware_chain_for_scheme_with_request_id_and_delivery_correlat }; if has_stages && let Some(correlation) = correlation { correlation - .run_stage( + .run_stage_classified( RequestStage::RequestMiddleware, super::correlation::HttpOutcome::PlatformError, "request_middleware_failed", apply, + |outcome| match outcome { + MiddlewareApplyResult::Denied { denial: None, .. } => { + Some("request_middleware_failed") + } + MiddlewareApplyResult::AdmissionExhausted => { + Some("middleware_admission_exhausted") + } + MiddlewareApplyResult::RequestTimeout => Some("request_timeout"), + MiddlewareApplyResult::Allowed(_) + | MiddlewareApplyResult::Streamed { .. } + | MiddlewareApplyResult::Denied { + denial: Some(_), .. + } => None, + }, ) .await } else { diff --git a/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs index 92063327fa..4128083134 100644 --- a/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs +++ b/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs @@ -154,6 +154,22 @@ struct Fixture { ctx: L7EvalContext, } +impl Fixture { + fn with_middleware_runner( + mut self, + runner: openshell_supervisor_middleware::ChainRunner, + ) -> Self { + self.middleware_engine + .set_middleware_runner_for_tests(runner); + // Tunnel engines snapshot the runner alongside the policy generation. + self.tunnel_engine = self + .middleware_engine + .clone_engine_for_tunnel(self.middleware_engine.current_generation()) + .expect("pin injected middleware runner"); + self + } +} + struct Exchange { response: Vec, forwarded: Vec, @@ -401,7 +417,10 @@ fn request(host: &str, path: &str, extra_headers: &str, body: &str) -> Vec { .into_bytes() } -struct BlockingRequestMiddleware; +#[derive(Default)] +struct BlockingRequestMiddleware { + http_hooks: bool, +} #[tonic::async_trait] impl openshell_supervisor_middleware::InProcessMiddleware for BlockingRequestMiddleware { @@ -409,8 +428,13 @@ impl openshell_supervisor_middleware::InProcessMiddleware for BlockingRequestMid use openshell_core::proto::{ MiddlewareBinding, SupervisorMiddlewareOperation, SupervisorMiddlewarePhase, }; + let name = if self.http_hooks { + "test/block-request-hooks" + } else { + "test/block-request" + }; openshell_core::proto::MiddlewareManifest { - name: "test/block-request".into(), + name: name.into(), service_version: "test".into(), bindings: vec![MiddlewareBinding { operation: SupervisorMiddlewareOperation::HttpRequest as i32, @@ -418,12 +442,16 @@ impl openshell_supervisor_middleware::InProcessMiddleware for BlockingRequestMid max_payload_bytes: 4096, request_timeout: None, }], - extension: Some(openshell_core::extension_protocol::extension_metadata( - openshell_core::extension_protocol::ExtensionFamily::SupervisorMiddleware, - "openshell/test-block-request", - "test", - [], - )), + extension: Some(if self.http_hooks { + openshell_core::extension_protocol::http_hooks_middleware_metadata(name, "test") + } else { + openshell_core::extension_protocol::extension_metadata( + openshell_core::extension_protocol::ExtensionFamily::SupervisorMiddleware, + name, + "test", + [], + ) + }), ..Default::default() } } @@ -450,7 +478,18 @@ fn rest_fixture(method: &str, block_middleware: bool) -> Fixture { } else { "" }; - let fixture = fixture( + let fixture = rest_fixture_with_middlewares(method, middleware); + if block_middleware { + fixture.with_middleware_runner(openshell_supervisor_middleware::ChainRunner::new(Arc::new( + BlockingRequestMiddleware::default(), + ))) + } else { + fixture + } +} + +fn rest_fixture_with_middlewares(method: &str, middleware: &str) -> Fixture { + fixture( &format!( r"{middleware} network_policies: @@ -470,13 +509,7 @@ network_policies: ), "rest.example.test", "rest_api", - ); - if block_middleware { - fixture.middleware_engine.set_middleware_runner_for_tests( - openshell_supervisor_middleware::ChainRunner::new(Arc::new(BlockingRequestMiddleware)), - ); - } - fixture + ) } #[tokio::test] @@ -528,11 +561,17 @@ async fn inspected_allow_creates_only_the_stages_that_run() { #[tokio::test] async fn request_middleware_block_never_creates_a_denial_only_span() { - for sampled in [true, false] { + for (sampled, service_registered) in [(true, false), (false, false), (true, true)] { + let fixture = rest_fixture("POST", true); + let fixture = if service_registered { + fixture + } else { + fixture.with_middleware_runner(openshell_supervisor_middleware::ChainRunner::default()) + }; let evidence = capture_request_spans( sampled, Box::pin(run_exchange( - rest_fixture("POST", true), + fixture, request("rest.example.test", "/item", "", "{}"), None, )), @@ -547,13 +586,35 @@ async fn request_middleware_block_never_creates_a_denial_only_span() { .starts_with(b"HTTP/1.1 403 Forbidden\r\n") ); if sampled { + let middleware: Vec<_> = evidence + .spans + .iter() + .filter(|span| span.name == "openshell.middleware.request") + .collect(); + assert_eq!(middleware.len(), 1); + if service_registered { + assert_eq!(middleware[0].status, opentelemetry::trace::Status::Ok); + } else { + assert!(matches!( + middleware[0].status, + opentelemetry::trace::Status::Error { .. } + )); + } + let reason = (!service_registered).then_some("request_middleware_failed"); assert_eq!( - evidence - .spans - .iter() - .filter(|span| span.name == "openshell.middleware.request") - .count(), - 1 + span_attribute(middleware[0], "openshell.request.failure_reason") + .map(|value| value.as_str()), + reason.map(std::borrow::Cow::Borrowed) + ); + assert_request_outcome( + &evidence, + request_id(&evidence.output.response), + if service_registered { + "denied" + } else { + "failed" + }, + reason, ); assert!( evidence @@ -565,7 +626,11 @@ async fn request_middleware_block_never_creates_a_denial_only_span() { let records = assert_control_records( &evidence, &[("l7", "Allowed"), ("middleware", "Denied")], - &["openshell.middleware.failure"], + if service_registered { + &[] + } else { + &["openshell.middleware.failure"] + }, "/item", ); assert_eq!(records[0]["unmapped"]["http_event_kind"], "decision"); @@ -581,7 +646,16 @@ async fn request_middleware_block_never_creates_a_denial_only_span() { "middleware_invocation" ); assert_eq!(records[1]["unmapped"]["middleware_phase"], "request"); + assert_eq!(records[1]["unmapped"]["failed"], !service_registered); assert_eq!(records[1]["status"], "Failure"); + assert_eq!( + records[1]["status_detail"], + if service_registered { + "middleware_denied:request:test_block" + } else { + "middleware_failed: binding_not_described" + } + ); assert!( records[1]["message"] .as_str() @@ -591,6 +665,89 @@ async fn request_middleware_block_never_creates_a_denial_only_span() { } } +#[tokio::test] +async fn mixed_request_middleware_hooks_fail_the_stage_and_request() { + let fixture = rest_fixture_with_middlewares( + "POST", + r"network_middlewares: + legacy: + middleware: test/block-request + order: 0 + on_error: fail_closed + endpoints: + include: [rest.example.test] + current: + middleware: test/block-request-hooks + order: 1 + on_error: fail_closed + endpoints: + include: [rest.example.test] +", + ); + let registry = openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + vec![ + Arc::new(BlockingRequestMiddleware::default()), + Arc::new(BlockingRequestMiddleware { http_hooks: true }), + ], + vec![], + ) + .await + .expect("register both hook versions"); + let fixture = fixture.with_middleware_runner( + openshell_supervisor_middleware::ChainRunner::from_registry(registry), + ); + let evidence = capture_request_spans( + true, + Box::pin(run_exchange( + fixture, + request("rest.example.test", "/item", "", "{}"), + None, + )), + ) + .await; + + assert!(evidence.output.forwarded.is_empty()); + assert!( + evidence + .output + .response + .starts_with(b"HTTP/1.1 403 Forbidden\r\n"), + "response: {}", + String::from_utf8_lossy(&evidence.output.response) + ); + assert_request_outcome( + &evidence, + request_id(&evidence.output.response), + "failed", + Some("request_middleware_failed"), + ); + let records = assert_control_records( + &evidence, + &[("l7", "Allowed"), ("middleware", "Denied")], + &["openshell.middleware.failure"], + "/item", + ); + assert_eq!( + records[1]["status_detail"], + "middleware_failed: middleware_hook_versions_mixed" + ); + let middleware = evidence + .spans + .iter() + .find(|span| span.name == "openshell.middleware.request") + .expect("request middleware stage"); + assert!(matches!( + middleware.status, + opentelemetry::trace::Status::Error { .. } + )); + assert_eq!( + span_attribute(middleware, "openshell.request.failure_reason") + .unwrap() + .as_str(), + "request_middleware_failed" + ); +} + #[tokio::test] async fn inspected_keep_alive_requests_have_distinct_receipts_without_tracing() { let Fixture { diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs index 3ceb430676..85e3a602b1 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs @@ -494,6 +494,12 @@ where } } +/// Preserve the stage's failure when its local 502 was delivered successfully. +enum ResponseMiddlewareResult { + Relayed(Option), + Failed(&'static str), +} + #[allow(clippy::too_many_arguments)] async fn relay_response_through_middleware( request_method: &str, @@ -546,17 +552,25 @@ where response_correlation, parsed, ); - if let Some(correlation) = response_correlation { - Box::pin(correlation.run_stage( + let response = if let Some(correlation) = response_correlation { + Box::pin(correlation.run_stage_classified( RequestStage::ResponseMiddleware, HttpOutcome::PlatformError, "response_middleware_failed", response, + |result| match result { + ResponseMiddlewareResult::Relayed(_) => None, + ResponseMiddlewareResult::Failed(reason) => Some(*reason), + }, )) .await } else { response.await - } + }?; + Ok(match response { + ResponseMiddlewareResult::Relayed(outcome) => outcome, + ResponseMiddlewareResult::Failed(_) => Some(RelayOutcome::Consumed), + }) } #[allow(clippy::too_many_arguments)] @@ -576,7 +590,7 @@ async fn relay_described_response_through_middleware( Vec, ParsedResponseHead, )>, -) -> Result> +) -> Result where U: AsyncRead + Unpin, C: AsyncWrite + Unpin, @@ -607,7 +621,9 @@ where response_correlation, ) .await?; - return Ok(Some(RelayOutcome::Consumed)); + return Ok(ResponseMiddlewareResult::Failed( + "response_middleware_headers", + )); } }; match openshell_supervisor_middleware::chain_http_hook_version(&described) { @@ -653,7 +669,9 @@ where response_correlation, ) .await?; - return Ok(Some(RelayOutcome::Consumed)); + return Ok(ResponseMiddlewareResult::Failed( + "response_middleware_hook_version", + )); } } let original_headers = parsed.headers.clone(); @@ -705,7 +723,9 @@ where response_correlation, ) .await?; - return Ok(Some(RelayOutcome::Consumed)); + return Ok(ResponseMiddlewareResult::Failed( + "response_middleware_preflight", + )); } }; if let Some(guard) = middleware.generation_guard @@ -769,7 +789,11 @@ where ) .await?; } - return Ok(Some(RelayOutcome::Consumed)); + return Ok(if preflight.denial.is_some() { + ResponseMiddlewareResult::Relayed(Some(RelayOutcome::Consumed)) + } else { + ResponseMiddlewareResult::Failed("response_middleware_preflight") + }); } crate::l7::correlation::emit_request_events( response_correlation, @@ -783,7 +807,7 @@ where let Some(mut session) = preflight.session else { if preflight.headers == original_headers { - return Ok(None); + return Ok(ResponseMiddlewareResult::Relayed(None)); } let status_line = response_status_line(header_bytes)?; let outcome = relay_headers_only_response( @@ -803,7 +827,7 @@ where ) .await?; record_allowed_response(response_correlation); - return Ok(Some(outcome)); + return Ok(ResponseMiddlewareResult::Relayed(Some(outcome))); }; let status_line = response_status_line(header_bytes)?; @@ -840,7 +864,9 @@ where response_correlation, ) .await?; - return Ok(Some(RelayOutcome::Consumed)); + return Ok(ResponseMiddlewareResult::Failed( + "response_middleware_finish", + )); } }; if let Some(guard) = middleware.generation_guard { @@ -872,11 +898,13 @@ where client.write_all(&head).await.into_diagnostic()?; client.flush().await.into_diagnostic()?; record_allowed_response(response_correlation); - return Ok(Some(if server_wants_close { - RelayOutcome::Consumed - } else { - RelayOutcome::Reusable - })); + return Ok(ResponseMiddlewareResult::Relayed(Some( + if server_wants_close { + RelayOutcome::Consumed + } else { + RelayOutcome::Reusable + }, + ))); } let whole_body = session.requires_whole_body(); @@ -1024,7 +1052,13 @@ where ) .await?; } - return Ok(Some(RelayOutcome::Consumed)); + return Ok( + if middleware_stop.is_some_and(|stop| stop.failure.denial.is_some()) { + ResponseMiddlewareResult::Relayed(Some(RelayOutcome::Consumed)) + } else { + ResponseMiddlewareResult::Failed("response_middleware_body") + }, + ); } }; @@ -1100,7 +1134,11 @@ where ) .await?; } - return Ok(Some(RelayOutcome::Consumed)); + return Ok(if error.denial.is_some() { + ResponseMiddlewareResult::Relayed(Some(RelayOutcome::Consumed)) + } else { + ResponseMiddlewareResult::Failed("response_middleware_finish") + }); } }; if let Some(guard) = middleware.generation_guard { @@ -1175,7 +1213,7 @@ where } client.flush().await.into_diagnostic()?; record_allowed_response(response_correlation); - Ok(Some( + Ok(ResponseMiddlewareResult::Relayed(Some( if server_wants_close || (committed && close_delimited_output) || (matches!(body_length, BodyLength::None) && event_stream) @@ -1184,7 +1222,7 @@ where } else { RelayOutcome::Reusable }, - )) + ))) } #[allow(clippy::too_many_arguments)] @@ -3095,6 +3133,7 @@ mod tests { #[tokio::test] async fn response_middleware_block_never_creates_a_denial_only_span() { use crate::l7::relay::correlation_dispatch_tests::capture_request_spans; + use crate::test_support::request_tracing::span_attribute; for sampled in [true, false] { let evidence = Box::pin(capture_request_spans( @@ -3105,18 +3144,81 @@ mod tests { evidence.assert_no_deny_span(sampled); assert_response_middleware_block_record(&evidence.events, sampled); if sampled { - assert_eq!( - evidence - .spans - .iter() - .filter(|span| span.name == "openshell.middleware.response") - .count(), - 1 + let middleware: Vec<_> = evidence + .spans + .iter() + .filter(|span| span.name == "openshell.middleware.response") + .collect(); + assert_eq!(middleware.len(), 1); + assert_eq!(middleware[0].status, opentelemetry::trace::Status::Ok); + assert!( + span_attribute(middleware[0], "openshell.request.failure_reason").is_none() ); } } } + #[tokio::test] + async fn response_middleware_operational_failure_marks_child_error_after_delivering_502() { + use crate::l7::correlation::RequestStage; + use crate::l7::relay::correlation_dispatch_tests::capture_request_spans; + use crate::test_support::request_tracing::span_attribute; + + let evidence = Box::pin(capture_request_spans(true, async { + let correlation = test_correlation("response-failed"); + let (runner, chain) = + response_middleware_fixture(ResponseRelayScript::InvalidWholeBodySequence); + let middleware = response_middleware_context(&runner, &chain, "GET"); + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody"[..]; + let mut client = Vec::new(); + let outcome = Box::pin(correlation.run_stage( + RequestStage::Response, + HttpOutcome::PlatformError, + "response_relay_failed", + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ), + )) + .await + .expect("local failure response is delivered"); + assert!(matches!(outcome, RelayOutcome::Consumed)); + assert!(client.starts_with(b"HTTP/1.1 502 Bad Gateway\r\n")); + })) + .await; + + evidence.assert_no_deny_span(true); + let middleware: Vec<_> = evidence + .spans + .iter() + .filter(|span| span.name == "openshell.middleware.response") + .collect(); + assert_eq!(middleware.len(), 1); + assert!(matches!( + middleware[0].status, + opentelemetry::trace::Status::Error { .. } + )); + assert_eq!( + span_attribute(middleware[0], "openshell.request.failure_reason") + .unwrap() + .as_str(), + "response_middleware_finish" + ); + let response = evidence + .spans + .iter() + .find(|span| span.name == "openshell.http.response") + .expect("response delivery stage"); + assert_eq!(response.status, opentelemetry::trace::Status::Ok); + assert!(span_attribute(response, "openshell.request.failure_reason").is_none()); + } + #[tokio::test] async fn upstream_receipt_is_replaced_by_platform_receipt() { let correlation = test_correlation("trusted"); diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs index 2c6efa78fb..3e638c042a 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response/http_hooks.rs @@ -109,7 +109,7 @@ pub(super) async fn relay_response_through_pipeline( server_wants_close: bool, event_stream: bool, response_correlation: Option<&RequestCorrelation>, -) -> Result> +) -> Result where U: AsyncRead + Unpin, C: AsyncWrite + Unpin, @@ -179,7 +179,9 @@ where response_correlation, ) .await?; - return Ok(Some(RelayOutcome::Consumed)); + return Ok(ResponseMiddlewareResult::Failed( + "middleware_response_preflight", + )); } }; if let Some(guard) = middleware.generation_guard @@ -235,12 +237,16 @@ where ) .await?; } - return Ok(Some(RelayOutcome::Consumed)); + return Ok(if preflight.denial.is_some() { + ResponseMiddlewareResult::Relayed(Some(RelayOutcome::Consumed)) + } else { + ResponseMiddlewareResult::Failed("middleware_response_preflight") + }); } let Some(session) = preflight.pipeline.take() else { if preflight.headers == original_headers { - return Ok(None); + return Ok(ResponseMiddlewareResult::Relayed(None)); } let outcome = relay_headers_only_response( request_method, @@ -259,7 +265,7 @@ where ) .await?; record_allowed_response(response_correlation); - return Ok(Some(outcome)); + return Ok(ResponseMiddlewareResult::Relayed(Some(outcome))); }; // Eligibility offers a bodyless response no body mode, so no stage can // have selected one. @@ -285,7 +291,9 @@ where response_correlation, ) .await?; - return Ok(Some(RelayOutcome::Consumed)); + return Ok(ResponseMiddlewareResult::Failed( + "middleware_response_preflight", + )); } let unit_limit = session.input_unit_limit(); @@ -362,7 +370,7 @@ where })?; } record_allowed_response(response_correlation); - return Ok(Some( + return Ok(ResponseMiddlewareResult::Relayed(Some( if server_wants_close || framing == Some(OutputFraming::CloseDelimited) || (matches!(body_length, BodyLength::None) && event_stream) @@ -371,7 +379,7 @@ where } else { RelayOutcome::Reusable }, - )); + ))); } (Err(failure), relayed) => { emit_http_response_stage_events( @@ -437,7 +445,7 @@ where response_correlation, ) .await?; - Ok(Some(RelayOutcome::Consumed)) + Ok(ResponseMiddlewareResult::Failed(stage)) } Ok(failure) if committed => { if let Some(correlation) = response_correlation { @@ -496,7 +504,11 @@ where ) .await?; } - Ok(Some(RelayOutcome::Consumed)) + Ok(if failure.denial.is_some() { + ResponseMiddlewareResult::Relayed(Some(RelayOutcome::Consumed)) + } else { + ResponseMiddlewareResult::Failed("middleware_response_finalize") + }) } } } From d6784dd223d3421727055e5e01f5624500930354 Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Mon, 16 Nov 2026 17:30:39 -0700 Subject: [PATCH 7/7] fix(observability): complete proxy request correlation paths Signed-off-by: Alex Fournier --- crates/openshell-supervisor-network/README.md | 18 +- .../src/l7/correlation.rs | 27 ++ .../src/l7/correlation/tests.rs | 101 +++++ .../src/l7/relay.rs | 41 +- .../openshell-supervisor-network/src/proxy.rs | 20 +- .../src/proxy/tests/request_trace.rs | 372 +++++++++++++++++- .../openshell-supervisor-network/src/run.rs | 1 - docs/observability/ocsf-json-export.mdx | 33 +- 8 files changed, 562 insertions(+), 51 deletions(-) diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index ca8520f571..dd8af76169 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -66,11 +66,10 @@ L4 and passthrough traffic do not provide this header guarantee. The emission hook adds request correlation to every OCSF class in the current request scope, including findings. Policy, middleware, token-grant and finding -records retain their original content, event IDs, timestamps and classification; -they are not merged or delayed. `unmapped.http_event_kind` distinguishes decisions, -findings, connection events and middleware invocations. Explicit control kinds, -such as `token_grant`, remain unchanged. Middleware records retain -`unmapped.middleware_phase`, including HTTP-hook preflight and completion. +records are individual events with their own timestamps and classification. +`unmapped.http_event_kind` identifies decisions, findings, connection events, +middleware invocations and token grants. Middleware invocations use +`unmapped.middleware_phase` for request, response and WebSocket phases. `unmapped.request_sequence` starts at 1 per request and describes local emission order, not downstream delivery order. Count policy outcomes by selecting the actual policy/firewall control record within a receipt group, not every @@ -80,10 +79,11 @@ Request IDs and audit outcomes do not depend on OpenTelemetry, sampling or collector availability. Inspected requests have a supervisor-owned request span and children around stages that actually run, with valid workload W3C context only as an untrusted link. Their records never create an extra denial-only span, -even if the request span is disabled or unsampled. Other egress paths retain the -bounded deny-span behavior. Both HTTP tracing and the request's ambient -audit scope end before an upgraded session; later WebSocket frames do not reuse -the handshake receipt. Terminal errors retain their request context explicitly, +even if the request span is disabled or unsampled. Uninspected HTTP denials use +bounded `supervisor.egress.deny` spans. The ambient HTTP audit scope ends before +protocol-start validation; one-shot attribution joins the existing upgrade +handoff record to the handshake and releases the request span before frames. +Later WebSocket frames do not reuse the handshake receipt. Terminal errors carry their request context explicitly, and policy-reload cancellation records the active request before dropping it. Idle connection closures have no HTTP receipt. The request span ends after its owned stages, not when response headers are logged. diff --git a/crates/openshell-supervisor-network/src/l7/correlation.rs b/crates/openshell-supervisor-network/src/l7/correlation.rs index a97f67f966..1d4cfe36fa 100644 --- a/crates/openshell-supervisor-network/src/l7/correlation.rs +++ b/crates/openshell-supervisor-network/src/l7/correlation.rs @@ -177,6 +177,22 @@ pub fn emit_error_event(event: OcsfEvent, error: &miette::Report) { } } +/// Retain attribution for one explicit handoff record, not ambient protocol +/// traffic. The request span stays alive only until this value emits or drops. +#[derive(Debug)] +pub struct DetachedRequestEvent { + scope: Arc, + span: tracing::Span, +} + +impl DetachedRequestEvent { + pub(crate) fn emit(self, event: OcsfEvent) { + in_request_scope(&self.scope, || { + crate::telemetry::emit_inspected_decision([event], Some(&self.span)); + }); + } +} + #[derive(Debug)] pub struct RequestCorrelation { scope: Arc, @@ -360,6 +376,17 @@ impl RequestCorrelation { drop(self); } + /// Restore the ambient scope now, retaining the request span only for the + /// existing upgrade handoff record after protocol-start validation. + pub(crate) fn finish_http_request_for_event(self) -> Option { + let event = self.scope.inspected.then(|| DetachedRequestEvent { + scope: Arc::clone(&self.scope), + span: self.scope.span(), + }); + self.finish_http_request(); + event + } + fn finish_http_trace(&self) { let trace = self .scope diff --git a/crates/openshell-supervisor-network/src/l7/correlation/tests.rs b/crates/openshell-supervisor-network/src/l7/correlation/tests.rs index d394c69ba6..b7c7dca6cb 100644 --- a/crates/openshell-supervisor-network/src/l7/correlation/tests.rs +++ b/crates/openshell-supervisor-network/src/l7/correlation/tests.rs @@ -313,6 +313,107 @@ async fn finishing_http_restores_the_previous_scope_before_protocol_relay() { assert!(records[2]["unmapped"]["request_sequence"].is_null()); } +#[tokio::test] +async fn detached_handoff_preserves_only_its_explicit_record_and_ends_before_frames() { + let capture = RequestTracing::new(Some(opentelemetry_sdk::trace::Sampler::AlwaysOn)); + scope_connection(async { + let correlation = + RequestCorrelation::with_request_id(&request(), &context(), "handshake".into()); + correlation.emit(allowed()); + correlation.record_response_status(101); + let handoff = correlation.finish_http_request_for_event().unwrap(); + assert!(!is_inspected_request()); + assert!(current_request_span().is_none()); + + // Protocol-start work must not inherit the detached HTTP context. + crate::telemetry::emit_decision([allowed()], None); + tokio::task::yield_now().await; + handoff.emit(allowed()); + assert!(!is_inspected_request()); + assert!(current_request_span().is_none()); + + // The retained root is exported before any longer-lived frame relay. + let spans = capture.spans(); + let root = spans + .iter() + .find(|span| span.name == "openshell.http.request") + .unwrap(); + assert_eq!( + span_attribute(root, "openshell.request.outcome") + .unwrap() + .as_str(), + "upgraded" + ); + crate::telemetry::emit_decision([allowed()], None); + }) + .with_subscriber(capture.subscriber()) + .await; + let events = capture.events.json_events(); + assert_eq!(events.len(), 4); + assert_eq!(events[0]["metadata"]["correlation_uid"], "handshake"); + assert_eq!(events[0]["unmapped"]["request_sequence"], 1); + assert_eq!(events[2]["metadata"]["correlation_uid"], "handshake"); + assert_eq!(events[2]["unmapped"]["request_sequence"], 2); + assert!(events[0]["trace"]["uid"].is_string()); + assert_eq!(events[0]["trace"], events[2]["trace"]); + for index in [1, 3] { + assert!(events[index]["metadata"]["correlation_uid"].is_null()); + assert!(events[index]["unmapped"]["request_sequence"].is_null()); + assert!(events[index]["unmapped"]["http_event_kind"].is_null()); + assert!(events[index]["trace"].is_null()); + } + assert_eq!(capture.created_span_names(), ["openshell.http.request"]); + capture.shutdown(); +} + +#[tokio::test] +async fn cancelling_a_detached_handoff_ends_the_root_without_emitting_or_leaking() { + let capture = RequestTracing::new(Some(opentelemetry_sdk::trace::Sampler::AlwaysOn)); + let mut pending = Box::pin( + scope_connection(async { + let correlation = RequestCorrelation::new(&request(), &context()); + correlation.record_response_status(101); + let handoff = correlation.finish_http_request_for_event().unwrap(); + assert!(!is_inspected_request()); + std::future::pending::<()>().await; + handoff.emit(allowed()); + }) + .with_subscriber(capture.subscriber()), + ); + assert!(futures::poll!(&mut pending).is_pending()); + assert!(!is_inspected_request()); + drop(pending); + assert_eq!(capture.spans().len(), 1); + assert!(capture.events.json_events().is_empty()); + tracing::subscriber::with_default(capture.subscriber(), || { + crate::telemetry::emit_decision([allowed()], None); + }); + let events = capture.events.json_events(); + assert_eq!(events.len(), 1); + assert!(events[0]["metadata"]["correlation_uid"].is_null()); + assert!(events[0]["unmapped"]["request_sequence"].is_null()); + assert!(events[0]["trace"].is_null()); + capture.shutdown(); +} + +#[tokio::test] +async fn an_uninspected_handoff_has_no_detached_attribution() { + let capture = OcsfCapture::default(); + let _subscriber = capture.install(); + scope_connection(async { + let correlation = RequestCorrelation::uninspected(); + assert!(correlation.finish_http_request_for_event().is_none()); + assert!(!is_inspected_request()); + crate::telemetry::emit_decision([allowed()], None); + }) + .await; + let events = capture.json_events(); + assert_eq!(events.len(), 1); + assert!(events[0]["metadata"]["correlation_uid"].is_null()); + assert!(events[0]["unmapped"]["request_sequence"].is_null()); + assert!(events[0]["unmapped"]["http_event_kind"].is_null()); +} + #[tokio::test] async fn stage_results_record_bounded_errors_and_preserve_a_failed_root() { use opentelemetry::trace::Status; diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index f3f47c7555..9725ef0ea5 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -7,7 +7,7 @@ //! Parses each request within the tunnel, evaluates it against OPA policy, //! and either forwards or denies the request. -use crate::l7::correlation::{HttpOutcome, RequestCorrelation, RequestStage}; +use crate::l7::correlation::{DetachedRequestEvent, HttpOutcome, RequestCorrelation, RequestStage}; use crate::l7::middleware::{ MiddlewareApplyResult, RequestBodyDelivery, UninspectableTrafficGate, apply_middleware_chain_with_request_id_and_delivery_correlated, @@ -1272,6 +1272,7 @@ pub(crate) struct UpgradeRelayOptions<'a> { pub(crate) policy_name: String, pub(crate) middleware_session: Option, pub(crate) selected_subprotocol: Option, + pub(crate) handshake_event: Option, } #[derive(Default)] @@ -2094,7 +2095,7 @@ where options.websocket.permessage_deflate = websocket_permessage_deflate; options.middleware_session = middleware_session.take(); options.selected_subprotocol = websocket_subprotocol; - correlation.finish_http_request(); + options.handshake_event = correlation.finish_http_request_for_event(); return handle_upgrade( client, upstream, overflow, &ctx.host, ctx.port, options, ) @@ -2325,18 +2326,21 @@ where } else { "raw bidirectional relay (L7 enforcement no longer active)" }; - egress_emit!( - NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .activity_name("Upgrade") - .severity(SeverityId::Informational) - .dst_endpoint(Endpoint::from_domain(host, port)) - .message(format!( - "101 Switching Protocols — {relay_mode} [host:{host} port:{port} overflow_bytes:{}]", - overflow.len() - )) - .build() - ); + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Other) + .activity_name("Upgrade") + .severity(SeverityId::Informational) + .dst_endpoint(Endpoint::from_domain(host, port)) + .message(format!( + "101 Switching Protocols — {relay_mode} [host:{host} port:{port} overflow_bytes:{}]", + overflow.len() + )) + .build(); + if let Some(handshake_event) = options.handshake_event.take() { + handshake_event.emit(event); + } else { + egress_emit!(event); + } if use_websocket_relay { let assembly_budget = options.assembly_budget.take().unwrap_or_default(); let resolver = if options.websocket.credential_rewrite { @@ -2489,6 +2493,7 @@ pub(crate) fn upgrade_options<'a>( policy_name: ctx.policy_name.clone(), middleware_session: None, selected_subprotocol: None, + handshake_event: None, } } @@ -2961,7 +2966,7 @@ where options.websocket.permessage_deflate = websocket_permessage_deflate; options.middleware_session = middleware_session.take(); options.selected_subprotocol = websocket_subprotocol; - correlation.finish_http_request(); + options.handshake_event = correlation.finish_http_request_for_event(); return handle_upgrade( client, upstream, overflow, &ctx.host, ctx.port, options, ) @@ -4660,14 +4665,16 @@ where RelayOutcome::Reusable => {} // continue loop RelayOutcome::Consumed => break, RelayOutcome::Upgraded { overflow, .. } => { - correlation.finish_http_request(); return handle_upgrade( client, upstream, overflow, &ctx.host, ctx.port, - UpgradeRelayOptions::default(), + UpgradeRelayOptions { + handshake_event: correlation.finish_http_request_for_event(), + ..Default::default() + }, ) .await; } diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index fe5067a77e..be255a423b 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -323,6 +323,8 @@ impl ProxyHandle { required_proxy_authorization: Option>, reserved_destination: Option, ) -> Result { + crate::l7::correlation::init_request_correlation()?; + // Use override bind_addr, fall back to policy http_addr, then default // to loopback:3128. The default allows the proxy to function when no // network namespace is available (e.g. missing CAP_NET_ADMIN) and the @@ -589,11 +591,14 @@ impl ProxyHandle { ) .await { - egress_emit!(build_proxy_connection_error_event( - peer_addr, - transparent_destination_addr, - format!("Proxy connection error: {err}"), - )); + crate::l7::correlation::emit_error_event( + build_proxy_connection_error_event( + peer_addr, + transparent_destination_addr, + format!("Proxy connection error: {err}"), + ), + &err, + ); } }); } @@ -6693,7 +6698,7 @@ async fn handle_forward_proxy_inner( } else { HttpOutcome::PlatformError }, - Some("request_middleware_denied"), + denial.is_none().then_some("request_middleware_failed"), ); let response = denial.as_ref().map_or_else( || { @@ -7304,7 +7309,7 @@ async fn handle_forward_proxy_inner( websocket_permessage_deflate, websocket_subprotocol, } => { - correlation.finish_http_request(); + let handshake_event = correlation.finish_http_request_for_event(); // Protocols whose rules apply to individual HTTP requests never // upgrade (see `upgrade_refusal_for_protocol`). No current path // forwards upgrade headers for them: the request-side refusal @@ -7347,6 +7352,7 @@ async fn handle_forward_proxy_inner( ..Default::default() } }; + upgrade_options.handshake_event = handshake_event; upgrade_options.generation_guard = Some(&forward_generation_guard); upgrade_options.assembly_budget = Some(opa_engine.websocket_assembly_budget()); upgrade_options.websocket.permessage_deflate = websocket_permessage_deflate; diff --git a/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs b/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs index e1a6d88820..86aa709cec 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs @@ -39,6 +39,294 @@ fn assert_request_record_sequence(records: &[serde_json::Value], receipt: &str) } } +#[tokio::test] +async fn shared_proxy_startup_registers_correlation_and_terminal_errors() { + use crate::l7::tls::{CertCache, SandboxCa, tls_terminate_client}; + use openshell_isolation_interface::contract::DuplexStream; + use rustls::pki_types::{CertificateDer, ServerName, pem::PemObject}; + + const CHILD: &str = "OPENSHELL_REQUEST_CORRELATION_STARTUP_CHILD"; + const COMPLETED: &str = "shared proxy startup and terminal regressions completed"; + if std::env::var_os(CHILD).is_none() { + // A fresh process prevents scope_connection's cfg(test) auto-init (or + // another fixture) from hiding a missing production startup hook. + let module = module_path!().split_once("::").unwrap().1; + let output = tokio::time::timeout( + std::time::Duration::from_secs(20), + tokio::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + &format!( + "{module}::shared_proxy_startup_registers_correlation_and_terminal_errors" + ), + "--nocapture", + ]) + .env(CHILD, "1") + .kill_on_drop(true) + .output(), + ) + .await + .expect("isolated startup regression must finish") + .expect("start isolated startup regression"); + assert!( + output.status.success() && String::from_utf8_lossy(&output.stdout).contains(COMPLETED), + "isolated startup regression failed:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + return; + } + + let ip = non_loopback_test_ipv4() + .expect("shared proxy regression requires a non-loopback IPv4 address"); + let listener = TcpListener::bind((ip, 0)).await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let engine = Arc::new( + native_identity_engine( + include_str!("../../../data/sandbox-policy.rego"), + &format!( + r#" +network_policies: + traced_api: + name: traced_api + endpoints: + - host: "{ip}" + port: {port} + allowed_ips: ["{ip}/32"] + protocol: rest + enforcement: enforce + rules: + - allow: {{ method: GET, path: /allowed }} + binaries: + - path: /usr/bin/python3 +"# + ), + ) + .unwrap(), + ); + let ca = SandboxCa::generate().unwrap(); + let mut roots = rustls::RootCertStore::empty(); + for cert in CertificateDer::pem_reader_iter(&mut ca.cert_pem().as_bytes()) { + roots.add(cert.unwrap()).unwrap(); + } + let client_config = Arc::new( + openshell_crypto::tls::client_builder() + .with_root_certificates(roots) + .with_no_client_auth(), + ); + let tls = Arc::new(ProxyTlsState::new( + CertCache::new(ca), + client_config.clone(), + )); + let capture = RequestTracing::new(Some(Sampler::AlwaysOn)); + // Production accept/connection tasks are spawned independently; the + // isolated process gives them the same subscriber without test wrappers. + tracing::subscriber::set_global_default( + capture + .subscriber() + .with(openshell_otel::supervisor_span_filter("info")), + ) + .unwrap(); + let (ready_tx, ready_rx) = tokio::sync::watch::channel(false); + let proxy = ProxyHandle::start_with_bind_addr( + &ProxyPolicy { http_addr: None }, + Some(([127, 0, 0, 1], 0).into()), + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(0)), + Some(tls.clone()), + None, + None, + None, + None, + None, + ready_rx, + &upstream_proxy::UpstreamProxyArgs::default(), + None, + None, + None, + Some(ContractBinaryIdentity { + executable: ContractExecutableIdentity { + path: executable_path("/usr/bin/python3"), + digest: Some("00".repeat(32).parse().unwrap()), + }, + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + }), + None, + None, + ) + .await + .unwrap(); + assert!( + !openshell_ocsf::register_event_enricher(|_| {}), + "shared proxy startup must register correlation before the first request" + ); + ready_tx.send(true).unwrap(); + + let scenario = async { + // One exchange per real terminal emission boundary, not a sampler or + // transport matrix. Each fails only after the client receives its ID. + for (connect, encrypted, terminal_prefix) in [ + (false, false, "Proxy connection error: "), + (true, false, "L7 relay error: "), + (true, true, "TLS relay error: "), + ] { + let (release, released) = tokio::sync::oneshot::channel(); + let serve = async { + let (upstream, _) = listener.accept().await.unwrap(); + let hostname = ip.to_string(); + let mut upstream: Box = if encrypted { + Box::new( + tls_terminate_client(upstream, &tls, &hostname) + .await + .unwrap(), + ) + } else { + Box::new(upstream) + }; + let request = read_http_headers(&mut upstream).await; + assert!(String::from_utf8_lossy(&request).starts_with("GET /allowed HTTP/1.1\r\n")); + upstream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\n") + .await + .unwrap(); + upstream.flush().await.unwrap(); + released.await.unwrap(); + upstream.write_all(b"ok").await.unwrap(); + // Clean TLS EOF too: the HTTP declared-length failure must + // reach the non-benign outer relay-error callsite. + upstream.shutdown().await.unwrap(); + }; + let exchange = async { + let mut client = connect_tcp_nodelay_best_effort(proxy.http_addr().unwrap()) + .await + .unwrap(); + if connect { + client + .write_all( + format!("CONNECT {ip}:{port} HTTP/1.1\r\nHost: {ip}:{port}\r\n\r\n") + .as_bytes(), + ) + .await + .unwrap(); + let established = read_http_headers(&mut client).await; + assert!(established.starts_with(b"HTTP/1.1 200 Connection Established\r\n")); + } + let mut client: BoundaryDuplexStream = if encrypted { + Box::new( + tokio_rustls::TlsConnector::from(client_config.clone()) + .connect(ServerName::try_from(ip.to_string()).unwrap(), client) + .await + .unwrap(), + ) + } else { + Box::new(client) + }; + let target = if connect { + "/allowed".to_string() + } else { + format!("http://{ip}:{port}/allowed") + }; + client + .write_all( + format!("GET {target} HTTP/1.1\r\nHost: {ip}:{port}\r\ntraceparent: 00-{WORKLOAD_TRACE_ID}-{WORKLOAD_SPAN_ID}-01\r\nx-openshell-request-id: forged-receipt\r\nConnection: close\r\n\r\n").as_bytes(), + ) + .await + .unwrap(); + client.flush().await.unwrap(); + let response = read_http_headers(&mut client).await; + assert!(response.starts_with(b"HTTP/1.1 200 OK\r\n")); + let receipt = response_request_id(&response).to_string(); + assert!(uuid::Uuid::parse_str(&receipt).is_ok()); + release.send(()).unwrap(); + let mut body = Vec::new(); + // The downstream TLS stream may omit close_notify when its + // failed HTTP relay drops it. The captured event is the oracle. + let _ = client.read_to_end(&mut body).await; + assert_eq!(body, b"ok"); + receipt + }; + let ((), receipt) = tokio::join!(serve, exchange); + let (records, root) = tokio::time::timeout(std::time::Duration::from_secs(2), async { + loop { + let records = capture.events.json_events(); + let root = capture.spans().into_iter().find(|span| { + span.name == "openshell.http.request" + && span_attribute(span, "openshell.request.id") + .is_some_and(|value| value.as_str() == receipt) + }); + if let Some(root) = root + && records.iter().any(|record| { + record["message"] + .as_str() + .is_some_and(|message| message.starts_with(terminal_prefix)) + && record["metadata"]["correlation_uid"] == receipt + }) + { + break (records, root); + } + tokio::task::yield_now().await; + } + }) + .await + .expect("actual outer terminal record must retain the failed request receipt"); + let correlated = records + .iter() + .filter(|record| record["metadata"]["correlation_uid"] == receipt) + .collect::>(); + assert_eq!(correlated.len(), 2, "{records:?}"); + for (index, record) in correlated.iter().enumerate() { + assert_eq!( + record["unmapped"]["request_sequence"], + index + 1, + "{records:?}" + ); + } + let terminal = correlated.last().unwrap(); + assert!( + terminal["message"] + .as_str() + .unwrap() + .starts_with(terminal_prefix) + ); + assert!( + terminal["message"] + .as_str() + .unwrap() + .contains("2 bytes remaining") + ); + assert_eq!(terminal["class_uid"], 4001); + assert_eq!(terminal["unmapped"]["http_event_kind"], "connection"); + assert_ne!(receipt, "forged-receipt"); + assert_ne!(root.span_context.trace_id().to_string(), WORKLOAD_TRACE_ID); + assert_eq!(root.parent_span_id, opentelemetry::trace::SpanId::INVALID); + assert_eq!(root.links.links.len(), 1); + assert!(matches!( + root.status, + opentelemetry::trace::Status::Error { .. } + )); + for decision in &correlated[..correlated.len() - 1] { + assert_eq!( + decision["trace"]["uid"], + root.span_context.trace_id().to_string() + ); + } + } + }; + tokio::time::timeout(std::time::Duration::from_secs(10), scenario) + .await + .expect("real proxy terminal-error exchanges must finish"); + assert!( + !capture + .created_span_names() + .contains(&"supervisor.egress.deny") + ); + drop(proxy); + capture.shutdown(); + println!("{COMPLETED}"); +} + async fn forward_case(ip: Ipv4Addr, outcome: Outcome, sampler: Option) { let listener = TcpListener::bind((ip, 0)).await.unwrap(); let port = listener.local_addr().unwrap().port(); @@ -476,11 +764,25 @@ fn inspected_forward_error_receipts_preserve_json_bodies() { #[tokio::test] async fn inspected_forward_http_trace_ends_before_websocket_session() { + forward_websocket_trace_case(true).await; +} + +#[tokio::test] +async fn inspected_forward_websocket_frame_denial_stays_outside_http_scope() { + forward_websocket_trace_case(false).await; +} + +async fn forward_websocket_trace_case(allow_frame: bool) { let _lock = openshell_otel_test_support::tracing_test_lock().await; let ip = non_loopback_test_ipv4() .expect("FORWARD trace qualification requires a non-loopback IPv4 address"); let listener = TcpListener::bind((ip, 0)).await.unwrap(); let port = listener.local_addr().unwrap().port(); + let frame_rule = if allow_frame { + " - allow: { method: WEBSOCKET_TEXT, path: /ws }" + } else { + "" + }; let engine = Arc::new( native_identity_engine( include_str!("../../../data/sandbox-policy.rego"), @@ -497,7 +799,7 @@ network_policies: enforcement: enforce rules: - allow: {{ method: GET, path: /ws }} - - allow: {{ method: WEBSOCKET_TEXT, path: /ws }} +{frame_rule} binaries: - path: /usr/bin/python3 "# @@ -566,9 +868,19 @@ network_policies: .unwrap(); handshake.send(()).unwrap(); let frame = crate::l7::websocket::read_frame_for_test(&mut upstream).await; - assert_eq!(frame, masked_text_frame(b"after-http-root")); + if allow_frame { + assert_eq!(frame, masked_text_frame(b"after-http-root")); + upstream.write_all(b"\x81\x02ok").await.unwrap(); + } else { + assert_eq!(masked_close_code(&frame), 1008); + let mut remaining = Vec::new(); + upstream.read_to_end(&mut remaining).await.unwrap(); + assert!( + remaining.is_empty(), + "blocked text must never reach upstream, before or after the policy close" + ); + } activity.send(std::time::SystemTime::now()).unwrap(); - upstream.write_all(b"\x81\x02ok").await.unwrap(); released.await.unwrap(); }); let scenario = async { @@ -608,12 +920,26 @@ network_policies: assert!(root.end_time <= observed_activity.await.unwrap()); let mut frame = [0; 4]; app.read_exact(&mut frame).await.unwrap(); - assert_eq!(&frame, b"\x81\x02ok"); - assert!(!handler.is_finished()); + if allow_frame { + assert_eq!(&frame, b"\x81\x02ok"); + assert!(!handler.is_finished()); + } else { + assert_eq!(&frame, b"\x88\x02\x03\xf0"); + } release.send(()).unwrap(); drop(app); upstream.await.unwrap(); - handler.await.unwrap().unwrap(); + let result = handler.await.unwrap(); + if allow_frame { + result.unwrap(); + } else { + assert!( + result + .expect_err("denied frame must terminate the relay") + .to_string() + .contains("websocket text message denied by policy") + ); + } let records = capture.events.json_events(); assert_eq!(records.len(), 4, "{records:?}"); assert_request_record_sequence(&records[..2], receipt); @@ -653,20 +979,50 @@ network_policies: ); assert!(records[2].get("http_response").is_none()); assert!(records[2].get("http_request").is_none()); + assert_eq!(records[2]["metadata"]["correlation_uid"], receipt); + assert_eq!(records[2]["unmapped"]["request_sequence"], 3); + assert_eq!(records[2]["unmapped"]["http_event_kind"], "connection"); + // NetworkActivity has no trace field; the receipt ties the unchanged + // 101 record to the completed HTTP root without changing its schema. + assert!(records[2].get("trace").is_none()); + assert_eq!( + span_attribute(&root, "openshell.request.id") + .unwrap() + .as_str(), + receipt + ); assert_eq!(records[3]["class_uid"], 4001); assert!( records[3]["message"] .as_str() .unwrap() - .starts_with("WEBSOCKET_L7_REQUEST allow WEBSOCKET_TEXT ") + .starts_with(if allow_frame { + "WEBSOCKET_L7_REQUEST allow WEBSOCKET_TEXT " + } else { + "WEBSOCKET_L7_REQUEST deny WEBSOCKET_TEXT " + }) ); assert_eq!(records[3]["firewall_rule"]["type"], "l7-websocket"); - for record in &records[2..] { + assert_eq!(records[3]["action_id"], if allow_frame { 1 } else { 2 }); + for record in &records[3..] { assert!(record["metadata"].get("correlation_uid").is_none()); assert!(record["unmapped"].get("request_sequence").is_none()); assert!(record["unmapped"].get("http_event_kind").is_none()); assert!(record.get("trace").is_none()); } + let created = capture.created_span_names(); + assert_eq!( + created + .iter() + .filter(|name| **name == "openshell.http.request") + .count(), + 1, + "WebSocket frames must not create HTTP request spans" + ); + assert!( + !created.contains(&"supervisor.egress.deny"), + "WebSocket NetworkActivity decisions must not gain HTTP denial spans" + ); assert!(!format!("{records:?}").contains("after-http-root")); }; tokio::time::timeout(std::time::Duration::from_secs(5), scenario) diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 8a151c4090..0c342c6bab 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -298,7 +298,6 @@ pub async fn run_networking( proxy_listener: Option, reserved_destination: Option, ) -> Result { - crate::l7::correlation::init_request_correlation()?; // Build the policy-local route context. The orchestrator's policy poll // loop also holds an `Arc` clone (via `Networking::policy_local_ctx`) so // it can publish updated policy snapshots after a successful reload. diff --git a/docs/observability/ocsf-json-export.mdx b/docs/observability/ocsf-json-export.mdx index f0092e98da..42ab054bd1 100644 --- a/docs/observability/ocsf-json-export.mdx +++ b/docs/observability/ocsf-json-export.mdx @@ -317,7 +317,7 @@ The request span ends after its owned stages, not when response headers are logg The supervisor owns the request trace. A valid workload `traceparent`, sampled or unsampled, becomes an explicitly untrusted link, never its parent. Invalid or duplicate context is ignored; workload `tracestate` is not adopted. Your backend may show the agent and supervisor as separate linked traces rather than one nested tree. -Inspected requests do not also create a denial-only `supervisor.egress.deny` span. Non-inspected denials use that span: a root at the default log level, or a child of an available connect span at debug level. Shared supervisor hosts do not initialize OTLP exporters, so their records carry no trace correlation. +Inspected requests do not also create a denial-only `supervisor.egress.deny` span. Non-inspected HTTP denials use that span: a root at the default log level, or a child of an available connect span at debug level. Shared supervisor hosts do not initialize OTLP exporters, so their records have no supervisor trace IDs. Request ID correlation works independently of those exporters. The non-inspected denial-span limiter allows at most 10 spans per destination at once and one more per second after that. A process-wide budget also allows at most 100 spans at once and refills at 100 per second across destinations. Rate-limited decisions create no additional deny span, while all audit records are still emitted. These denial-specific budgets do not select inspected request spans, which cover every outcome rather than denials alone. @@ -337,13 +337,27 @@ authorization or downstream execution. L4 and uninspected traffic do not provide this receipt guarantee. OpenShell does not add the receipt to JSON error bodies. All OCSF event classes emitted in the inspected request scope carry the request -ID, including Detection Findings. Policy, middleware, token-grant and finding -records remain separate, with their original event IDs, timestamps, -classification and decision fields. The additive `unmapped.http_event_kind` -field distinguishes decisions, findings, connection events and middleware -invocations. Explicit control kinds such as `token_grant` remain unchanged. -Middleware records retain `unmapped.middleware_phase` and their existing -attribution and transformation details. +ID, including Detection Findings. Each control emits its own records with event +IDs, timestamps, classification and decision fields. Use +`unmapped.http_event_kind` to select the record kind: + +| Value | Record kind | +|---|---| +| `decision` | HTTP control decision. | +| `middleware_invocation` | Request, response or WebSocket middleware invocation. | +| `token_grant` | Provider token grant. | +| `finding` | Detection Finding. | +| `connection` | Network or SSH activity, including the HTTP upgrade handoff. | +| `lifecycle` | Process or application lifecycle. | +| `configuration` | Configuration state change. | +| `authentication` | Authentication activity. | +| `event` | Other event classes. | + +Middleware invocation records use `unmapped.middleware_phase` with values +`request`, `response`, `websocket_preflight`, `websocket_session_start` or +`websocket_message`. This field is separate from `unmapped.phase`, which describes +a finding's phase, such as `pre_return`. Attribution and transformation details +are recorded on the individual invocation. `unmapped.request_sequence` starts at 1 for each request and records local emission order. It does not guarantee downstream delivery order or complete @@ -353,7 +367,8 @@ Activity or `decision` record as another request. Errors that end an inspected request retain that request's ID. A policy reload records the active request before closing it; an idle connection has no request -ID. After a WebSocket upgrade, frame events do not reuse the HTTP handshake ID. +ID. The `101` upgrade handoff record shares the HTTP handshake ID. Later +WebSocket frame events do not reuse that ID. The request ID join and enforcement work without tracing, when spans are not sampled, and when the collector is unavailable. Sampled OCSF trace fields identify