Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions crates/openshell-supervisor-network/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,3 +94,9 @@ filtering preserves raw HTTP bytes
and borrows unchanged header blocks. It removes only protected receipt fields
and their `Connection` or `Trailer` nominations, including folded continuations,
without adding general response-header validation.

The existing response stage observes accepted wire bytes, downstream I/O
failures and cancellation without changing relay results or synthesizing audit
records. `completed` requires observed framing completion or close-delimited EOF;
ambiguous successful returns, including middleware replacement paths, remain
`returned`. Accepted writes include headers and framing, not proof of peer receipt.
14 changes: 14 additions & 0 deletions crates/openshell-supervisor-network/src/l7/request_trace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,20 @@ impl RequestTrace {
if self.span.is_disabled() || !self.span.context().span().span_context().is_valid() {
return Span::none();
}
if matches!(stage, RequestStage::Response) {
return tracing::info_span!(
parent: &self.span,
"openshell.http.stage",
otel.name = stage.name(),
otel.status_code = tracing::field::Empty,
openshell.request.failure_reason = tracing::field::Empty,
openshell.response.wire_bytes_written = tracing::field::Empty,
openshell.response.delivery_outcome = tracing::field::Empty,
openshell.response.io_error_operation = tracing::field::Empty,
openshell.response.upstream_status_code = tracing::field::Empty,
openshell.response.selected_status_code = tracing::field::Empty,
);
}
tracing::info_span!(
parent: &self.span,
"openshell.http.stage",
Expand Down
51 changes: 50 additions & 1 deletion crates/openshell-supervisor-network/src/l7/rest/http_response.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@

//! HTTP response relay and pre-return middleware integration.

mod delivery;
#[cfg(test)]
mod delivery_tests;
mod http_hooks;

use super::*;
Expand Down Expand Up @@ -76,6 +79,34 @@ pub(super) async fn relay_response<U, C>(
options: RelayResponseOptions<'_>,
response_middleware: Option<HttpResponseMiddlewareRelay<'_>>,
) -> Result<RelayOutcome>
where
U: AsyncRead + Unpin,
C: AsyncWrite + Unpin,
{
let span = options
.correlation
.map_or_else(tracing::Span::none, |_| tracing::Span::current());
let mut upstream = delivery::ResponseReader::new(upstream);
let mut client = delivery::ResponseWriter::new(client, span);
let result = relay_response_inner(
request_method,
&mut upstream,
&mut client,
options,
response_middleware,
)
.await;
client.finish(result.is_err());
result
}

async fn relay_response_inner<U, C>(
request_method: &str,
upstream: &mut delivery::ResponseReader<'_, U>,
client: &mut delivery::ResponseWriter<'_, C>,
options: RelayResponseOptions<'_>,
response_middleware: Option<HttpResponseMiddlewareRelay<'_>>,
) -> Result<RelayOutcome>
where
U: AsyncRead + Unpin,
C: AsyncWrite + Unpin,
Expand Down Expand Up @@ -131,6 +162,7 @@ where
if !buf.is_empty() {
client.write_all(&buf).await.into_diagnostic()?;
}
client.outcome("incomplete");
return Ok(RelayOutcome::Consumed);
}
buf.extend_from_slice(&tmp[..n]);
Expand All @@ -149,6 +181,7 @@ where
// Parse response framing
let header_str = String::from_utf8_lossy(&buf[..header_end]);
let observed_status = parse_observed_http_status_code(&header_str);
client.upstream_status(observed_status);
if let Some(observer) = options.observer.as_ref() {
match observed_status {
Some(status_code) if status_code < 400 => {
Expand Down Expand Up @@ -181,6 +214,7 @@ where
// upgraded protocol and must be forwarded before switching.
if status_code == 101 {
if !options.client_requested_upgrade {
client.outcome("incomplete");
return Ok(RelayOutcome::Consumed);
}
let (websocket_permessage_deflate, websocket_subprotocol) = validate_websocket_response(
Expand All @@ -189,6 +223,7 @@ where
options.websocket.as_ref(),
)?;
let final_head = final_response_head(&buf[..header_end], options.correlation);
client.selected_status(observed_status);
client.write_all(&final_head).await.into_diagnostic()?;
client.flush().await.into_diagnostic()?;
let overflow = buf[header_end..].to_vec();
Expand All @@ -198,6 +233,7 @@ where
"101 Switching Protocols — signaling protocol upgrade"
);
record_allowed_response(options.correlation);
client.outcome("upgraded");
return Ok(RelayOutcome::Upgraded {
overflow,
websocket_permessage_deflate,
Expand Down Expand Up @@ -229,13 +265,19 @@ where
}

let final_head = final_response_head(&buf[..header_end], options.correlation);
client.selected_status(observed_status);

// Bodiless responses (HEAD, 1xx, 204, 304): forward headers only, skip body
if is_bodiless_response(request_method, status_code) {
client.write_all(&final_head).await.into_diagnostic()?;
let outcome =
finish_response(client, server_wants_close || http_10_closes_by_default).await?;
record_allowed_response(options.correlation);
client.outcome(if status_code >= 200 {
"completed"
} else {
"returned"
});
return Ok(outcome);
}

Expand Down Expand Up @@ -268,6 +310,11 @@ where
client.flush().await.into_diagnostic()?;
client.shutdown().await.into_diagnostic()?;
record_allowed_response(options.correlation);
client.outcome(if upstream.saw_eof() {
"completed"
} else {
"incomplete"
});
return Ok(RelayOutcome::Consumed);
}
// No Connection: close — an HTTP/1.1 keep-alive server that omits
Expand All @@ -277,6 +324,7 @@ where
client.write_all(&final_head).await.into_diagnostic()?;
client.flush().await.into_diagnostic()?;
record_allowed_response(options.correlation);
client.outcome("returned");
return Ok(RelayOutcome::Reusable);
}

Expand Down Expand Up @@ -317,6 +365,7 @@ where

let outcome = finish_response(client, server_wants_close || http_10_closes_by_default).await?;
record_allowed_response(options.correlation);
client.outcome("completed");
Ok(outcome)
}

Expand Down Expand Up @@ -2878,7 +2927,7 @@ mod tests {
)
}

fn test_correlation(request_id: &str) -> crate::l7::correlation::RequestCorrelation {
pub(super) fn test_correlation(request_id: &str) -> crate::l7::correlation::RequestCorrelation {
test_correlation_for_method(request_id, "GET")
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

use std::io::{IoSlice, Result};
use std::pin::Pin;
use std::task::{Context, Poll};

use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
use tracing::Span;

pub(super) struct ResponseReader<'a, R> {
inner: &'a mut R,
eof: bool,
}

impl<'a, R> ResponseReader<'a, R> {
pub(super) fn new(inner: &'a mut R) -> Self {
Self { inner, eof: false }
}

pub(super) fn saw_eof(&self) -> bool {
self.eof
}
}

impl<R: AsyncRead + Unpin> AsyncRead for ResponseReader<'_, R> {
fn poll_read(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
buffer: &mut ReadBuf<'_>,
) -> Poll<Result<()>> {
let this = self.get_mut();
let before = buffer.filled().len();
let can_read = buffer.remaining() > 0;
let result = Pin::new(&mut *this.inner).poll_read(cx, buffer);
if can_read && matches!(result, Poll::Ready(Ok(()))) && buffer.filled().len() == before {
this.eof = true;
}
result
}
}

/// Observe accepted wire bytes without changing or interpreting downstream I/O.
pub(super) struct ResponseWriter<'a, W> {
inner: &'a mut W,
span: Span,
wire_bytes: u64,
io_error: Option<&'static str>,
outcome: &'static str,
}

impl<'a, W> ResponseWriter<'a, W> {
pub(super) fn new(inner: &'a mut W, span: Span) -> Self {
Self {
inner,
span,
wire_bytes: 0,
io_error: None,
outcome: "cancelled",
}
}

pub(super) fn outcome(&mut self, outcome: &'static str) {
self.outcome = outcome;
}

pub(super) fn upstream_status(&self, status: Option<u16>) {
if let Some(status) = status {
self.span
.record("openshell.response.upstream_status_code", i64::from(status));
}
}

pub(super) fn selected_status(&self, status: Option<u16>) {
if let Some(status) = status {
self.span
.record("openshell.response.selected_status_code", i64::from(status));
}
}

pub(super) fn finish(&mut self, failed: bool) {
if self.io_error.is_some() {
self.outcome = "downstream_failed";
} else if failed {
self.outcome = "failed";
} else if self.outcome == "cancelled" {
// Middleware may return after delivering a replacement or stopping
// a stream. Returning alone does not establish complete framing.
self.outcome = "returned";
}
}

fn record_write(&mut self, result: &Poll<Result<usize>>, nonempty: bool) {
match result {
Poll::Ready(Ok(written)) => {
self.wire_bytes = self.wire_bytes.saturating_add(*written as u64);
if *written == 0 && nonempty {
self.io_error.get_or_insert("write");
}
}
Poll::Ready(Err(_)) => {
self.io_error.get_or_insert("write");
}
Poll::Pending => {}
}
}
}

impl<W: AsyncWrite + Unpin> AsyncWrite for ResponseWriter<'_, W> {
fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, bytes: &[u8]) -> Poll<Result<usize>> {
let this = self.get_mut();
let result = Pin::new(&mut *this.inner).poll_write(cx, bytes);
this.record_write(&result, !bytes.is_empty());
result
}

fn poll_write_vectored(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
buffers: &[IoSlice<'_>],
) -> Poll<Result<usize>> {
let this = self.get_mut();
let result = Pin::new(&mut *this.inner).poll_write_vectored(cx, buffers);
this.record_write(&result, buffers.iter().any(|buffer| !buffer.is_empty()));
result
}

fn is_write_vectored(&self) -> bool {
self.inner.is_write_vectored()
}

fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<()>> {
let this = self.get_mut();
let result = Pin::new(&mut *this.inner).poll_flush(cx);
if matches!(result, Poll::Ready(Err(_))) {
this.io_error.get_or_insert("flush");
}
result
}

fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<()>> {
let this = self.get_mut();
let result = Pin::new(&mut *this.inner).poll_shutdown(cx);
if matches!(result, Poll::Ready(Err(_))) {
this.io_error.get_or_insert("shutdown");
}
result
}
}

impl<W> Drop for ResponseWriter<'_, W> {
fn drop(&mut self) {
self.span
.record("openshell.response.wire_bytes_written", self.wire_bytes);
self.span.record(
"openshell.response.delivery_outcome",
if self.io_error.is_some() {
"downstream_failed"
} else {
self.outcome
},
);
if let Some(operation) = self.io_error {
self.span
.record("openshell.response.io_error_operation", operation);
}
}
}
Loading