From 5e7eacaaedede863ec39ef619711eb03d5a56293 Mon Sep 17 00:00:00 2001 From: Shiju Date: Tue, 17 Nov 2026 16:26:20 +0530 Subject: [PATCH 1/2] fix(proxy): handle upstream HTTP connection closure Signed-off-by: Shiju --- .../src/l7/relay.rs | 307 +++++++++--- .../src/l7/relay/upstream_close_tests.rs | 441 ++++++++++++++++++ .../src/l7/rest.rs | 328 +++++++++++-- .../src/l7/rest/http_response.rs | 22 +- .../openshell-supervisor-network/src/proxy.rs | 156 ++++++- docs/how-it-works/policies/network-rules.mdx | 15 + 6 files changed, 1124 insertions(+), 145 deletions(-) create mode 100644 crates/openshell-supervisor-network/src/l7/relay/upstream_close_tests.rs diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 9725ef0ea5..ef17a9b867 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -25,6 +25,7 @@ use crate::l7::rest::WebSocketExtensionMode; use crate::l7::{EndpointObserver, EnforcementMode, L7EndpointConfig, L7Protocol, L7RequestInfo}; use crate::opa::{PolicyGenerationGuard, TunnelPolicyEngine}; use crate::telemetry::egress_emit; +use futures::FutureExt; use miette::{IntoDiagnostic, Result, miette}; use openshell_core::activity::{ActivitySender, try_record_activity}; use openshell_core::endpoint_status::{EndpointObservationSender, EndpointResult}; @@ -37,13 +38,15 @@ use openshell_ocsf::{ use std::collections::BTreeMap; use std::collections::HashSet; use std::sync::Arc; -use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tracing::{debug, warn}; #[cfg(test)] pub(crate) mod correlation_dispatch_tests; #[cfg(test)] mod token_grant_ownership_tests; +#[cfg(test)] +mod upstream_close_tests; const CONNECTION_READ_AHEAD_BYTES: usize = 8 * 1024; @@ -888,6 +891,7 @@ where client, upstream, crate::l7::rest::RelayRequestOptions { + request_id: Some(request_id), resolver: ctx.secret_resolver.as_deref(), body_classifier: ctx.body_classifier.as_deref(), mcp_request_validation: (config.protocol == L7Protocol::Mcp).then_some( @@ -956,6 +960,62 @@ where .await } +fn build_upstream_http_failure_event( + ctx: &L7EvalContext, + method: &str, + request_id: Option<&str>, + response_code: Option, +) -> openshell_ocsf::OcsfEvent { + let mut event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::for_http_method(method)) + .http_request(method_only_request(method)) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(SeverityId::Low) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) + .message("HTTP upstream connection closed or failed") + .status_detail("upstream_http_transport_failure"); + if let Some(request_id) = request_id { + event = event.unmapped("request_id", request_id); + } + if let Some(code) = response_code { + event = event.http_response(openshell_ocsf::HttpResponse { code }); + } + event.build() +} + +pub(crate) async fn handle_upstream_http_error( + client: &mut C, + upstream: &mut U, + ctx: &L7EvalContext, + method: &str, + request_id: Option<&str>, + error: &crate::l7::rest::UpstreamHttpError, +) -> Result<()> +where + C: AsyncWrite + Unpin, + U: AsyncWrite + Unpin, +{ + debug!(error = ?error, "HTTP upstream transport failure"); + let _ = upstream.shutdown().await; + // An unanswered request may already have executed. Never replay it or + // append a new status after the client stream accepted response bytes. + let delivery = if error.response_started { + Ok(()) + } else { + write_bad_gateway_response(client, request_id).await + }; + let response_code = (!error.response_started && delivery.is_ok()).then_some(502); + egress_emit!(build_upstream_http_failure_event( + ctx, + method, + request_id, + response_code + )); + delivery +} + #[derive(Clone, Copy, Default)] struct RelayHttpInstrumentation<'a> { observer: Option<&'a EndpointObserver>, @@ -1011,20 +1071,44 @@ where observer, correlation, } = instrumentation; - match Box::pin( - crate::l7::rest::relay_http_request_with_body_guarded_observed_correlated( - request, - client, - upstream, - options, - live_body, - response_middleware, - observer, - correlation, - ), - ) - .await + // The ready request has passed policy evaluation, but data sent before + // forwarding it cannot be treated as its response. + let mut byte = [0_u8; 1]; + let probe_error = match upstream + .read(&mut byte) + .now_or_never() + .map(IntoDiagnostic::into_diagnostic) { + Some(Ok(1..)) => Some(miette!( + "HTTP upstream sent unsolicited data before the request" + )), + Some(Err(error)) => Some(error), + _ => None, + }; + let result = if let Some(source) = probe_error { + if let Some(observer) = observer { + observer.observe(EndpointResult::TransportFailed); + } + Err(miette::Report::new(crate::l7::rest::UpstreamHttpError { + response_started: false, + source, + })) + } else { + Box::pin( + crate::l7::rest::relay_http_request_with_body_guarded_observed_correlated( + request, + client, + upstream, + options, + live_body, + response_middleware, + observer, + correlation, + ), + ) + .await + }; + match result { Ok(outcome) => Ok(Some(outcome)), Err(report) => { if let Some(failure) = report.downcast_ref::() { @@ -1033,7 +1117,30 @@ where .await?; return Ok(None); } - if let Some(error) = report.downcast_ref::() { + if let Some(error) = report.downcast_ref::() { + if let Some(correlation) = correlation { + correlation.record_failure( + HttpOutcome::UpstreamError, + "upstream_http_transport_failure", + ); + } + handle_upstream_http_error( + client, + upstream, + ctx, + &request.action, + options.request_id, + error, + ) + .await?; + if !error.response_started + && let Some(correlation) = correlation + { + correlation.record_response_status(502); + } + Ok(Some(RelayOutcome::Consumed)) + } else if let Some(error) = report.downcast_ref::() + { // Body classification includes credentials for other destinations, // so denial does not establish that this endpoint lacks credentials. // Record the local decision before cleanup or client I/O can fail. @@ -1540,25 +1647,26 @@ where .endpoint_observation_tx .as_ref() .and_then(EndpointObservationSender::capture); - let mut req = match provider.parse_request(client).await { - Ok(Some(req)) => req, - Ok(None) => return Ok(()), - Err(e) => { - if is_benign_connection_error(&e) { - debug!( - host = %ctx.host, - port = ctx.port, - error = %e, - "L7 route-selected connection closed" - ); - } else { - let detail = - parse_rejection_detail(&e.to_string(), ParseRejectionMode::L7Endpoint); - emit_parse_rejection(ctx, &detail, "l7"); + let mut req = + match parse_next_http_request(provider.parse_request(client), upstream, ctx).await { + Ok(Some(req)) => req, + Ok(None) => return Ok(()), + Err(e) => { + if is_benign_connection_error(&e) { + debug!( + host = %ctx.host, + port = ctx.port, + error = %e, + "L7 route-selected connection closed" + ); + } else { + let detail = + parse_rejection_detail(&e.to_string(), ParseRejectionMode::L7Endpoint); + emit_parse_rejection(ctx, &detail, "l7"); + } + return Ok(()); } - return Ok(()); - } - }; + }; let correlation = RequestCorrelation::new(&req, ctx); let capture_error = |error| correlation.capture_error(error); if !request_authority_matches_endpoint(&req, ctx) { @@ -2566,9 +2674,10 @@ where .filter(|_| ctx.token_grant_resolver.is_some()) .and_then(EndpointObservationSender::capture); // Parse one HTTP request from client - let req = match provider.parse_request(client).await { + let req = match parse_next_http_request(provider.parse_request(client), upstream, ctx).await + { Ok(Some(req)) => req, - Ok(None) => return Ok(()), // Client closed connection + Ok(None) => return Ok(()), // The HTTP connection is no longer reusable. Err(e) => { if is_benign_connection_error(&e) { debug!( @@ -3109,14 +3218,18 @@ where .endpoint_observation_tx .as_ref() .and_then(EndpointObservationSender::capture); - let parsed = match crate::l7::jsonrpc::parse_jsonrpc_http_request( - client, - config.json_rpc_max_body_bytes, - crate::l7::path::CanonicalizeOptions { - allow_encoded_slash: config.allow_encoded_slash, - ..Default::default() - }, - crate::l7::jsonrpc::JsonRpcInspectionOptions::for_config(config), + let parsed = match parse_next_http_request( + Box::pin(crate::l7::jsonrpc::parse_jsonrpc_http_request( + client, + config.json_rpc_max_body_bytes, + crate::l7::path::CanonicalizeOptions { + allow_encoded_slash: config.allow_encoded_slash, + ..Default::default() + }, + crate::l7::jsonrpc::JsonRpcInspectionOptions::for_config(config), + )), + upstream, + ctx, ) .await { @@ -3494,25 +3607,26 @@ where // Validate the head, including body framing, before deciding whether // this endpoint can inspect the requested protocol. Upgrade refusal // must not wait for a body or depend on its inspection size limit. - let mut req = match provider.parse_request(client).await { - Ok(Some(req)) => req, - Ok(None) => return Ok(()), - Err(e) => { - if is_benign_connection_error(&e) { - debug!( - host = %ctx.host, - port = ctx.port, - error = %e, - "GraphQL L7 connection closed" - ); - } else { - let detail = - parse_rejection_detail(&e.to_string(), ParseRejectionMode::L7Endpoint); - emit_parse_rejection(ctx, &detail, "l7-graphql"); + let mut req = + match parse_next_http_request(provider.parse_request(client), upstream, ctx).await { + Ok(Some(req)) => req, + Ok(None) => return Ok(()), + Err(e) => { + if is_benign_connection_error(&e) { + debug!( + host = %ctx.host, + port = ctx.port, + error = %e, + "GraphQL L7 connection closed" + ); + } else { + let detail = + parse_rejection_detail(&e.to_string(), ParseRejectionMode::L7Endpoint); + emit_parse_rejection(ctx, &detail, "l7-graphql"); + } + return Ok(()); } - return Ok(()); - } - }; + }; let correlation = RequestCorrelation::new(&req, ctx); let capture_error = |error| correlation.capture_error(error); @@ -4353,6 +4467,62 @@ fn evaluate_l7_request_once( Ok((allowed, reason)) } +/// Read a request while watching the otherwise idle upstream connection. +/// +/// A closed upstream must not leave a live socket in the client's pool. If +/// both sides are ready, finish the request first so a forwarding failure can +/// produce an explicit HTTP error. If the upstream closes during partial +/// headers, close without a reply: no complete request is available to answer. +/// Ready requests are checked for unsolicited upstream data before forwarding. +async fn parse_next_http_request( + request: F, + upstream: &mut U, + ctx: &L7EvalContext, +) -> Result> +where + F: Future>>, + U: AsyncRead + Unpin, +{ + let mut byte = [0u8; 1]; + tokio::select! { + biased; + request = request => request, + result = upstream.read(&mut byte) => { + emit_idle_upstream_close(ctx, result); + Ok(None) + } + } +} + +fn emit_idle_upstream_close(ctx: &L7EvalContext, result: std::io::Result) { + let (detail, status, severity) = match result { + Ok(0) => { + debug!(host = %ctx.host, port = ctx.port, "HTTP upstream closed while idle"); + return; + } + Ok(_) => ( + "HTTP upstream sent unsolicited data while idle", + StatusId::Failure, + SeverityId::Low, + ), + Err(_) => ( + "HTTP upstream failed while idle", + StatusId::Failure, + SeverityId::Low, + ), + }; + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Close) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(severity) + .status(status) + .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) + .message(detail) + .build(); + egress_emit!(event); +} + /// Relay HTTP traffic with credential injection only (no L7 OPA evaluation). /// /// Used when TLS is auto-terminated but no L7 policy (`protocol` + `access`/`rules`) @@ -4415,14 +4585,16 @@ where Some(Ok((chain, _))) if !chain.is_empty() ); // Read next request from client. - let parsed = if inspected { - provider.parse_request(client).await - } else { - provider.parse_uninspected_request(client).await + let parsed = async { + if inspected { + provider.parse_request(client).await + } else { + provider.parse_uninspected_request(client).await + } }; - let req = match parsed { + let req = match parse_next_http_request(parsed, upstream, ctx).await { Ok(Some(req)) => req, - Ok(None) => break, // Client closed connection. + Ok(None) => break, // The HTTP connection is no longer reusable. Err(e) => { if is_benign_connection_error(&e) { break; @@ -4642,6 +4814,7 @@ where client, upstream, crate::l7::rest::RelayRequestOptions { + request_id: Some(&request_id), resolver, credential_generation: credential_generation_guard(ctx), generation_guard: Some(generation_guard), diff --git a/crates/openshell-supervisor-network/src/l7/relay/upstream_close_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/upstream_close_tests.rs new file mode 100644 index 0000000000..6e085bf59d --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/relay/upstream_close_tests.rs @@ -0,0 +1,441 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use crate::opa::{NetworkInput, OpaEngine}; +use std::path::PathBuf; +use std::time::Duration; +use tokio::net::{TcpListener, TcpStream}; +use tokio::time::timeout; + +const REQUEST: &[u8] = + b"POST /mcp HTTP/1.1\r\nHost: api.example.test\r\nContent-Length: 2\r\n\r\n{}"; +const RESPONSE: &[u8] = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK"; + +fn context() -> L7EvalContext { + L7EvalContext { + host: "api.example.test".into(), + port: 80, + request_default_port: Some(80), + policy_name: "rest_api".into(), + binary_path: "/usr/bin/curl".into(), + ..Default::default() + } +} + +fn without_request_id(bytes: &[u8]) -> Vec { + let Some(end) = bytes.windows(4).position(|part| part == b"\r\n\r\n") else { + return bytes.to_vec(); + }; + let (head, body) = bytes.split_at(end + 4); + let mut result = Vec::new(); + for line in head.split_inclusive(|byte| *byte == b'\n') { + if line.starts_with(b"x-openshell-request-id:") { + let id = std::str::from_utf8(line) + .unwrap() + .split_once(':') + .unwrap() + .1 + .trim(); + uuid::Uuid::parse_str(id).unwrap(); + } else { + result.extend_from_slice(line); + } + } + result.extend_from_slice(body); + result +} + +async fn read_exact_message(stream: &mut S, expected: &[u8]) { + let mut bytes = Vec::new(); + while !bytes.ends_with(b"\r\n\r\n") { + bytes.push(stream.read_u8().await.unwrap()); + } + let expected_head_end = expected + .windows(4) + .position(|part| part == b"\r\n\r\n") + .unwrap() + + 4; + let mut body = vec![0; expected.len() - expected_head_end]; + stream.read_exact(&mut body).await.unwrap(); + bytes.extend_from_slice(&body); + assert_eq!(without_request_id(&bytes), expected); +} + +async fn tcp_pair() -> (TcpStream, TcpStream) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let client = + openshell_core::net::connect_tcp_nodelay_best_effort(listener.local_addr().unwrap()) + .await + .unwrap(); + let (server, _) = listener.accept().await.unwrap(); + openshell_core::net::set_tcp_nodelay_best_effort(&client); + openshell_core::net::set_tcp_nodelay_best_effort(&server); + (client, server) +} + +async fn relay_fixture(protocol: &str, mut client: C, mut upstream: U) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send, + U: AsyncRead + AsyncWrite + Unpin + Send, +{ + let ctx = context(); + let policy_protocol = if protocol == "passthrough" { + "rest" + } else { + protocol + }; + let rule = if policy_protocol == "rest" { + serde_json::json!({"method":"POST", "path":"/mcp"}) + } else { + serde_json::json!({"method":"initialize"}) + }; + let data = serde_json::json!({"network_policies": {"rest_api": { + "name":"rest_api", "endpoints":[{"host":ctx.host, "port":ctx.port, + "path":"/mcp", "protocol":policy_protocol, "enforcement":"enforce", "rules":[{"allow":rule}]}], + "binaries":[{"path":ctx.binary_path}] + }}}); + let engine = OpaEngine::from_strings( + include_str!("../../../data/sandbox-policy.rego"), + &data.to_string(), + ) + .unwrap(); + if protocol == "passthrough" { + let guard = engine + .generation_guard(engine.current_generation()) + .unwrap(); + relay_passthrough_with_credentials(&mut client, &mut upstream, &ctx, &guard, Some(&engine)) + .await + } else { + let input = NetworkInput { + host: ctx.host.clone(), + port: ctx.port, + binary_path: PathBuf::from(&ctx.binary_path), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (config, generation) = engine + .query_endpoint_config_with_generation(&input) + .unwrap(); + let config = crate::l7::parse_l7_config(&config.unwrap()).unwrap(); + relay_with_inspection( + &config, + engine.clone_engine_for_tunnel(generation).unwrap(), + &mut client, + &mut upstream, + &ctx, + ) + .await + } +} + +fn assert_bad_gateway(bytes: &[u8]) { + let response = std::str::from_utf8(bytes).unwrap(); + assert!( + response.starts_with("HTTP/1.1 502 Bad Gateway\r\n"), + "{response:?}" + ); + assert!(response.contains("Content-Length: 0\r\n")); + assert!(response.ends_with("Connection: close\r\n\r\n")); + let prefix = format!( + "{}: ", + openshell_core::token_grant_failure::REQUEST_ID_HEADER + ); + let id = response + .lines() + .find_map(|line| line.strip_prefix(&prefix)) + .expect("correlation header"); + uuid::Uuid::parse_str(id).unwrap(); +} + +#[tokio::test] +async fn keep_alive_propagates_upstream_idle_close_over_tcp() { + timeout(Duration::from_secs(5), async { + for protocol in ["passthrough", "rest", "json-rpc", "mcp"] { + let body = if matches!(protocol, "json-rpc" | "mcp") { + r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"test","version":"1"}}}"# + } else { "{}" }; + let request = format!("POST /mcp HTTP/1.1\r\nHost: api.example.test\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", body.len()); + let (mut app, client) = tcp_pair().await; + let (relay_upstream, mut upstream) = tcp_pair().await; + let relay = tokio::spawn(relay_fixture(protocol, client, relay_upstream)); + for _ in 0..2 { + app.write_all(request.as_bytes()).await.unwrap(); + read_exact_message(&mut upstream, request.as_bytes()).await; + upstream.write_all(RESPONSE).await.unwrap(); + read_exact_message(&mut app, RESPONSE).await; + } + upstream.shutdown().await.unwrap(); + let mut rest = Vec::new(); + app.read_to_end(&mut rest).await.unwrap(); + assert!(rest.is_empty(), "idle close must not invent an HTTP response"); + relay.await.unwrap().unwrap(); + } + }).await.expect("idle upstream close must reach the client promptly"); +} + +#[tokio::test] +async fn queued_request_handles_upstream_close_without_replay() { + timeout(Duration::from_secs(5), async { + // None closes before request two is forwarded (BrokenPipe). The other + // cases accept it, then close before or during the response headers. + for protocol in ["passthrough", "rest"] { + for (response, expect_gateway) in [ + (None, true), + (Some(&b""[..]), true), + (Some(&b"HTTP/1.1 100 Continue\r\n\r\n"[..]), true), + (Some(&b"HTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 200"[..]), true), + (Some(&b"HTTP/1.1 200"[..]), true), + ( + Some(&b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nOK"[..]), + false, + ), + ( + Some(&b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nOK\r\n"[..]), + false, + ), + ] { + let (mut app, client) = tokio::io::duplex(8192); + let (relay_upstream, mut upstream) = tokio::io::duplex(8192); + // Buffer both requests so request two and the upstream close can + // become ready together after the first successful exchange. + app.write_all(&[REQUEST, REQUEST].concat()).await.unwrap(); + let relay = tokio::spawn(relay_fixture(protocol, client, relay_upstream)); + read_exact_message(&mut upstream, REQUEST).await; + upstream.write_all(RESPONSE).await.unwrap(); + let server = response.map(|response| { + tokio::spawn(async move { + read_exact_message(&mut upstream, REQUEST).await; + upstream.write_all(response).await.unwrap(); + upstream.shutdown().await.unwrap(); + let mut extra = Vec::new(); + upstream.read_to_end(&mut extra).await.unwrap(); + assert!(extra.is_empty(), "POST must not be replayed"); + }) + }); + read_exact_message(&mut app, RESPONSE).await; + let mut received = Vec::new(); + app.read_to_end(&mut received).await.unwrap(); + if expect_gateway { + let interim_end = response.and_then(|bytes| bytes.windows(4).rposition(|part| part == b"\r\n\r\n")).map_or(0, |end| end + 4); + assert_bad_gateway(&received[interim_end..]); + } else { + assert_eq!(without_request_id(&received), response.unwrap()); + } + relay.await.unwrap().unwrap(); + if let Some(server) = server { + server.await.unwrap(); + } + } + } + }) + .await + .expect("queued request must finish with a response or a clean close"); +} + +#[tokio::test] +async fn client_body_eof_is_not_an_upstream_failure() { + timeout(Duration::from_secs(5), async { + let engine = OpaEngine::from_strings( + include_str!("../../../data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + let guard = engine + .generation_guard(engine.current_generation()) + .unwrap(); + let (mut app, mut client) = tokio::io::duplex(8192); + let (mut relay_upstream, _upstream) = tokio::io::duplex(8192); + app.write_all(&REQUEST[..REQUEST.len() - 2]).await.unwrap(); + app.shutdown().await.unwrap(); + relay_passthrough_with_credentials( + &mut client, + &mut relay_upstream, + &context(), + &guard, + Some(&engine), + ) + .await + .expect_err("client did not provide its declared body"); + drop(client); + let mut response = Vec::new(); + app.read_to_end(&mut response).await.unwrap(); + assert!( + response.is_empty(), + "client error must not be reported as a 502" + ); + }) + .await + .expect("client EOF must terminate the exchange"); +} + +#[test] +fn upstream_failure_event_reports_502_without_request_secrets() { + let request_id = "9b8a7f6e-5d4c-4b3a-9f8e-7d6c5b4a3f2e"; + for (response_started, request_id) in [ + (false, Some(request_id)), + (true, Some(request_id)), + (false, None), + ] { + let event = build_upstream_http_failure_event( + &context(), + "POST", + request_id, + (!response_started).then_some(502), + ); + let json = event.to_json().unwrap(); + assert_eq!(json["class_uid"], 4002); + assert_eq!(json["status_detail"], "upstream_http_transport_failure"); + assert_eq!(json["http_request"]["http_method"], "POST"); + assert!(json["http_request"].get("url").is_none()); + assert_eq!( + json["unmapped"].get("request_id"), + request_id.map(serde_json::Value::from).as_ref() + ); + if response_started { + assert!(json.get("http_response").is_none()); + } else { + assert_eq!(json["http_response"]["code"], 502); + } + openshell_ocsf_validate::validate_event(&json).unwrap(); + } +} + +// A reset after a scripted prefix covers the error path separately from FIN. +struct ResetAfterPrefix(&'static [u8], bool, usize); +impl AsyncRead for ResetAfterPrefix { + fn poll_read( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let this = self.get_mut(); + if this.0.is_empty() { + std::task::Poll::Ready(Err(std::io::ErrorKind::ConnectionReset.into())) + } else { + let n = this.0.len().min(buf.remaining()); + buf.put_slice(&this.0[..n]); + this.0 = &this.0[n..]; + std::task::Poll::Ready(Ok(())) + } + } +} +impl AsyncWrite for ResetAfterPrefix { + fn poll_write( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + bytes: &[u8], + ) -> std::task::Poll> { + self.get_mut().2 += bytes.len(); + std::task::Poll::Ready(Ok(bytes.len())) + } + fn poll_flush( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + fn poll_shutdown( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.get_mut().1 = true; + std::task::Poll::Ready(Ok(())) + } +} + +#[tokio::test] +async fn upstream_reset_uses_client_commitment_and_preserves_cause() { + timeout(Duration::from_secs(5), async { + let (_, resolver) = SecretResolver::from_provider_env([ + ("AWS_ACCESS_KEY_ID".into(), "synthetic-access-key".into()), + ("AWS_SECRET_ACCESS_KEY".into(), "synthetic-signing-secret".into()), + ].into_iter().collect()); + for local_continue in [false, true] { + for (prefix, committed) in [ + (&b""[..], false), + (&b"HTTP/1.1 200"[..], false), + (&b"HTTP/1.1 100 Continue\r\n\r\n"[..], false), + (&b"HTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 200"[..], false), + (&b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nOK"[..], true), + ] { + let (mut app, mut client) = tokio::io::duplex(8192); + let mut upstream = ResetAfterPrefix(prefix, false, 0); + let raw = if local_continue { + b"POST /mcp HTTP/1.1\r\nHost: api.example.test\r\nExpect: 100-continue\r\nContent-Length: 2\r\n\r\n{}".to_vec() + } else { + REQUEST.to_vec() + }; + let request = crate::l7::rest::request_from_buffered_http("POST", "/mcp", "/mcp", raw).unwrap(); + let report = crate::l7::rest::relay_http_request_with_response_middleware_guarded_observed( + &request, &mut client, &mut upstream, + crate::l7::rest::RelayRequestOptions { + resolver: resolver.as_ref(), + credential_signing: if local_continue { crate::l7::CredentialSigning::SigV4NoBody } else { crate::l7::CredentialSigning::None }, + signing_service: "execute-api", signing_region: "us-east-1", host: "api.example.test", port: 443, + ..Default::default() + }, None, None, None, + ).await.expect_err("scripted upstream reset"); + assert!(report.chain().any(|cause| cause.to_string().to_lowercase().contains("reset")), + "original I/O cause must remain available: {report:?}"); + let error = report.downcast_ref::().unwrap(); + assert_eq!(error.response_started, committed); + let request_id = "9b8a7f6e-5d4c-4b3a-9f8e-7d6c5b4a3f2e"; + handle_upstream_http_error(&mut client, &mut upstream, &context(), "POST", Some(request_id), error).await.unwrap(); + assert!(upstream.1, "failed upstream must be shut down"); + drop(client); + let mut response = Vec::new(); + app.read_to_end(&mut response).await.unwrap(); + let local_interim = if local_continue { &b"HTTP/1.1 100 Continue\r\n\r\n"[..] } else { &b""[..] }; + assert!(response.starts_with(local_interim)); + let response = &response[local_interim.len()..]; + if committed { + assert_eq!(response, prefix); + } else { + let interim_end = prefix.windows(4).rposition(|part| part == b"\r\n\r\n").map_or(0, |end| end + 4); + assert_eq!(&response[..interim_end], &prefix[..interim_end]); + assert_bad_gateway(&response[interim_end..]); + assert!(std::str::from_utf8(response).unwrap().contains(request_id)); + } + } + } + }).await.expect("reset must terminate the exchange"); +} + +#[tokio::test] +async fn ready_request_does_not_consume_unsolicited_upstream_response() { + timeout(Duration::from_secs(5), async { + for protocol in ["passthrough", "rest"] { + let (mut app, client) = tokio::io::duplex(8192); + let (relay_upstream, mut server) = tokio::io::duplex(8192); + app.write_all(REQUEST).await.unwrap(); + server.write_all(RESPONSE).await.unwrap(); + let relay = tokio::spawn(relay_fixture(protocol, client, relay_upstream)); + let mut response = Vec::new(); + app.read_to_end(&mut response).await.unwrap(); + assert_bad_gateway(&response); + let mut forwarded = Vec::new(); + server.read_to_end(&mut forwarded).await.unwrap(); + assert!( + forwarded.is_empty(), + "request must not reach the invalid upstream" + ); + relay.await.unwrap().unwrap(); + + let (mut app, mut client) = tokio::io::duplex(8192); + app.write_all(REQUEST).await.unwrap(); + let mut reset = ResetAfterPrefix(b"", false, 0); + relay_fixture(protocol, &mut client, &mut reset) + .await + .unwrap(); + assert_eq!(reset.2, 0, "a ready upstream reset must prevent forwarding"); + drop(client); + let mut response = Vec::new(); + app.read_to_end(&mut response).await.unwrap(); + assert_bad_gateway(&response); + } + }) + .await + .expect("queued upstream failures must produce a final error"); +} diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index caec858eaf..bdcb406000 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -797,6 +797,7 @@ where client, upstream, RelayRequestOptions { + request_id: None, resolver, body_classifier: None, mcp_request_validation: None, @@ -825,6 +826,7 @@ pub(crate) enum WebSocketExtensionMode { #[derive(Clone, Copy, Default)] pub(crate) struct RelayRequestOptions<'a> { + pub(crate) request_id: Option<&'a str>, pub(crate) resolver: Option<&'a SecretResolver>, pub(crate) body_classifier: Option<&'a openshell_core::secrets::body::BodyCredentialClassifier>, /// Revalidate buffered MCP requests after header transformations. @@ -886,6 +888,33 @@ impl std::error::Error for CredentialUnavailableError {} impl miette::Diagnostic for CredentialUnavailableError {} +/// A body ended before its declared framing was complete. The upstream +/// adapter identifies the origin when the same readers handle request bodies. +#[derive(Debug, thiserror::Error)] +#[error("{0}")] +struct IncompleteHttpBody(String); +impl miette::Diagnostic for IncompleteHttpBody {} + +/// An upstream transport failure, distinct from client I/O and credential refusals. +#[derive(Debug)] +pub(crate) struct UpstreamHttpError { + pub(crate) response_started: bool, + pub(super) source: miette::Report, +} + +impl fmt::Display for UpstreamHttpError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("HTTP upstream connection closed or failed") + } +} + +impl std::error::Error for UpstreamHttpError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&*self.source) + } +} +impl miette::Diagnostic for UpstreamHttpError {} + impl<'a> CredentialGenerationGuard<'a> { pub(crate) fn new( state: &'a openshell_core::provider_credentials::ProviderCredentialState, @@ -994,8 +1023,63 @@ where correlation, request_span: None, track_request_span: live_body.is_none(), + transport_failed: false, + read_eof: false, }; - let upstream = &mut observed_upstream; + let mut client = ResponseProgress { + client, + response_started: false, + informational_remaining: 0, + }; + let result = Box::pin(relay_http_exchange( + req, + &mut client, + &mut observed_upstream, + options, + live_body, + response_middleware, + observer, + correlation, + )) + .await; + // Only writes accepted by the client commit the final response. Normal + // close-delimited body EOF must not reclassify a later middleware error. + match result { + Err(mut report) => { + if let Some(error) = report.downcast_mut::() { + error.response_started = client.response_started; + Err(report) + } else if observed_upstream.transport_failed + || (observed_upstream.read_eof + && report.downcast_ref::().is_some()) + { + Err(miette::Report::new(UpstreamHttpError { + response_started: client.response_started, + source: report, + })) + } else { + Err(report) + } + } + result => result, + } +} + +#[allow(clippy::too_many_arguments)] +async fn relay_http_exchange( + req: &L7Request, + client: &mut C, + upstream: &mut ObservedUpstream<'_, U>, + options: RelayRequestOptions<'_>, + live_body: Option<&mut crate::l7::middleware::RequestBodyStream>, + response_middleware: Option>, + observer: Option<&EndpointObserver>, + correlation: Option<&RequestCorrelation>, +) -> Result +where + C: AsyncRead + AsyncWrite + Unpin, + U: AsyncRead + AsyncWrite + Unpin, +{ let mut options = options; if let Some(middleware) = response_middleware.as_ref() { options.identity_response_coding = middleware.inspects_bodies().await; @@ -1494,6 +1578,69 @@ fn ensure_body_generation_current(options: RelayRequestOptions<'_>) -> Result<() Ok(()) } +/// Track final response bytes accepted by the client, excluding interim heads. +struct ResponseProgress<'a, C> { + client: &'a mut C, + response_started: bool, + informational_remaining: usize, +} + +impl AsyncRead for ResponseProgress<'_, C> { + fn poll_read( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let this = self.get_mut(); + std::pin::Pin::new(&mut *this.client).poll_read(cx, buf) + } +} + +impl AsyncWrite for ResponseProgress<'_, C> { + fn poll_write( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> std::task::Poll> { + let this = self.get_mut(); + // Interim heads are written separately, both by the response relay + // and by local Expect handling. Retain the remaining count when the + // client accepts only part of a head. A 101 commits the upgrade. + if !this.response_started + && this.informational_remaining == 0 + && let Some(end) = buf.windows(4).position(|bytes| bytes == b"\r\n\r\n") + && matches!( + http_response::parse_observed_http_status_code(&String::from_utf8_lossy( + &buf[..end + 4] + )), + Some(100 | 102..=199) + ) + { + this.informational_remaining = end + 4; + } + let result = std::pin::Pin::new(&mut *this.client).poll_write(cx, buf); + if let std::task::Poll::Ready(Ok(n)) = result { + this.response_started |= n > this.informational_remaining; + this.informational_remaining = this.informational_remaining.saturating_sub(n); + } + result + } + fn poll_flush( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + let this = self.get_mut(); + std::pin::Pin::new(&mut *this.client).poll_flush(cx) + } + fn poll_shutdown( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + let this = self.get_mut(); + std::pin::Pin::new(&mut *this.client).poll_shutdown(cx) + } +} + /// Upstream-only I/O adapter that cannot accidentally classify client errors. /// /// Once a valid response status consumes the observation token, later stream @@ -1501,6 +1648,8 @@ fn ensure_body_generation_current(options: RelayRequestOptions<'_>) -> Result<() struct ObservedUpstream<'a, U> { upstream: &'a mut U, observer: Option<&'a EndpointObserver>, + transport_failed: bool, + read_eof: bool, correlation: Option<&'a RequestCorrelation>, request_span: Option, track_request_span: bool, @@ -1516,6 +1665,14 @@ impl ObservedUpstream<'_, U> { correlation.record_failure(HttpOutcome::UpstreamError, "upstream_send_failed"); } } + + fn transport_error(&mut self, error: std::io::Error) -> std::io::Error { + self.transport_failed = true; + if let Some(observer) = self.observer { + observer.observe(EndpointResult::TransportFailed); + } + error + } } impl AsyncRead for ObservedUpstream<'_, U> { @@ -1525,13 +1682,18 @@ impl AsyncRead for ObservedUpstream<'_, U> { buf: &mut tokio::io::ReadBuf<'_>, ) -> std::task::Poll> { let this = self.get_mut(); + let before = buf.filled().len(); + let can_read = buf.remaining() > 0; let result = std::pin::Pin::new(&mut *this.upstream).poll_read(cx, buf); - if matches!(result, std::task::Poll::Ready(Err(_))) - && let Some(observer) = this.observer - { - observer.observe(EndpointResult::TransportFailed); + this.read_eof |= matches!(result, std::task::Poll::Ready(Ok(()))) + && can_read + && buf.filled().len() == before; + match result { + std::task::Poll::Ready(Err(error)) => { + std::task::Poll::Ready(Err(this.transport_error(error))) + } + result => result, } - result } } @@ -1553,13 +1715,19 @@ impl AsyncWrite for ObservedUpstream<'_, U> { } else { std::pin::Pin::new(&mut *this.upstream).poll_write(cx, buf) }; - if matches!(result, std::task::Poll::Ready(Err(_))) { - this.record_write_failure(); - if let Some(observer) = this.observer { - observer.observe(EndpointResult::TransportFailed); + let result = match result { + std::task::Poll::Ready(Ok(0)) if !buf.is_empty() => { + std::task::Poll::Ready(Err(std::io::ErrorKind::WriteZero.into())) + } + result => result, + }; + match result { + std::task::Poll::Ready(Err(error)) => { + this.record_write_failure(); + std::task::Poll::Ready(Err(this.transport_error(error))) } + result => result, } - result } fn poll_flush( @@ -1572,17 +1740,20 @@ impl AsyncWrite for ObservedUpstream<'_, U> { } else { std::pin::Pin::new(&mut *this.upstream).poll_flush(cx) }; - if matches!(result, std::task::Poll::Ready(Err(_))) { - this.record_write_failure(); - if let Some(observer) = this.observer { - observer.observe(EndpointResult::TransportFailed); + match result { + std::task::Poll::Ready(Err(error)) => { + this.record_write_failure(); + std::task::Poll::Ready(Err(this.transport_error(error))) + } + result => { + if matches!(result, std::task::Poll::Ready(Ok(()))) + && let Some(span) = &this.request_span + { + span.record("otel.status_code", "OK"); + } + result } - } else if matches!(result, std::task::Poll::Ready(Ok(()))) - && let Some(span) = &this.request_span - { - span.record("otel.status_code", "OK"); } - result } fn poll_shutdown( @@ -3703,9 +3874,9 @@ where .min(buf.len()); let n = reader.read(&mut buf[..to_read]).await.into_diagnostic()?; if n == 0 { - return Err(miette!( + return Err(miette::Report::new(IncompleteHttpBody(format!( "Connection closed with {remaining} bytes remaining" - )); + )))); } if let Some(guard) = generation_guard { guard.ensure_current()?; @@ -3903,7 +4074,9 @@ impl<'a> ChunkedRelayInput<'a> { .into_diagnostic()?; if read == 0 { self.pending.truncate(start); - return Err(miette!(eof_message)); + return Err(miette::Report::new(IncompleteHttpBody( + eof_message.to_string(), + ))); } self.pending.truncate(start + read); if let Some(guard) = generation_guard { @@ -3927,7 +4100,9 @@ impl<'a> ChunkedRelayInput<'a> { let byte = match reader.read_u8().await { Ok(byte) => byte, Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => { - return Err(miette!(eof_message)); + return Err(miette::Report::new(IncompleteHttpBody( + eof_message.to_string(), + ))); } Err(error) => return Err(error).into_diagnostic(), }; @@ -6899,7 +7074,10 @@ mod tests { } } - async fn assert_observed_response_result(response: &'static [u8], expected: EndpointResult) { + async fn assert_observed_response_result( + response: &'static [u8], + expected: EndpointResult, + ) -> Result { let (observer, mut receiver) = test_endpoint_observer().await; let (mut upstream, mut upstream_peer) = tokio::io::duplex(4096); let writer = tokio::spawn(async move { @@ -6909,7 +7087,7 @@ mod tests { .expect("write response"); }); let mut client = tokio::io::sink(); - relay_response( + let result = relay_response( "POST", &mut upstream, &mut client, @@ -6919,14 +7097,14 @@ mod tests { }, None, ) - .await - .expect("relay observed response"); + .await; writer.await.expect("join upstream writer"); let command = receiver.recv().await.expect("endpoint observation"); assert!(matches!( command, EndpointStatusCommand::Observe { result, .. } if result == expected )); + result } #[tokio::test] @@ -6935,7 +7113,8 @@ mod tests { b"HTTP/1.1 204 No Content\r\n\r\n", EndpointResult::HttpResponseReceived, ) - .await; + .await + .unwrap(); } #[tokio::test] @@ -6944,7 +7123,9 @@ mod tests { &b"HTTP/1.1 401 Unauthorized\r\nContent-Length: 0\r\n\r\n"[..], &b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\n\r\n"[..], ] { - assert_observed_response_result(response, EndpointResult::UpstreamRejected).await; + assert_observed_response_result(response, EndpointResult::UpstreamRejected) + .await + .unwrap(); } } @@ -7023,7 +7204,15 @@ mod tests { &b"HTTP/1.1 100 Continue\r\n\r\n"[..], &b"HTTP/1.1 103 Early Hints\r\n\r\n"[..], ] { - assert_observed_response_result(response, EndpointResult::TransportFailed).await; + let error = assert_observed_response_result(response, EndpointResult::TransportFailed) + .await + .expect_err("interim response without a final response"); + assert!( + error + .downcast_ref::() + .unwrap() + .response_started + ); } } @@ -7095,6 +7284,8 @@ mod tests { correlation: None, request_span: None, track_request_span: true, + transport_failed: false, + read_eof: false, }; let mut client = Vec::new(); relay_response( @@ -7127,7 +7318,8 @@ mod tests { b"not-http 200 nope\r\nContent-Length: 0\r\n\r\n", EndpointResult::TransportFailed, ) - .await; + .await + .unwrap(); } #[tokio::test] @@ -7967,6 +8159,7 @@ mod tests { async fn run_response_relay_across_policy_reload( script: ResponseRelayScript, + close_delimited: bool, ) -> (Result, Vec) { let entered = Arc::new(tokio::sync::Notify::new()); let release = Arc::new(tokio::sync::Notify::new()); @@ -7993,14 +8186,29 @@ mod tests { .expect("initial generation guard"); let mut middleware = response_middleware_context(&runner, &chain, "GET"); middleware.generation_guard = Some(&guard); - let mut upstream = b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello".as_slice(); - let mut delivered = Vec::new(); - let relay = relay_response( - "GET", + let (mut upstream, mut server) = tokio::io::duplex(8192); + let (mut app, mut client) = tokio::io::duplex(8192); + let raw = b"GET / HTTP/1.1\r\nHost: example.test\r\n\r\n"; + let request = request_from_buffered_http("GET", "/", "/", raw.to_vec()).unwrap(); + let response = if close_delimited { + &b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nhello"[..] + } else { + &b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello"[..] + }; + let writer = tokio::spawn(async move { + let mut received = vec![0; raw.len()]; + server.read_exact(&mut received).await.unwrap(); + assert_eq!(received, raw); + server.write_all(response).await.unwrap(); + }); + let relay = relay_http_request_with_response_middleware_guarded_observed( + &request, + &mut client, &mut upstream, - &mut delivered, - RelayResponseOptions::default(), + RelayRequestOptions::default(), Some(middleware), + None, + None, ); let reload = async { entered.notified().await; @@ -8010,28 +8218,50 @@ mod tests { release.notify_one(); }; let (outcome, ()) = tokio::join!(relay, reload); + writer.await.unwrap(); + drop(client); + let mut delivered = Vec::new(); + app.read_to_end(&mut delivered).await.unwrap(); (outcome, delivered) } #[tokio::test] async fn response_middleware_rechecks_generation_after_stream_exchange() { - let (outcome, delivered) = - run_response_relay_across_policy_reload(ResponseRelayScript::Stream).await; + for close_delimited in [false, true] { + let (outcome, delivered) = run_response_relay_across_policy_reload( + ResponseRelayScript::Stream, + close_delimited, + ) + .await; - let error = outcome.expect_err("stale stream output must not be delivered"); - assert!(error.to_string().contains("policy generation is stale")); - assert!(delivered.ends_with(b"\r\n\r\n")); - assert!(!delivered.windows(5).any(|window| window == b"HELLO")); + let error = outcome.expect_err("stale stream output must not be delivered"); + assert!( + error.downcast_ref::().is_none(), + "normal EOF then policy reload is not an upstream failure" + ); + assert!(error.to_string().contains("policy generation is stale")); + assert!(delivered.ends_with(b"\r\n\r\n")); + assert!(!delivered.windows(5).any(|window| window == b"HELLO")); + } } #[tokio::test] async fn response_middleware_rechecks_generation_after_whole_body_finish() { - let (outcome, delivered) = - run_response_relay_across_policy_reload(ResponseRelayScript::WholeBody).await; + for close_delimited in [false, true] { + let (outcome, delivered) = run_response_relay_across_policy_reload( + ResponseRelayScript::WholeBody, + close_delimited, + ) + .await; - let error = outcome.expect_err("stale whole-body output must not be delivered"); - assert!(error.to_string().contains("policy generation is stale")); - assert!(delivered.is_empty()); + let error = outcome.expect_err("stale whole-body output must not be delivered"); + assert!( + error.downcast_ref::().is_none(), + "normal EOF then policy reload is not an upstream failure" + ); + assert!(error.to_string().contains("policy generation is stale")); + assert!(delivered.is_empty()); + } } /// HTTP hook response stage that streams its input back and diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs index 85e3a602b1..fa33090291 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs @@ -128,10 +128,10 @@ where observer.observe(EndpointResult::TransportFailed); } record_upstream_response_error(options.correlation, "upstream_response_incomplete"); - if !buf.is_empty() { - client.write_all(&buf).await.into_diagnostic()?; - } - return Ok(RelayOutcome::Consumed); + return Err(miette::Report::new(UpstreamHttpError { + response_started: informational_header_bytes > 0, + source: miette!("HTTP upstream closed before a complete response header"), + })); } buf.extend_from_slice(&tmp[..n]); }; @@ -1855,7 +1855,9 @@ impl<'a, R: AsyncRead + Unpin> BufferedResponseReader<'a, R> { while self.exact_buffer.len() < length { let remaining = length - self.exact_buffer.len(); let Some(data) = self.read_some(remaining).await? else { - return Err(miette!("HTTP response body ended unexpectedly")); + return Err(miette::Report::new(IncompleteHttpBody( + "HTTP response body ended unexpectedly".into(), + ))); }; self.exact_buffer.extend_from_slice(&data); } @@ -1866,7 +1868,9 @@ impl<'a, R: AsyncRead + Unpin> BufferedResponseReader<'a, R> { async fn read_line(&mut self) -> Result> { loop { let Some(byte) = self.read_some(1).await? else { - return Err(miette!("HTTP response ended before line terminator")); + return Err(miette::Report::new(IncompleteHttpBody( + "HTTP response ended before line terminator".into(), + ))); }; self.line_buffer.push(byte[0]); if self.line_buffer.len() > MAX_HEADER_BYTES { @@ -2230,7 +2234,11 @@ where } else { read.await }; - let data = result?.ok_or_else(|| miette!("HTTP response body ended unexpectedly"))?; + let data = result?.ok_or_else(|| { + miette::Report::new(IncompleteHttpBody( + "HTTP response body ended unexpectedly".into(), + )) + })?; payload.extend_from_slice(&data); if payload.len() == limit { return Ok(payload); diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index be255a423b..aeef6c5dc4 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -5465,6 +5465,7 @@ where client, upstream, crate::l7::rest::RelayRequestOptions { + request_id: None, resolver: options.secret_resolver, body_classifier: options.body_classifier, mcp_request_validation: None, @@ -7231,39 +7232,64 @@ async fn handle_forward_proxy_inner( .await; let outcome_result = match outcome_result { Err(report) => { - if let Some(error) = report.downcast_ref::() { - if let Some(session) = middleware_session.take() { - session - .end(openshell_core::proto::MiddlewareSessionEndReason::Cancellation) - .await; - } - let _ = upstream.shutdown().await; - crate::l7::relay::reject_body_credential( - client, - *error, - &l7_ctx, - request_correlation, - ) - .await?; - return Ok(()); - } - if let Some(error) = report.downcast_ref::() { + if let Some(error) = report.downcast_ref::() { + correlation.record_failure( + HttpOutcome::UpstreamError, + "upstream_http_transport_failure", + ); if let Some(session) = middleware_session.take() { session - .end(openshell_core::proto::MiddlewareSessionEndReason::Cancellation) + .end(openshell_core::proto::MiddlewareSessionEndReason::UpstreamFailure) .await; } - crate::l7::relay::reject_credential_resolution( + crate::l7::relay::handle_upstream_http_error( client, + &mut upstream, &l7_ctx, method, + Some(&request_id), error, - request_correlation, ) .await?; - return Ok(()); + if !error.response_started { + correlation.record_response_status(502); + } + Ok(crate::l7::provider::RelayOutcome::Consumed) + } else { + if let Some(error) = report.downcast_ref::() { + if let Some(session) = middleware_session.take() { + session + .end(openshell_core::proto::MiddlewareSessionEndReason::Cancellation) + .await; + } + let _ = upstream.shutdown().await; + crate::l7::relay::reject_body_credential( + client, + *error, + &l7_ctx, + request_correlation, + ) + .await?; + return Ok(()); + } + if let Some(error) = report.downcast_ref::() { + if let Some(session) = middleware_session.take() { + session + .end(openshell_core::proto::MiddlewareSessionEndReason::Cancellation) + .await; + } + crate::l7::relay::reject_credential_resolution( + client, + &l7_ctx, + method, + error, + request_correlation, + ) + .await?; + return Ok(()); + } + Err(report) } - Err(report) } outcome => outcome, }; @@ -10154,6 +10180,92 @@ network_policies: assert!(!record.to_string().contains("not-logged")); } + #[tokio::test] + async fn forward_upstream_eof_returns_502_and_preserves_allow_activity() { + let Some(ip) = non_loopback_test_ipv4() else { + eprintln!("skipping: no non-loopback IPv4 address"); + return; + }; + tokio::time::timeout(std::time::Duration::from_secs(5), async { + let listener = TcpListener::bind((ip, 0)).await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let data = serde_json::json!({"network_policies":{"upstream":{ + "name":"upstream", "endpoints":[{"host":ip.to_string(),"port":port,"allowed_ips":[format!("{ip}/32")]}], + "binaries":[{"path":"/usr/bin/python3"}] + }}}); + let engine = Arc::new( + native_identity_engine( + include_str!("../data/sandbox-policy.rego"), + &data.to_string(), + ) + .unwrap(), + ); + let identity = Ok(ContractBinaryIdentity { + executable: ContractExecutableIdentity { + path: executable_path("/usr/bin/python3"), + digest: Some("00".repeat(32).parse().unwrap()), + }, + ancestors: vec![], + cmdline_paths: vec![], + }); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + let mut byte = [0u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + stream.read_exact(&mut byte).await.unwrap(); + request.push(byte[0]); + } + assert!(request.starts_with(b"GET /mcp HTTP/1.1")); + // Accept the allowed request, then close before its response. + }); + let target = format!("http://{ip}:{port}/mcp"); + let request = format!("GET {target} HTTP/1.1\r\nHost: {ip}:{port}\r\n\r\n"); + let (mut app, client) = tokio::io::duplex(8192); + let mut client: ProxyClient = tokio::io::BufReader::new(Box::new(client)); + let (activity, mut events) = mpsc::channel(4); + Box::pin(handle_forward_proxy( + "GET", + &target, + request.as_bytes(), + request.len(), + &mut client, + Some(&identity), + None, + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(0)), + None, + AgentProposals::default(), + Arc::new(None), + Arc::new(None), + None, + None, + None, + Some(&activity), + None, + )) + .await + .unwrap(); + drop(client); + let mut response = String::new(); + app.read_to_string(&mut response).await.unwrap(); + assert!( + response.starts_with("HTTP/1.1 502 Bad Gateway\r\n"), + "{response}" + ); + assert!(response.contains("x-openshell-request-id:")); + assert!(response.contains("Connection: close\r\n")); + assert!( + !events.try_recv().unwrap().denied, + "allowed decision must survive upstream EOF" + ); + server.await.unwrap(); + }) + .await + .expect("forward EOF must yield a response"); + } + #[tokio::test] async fn plaintext_mcp_forwarding_preserves_initialization_and_selected_revision() { if !cfg!(target_os = "linux") { diff --git a/docs/how-it-works/policies/network-rules.mdx b/docs/how-it-works/policies/network-rules.mdx index 119818a395..3cb8b2f2a1 100644 --- a/docs/how-it-works/policies/network-rules.mdx +++ b/docs/how-it-works/policies/network-rules.mdx @@ -115,6 +115,21 @@ it. For a protocol in which the server sends first, such as SMTP, set Use an inspected endpoint when the request, not only the destination, determines the risk, such as allowing reads but not writes on an API. +### HTTP Connection Failures + +When an HTTP upstream closes an idle connection, OpenShell closes the client +connection so clients can retire it from their pools. If an allowed request +encounters an upstream failure before its final response starts, OpenShell +returns `502 Bad Gateway` with `Connection: close` and a request ID. A completed +informational response, such as `100 Continue`, still permits a final `502`. +After any final response bytes reach the client, OpenShell closes the connection +without appending another status. + +This applies to every HTTP method, including `GET`. Clients that automatically +retry a closed connection may treat the `502` as a response and leave retry +decisions to the application. OpenShell does not reconnect or replay requests; +an upstream may have processed a request even when its response was lost. + ### Enforcement The `enforcement` field on an inspected endpoint decides what happens when a From edce16d40a12354401a4bfb458435923fa95dcf5 Mon Sep 17 00:00:00 2001 From: Shiju Date: Tue, 17 Nov 2026 18:58:13 +0530 Subject: [PATCH 2/2] test(proxy): align transport failure and trace expectations Signed-off-by: Shiju --- .../src/l7/rest/http_response.rs | 9 +- .../src/proxy/tests/request_trace.rs | 95 ++++++++++--------- 2 files changed, 55 insertions(+), 49 deletions(-) diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs index fa33090291..f118ae6daf 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs @@ -2999,10 +2999,11 @@ mod tests { let _subscriber = capture.install(); let correlation = test_correlation("eof-request"); let (outcome, client) = relay_bytes(b"", Some(&correlation)).await; - assert!(matches!( - outcome.expect("upstream close"), - RelayOutcome::Consumed - )); + let error = outcome.expect_err("upstream EOF must reach the failure handler"); + let error = error + .downcast_ref::() + .expect("typed upstream failure"); + assert!(!error.response_started); assert!( client.is_empty(), "EOF must not become a synthetic HTTP response" diff --git a/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs b/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs index 86aa709cec..dad2d67145 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs @@ -188,14 +188,14 @@ network_policies: let request = read_http_headers(&mut upstream).await; assert!(String::from_utf8_lossy(&request).starts_with("GET /allowed HTTP/1.1\r\n")); upstream - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\n") + .write_all(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n") .await .unwrap(); upstream.flush().await.unwrap(); released.await.unwrap(); - upstream.write_all(b"ok").await.unwrap(); - // Clean TLS EOF too: the HTTP declared-length failure must - // reach the non-benign outer relay-error callsite. + upstream.write_all(b"invalid\r\n").await.unwrap(); + // Invalid chunk framing reaches the non-benign outer error + // callsite; transport EOF is handled by the relay itself. upstream.shutdown().await.unwrap(); }; let exchange = async { @@ -244,7 +244,10 @@ network_policies: // The downstream TLS stream may omit close_notify when its // failed HTTP relay drops it. The captured event is the oracle. let _ = client.read_to_end(&mut body).await; - assert_eq!(body, b"ok"); + assert!( + body.is_empty(), + "malformed chunk framing must not be forwarded" + ); receipt }; let ((), receipt) = tokio::join!(serve, exchange); @@ -294,7 +297,7 @@ network_policies: terminal["message"] .as_str() .unwrap() - .contains("2 bytes remaining") + .contains("Invalid HTTP chunk size token") ); assert_eq!(terminal["class_uid"], 4001); assert_eq!(terminal["unmapped"]["http_event_kind"], "connection"); @@ -477,17 +480,22 @@ network_policies: .unwrap(); result.unwrap(); let spans = capture.spans(); - let expected = (!matches!(outcome, Outcome::UpstreamClosed)).then_some(if policy_denied { + let upstream_failure = matches!(outcome, Outcome::UpstreamClosed); + let expected = if policy_denied { 403 } else if credential_failure { 500 + } else if upstream_failure { + 502 } else { 200 - }); + }; let response = String::from_utf8(response).unwrap(); let records = capture.events.json_events(); let expected_records = if policy_denied || credential_failure { 1 + } else if upstream_failure { + 3 } else { 2 }; @@ -508,12 +516,20 @@ network_policies: assert_eq!(record["firewall_rule"]["type"], "l7"); assert!(record.get("status_id").is_none()); assert!(record.get("status_detail").is_none()); - for control in &records { - assert_eq!(control["http_request"]["url"]["path"], path); - assert!(control.get("http_response").is_none()); + for (index, control) in records.iter().enumerate() { + if upstream_failure && index == 1 { + assert_eq!(control["status_detail"], "upstream_http_transport_failure"); + assert_eq!(control["status_id"], 2); + assert_eq!(control["http_request"]["http_method"], "GET"); + assert!(control["http_request"].get("url").is_none()); + assert_eq!(control["http_response"]["code"], 502); + } else { + assert_eq!(control["http_request"]["url"]["path"], path); + assert!(control.get("http_response").is_none()); + } } if !policy_denied && !credential_failure { - let forwarded = &records[1]; + let forwarded = records.last().unwrap(); assert!( forwarded["message"] .as_str() @@ -526,33 +542,30 @@ network_policies: } let receipt = record["metadata"]["correlation_uid"].as_str().unwrap(); assert!(uuid::Uuid::parse_str(receipt).is_ok()); + if upstream_failure { + assert_eq!(records[1]["unmapped"]["request_id"], receipt); + assert!(response.contains("Connection: close\r\n")); + } assert_request_record_sequence(&records, receipt); assert!(!format!("{records:?}").contains("SENTINEL")); assert!(!format!("{records:?}").contains("forged-receipt")); assert!(!format!("{records:?}").contains("%ff")); - if let Some(expected) = expected { - assert!( - response.starts_with(&format!("HTTP/1.1 {expected} ")), - "{response}" - ); - assert_eq!(response_request_id(response.as_bytes()), receipt); - if policy_denied { - let (_, body) = response.split_once("\r\n\r\n").unwrap(); - let body: serde_json::Value = serde_json::from_str(body).unwrap(); - assert_eq!(body["error"], "policy_denied"); - assert!( - body["detail"] - .as_str() - .unwrap() - .starts_with(&format!("GET {ip}:{port}{path} denied by L7 policy: ")) - ); - assert_eq!(body.as_object().unwrap().len(), 2); - } - } else { + assert!( + response.starts_with(&format!("HTTP/1.1 {expected} ")), + "{response}" + ); + assert_eq!(response_request_id(response.as_bytes()), receipt); + if policy_denied { + let (_, body) = response.split_once("\r\n\r\n").unwrap(); + let body: serde_json::Value = serde_json::from_str(body).unwrap(); + assert_eq!(body["error"], "policy_denied"); assert!( - response.is_empty(), - "upstream EOF must not synthesize a response" + body["detail"] + .as_str() + .unwrap() + .starts_with(&format!("GET {ip}:{port}{path} denied by L7 policy: ")) ); + assert_eq!(body.as_object().unwrap().len(), 2); } assert_eq!(record["action_id"], if policy_denied { 2 } else { 1 }); let created = capture.created_span_names(); @@ -606,18 +619,10 @@ network_policies: root.span_context.trace_id().to_string() ); } - if let Some(expected) = expected { - assert!(root.attributes.iter().any(|attribute| { - attribute.key.as_str() == "http.response.status_code" - && attribute.value == opentelemetry::Value::I64(expected) - })); - } else { - assert!( - root.attributes - .iter() - .all(|attribute| attribute.key.as_str() != "http.response.status_code") - ); - } + assert!(root.attributes.iter().any(|attribute| { + attribute.key.as_str() == "http.response.status_code" + && attribute.value == opentelemetry::Value::I64(expected) + })); assert!(root.attributes.iter().any(|attribute| { attribute.key.as_str() == "openshell.request.id" && attribute.value.as_str() == receipt }));