From e888964ca1a7b689c168cded81f1ffa0710aab31 Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Tue, 17 Nov 2026 17:08:20 -0700 Subject: [PATCH] feat(observability): observe terminal HTTP response delivery Signed-off-by: Alex Fournier --- crates/openshell-supervisor-network/README.md | 6 + .../src/l7/request_trace.rs | 14 + .../src/l7/rest/http_response.rs | 51 ++- .../src/l7/rest/http_response/delivery.rs | 168 +++++++++ .../l7/rest/http_response/delivery_tests.rs | 320 ++++++++++++++++++ docs/observability/traces.mdx | 13 + 6 files changed, 571 insertions(+), 1 deletion(-) create mode 100644 crates/openshell-supervisor-network/src/l7/rest/http_response/delivery.rs create mode 100644 crates/openshell-supervisor-network/src/l7/rest/http_response/delivery_tests.rs diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index dd8af76169..b9279baa04 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -94,3 +94,9 @@ filtering preserves raw HTTP bytes and borrows unchanged header blocks. It removes only protected receipt fields and their `Connection` or `Trailer` nominations, including folded continuations, without adding general response-header validation. + +The existing response stage observes accepted wire bytes, downstream I/O +failures and cancellation without changing relay results or synthesizing audit +records. `completed` requires observed framing completion or close-delimited EOF; +ambiguous successful returns, including middleware replacement paths, remain +`returned`. Accepted writes include headers and framing, not proof of peer receipt. diff --git a/crates/openshell-supervisor-network/src/l7/request_trace.rs b/crates/openshell-supervisor-network/src/l7/request_trace.rs index 1d9c36a509..a6fdf2752e 100644 --- a/crates/openshell-supervisor-network/src/l7/request_trace.rs +++ b/crates/openshell-supervisor-network/src/l7/request_trace.rs @@ -95,6 +95,20 @@ impl RequestTrace { if self.span.is_disabled() || !self.span.context().span().span_context().is_valid() { return Span::none(); } + if matches!(stage, RequestStage::Response) { + return tracing::info_span!( + parent: &self.span, + "openshell.http.stage", + otel.name = stage.name(), + otel.status_code = tracing::field::Empty, + openshell.request.failure_reason = tracing::field::Empty, + openshell.response.wire_bytes_written = tracing::field::Empty, + openshell.response.delivery_outcome = tracing::field::Empty, + openshell.response.io_error_operation = tracing::field::Empty, + openshell.response.upstream_status_code = tracing::field::Empty, + openshell.response.selected_status_code = tracing::field::Empty, + ); + } tracing::info_span!( parent: &self.span, "openshell.http.stage", diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs index 85e3a602b1..e30497853e 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs @@ -3,6 +3,9 @@ //! HTTP response relay and pre-return middleware integration. +mod delivery; +#[cfg(test)] +mod delivery_tests; mod http_hooks; use super::*; @@ -76,6 +79,34 @@ pub(super) async fn relay_response( options: RelayResponseOptions<'_>, response_middleware: Option>, ) -> Result +where + U: AsyncRead + Unpin, + C: AsyncWrite + Unpin, +{ + let span = options + .correlation + .map_or_else(tracing::Span::none, |_| tracing::Span::current()); + let mut upstream = delivery::ResponseReader::new(upstream); + let mut client = delivery::ResponseWriter::new(client, span); + let result = relay_response_inner( + request_method, + &mut upstream, + &mut client, + options, + response_middleware, + ) + .await; + client.finish(result.is_err()); + result +} + +async fn relay_response_inner( + request_method: &str, + upstream: &mut delivery::ResponseReader<'_, U>, + client: &mut delivery::ResponseWriter<'_, C>, + options: RelayResponseOptions<'_>, + response_middleware: Option>, +) -> Result where U: AsyncRead + Unpin, C: AsyncWrite + Unpin, @@ -131,6 +162,7 @@ where if !buf.is_empty() { client.write_all(&buf).await.into_diagnostic()?; } + client.outcome("incomplete"); return Ok(RelayOutcome::Consumed); } buf.extend_from_slice(&tmp[..n]); @@ -149,6 +181,7 @@ where // Parse response framing let header_str = String::from_utf8_lossy(&buf[..header_end]); let observed_status = parse_observed_http_status_code(&header_str); + client.upstream_status(observed_status); if let Some(observer) = options.observer.as_ref() { match observed_status { Some(status_code) if status_code < 400 => { @@ -181,6 +214,7 @@ where // upgraded protocol and must be forwarded before switching. if status_code == 101 { if !options.client_requested_upgrade { + client.outcome("incomplete"); return Ok(RelayOutcome::Consumed); } let (websocket_permessage_deflate, websocket_subprotocol) = validate_websocket_response( @@ -189,6 +223,7 @@ where options.websocket.as_ref(), )?; let final_head = final_response_head(&buf[..header_end], options.correlation); + client.selected_status(observed_status); client.write_all(&final_head).await.into_diagnostic()?; client.flush().await.into_diagnostic()?; let overflow = buf[header_end..].to_vec(); @@ -198,6 +233,7 @@ where "101 Switching Protocols — signaling protocol upgrade" ); record_allowed_response(options.correlation); + client.outcome("upgraded"); return Ok(RelayOutcome::Upgraded { overflow, websocket_permessage_deflate, @@ -229,6 +265,7 @@ where } let final_head = final_response_head(&buf[..header_end], options.correlation); + client.selected_status(observed_status); // Bodiless responses (HEAD, 1xx, 204, 304): forward headers only, skip body if is_bodiless_response(request_method, status_code) { @@ -236,6 +273,11 @@ where let outcome = finish_response(client, server_wants_close || http_10_closes_by_default).await?; record_allowed_response(options.correlation); + client.outcome(if status_code >= 200 { + "completed" + } else { + "returned" + }); return Ok(outcome); } @@ -268,6 +310,11 @@ where client.flush().await.into_diagnostic()?; client.shutdown().await.into_diagnostic()?; record_allowed_response(options.correlation); + client.outcome(if upstream.saw_eof() { + "completed" + } else { + "incomplete" + }); return Ok(RelayOutcome::Consumed); } // No Connection: close — an HTTP/1.1 keep-alive server that omits @@ -277,6 +324,7 @@ where client.write_all(&final_head).await.into_diagnostic()?; client.flush().await.into_diagnostic()?; record_allowed_response(options.correlation); + client.outcome("returned"); return Ok(RelayOutcome::Reusable); } @@ -317,6 +365,7 @@ where let outcome = finish_response(client, server_wants_close || http_10_closes_by_default).await?; record_allowed_response(options.correlation); + client.outcome("completed"); Ok(outcome) } @@ -2878,7 +2927,7 @@ mod tests { ) } - fn test_correlation(request_id: &str) -> crate::l7::correlation::RequestCorrelation { + pub(super) fn test_correlation(request_id: &str) -> crate::l7::correlation::RequestCorrelation { test_correlation_for_method(request_id, "GET") } diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response/delivery.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response/delivery.rs new file mode 100644 index 0000000000..3829e1a212 --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response/delivery.rs @@ -0,0 +1,168 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::io::{IoSlice, Result}; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use tracing::Span; + +pub(super) struct ResponseReader<'a, R> { + inner: &'a mut R, + eof: bool, +} + +impl<'a, R> ResponseReader<'a, R> { + pub(super) fn new(inner: &'a mut R) -> Self { + Self { inner, eof: false } + } + + pub(super) fn saw_eof(&self) -> bool { + self.eof + } +} + +impl AsyncRead for ResponseReader<'_, R> { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buffer: &mut ReadBuf<'_>, + ) -> Poll> { + let this = self.get_mut(); + let before = buffer.filled().len(); + let can_read = buffer.remaining() > 0; + let result = Pin::new(&mut *this.inner).poll_read(cx, buffer); + if can_read && matches!(result, Poll::Ready(Ok(()))) && buffer.filled().len() == before { + this.eof = true; + } + result + } +} + +/// Observe accepted wire bytes without changing or interpreting downstream I/O. +pub(super) struct ResponseWriter<'a, W> { + inner: &'a mut W, + span: Span, + wire_bytes: u64, + io_error: Option<&'static str>, + outcome: &'static str, +} + +impl<'a, W> ResponseWriter<'a, W> { + pub(super) fn new(inner: &'a mut W, span: Span) -> Self { + Self { + inner, + span, + wire_bytes: 0, + io_error: None, + outcome: "cancelled", + } + } + + pub(super) fn outcome(&mut self, outcome: &'static str) { + self.outcome = outcome; + } + + pub(super) fn upstream_status(&self, status: Option) { + if let Some(status) = status { + self.span + .record("openshell.response.upstream_status_code", i64::from(status)); + } + } + + pub(super) fn selected_status(&self, status: Option) { + if let Some(status) = status { + self.span + .record("openshell.response.selected_status_code", i64::from(status)); + } + } + + pub(super) fn finish(&mut self, failed: bool) { + if self.io_error.is_some() { + self.outcome = "downstream_failed"; + } else if failed { + self.outcome = "failed"; + } else if self.outcome == "cancelled" { + // Middleware may return after delivering a replacement or stopping + // a stream. Returning alone does not establish complete framing. + self.outcome = "returned"; + } + } + + fn record_write(&mut self, result: &Poll>, nonempty: bool) { + match result { + Poll::Ready(Ok(written)) => { + self.wire_bytes = self.wire_bytes.saturating_add(*written as u64); + if *written == 0 && nonempty { + self.io_error.get_or_insert("write"); + } + } + Poll::Ready(Err(_)) => { + self.io_error.get_or_insert("write"); + } + Poll::Pending => {} + } + } +} + +impl AsyncWrite for ResponseWriter<'_, W> { + fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, bytes: &[u8]) -> Poll> { + let this = self.get_mut(); + let result = Pin::new(&mut *this.inner).poll_write(cx, bytes); + this.record_write(&result, !bytes.is_empty()); + result + } + + fn poll_write_vectored( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buffers: &[IoSlice<'_>], + ) -> Poll> { + let this = self.get_mut(); + let result = Pin::new(&mut *this.inner).poll_write_vectored(cx, buffers); + this.record_write(&result, buffers.iter().any(|buffer| !buffer.is_empty())); + result + } + + fn is_write_vectored(&self) -> bool { + self.inner.is_write_vectored() + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + let result = Pin::new(&mut *this.inner).poll_flush(cx); + if matches!(result, Poll::Ready(Err(_))) { + this.io_error.get_or_insert("flush"); + } + result + } + + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + let result = Pin::new(&mut *this.inner).poll_shutdown(cx); + if matches!(result, Poll::Ready(Err(_))) { + this.io_error.get_or_insert("shutdown"); + } + result + } +} + +impl Drop for ResponseWriter<'_, W> { + fn drop(&mut self) { + self.span + .record("openshell.response.wire_bytes_written", self.wire_bytes); + self.span.record( + "openshell.response.delivery_outcome", + if self.io_error.is_some() { + "downstream_failed" + } else { + self.outcome + }, + ); + if let Some(operation) = self.io_error { + self.span + .record("openshell.response.io_error_operation", operation); + } + } +} diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response/delivery_tests.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response/delivery_tests.rs new file mode 100644 index 0000000000..268d9d7085 --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response/delivery_tests.rs @@ -0,0 +1,320 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::io; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use tokio::io::{AsyncWrite, AsyncWriteExt}; + +use super::{RelayOutcome, RelayResponseOptions, relay_response, tests::test_correlation}; +use crate::l7::correlation::{HttpOutcome, RequestStage}; +use crate::l7::relay::correlation_dispatch_tests::{TracedOperation, capture_request_spans}; +use crate::l7::rest::tests::{ + ResponseRelayScript, response_middleware_context, response_middleware_fixture, +}; +use crate::test_support::request_tracing::span_attribute; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Failure { + Write, + WriteZero, + Flush, + Shutdown, + Pending, +} + +#[derive(Default)] +struct Writer { + bytes: Vec, + failure: Option, +} + +impl AsyncWrite for Writer { + fn poll_write( + mut self: Pin<&mut Self>, + _: &mut Context<'_>, + bytes: &[u8], + ) -> Poll> { + if self.failure == Some(Failure::Pending) { + return Poll::Pending; + } + if self.failure == Some(Failure::WriteZero) { + return Poll::Ready(Ok(0)); + } + let remaining = if self.failure == Some(Failure::Write) { + 8_usize.saturating_sub(self.bytes.len()) + } else { + usize::MAX + }; + if remaining == 0 { + return Poll::Ready(Err(io::Error::other("write sentinel"))); + } + let accepted = bytes.len().min(3).min(remaining); + self.bytes.extend_from_slice(&bytes[..accepted]); + Poll::Ready(Ok(accepted)) + } + + fn poll_flush(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + Poll::Ready(if self.failure == Some(Failure::Flush) { + Err(io::Error::other("flush sentinel")) + } else { + Ok(()) + }) + } + + fn poll_shutdown(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + Poll::Ready(if self.failure == Some(Failure::Shutdown) { + Err(io::Error::other("shutdown sentinel")) + } else { + Ok(()) + }) + } +} + +async fn observe( + mut upstream: &[u8], + failure: Option, + requested_upgrade: bool, +) -> TracedOperation<(miette::Result, Writer)> { + capture_request_spans(true, async { + let correlation = test_correlation("delivery-observation"); + let mut writer = Writer { + failure, + ..Default::default() + }; + let result = Box::pin(correlation.run_stage( + RequestStage::Response, + HttpOutcome::UpstreamError, + "response_failed", + relay_response( + "GET", + &mut upstream, + &mut writer, + RelayResponseOptions { + client_requested_upgrade: requested_upgrade, + correlation: Some(&correlation), + ..Default::default() + }, + None, + ), + )) + .await; + (result, writer) + }) + .await +} + +#[tokio::test] +async fn delivery_observation_counts_partial_writes_and_requires_framing_completion() { + for (bytes, upgrade, expected, error) in [ + (&b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody"[..], false, "completed", false), + (&b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n"[..], false, "completed", false), + (&b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nbody"[..], false, "completed", false), + (&b"HTTP/1.1 204 No Content\r\n\r\n"[..], false, "completed", false), + (&b"HTTP/1.1 200 OK\r\n\r\n"[..], false, "returned", false), + (&b"HTTP/1.1 100 Continue\r\n\r\n"[..], false, "incomplete", false), + (&b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbo"[..], false, "failed", true), + (&b"HTTP/1.1 200"[..], false, "incomplete", false), + (&b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: example\r\n\r\nnext"[..], false, "incomplete", false), + (&b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: example\r\n\r\nnext"[..], true, "upgraded", false), + ] { + let evidence = observe(bytes, None, upgrade).await; + assert_eq!(evidence.output.0.is_err(), error); + if error { + assert!(evidence.output.0.unwrap_err().to_string().contains("2 bytes remaining")); + } + let response = evidence.spans.iter().find(|span| span.name == "openshell.http.response").unwrap(); + assert_eq!(evidence.spans.iter().filter(|span| span.name == "openshell.http.response").count(), 1); + assert_eq!(span_attribute(response, "openshell.response.delivery_outcome").unwrap().as_str(), expected); + assert_eq!(span_attribute(response, "openshell.response.wire_bytes_written").unwrap().to_string(), evidence.output.1.bytes.len().to_string()); + assert!(span_attribute(response, "openshell.response.io_error_operation").is_none()); + assert!(evidence.events.is_empty(), "observation must not synthesize audit records"); + } +} + +#[tokio::test] +async fn delivery_observation_middleware_replacement_does_not_claim_completion() { + let evidence = capture_request_spans(true, async { + let correlation = test_correlation("replacement-delivery"); + let (runner, chain) = response_middleware_fixture(ResponseRelayScript::WholeBody); + let middleware = response_middleware_context(&runner, &chain, "GET"); + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nETag: stale\r\n\r\nhello"[..]; + let mut client = Vec::new(); + let outcome = Box::pin(correlation.run_stage( + RequestStage::Response, + HttpOutcome::UpstreamError, + "response_failed", + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ), + )) + .await; + (outcome, client) + }) + .await; + + assert!(matches!(evidence.output.0.unwrap(), RelayOutcome::Reusable)); + let delivered = String::from_utf8(evidence.output.1).unwrap(); + assert!(delivered.starts_with("HTTP/1.1 200 OK\r\n")); + assert!(delivered.contains("Content-Length: 11\r\n")); + assert!(!delivered.to_ascii_lowercase().contains("etag:")); + assert!(delivered.ends_with("\r\n\r\nwhole:hello")); + let response = evidence + .spans + .iter() + .find(|span| span.name == "openshell.http.response") + .unwrap(); + assert_eq!( + span_attribute(response, "openshell.response.delivery_outcome") + .unwrap() + .as_str(), + "returned" + ); + assert_eq!( + span_attribute(response, "openshell.response.wire_bytes_written") + .unwrap() + .to_string(), + delivered.len().to_string() + ); +} + +#[tokio::test] +async fn delivery_observation_retains_downstream_error_operations_and_original_errors() { + for (failure, operation, message) in [ + (Failure::Write, "write", "write sentinel"), + (Failure::WriteZero, "write", "write"), + (Failure::Flush, "flush", "flush sentinel"), + (Failure::Shutdown, "shutdown", "shutdown sentinel"), + ] { + let evidence = observe( + b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\nbody", + Some(failure), + false, + ) + .await; + assert!(evidence.output.0.unwrap_err().to_string().contains(message)); + let response = evidence + .spans + .iter() + .find(|span| span.name == "openshell.http.response") + .unwrap(); + assert_eq!( + span_attribute(response, "openshell.response.delivery_outcome") + .unwrap() + .as_str(), + "downstream_failed" + ); + assert_eq!( + span_attribute(response, "openshell.response.io_error_operation") + .unwrap() + .as_str(), + operation + ); + assert_eq!( + span_attribute(response, "openshell.response.wire_bytes_written") + .unwrap() + .to_string(), + evidence.output.1.bytes.len().to_string() + ); + assert!(evidence.events.is_empty()); + assert!(!format!("{:?}", evidence.spans).contains("sentinel")); + } +} + +#[tokio::test] +async fn delivery_observation_cancellation_records_no_success_or_audit_record() { + let evidence = capture_request_spans(true, async { + let correlation = test_correlation("cancelled-delivery"); + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody"[..]; + let mut writer = Writer { + failure: Some(Failure::Pending), + ..Default::default() + }; + let mut response = Box::pin(correlation.run_stage( + RequestStage::Response, + HttpOutcome::UpstreamError, + "response_failed", + relay_response( + "GET", + &mut upstream, + &mut writer, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ), + )); + assert!(futures::poll!(&mut response).is_pending()); + drop(response); + assert!(writer.bytes.is_empty()); + }) + .await; + let response = evidence + .spans + .iter() + .find(|span| span.name == "openshell.http.response") + .unwrap(); + assert_eq!( + span_attribute(response, "openshell.response.delivery_outcome") + .unwrap() + .as_str(), + "cancelled" + ); + assert!(matches!( + response.status, + opentelemetry::trace::Status::Error { .. } + )); + assert!(evidence.events.is_empty()); +} + +#[tokio::test(start_paused = true)] +async fn delivery_observation_does_not_treat_close_delimited_idle_timeout_as_eof() { + let evidence = capture_request_spans(true, async { + let correlation = test_correlation("idle-delivery"); + let (mut upstream, mut peer) = tokio::io::duplex(1024); + peer.write_all(b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nbody") + .await + .unwrap(); + let mut client = Vec::new(); + let result = Box::pin(correlation.run_stage( + RequestStage::Response, + HttpOutcome::UpstreamError, + "response_failed", + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ), + )) + .await; + assert!(matches!(result.unwrap(), RelayOutcome::Consumed)); + drop(peer); + }) + .await; + let response = evidence + .spans + .iter() + .find(|span| span.name == "openshell.http.response") + .unwrap(); + assert_eq!( + span_attribute(response, "openshell.response.delivery_outcome") + .unwrap() + .as_str(), + "incomplete" + ); + assert!(evidence.events.is_empty()); +} diff --git a/docs/observability/traces.mdx b/docs/observability/traces.mdx index abe075e27b..8c7bb3254d 100644 --- a/docs/observability/traces.mdx +++ b/docs/observability/traces.mdx @@ -75,6 +75,19 @@ The CLI forwards `TRACEPARENT` and `TRACESTATE` from its environment on gateway The gateway also passes its trace context to compute drivers, so driver spans appear in the same trace as the gateway request that caused them. Built-in drivers export their spans through the gateway's process. Managed external drivers receive the trace endpoint, gateway name, and compute driver from the gateway. Drivers that you run yourself own their telemetry configuration. +## Response Delivery + +The `openshell.http.response` stage records accepted wire bytes, including +headers and framing, in `openshell.response.wire_bytes_written`. This is not +proof that the peer received the response. `openshell.response.delivery_outcome` +is `completed` only when framing or close-delimited EOF establishes completion; +`returned` means the relay returned without that proof, including middleware +replacement paths. Other outcomes distinguish incomplete responses, failures, +downstream failures, cancellation and upgrades. The first downstream I/O failure +records `write`, `flush` or `shutdown` in `openshell.response.io_error_operation`. +Upstream and selected status codes are recorded when known. These observations +do not replace the request's policy outcome or add security log records. + ## Next Steps - Configure [OTLP export](/how-it-works/gateways/configuration#otlp-export), including sampling and the `OTEL_*` variables the gateway honors.