From 11c6d4e847c437a9ffb9e1a2506f28993bf72554 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Mon, 16 Nov 2026 15:57:35 -0800 Subject: [PATCH 1/5] feat(server): authenticate postgres with azure workload identity Signed-off-by: Taylor Mutch --- Cargo.lock | 1 + crates/openshell-core/src/config.rs | 40 ++ crates/openshell-server/Cargo.toml | 1 + crates/openshell-server/src/cli.rs | 40 +- crates/openshell-server/src/config_file.rs | 3 + crates/openshell-server/src/lib.rs | 3 +- .../persistence/azure_workload_identity.rs | 397 ++++++++++++++++++ .../azure_workload_identity_tests.rs | 320 ++++++++++++++ .../openshell-server/src/persistence/mod.rs | 46 +- .../src/persistence/postgres.rs | 68 ++- docs/how-it-works/gateways/configuration.mdx | 49 +++ skills/debug-openshell-cluster/SKILL.md | 18 + 12 files changed, 967 insertions(+), 19 deletions(-) create mode 100644 crates/openshell-server/src/persistence/azure_workload_identity.rs create mode 100644 crates/openshell-server/src/persistence/azure_workload_identity_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 8fb7460537..391eb27e13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5111,6 +5111,7 @@ version = "0.0.0" dependencies = [ "anyhow", "arc-swap", + "async-stream", "async-trait", "axum", "base64 0.22.1", diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 567ad45341..6ffeaf1dce 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -13,6 +13,35 @@ use std::path::PathBuf; use std::str::FromStr; use std::time::Duration; +/// Database credential source. The default preserves URL/SQLite behavior. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum DatabaseAuthMode { + #[default] + Default, + AzureWorkloadIdentity, +} + +impl fmt::Display for DatabaseAuthMode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Default => "default", + Self::AzureWorkloadIdentity => "azure-workload-identity", + }) + } +} + +impl FromStr for DatabaseAuthMode { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "default" => Ok(Self::Default), + "azure-workload-identity" => Ok(Self::AzureWorkloadIdentity), + _ => Err("database authentication must be default or azure-workload-identity"), + } + } +} + // ── Public default constants ──────────────────────────────────────────── // // Canonical source for default values used across multiple crates. @@ -301,6 +330,9 @@ pub struct Config { /// Database URL for persistence. pub database_url: String, + /// Explicit database identity mode; independent of provider credentials. + pub database_auth: DatabaseAuthMode, + /// Connection ceiling for the persistence pool. /// /// `None` leaves the backend's built-in default in place. The right @@ -1084,6 +1116,7 @@ impl Config { mtls_auth: MtlsAuthConfig::default(), gateway_jwt: None, database_url: String::new(), + database_auth: DatabaseAuthMode::default(), database_max_connections: None, database_lock_max_connections: None, compute_driver: None, @@ -1144,6 +1177,13 @@ impl Config { self } + /// Select database credentials independently of the connection URL. + #[must_use] + pub fn with_database_auth(mut self, mode: DatabaseAuthMode) -> Self { + self.database_auth = mode; + self + } + /// Create a new configuration with a database URL. #[must_use] pub fn with_database_url(mut self, url: impl Into) -> Self { diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index a7c13be2cb..7b05da741c 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -99,6 +99,7 @@ sha2 = { workspace = true } jsonwebtoken = { workspace = true } spiffe = { workspace = true } async-trait = "0.1" +async-stream = "0.3" url = { workspace = true } glob = { workspace = true } hex = "0.4" diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 8c6a2d2c3f..b1e015fa72 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -154,6 +154,10 @@ struct RunArgs { #[arg(long, env = "OPENSHELL_DB_URL")] db_url: Option, + /// Database identity source: default or azure-workload-identity. + #[arg(long, env = "OPENSHELL_DB_AUTH")] + db_auth: Option, + /// Connection ceiling for the database pool, shared by every /// database-backed request. /// @@ -629,6 +633,7 @@ fn prepare_server_config_with_drivers( config = config .with_database_url(db_url) + .with_database_auth(args.db_auth.unwrap_or_default()) .with_database_max_connections(args.db_max_connections) .with_database_lock_max_connections(args.db_lock_max_connections); if let Some(driver) = &args.compute_driver { @@ -1698,6 +1703,12 @@ fn merge_file_into_args(args: &mut RunArgs, file: &GatewayFileSection, matches: { args.grpc_rate_limit_window_seconds = Some(window); } + if let Some(mode) = file.database_auth + && args.db_auth.is_none() + && arg_defaulted(matches, "db_auth") + { + args.db_auth = Some(mode); + } if let Some(max_connections) = file.database_max_connections && args.db_max_connections.is_none() && arg_defaulted(matches, "db_max_connections") @@ -1801,7 +1812,7 @@ mod tests { use super::{Cli, command}; use crate::test_environment::Environment; use clap::Parser; - use openshell_core::config::ServiceHostnameFormat; + use openshell_core::config::{DatabaseAuthMode, ServiceHostnameFormat}; use std::net::{IpAddr, Ipv4Addr}; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -4468,6 +4479,33 @@ grpc_rate_limit_window_seconds = 30 assert_eq!(args.grpc_rate_limit_window_seconds, Some(30)); } + #[test] + fn database_auth_file_and_explicit_overrides() { + Environment::new().remove("OPENSHELL_DB_AUTH").run(|| { + let file = config_file_from_toml( + "[openshell.gateway]\ndatabase_auth = \"azure-workload-identity\"\n", + ); + let (mut args, matches) = parse_with_args(&["openshell-gateway"]); + assert_eq!(args.db_auth, None); + merge_file_into_args(&mut args, &file.openshell.gateway, &matches); + assert_eq!(args.db_auth, Some(DatabaseAuthMode::AzureWorkloadIdentity)); + let (mut args, matches) = + parse_with_args(&["openshell-gateway", "--db-auth", "default"]); + merge_file_into_args(&mut args, &file.openshell.gateway, &matches); + assert_eq!(args.db_auth, Some(DatabaseAuthMode::Default)); + }); + Environment::new() + .set("OPENSHELL_DB_AUTH", "default") + .run(|| { + let file = config_file_from_toml( + "[openshell.gateway]\ndatabase_auth = \"azure-workload-identity\"\n", + ); + let (mut args, matches) = parse_with_args(&["openshell-gateway"]); + merge_file_into_args(&mut args, &file.openshell.gateway, &matches); + assert_eq!(args.db_auth, Some(DatabaseAuthMode::Default)); + }); + } + #[test] fn file_db_max_connections_populates_args_when_cli_omits() { Environment::new() diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 9e9c01f5e9..b541762081 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -111,6 +111,9 @@ pub struct GatewayFileSection { pub log_level: Option, // ── Database ───────────────────────────────────────────────────────── + /// Explicit identity source; connection URL stays environment-only. + #[serde(default)] + pub database_auth: Option, /// Connection ceiling for the persistence pool. One shared pool serves /// every database-backed RPC, so this is also the gateway's ceiling on /// concurrent database work. On Postgres, each replica also opens up to diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 248f034a09..fbb80851c3 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -1631,10 +1631,11 @@ async fn connect_store( let mut delay = DATABASE_CONNECT_RETRY_INITIAL_DELAY; let mut waiting = false; loop { - let attempt = Store::try_connect_with_pool_sizes( + let attempt = Store::try_connect_with_database_auth( database_url, config.database_max_connections, config.database_lock_max_connections, + config.database_auth, ); let result = if waiting { tokio::select! { diff --git a/crates/openshell-server/src/persistence/azure_workload_identity.rs b/crates/openshell-server/src/persistence/azure_workload_identity.rs new file mode 100644 index 0000000000..b4b37e4d4f --- /dev/null +++ b/crates/openshell-server/src/persistence/azure_workload_identity.rs @@ -0,0 +1,397 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Explicit AKS federation and credential-aware `PostgreSQL` pool acquisitions. +//! +//! `SQLx` 0.9 has no before-connect callback. This adapter keeps `min_connections` +//! at zero and installs a sufficiently long-lived credential before entering a +//! bounded `SQLx` acquisition. Thus every physical connection attempt sees a valid +//! credential, including `SQLx`'s internal retries. Idle sessions do not need a +//! new credential and remain usable during an identity-service outage. + +use futures::TryStreamExt; +use futures::future::BoxFuture; +use futures::stream::BoxStream; +use serde::Deserialize; +use sqlx::pool::{MaybePoolConnection, PoolConnection}; +use sqlx::postgres::{PgConnectOptions, PgPoolOptions, PgRow, PgSslMode, PgStatement, PgTypeInfo}; +use sqlx::{Acquire, Connection, Either, Execute, Executor, PgPool, Postgres, Transaction}; +use sqlx_core::sql_str::SqlStr; +use std::fmt; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; +use tokio::io::AsyncReadExt; +use tokio::sync::Mutex; +use tokio::time::Instant; + +const POSTGRES_SCOPE: &str = "https://ossrdbms-aad.database.windows.net/.default"; +const MAX_CREDENTIAL_BYTES: usize = 64 * 1024; +const CLOCK_SKEW: Duration = Duration::from_secs(30); + +fn config_error(message: &'static str) -> sqlx::Error { + sqlx::Error::Configuration(message.into()) +} + +fn unavailable(message: &'static str) -> sqlx::Error { + sqlx::Error::Io(std::io::Error::new(std::io::ErrorKind::TimedOut, message)) +} + +#[derive(Deserialize)] +struct TokenResponse { + access_token: String, + expires_in: u64, + token_type: String, +} + +struct CachedToken { + value: String, + expires_at: Instant, +} + +/// Deliberately has no `DefaultAzureCredential` chain or configurable scope. +pub(super) struct AzureWorkloadIdentity { + client_id: String, + token_file: PathBuf, + endpoint: String, + client: reqwest::Client, + token: Mutex>, +} + +impl fmt::Debug for AzureWorkloadIdentity { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("AzureWorkloadIdentity { credentials: [REDACTED] }") + } +} + +impl AzureWorkloadIdentity { + pub(super) fn from_env() -> Result, sqlx::Error> { + let tenant = std::env::var("AZURE_TENANT_ID") + .map_err(|_| config_error("Azure database authentication requires AZURE_TENANT_ID"))?; + let client_id = std::env::var("AZURE_CLIENT_ID") + .map_err(|_| config_error("Azure database authentication requires AZURE_CLIENT_ID"))?; + if uuid::Uuid::parse_str(&tenant).is_err() || uuid::Uuid::parse_str(&client_id).is_err() { + return Err(config_error( + "AZURE_TENANT_ID and AZURE_CLIENT_ID must be UUIDs", + )); + } + let authority = std::env::var("AZURE_AUTHORITY_HOST") + .unwrap_or_else(|_| "https://login.microsoftonline.com/".into()); + if authority.trim_end_matches('/') != "https://login.microsoftonline.com" { + return Err(config_error( + "Azure database authentication requires the public-cloud AZURE_AUTHORITY_HOST https://login.microsoftonline.com/", + )); + } + let token_file = + PathBuf::from(std::env::var("AZURE_FEDERATED_TOKEN_FILE").map_err(|_| { + config_error("Azure database authentication requires AZURE_FEDERATED_TOKEN_FILE") + })?); + if !token_file.is_absolute() { + return Err(config_error( + "AZURE_FEDERATED_TOKEN_FILE must be an absolute file path", + )); + } + openshell_crypto::tls::ensure_default_provider(); + let client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(3)) + .timeout(Duration::from_secs(5)) + .build() + .map_err(|_| { + config_error("could not initialize Azure database identity HTTP client") + })?; + Ok(Arc::new(Self { + client_id, + token_file, + endpoint: format!("https://login.microsoftonline.com/{tenant}/oauth2/v2.0/token"), + client, + token: Mutex::new(None), + })) + } + + async fn access_token(&self, connection_budget: Duration) -> Result { + // One refresh per replica, shared by data and advisory-lock pools. + let mut cached = self.token.lock().await; + let minimum = connection_budget + CLOCK_SKEW; + if let Some(token) = &*cached + && token.expires_at > Instant::now() + minimum + { + return Ok(token.value.clone()); + } + // AKS updates this symlink atomically. Reopen it for every refresh; + // retaining a file descriptor would pin the previous projected token. + let mut file = tokio::fs::File::open(&self.token_file).await.map_err(|_| { + unavailable("cannot read AZURE_FEDERATED_TOKEN_FILE; check the gateway ServiceAccount and workload identity projection") + })?; + if !file + .metadata() + .await + .map_err(|_| unavailable("cannot inspect Azure projected identity token"))? + .is_file() + { + return Err(config_error( + "AZURE_FEDERATED_TOKEN_FILE must reference a regular file", + )); + } + let mut assertion = String::new(); + (&mut file) + .take((MAX_CREDENTIAL_BYTES + 1) as u64) + .read_to_string(&mut assertion) + .await + .map_err(|_| unavailable("cannot read Azure projected identity token"))?; + let assertion = assertion.trim(); + if assertion.is_empty() || assertion.len() > MAX_CREDENTIAL_BYTES { + return Err(config_error( + "Azure projected identity token is empty or exceeds 64 KiB", + )); + } + let started = Instant::now(); + let mut response = self + .client + .post(&self.endpoint) + .form(&[ + ("grant_type", "client_credentials"), + ("client_id", self.client_id.as_str()), + ( + "client_assertion_type", + "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", + ), + ("client_assertion", assertion), + ("scope", POSTGRES_SCOPE), + ]) + .send() + .await + .map_err(|_| { + unavailable("Azure database token exchange unavailable; check Entra connectivity") + })?; + if !response.status().is_success() { + // Never display an upstream error body: it may echo an assertion. + return Err(unavailable( + "Azure database token exchange rejected; check tenant, client ID, and the ServiceAccount federated credential", + )); + } + let mut body = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|_| unavailable("Azure database token response could not be read"))? + { + if body.len() + chunk.len() > MAX_CREDENTIAL_BYTES { + return Err(unavailable("Azure database token response exceeds 64 KiB")); + } + body.extend_from_slice(&chunk); + } + let response: TokenResponse = serde_json::from_slice(&body) + .map_err(|_| unavailable("Azure database token response is invalid"))?; + if response.access_token.is_empty() + || response.access_token.len() > MAX_CREDENTIAL_BYTES + || response.access_token.chars().any(char::is_whitespace) + || !response.token_type.eq_ignore_ascii_case("Bearer") + || response.expires_in > 7 * 24 * 3600 + { + return Err(unavailable( + "Azure database token response has invalid credential metadata", + )); + } + let expires_at = started + Duration::from_secs(response.expires_in); + if expires_at <= Instant::now() + minimum { + return Err(unavailable( + "Azure database token expires before a connection can safely complete", + )); + } + let value = response.access_token; + *cached = Some(CachedToken { + value: value.clone(), + expires_at, + }); + Ok(value) + } +} + +pub(super) fn validate_url(url: &str) -> Result { + let parsed = url::Url::parse(url) + .map_err(|_| config_error("Azure database authentication requires a PostgreSQL URL"))?; + if !matches!(parsed.scheme(), "postgres" | "postgresql") + || parsed.host_str().is_none() + || parsed.username().is_empty() + || parsed.password().is_some() + || parsed + .query_pairs() + .any(|(key, _)| matches!(key.as_ref(), "password" | "passfile" | "host" | "hostaddr")) + { + return Err(config_error( + "Azure database authentication requires an explicit PostgreSQL host/user and a password-free URL without host overrides or passfile", + )); + } + let options: PgConnectOptions = url + .parse() + .map_err(|_| config_error("Azure database connection options are invalid"))?; + if !matches!(options.get_ssl_mode(), PgSslMode::VerifyFull) { + return Err(config_error( + "Azure database authentication requires sslmode=verify-full", + )); + } + // Overwrite SQLx's ambient PGPASSWORD/.pgpass even before the first token. + Ok(options.password("")) +} + +/// All access to the underlying pool stays here: no caller can bypass refresh. +#[derive(Clone)] +pub(super) struct CredentialPool { + inner: PgPool, + identity: Option>, +} + +impl fmt::Debug for CredentialPool { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("CredentialPool") + .field("azure", &self.identity.is_some()) + .finish_non_exhaustive() + } +} + +impl From for CredentialPool { + fn from(inner: PgPool) -> Self { + Self { + inner, + identity: None, + } + } +} + +impl CredentialPool { + pub(super) fn azure( + options: PgPoolOptions, + connect: PgConnectOptions, + identity: Arc, + ) -> Self { + Self { + // Maintenance must not open a connection behind the credential guard. + inner: options + .min_connections(0) + .max_lifetime(None) + .connect_lazy_with(connect), + identity: Some(identity), + } + } + + pub(super) fn acquire(&self) -> BoxFuture<'_, Result, sqlx::Error>> { + Box::pin(async move { + let Some(identity) = &self.identity else { + return self.inner.acquire().await; + }; + let budget = self.inner.options().get_acquire_timeout(); + let acquired = tokio::time::timeout(budget, async { + if let Some(mut connection) = self.inner.try_acquire() { + if connection.ping().await.is_ok() { + return Ok(connection); + } + let _ = connection.close().await; + } + let token = identity.access_token(budget).await?; + let options = (*self.inner.connect_options()).clone().password(&token); + self.inner.set_connect_options(options); + self.inner.acquire().await + }) + .await; + acquired.map_err(|_| unavailable( + "Azure database connection acquisition timed out; check workload identity and PostgreSQL connectivity", + ))? + }) + } + + pub(super) async fn begin(&self) -> Result, sqlx::Error> { + Transaction::begin( + MaybePoolConnection::PoolConnection(self.acquire().await?), + None, + ) + .await + } + pub(super) fn options(&self) -> &PgPoolOptions { + self.inner.options() + } + #[cfg(test)] + pub(super) fn size(&self) -> u32 { + self.inner.size() + } + #[cfg(test)] + pub(super) fn num_idle(&self) -> usize { + self.inner.num_idle() + } + #[cfg(any(test, feature = "test-support"))] + pub(super) async fn close(&self) { + self.inner.close().await; + } +} + +impl<'a> Acquire<'a> for &'a CredentialPool { + type Database = Postgres; + type Connection = PoolConnection; + fn acquire(self) -> BoxFuture<'a, Result> { + CredentialPool::acquire(self) + } + fn begin(self) -> BoxFuture<'a, Result, sqlx::Error>> { + Box::pin(async move { + Transaction::begin( + MaybePoolConnection::PoolConnection(self.acquire().await?), + None, + ) + .await + }) + } +} + +impl<'p> Executor<'p> for &CredentialPool { + type Database = Postgres; + + fn fetch_many<'e, 'q: 'e, E>( + self, + query: E, + ) -> BoxStream<'e, Result, sqlx::Error>> + where + E: 'q + Execute<'q, Postgres>, + { + let pool = self.clone(); + Box::pin(async_stream::try_stream! { + let mut connection = pool.acquire().await?; + let mut rows = connection.fetch_many(query); + while let Some(row) = rows.try_next().await? { yield row; } + }) + } + + fn fetch_optional<'e, 'q: 'e, E>( + self, + query: E, + ) -> BoxFuture<'e, Result, sqlx::Error>> + where + E: 'q + Execute<'q, Postgres>, + { + let pool = self.clone(); + Box::pin(async move { pool.acquire().await?.fetch_optional(query).await }) + } + + fn prepare_with<'e>( + self, + sql: SqlStr, + parameters: &'e [PgTypeInfo], + ) -> BoxFuture<'e, Result> + where + 'p: 'e, + { + let pool = self.clone(); + Box::pin(async move { pool.acquire().await?.prepare_with(sql, parameters).await }) + } + + fn describe<'e>( + self, + sql: SqlStr, + ) -> BoxFuture<'e, Result, sqlx::Error>> { + let pool = self.clone(); + Box::pin(async move { pool.acquire().await?.describe(sql).await }) + } +} + +#[cfg(test)] +#[path = "azure_workload_identity_tests.rs"] +mod tests; diff --git a/crates/openshell-server/src/persistence/azure_workload_identity_tests.rs b/crates/openshell-server/src/persistence/azure_workload_identity_tests.rs new file mode 100644 index 0000000000..39eed0e262 --- /dev/null +++ b/crates/openshell-server/src/persistence/azure_workload_identity_tests.rs @@ -0,0 +1,320 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +async fn fixture() -> (tempfile::TempDir, MockServer, Arc) { + openshell_crypto::tls::ensure_default_provider(); + let directory = tempfile::tempdir().unwrap(); + let token_file = directory.path().join("projected-token"); + tokio::fs::write(&token_file, "first-projected-assertion") + .await + .unwrap(); + let server = MockServer::start().await; + let identity = Arc::new(AzureWorkloadIdentity { + client_id: "11111111-1111-1111-1111-111111111111".into(), + token_file, + endpoint: format!("{}/token", server.uri()), + client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap(), + token: Mutex::new(None), + }); + (directory, server, identity) +} + +async fn token_response(server: &MockServer, value: &str) { + Mock::given(method("POST")) + .and(path("/token")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "access_token": value, "expires_in": 3600, "token_type": "Bearer" + }))) + .expect(1) + .mount(server) + .await; +} + +#[tokio::test] +async fn concurrent_acquisitions_share_one_cached_token() { + let (_directory, server, identity) = fixture().await; + token_response(&server, "first-access-token").await; + let calls = (0..12).map(|_| { + let identity = Arc::clone(&identity); + async move { identity.access_token(Duration::from_secs(5)).await.unwrap() } + }); + for token in futures::future::join_all(calls).await { + assert_eq!(token, "first-access-token"); + } + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 1); + let fields: std::collections::HashMap<_, _> = url::form_urlencoded::parse(&requests[0].body) + .into_owned() + .collect(); + assert_eq!(fields["client_assertion"], "first-projected-assertion"); + assert_eq!(fields["scope"], POSTGRES_SCOPE); + assert_eq!(fields["grant_type"], "client_credentials"); + assert!(!format!("{identity:?}").contains("first-access-token")); +} + +#[tokio::test] +async fn expired_cache_reopens_rotated_projected_token() { + let (directory, server, identity) = fixture().await; + token_response(&server, "first-access-token").await; + identity.access_token(Duration::from_secs(5)).await.unwrap(); + identity.token.lock().await.as_mut().unwrap().expires_at = Instant::now(); + let next = directory.path().join("next-token"); + tokio::fs::write(&next, "second-projected-assertion") + .await + .unwrap(); + tokio::fs::rename(&next, &identity.token_file) + .await + .unwrap(); + server.reset().await; + token_response(&server, "second-access-token").await; + assert_eq!( + identity.access_token(Duration::from_secs(5)).await.unwrap(), + "second-access-token" + ); + let requests = server.received_requests().await.unwrap(); + let fields: std::collections::HashMap<_, _> = url::form_urlencoded::parse(&requests[0].body) + .into_owned() + .collect(); + assert_eq!(fields["client_assertion"], "second-projected-assertion"); +} + +#[tokio::test] +async fn failed_refresh_is_redacted_and_recovers_without_fallback() { + let (_directory, server, identity) = fixture().await; + Mock::given(method("POST")) + .respond_with( + ResponseTemplate::new(503) + .set_body_string("first-projected-assertion SECRET-UPSTREAM-TOKEN"), + ) + .expect(1) + .mount(&server) + .await; + let error = identity + .access_token(Duration::from_secs(5)) + .await + .unwrap_err(); + let diagnostic = format!("{error:?} {error}"); + assert!(!diagnostic.contains("first-projected-assertion")); + assert!(!diagnostic.contains("SECRET-UPSTREAM-TOKEN")); + assert!(identity.token.lock().await.is_none()); + server.reset().await; + token_response(&server, "recovered-token").await; + assert_eq!( + identity.access_token(Duration::from_secs(5)).await.unwrap(), + "recovered-token" + ); +} + +#[tokio::test] +async fn short_lived_or_malformed_tokens_are_not_cached() { + let (_directory, server, identity) = fixture().await; + for response in [ + serde_json::json!({"access_token": "secret", "expires_in": 1, "token_type": "Bearer"}), + serde_json::json!({"access_token": "", "expires_in": 3600, "token_type": "Bearer"}), + serde_json::json!({"access_token": "secret", "expires_in": 3600, "token_type": "Other"}), + serde_json::json!({"access_token": "secret with whitespace", "expires_in": 3600, "token_type": "Bearer"}), + serde_json::json!({"access_token": "SECRET-MALFORMED", "expires_in": "not-a-number"}), + ] { + server.reset().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200).set_body_json(response)) + .mount(&server) + .await; + let error = identity + .access_token(Duration::from_secs(5)) + .await + .unwrap_err(); + assert!(!error.to_string().contains("SECRET-MALFORMED")); + assert!(identity.token.lock().await.is_none()); + } +} + +#[tokio::test] +async fn absent_or_oversized_assertion_never_reaches_entra() { + let (_directory, server, identity) = fixture().await; + for value in [String::new(), "x".repeat(MAX_CREDENTIAL_BYTES + 1)] { + tokio::fs::write(&identity.token_file, value).await.unwrap(); + assert!(identity.access_token(Duration::from_secs(5)).await.is_err()); + } + tokio::fs::remove_file(&identity.token_file).await.unwrap(); + assert!(identity.access_token(Duration::from_secs(5)).await.is_err()); + assert!(server.received_requests().await.unwrap().is_empty()); +} + +#[test] +fn azure_urls_require_verified_tls_and_never_accept_passwords() { + let base = "postgresql://mapped-role@server.postgres.database.azure.com/db"; + assert!(validate_url(&format!("{base}?sslmode=verify-full")).is_ok()); + for url in [ + "sqlite::memory:".to_string(), + base.to_string(), + format!("{base}?sslmode=require"), + format!("{base}?sslmode=disable"), + format!("{base}?sslmode=verify-full&password=URL-SECRET"), + format!("{base}?sslmode=verify-full&passfile=/tmp/pgpass"), + format!("{base}?sslmode=verify-full&host=override"), + "postgresql://role:URL-SECRET@server/db?sslmode=verify-full".into(), + ] { + let error = validate_url(&url).unwrap_err(); + assert!(!error.to_string().contains("URL-SECRET")); + } +} + +#[tokio::test] +async fn azure_mode_rejects_sqlite_before_any_identity_lookup() { + let result = super::super::Store::try_connect_with_database_auth( + "sqlite::memory:", + None, + None, + openshell_core::config::DatabaseAuthMode::AzureWorkloadIdentity, + ) + .await; + assert!( + result + .unwrap_err() + .to_string() + .contains("requires PostgreSQL") + ); +} + +/// Real `PostgreSQL` validates credentials; only the Entra HTTP endpoint is fake. +/// Controlled password changes prove both replica/pool pairs authenticate new +/// sessions with refreshed credentials rather than merely reusing live sessions. +#[tokio::test] +#[ignore = "requires a disposable PostgreSQL; run mise run test:rust:postgres"] +async fn postgres_azure_refresh_reconnects_both_pools_and_replicas() { + let url = std::env::var(super::super::test_postgres::TEST_POSTGRES_URL_ENV).unwrap(); + let admin = PgPool::connect(&url).await.unwrap(); + let role = format!("azure_wi_{}", uuid::Uuid::new_v4().simple()); + sqlx::query(sqlx::AssertSqlSafe(format!( + "CREATE ROLE {role} LOGIN PASSWORD 'first-access-token'" + ))) + .execute(&admin) + .await + .unwrap(); + let mut scoped = url::Url::parse(&url).unwrap(); + scoped.set_username(&role).unwrap(); + scoped.set_password(None).unwrap(); + let options: PgConnectOptions = scoped.as_str().parse().unwrap(); + let options = options.ssl_mode(PgSslMode::Disable).password(""); + let (directory, server, identity) = fixture().await; + token_response(&server, "first-access-token").await; + let pools: Vec<_> = (0..4) + .map(|index| { + CredentialPool::azure( + PgPoolOptions::new() + .max_connections(1) + .acquire_timeout(Duration::from_secs(3)), + options + .clone() + .application_name(&format!("azure-wi-test-{index}")), + Arc::clone(&identity), + ) + }) + .collect(); + for pool in &pools { + assert_eq!( + sqlx::query_scalar::<_, i32>("SELECT 1") + .fetch_one(pool) + .await + .unwrap(), + 1 + ); + } + for pool in [&pools[1], &pools[3]] { + let mut session = pool.acquire().await.unwrap(); + sqlx::query("SELECT pg_advisory_lock(5560)") + .execute(&mut *session) + .await + .unwrap(); + let locked: bool = sqlx::query_scalar("SELECT pg_advisory_unlock(5560)") + .fetch_one(&mut *session) + .await + .unwrap(); + assert!(locked); + } + server.reset().await; + token_response(&server, "second-access-token").await; + sqlx::query(sqlx::AssertSqlSafe(format!( + "ALTER ROLE {role} PASSWORD 'second-access-token'" + ))) + .execute(&admin) + .await + .unwrap(); + tokio::fs::write(directory.path().join("next"), "rotated-projected-assertion") + .await + .unwrap(); + tokio::fs::rename(directory.path().join("next"), &identity.token_file) + .await + .unwrap(); + identity.token.lock().await.as_mut().unwrap().expires_at = Instant::now(); + sqlx::query("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename = $1") + .bind(&role) + .execute(&admin) + .await + .unwrap(); + for pool in &pools { + assert_eq!( + sqlx::query_scalar::<_, i32>("SELECT 2") + .fetch_one(pool) + .await + .unwrap(), + 2 + ); + } + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 1); + let fields: std::collections::HashMap<_, _> = url::form_urlencoded::parse(&requests[0].body) + .into_owned() + .collect(); + assert_eq!(fields["client_assertion"], "rotated-projected-assertion"); + server.reset().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + identity.token.lock().await.as_mut().unwrap().expires_at = Instant::now(); + // Expiry alone must not interrupt healthy physical sessions. + assert_eq!( + sqlx::query_scalar::<_, i32>("SELECT 3") + .fetch_one(&pools[0]) + .await + .unwrap(), + 3 + ); + assert!(server.received_requests().await.unwrap().is_empty()); + sqlx::query("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename = $1") + .bind(&role) + .execute(&admin) + .await + .unwrap(); + let error = sqlx::query_scalar::<_, i32>("SELECT 4") + .fetch_one(&pools[0]) + .await + .unwrap_err(); + assert!(error.to_string().contains("Azure database token exchange")); + server.reset().await; + token_response(&server, "second-access-token").await; + assert_eq!( + sqlx::query_scalar::<_, i32>("SELECT 5") + .fetch_one(&pools[0]) + .await + .unwrap(), + 5 + ); + for pool in pools { + pool.close().await; + } + sqlx::query(sqlx::AssertSqlSafe(format!("DROP ROLE {role}"))) + .execute(&admin) + .await + .unwrap(); + admin.close().await; +} diff --git a/crates/openshell-server/src/persistence/mod.rs b/crates/openshell-server/src/persistence/mod.rs index 21de43daa2..d7b7530c3c 100644 --- a/crates/openshell-server/src/persistence/mod.rs +++ b/crates/openshell-server/src/persistence/mod.rs @@ -3,6 +3,7 @@ //! Persistence layer for `OpenShell` Server. +mod azure_workload_identity; mod image_activation; mod legacy_time_wire; pub mod lock_order; @@ -503,6 +504,28 @@ impl Store { max_connections: Option, lock_max_connections: Option, ) -> CoreResult> { + Self::try_connect_with_database_auth( + url, + max_connections, + lock_max_connections, + openshell_core::config::DatabaseAuthMode::Default, + ) + .await + } + + pub(crate) async fn try_connect_with_database_auth( + url: &str, + max_connections: Option, + lock_max_connections: Option, + auth: openshell_core::config::DatabaseAuthMode, + ) -> CoreResult> { + if auth == openshell_core::config::DatabaseAuthMode::AzureWorkloadIdentity + && !(url.starts_with("postgres://") || url.starts_with("postgresql://")) + { + return Err(CoreError::config( + "azure-workload-identity database authentication requires PostgreSQL", + )); + } if url.starts_with("postgres://") || url.starts_with("postgresql://") { if let Some(configured) = max_connections && configured < postgres::MIN_MAX_CONNECTIONS @@ -515,8 +538,23 @@ impl Store { ))); } Ok(async { - let store = - PostgresStore::connect(url, max_connections, lock_max_connections).await?; + let identity = + if auth == openshell_core::config::DatabaseAuthMode::AzureWorkloadIdentity { + Some( + azure_workload_identity::AzureWorkloadIdentity::from_env() + .map_err(|error| PersistenceError::Config(error.to_string()))?, + ) + } else { + None + }; + let store = PostgresStore::connect_with_identity( + url, + max_connections, + lock_max_connections + .unwrap_or(mutation_lock::MUTATION_LOCK_POOL_MAX_CONNECTIONS), + identity, + ) + .await?; store.migrate().await?; Ok(Self::Postgres(store)) } @@ -1548,7 +1586,9 @@ pub fn current_time_ms() -> i64 { } fn map_db_error(error: &sqlx::Error) -> PersistenceError { - if matches!(error, sqlx::Error::PoolTimedOut) { + if matches!(error, sqlx::Error::PoolTimedOut) + || matches!(error, sqlx::Error::Io(error) if error.kind() == std::io::ErrorKind::TimedOut) + { return PersistenceError::Unavailable(error.to_string()); } if let sqlx::Error::Database(db) = error diff --git a/crates/openshell-server/src/persistence/postgres.rs b/crates/openshell-server/src/persistence/postgres.rs index 2fe608dfa7..50ee6d66f7 100644 --- a/crates/openshell-server/src/persistence/postgres.rs +++ b/crates/openshell-server/src/persistence/postgres.rs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +use super::azure_workload_identity::{self, AzureWorkloadIdentity, CredentialPool}; use super::mutation_lock::{ LOCK_CONNECTION_MIN_BUDGET, LockMode, MUTATION_LOCK_POOL_MAX_CONNECTIONS, MUTATION_LOCK_TIMEOUT, MUTATION_LOCK_TIMEOUT_SETTING, MutationLockSet, @@ -21,7 +22,7 @@ use openshell_core::proto::Sandbox; use prost::Message; use sqlx::pool::PoolConnection; use sqlx::postgres::PgPoolOptions; -use sqlx::{Connection, PgPool, Postgres, QueryBuilder, Row}; +use sqlx::{Connection, Postgres, QueryBuilder, Row}; use std::sync::Arc; use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use std::time::Duration; @@ -47,9 +48,9 @@ use std::collections::HashSet; #[derive(Debug, Clone)] pub struct PostgresStore { - pub(super) pool: PgPool, + pub(super) pool: CredentialPool, /// Dedicated connections for session-level mutation advisory locks. - lock_pool: PgPool, + lock_pool: CredentialPool, /// How acquisitions and guards use the lock pool's connections. lock_usage: Arc, } @@ -342,6 +343,15 @@ impl PostgresStore { url: &str, max_connections: Option, lock_pool_size: u32, + ) -> PersistenceResult { + Self::connect_with_identity(url, max_connections, lock_pool_size, None).await + } + + pub(super) async fn connect_with_identity( + url: &str, + max_connections: Option, + lock_pool_size: u32, + identity: Option>, ) -> PersistenceResult { let max_connections = max_connections.unwrap_or(DEFAULT_MAX_CONNECTIONS); tracing::info!( @@ -349,13 +359,10 @@ impl PostgresStore { lock_max_connections = lock_pool_size, "sizing Postgres connection pools" ); - let pool = PgPoolOptions::new() + let data_options = PgPoolOptions::new() .max_connections(max_connections) - .acquire_timeout(DATA_POOL_ACQUIRE_TIMEOUT) - .connect(url) - .await - .map_err(|e| map_startup_error(&e))?; - let lock_pool = PgPoolOptions::new() + .acquire_timeout(DATA_POOL_ACQUIRE_TIMEOUT); + let lock_options = PgPoolOptions::new() .max_connections(lock_pool_size) .min_connections(0) // Backstop only; callers bound the acquire by their own deadline. @@ -380,9 +387,40 @@ impl PostgresStore { .await?; Ok(true) }) - }) - .connect_lazy(url) - .map_err(|e| map_db_error(&e))?; + }); + let (pool, lock_pool) = if let Some(identity) = identity { + let connect = azure_workload_identity::validate_url(url) + .map_err(|error| PersistenceError::Config(error.to_string()))?; + let replica = + std::env::var("OPENSHELL_REPLICA_ID").unwrap_or_else(|_| "gateway".into()); + let pool = CredentialPool::azure( + data_options, + connect + .clone() + .application_name(&format!("openshell-data/{replica}")), + Arc::clone(&identity), + ); + let lock_pool = CredentialPool::azure( + lock_options, + connect.application_name(&format!("openshell-lock/{replica}")), + identity, + ); + drop( + pool.acquire() + .await + .map_err(|error| map_startup_error(&error))?, + ); + (pool, lock_pool) + } else { + let pool = data_options + .connect(url) + .await + .map_err(|error| map_startup_error(&error))?; + let lock_pool = lock_options + .connect_lazy(url) + .map_err(|error| map_db_error(&error))?; + (pool.into(), lock_pool.into()) + }; Ok(Self { pool, @@ -2532,13 +2570,15 @@ impl PostgresStore { pool: PgPoolOptions::new() .max_connections(10) .connect_lazy(url) - .expect("build the lazy data pool"), + .expect("build the lazy data pool") + .into(), lock_pool: PgPoolOptions::new() .max_connections(lock_pool_size) .min_connections(0) .acquire_timeout(MUTATION_LOCK_TIMEOUT) .connect_lazy(url) - .expect("build the lazy lock pool"), + .expect("build the lazy lock pool") + .into(), lock_usage: Arc::new(LockPoolUsage::new(lock_pool_size)), } } diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index b486eaf165..b4c71ad8fd 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -681,6 +681,55 @@ an image update is pending or retained sandboxes require the private image revision protocol. SQLite supports one gateway process only; activation attests that the previous process has stopped, not that sharing a SQLite file enables HA. +## Azure PostgreSQL workload identity + +Set `database_auth = "azure-workload-identity"` under `[openshell.gateway]` to +use an AKS projected ServiceAccount token for Azure Database for PostgreSQL. +The equivalent flag is `--db-auth azure-workload-identity`; the environment +variable is `OPENSHELL_DB_AUTH`. The default, `default`, preserves SQLite and +password-based PostgreSQL authentication. + +Use a password-free database URL with an explicit mapped PostgreSQL role and +`sslmode=verify-full`. Configure the PostgreSQL Entra principal for the gateway +managed identity's object ID and grant it ownership of the OpenShell database +so it can run migrations. Connect directly to PostgreSQL: session advisory locks +require session affinity and cannot use transaction pooling. + +For Helm, set the following values and create `openshell-db` with a `uri` key +containing `postgresql://openshell_gateway@SERVER.postgres.database.azure.com/openshell?sslmode=verify-full`: + +```yaml +workload: + kind: deployment +replicaCount: 2 +serviceAccount: + name: openshell + annotations: + azure.workload.identity/client-id: MANAGED_IDENTITY_CLIENT_ID +podLabels: + azure.workload.identity/use: "true" +server: + externalDbSecret: openshell-db +gatewayConfig: + "openshell.gateway": + database_auth: azure-workload-identity +``` + +Enable AKS OIDC and workload identity, then federate only the gateway +ServiceAccount subject with audience `api://AzureADTokenExchange`. The AKS +webhook supplies `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, and +`AZURE_FEDERATED_TOKEN_FILE`. Only the public Azure authority is supported. +Sandbox ServiceAccounts do not need this federation or database credentials. + +The gateway caches access tokens until near their reported expiry and rereads +the projected token file on refresh. Both data and advisory-lock pools obtain +valid credentials before opening connections. Healthy sessions remain usable +through token expiry or an Entra outage; new connections fail within the pool +acquisition deadline if refresh fails. The gateway does not fall back to a +password, node identity, or Azure CLI credentials. Check the gateway identity, +federated subject, projected volume, and Entra connectivity when refresh fails; +never print token files or token responses while diagnosing authentication. + ## Database Pool `[openshell.gateway] database_max_connections` sets the connection ceiling for the gateway's database pool. One pool is shared by every database-backed RPC, so this value is also the gateway's ceiling on concurrent database work. When the pool is saturated, callers queue on connection acquisition and sqlx logs that the time to acquire a connection exceeded its slow threshold. The background readiness monitor draws from the same pool, so sustained saturation can push its database ping past the check timeout and flip `/readyz` to not-ready while the gateway is otherwise serving. On Postgres, a caller that gets no connection within 5 seconds fails instead of waiting for the database to answer, usually with a retryable `UNAVAILABLE` error whose reason is `DATABASE_UNAVAILABLE`. diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 1e8995fd8b..674475a0cb 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1759,3 +1759,21 @@ admits 16 unary requests (four per principal, eight per workspace), and 32 watch record/route/byte limits; `UNAVAILABLE` means storage or collection timeout. Watches recheck bearer credentials and membership every second. Allow one refresh interval for withdrawals; route lease expiry remains independently enforced. + +## Azure PostgreSQL workload identity + +When `database_auth = "azure-workload-identity"` is configured, verify the gateway +ServiceAccount client-ID annotation and gateway Pod label +`azure.workload.identity/use: "true"`. AKS must enable OIDC and workload identity; +the federated subject must match the gateway namespace and ServiceAccount with +audience `api://AzureADTokenExchange`. Check that the webhook injected +`AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, and `AZURE_FEDERATED_TOKEN_FILE`, without +printing the token file or environment secrets. The database URL must omit the +password, name the mapped PostgreSQL role, and use `sslmode=verify-full`. + +Both database pools refresh credentials before opening connections and retain +healthy sessions. A refresh outage can therefore affect new connections while +existing connections continue working. Check Entra egress, the federated +credential, and PostgreSQL role mapping; do not suggest CLI login, node identity, +or a password fallback. Sandbox ServiceAccounts must not carry gateway database +federation. See the published gateway configuration documentation for setup. From 99164a7f3eddd464db6a2fd3eb1a99a0268be231 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Mon, 16 Nov 2026 17:26:24 -0800 Subject: [PATCH 2/5] docs(gateway): clarify azure postgres schema grants Signed-off-by: Taylor Mutch --- docs/how-it-works/gateways/configuration.mdx | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index b4c71ad8fd..932f6da377 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -692,8 +692,17 @@ password-based PostgreSQL authentication. Use a password-free database URL with an explicit mapped PostgreSQL role and `sslmode=verify-full`. Configure the PostgreSQL Entra principal for the gateway managed identity's object ID and grant it ownership of the OpenShell database -so it can run migrations. Connect directly to PostgreSQL: session advisory locks -require session affinity and cannot use transaction pooling. +so it can run migrations. Also grant `USAGE` and `CREATE` on its `public` +schema; Azure PostgreSQL can retain separate schema ownership after you change +the database owner. Run this as the Entra administrator connected to the +OpenShell database, replacing the role name if needed: + +```sql +GRANT USAGE, CREATE ON SCHEMA public TO openshell_gateway; +``` + +Connect directly to PostgreSQL. Session advisory locks require session affinity +and cannot use transaction pooling. For Helm, set the following values and create `openshell-db` with a `uri` key containing `postgresql://openshell_gateway@SERVER.postgres.database.azure.com/openshell?sslmode=verify-full`: From 8a5bef038261f9028eb7776abe20f61b3dacdffe Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Tue, 17 Nov 2026 08:25:59 -0800 Subject: [PATCH 3/5] fix(server): reuse busy postgres sessions during entra outages Signed-off-by: Taylor Mutch --- .../persistence/azure_workload_identity.rs | 79 +++++++++-- .../azure_workload_identity_tests.rs | 131 ++++++++++++++++++ docs/how-it-works/gateways/configuration.mdx | 6 +- skills/debug-openshell-cluster/SKILL.md | 6 +- 4 files changed, 208 insertions(+), 14 deletions(-) diff --git a/crates/openshell-server/src/persistence/azure_workload_identity.rs b/crates/openshell-server/src/persistence/azure_workload_identity.rs index b4b37e4d4f..d74771ed9d 100644 --- a/crates/openshell-server/src/persistence/azure_workload_identity.rs +++ b/crates/openshell-server/src/persistence/azure_workload_identity.rs @@ -28,6 +28,9 @@ use tokio::time::Instant; const POSTGRES_SCOPE: &str = "https://ossrdbms-aad.database.windows.net/.default"; const MAX_CREDENTIAL_BYTES: usize = 64 * 1024; const CLOCK_SKEW: Duration = Duration::from_secs(30); +// SQLx exposes no notification for returned idle connections. Poll only while +// refresh is unavailable/in flight; a valid cached token keeps SQLx's native wait. +const REUSE_POLL_INTERVAL: Duration = Duration::from_millis(10); fn config_error(message: &'static str) -> sqlx::Error { sqlx::Error::Configuration(message.into()) @@ -109,6 +112,14 @@ impl AzureWorkloadIdentity { })) } + fn cached_access_token(&self, connection_budget: Duration) -> Option { + let cached = self.token.try_lock().ok()?; + cached + .as_ref() + .filter(|token| token.expires_at > Instant::now() + connection_budget + CLOCK_SKEW) + .map(|token| token.value.clone()) + } + async fn access_token(&self, connection_budget: Duration) -> Result { // One refresh per replica, shared by data and advisory-lock pools. let mut cached = self.token.lock().await; @@ -282,25 +293,73 @@ impl CredentialPool { return self.inner.acquire().await; }; let budget = self.inner.options().get_acquire_timeout(); + let mut refresh_error = None; let acquired = tokio::time::timeout(budget, async { - if let Some(mut connection) = self.inner.try_acquire() { - if connection.ping().await.is_ok() { - return Ok(connection); + if let Some(connection) = self.try_reuse().await? { + return Ok(connection); + } + let reusable = async { + loop { + if let Some(connection) = self.try_reuse().await? { + return Ok(connection); + } + tokio::time::sleep(REUSE_POLL_INTERVAL).await; } - let _ = connection.close().await; + }; + tokio::pin!(reusable); + let opening = async { + let token = loop { + // Preserve SQLx's native queuing when credentials are + // cached, including when every connection is checked out. + if let Some(token) = identity.cached_access_token(budget) { + break token; + } + // At capacity, only a returned session can serve us. + // Do not make healthy busy sessions depend on Entra. + if self.inner.size() < self.inner.options().get_max_connections() { + break identity.access_token(budget).await?; + } + tokio::time::sleep(REUSE_POLL_INTERVAL).await; + }; + let options = (*self.inner.connect_options()).clone().password(&token); + self.inner.set_connect_options(options); + self.inner.acquire().await + }; + tokio::select! { + biased; + result = &mut reusable => result, + result = opening => match result { + Ok(connection) => Ok(connection), + Err(error) if self.inner.size() == 0 => Err(error), + Err(error) => { + // A failed refresh must not preempt an existing + // healthy session that can return before the deadline. + refresh_error = Some(error); + reusable.await + } + }, } - let token = identity.access_token(budget).await?; - let options = (*self.inner.connect_options()).clone().password(&token); - self.inner.set_connect_options(options); - self.inner.acquire().await }) .await; - acquired.map_err(|_| unavailable( + acquired.unwrap_or_else(|_| Err(refresh_error.unwrap_or_else(|| unavailable( "Azure database connection acquisition timed out; check workload identity and PostgreSQL connectivity", - ))? + )))) }) } + async fn try_reuse(&self) -> Result>, sqlx::Error> { + if self.inner.is_closed() { + return Err(sqlx::Error::PoolClosed); + } + if let Some(mut connection) = self.inner.try_acquire() { + if connection.ping().await.is_ok() { + return Ok(Some(connection)); + } + let _ = connection.close().await; + } + Ok(None) + } + pub(super) async fn begin(&self) -> Result, sqlx::Error> { Transaction::begin( MaybePoolConnection::PoolConnection(self.acquire().await?), diff --git a/crates/openshell-server/src/persistence/azure_workload_identity_tests.rs b/crates/openshell-server/src/persistence/azure_workload_identity_tests.rs index 39eed0e262..3633b30b6e 100644 --- a/crates/openshell-server/src/persistence/azure_workload_identity_tests.rs +++ b/crates/openshell-server/src/persistence/azure_workload_identity_tests.rs @@ -318,3 +318,134 @@ async fn postgres_azure_refresh_reconnects_both_pools_and_replicas() { .unwrap(); admin.close().await; } + +/// Busy pools must wait for reusable sessions even when refresh fails or stalls. +#[tokio::test] +#[ignore = "requires a disposable PostgreSQL; run mise run test:rust:postgres"] +async fn postgres_azure_busy_pools_reuse_sessions_during_identity_outage() { + let url = std::env::var(super::super::test_postgres::TEST_POSTGRES_URL_ENV).unwrap(); + let admin = PgPool::connect(&url).await.unwrap(); + let role = format!("azure_busy_{}", uuid::Uuid::new_v4().simple()); + sqlx::query(sqlx::AssertSqlSafe(format!( + "CREATE ROLE {role} LOGIN PASSWORD 'busy-pool-token'" + ))) + .execute(&admin) + .await + .unwrap(); + let mut scoped = url::Url::parse(&url).unwrap(); + scoped.set_username(&role).unwrap(); + scoped.set_password(None).unwrap(); + let options: PgConnectOptions = scoped.as_str().parse().unwrap(); + let options = options.ssl_mode(PgSslMode::Disable).password(""); + + assert_busy_pool_reuse(&options, "busy-pool-token").await; + sqlx::query(sqlx::AssertSqlSafe(format!("DROP ROLE {role}"))) + .execute(&admin) + .await + .unwrap(); + admin.close().await; +} + +async fn assert_busy_pool_reuse(options: &PgConnectOptions, access_token: &str) { + for (capacity, delay) in [ + (1, Duration::ZERO), + (2, Duration::ZERO), + (2, Duration::from_secs(15)), + ] { + let (_directory, server, identity) = fixture().await; + token_response(&server, access_token).await; + // Both pool kinds on two replicas use the same acquisition adapter. + let pools: Vec<_> = (0..4) + .map(|_| { + CredentialPool::azure( + PgPoolOptions::new() + .max_connections(capacity) + .acquire_timeout(Duration::from_secs(5)), + options.clone(), + Arc::clone(&identity), + ) + }) + .collect(); + for pool in &pools { + drop(pool.acquire().await.unwrap()); + } + identity.token.lock().await.as_mut().unwrap().expires_at = Instant::now(); + server.reset().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(503).set_delay(delay)) + .mount(&server) + .await; + + for pool in &pools { + let mut held = pool.acquire().await.unwrap(); + let pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&mut *held) + .await + .unwrap(); + let pool = pool.clone(); + let mut waiter = tokio::spawn(async move { pool.acquire().await }); + // Poll the waiter before release: the original code fails here on 503. + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut waiter) + .await + .is_err(), + "busy acquisition must wait for a healthy session during an identity outage" + ); + held.ping().await.unwrap(); + drop(held); + let mut reused = tokio::time::timeout(Duration::from_secs(1), waiter) + .await + .unwrap() + .unwrap() + .unwrap(); + let actual: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&mut *reused) + .await + .unwrap(); + assert_eq!( + actual, pid, + "reuse the returned physical session without authentication" + ); + } + // If no session returns, retain the same finite acquisition deadline. + // The checked-out healthy connection must survive the waiter's failure. + let mut held = pools[0].acquire().await.unwrap(); + let started = Instant::now(); + let error = pools[0].acquire().await.unwrap_err(); + assert!(started.elapsed() >= Duration::from_secs(4)); + assert!(started.elapsed() < Duration::from_secs(7)); + assert!( + matches!(error, sqlx::Error::Io(ref error) if error.kind() == std::io::ErrorKind::TimedOut) + ); + held.ping().await.unwrap(); + drop(held); + if capacity == 1 { + assert!( + server.received_requests().await.unwrap().is_empty(), + "a saturated pool must not request a token while awaiting an existing session" + ); + } + for pool in pools { + pool.close().await; + } + } +} + +/// Run in an AKS Pod with gateway workload identity and its password-free URL. +/// Only initial credential acquisition uses real Entra. Expired cache entries +/// and an unavailable/stalled issuer reproduce the review scenario deterministically; +/// every physical database session uses the real Azure role and verified TLS. +#[tokio::test] +#[ignore = "requires AKS gateway workload identity and OPENSHELL_DB_URL"] +async fn azure_postgres_busy_pools_reuse_sessions_during_identity_outage() { + let url = std::env::var("OPENSHELL_DB_URL").unwrap(); + let options = validate_url(&url).unwrap(); + let identity = AzureWorkloadIdentity::from_env().unwrap(); + let token = identity + .access_token(Duration::from_secs(30)) + .await + .unwrap(); + // The mock issuer receives only fixture assertions, never the real projected + // token. Access tokens are used as database passwords and are never printed. + assert_busy_pool_reuse(&options, &token).await; +} diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 932f6da377..45f4f457fd 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -733,8 +733,10 @@ Sandbox ServiceAccounts do not need this federation or database credentials. The gateway caches access tokens until near their reported expiry and rereads the projected token file on refresh. Both data and advisory-lock pools obtain valid credentials before opening connections. Healthy sessions remain usable -through token expiry or an Entra outage; new connections fail within the pool -acquisition deadline if refresh fails. The gateway does not fall back to a +through token expiry or an Entra outage. Requests can wait for a busy healthy +session to return, even if token refresh fails or stalls. New physical +connections still require a valid token; acquisitions fail within the pool +deadline if neither credential acquisition nor session reuse succeeds. The gateway does not fall back to a password, node identity, or Azure CLI credentials. Check the gateway identity, federated subject, projected volume, and Entra connectivity when refresh fails; never print token files or token responses while diagnosing authentication. diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 674475a0cb..bf9e9e0b83 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1772,8 +1772,10 @@ printing the token file or environment secrets. The database URL must omit the password, name the mapped PostgreSQL role, and use `sslmode=verify-full`. Both database pools refresh credentials before opening connections and retain -healthy sessions. A refresh outage can therefore affect new connections while -existing connections continue working. Check Entra egress, the federated +healthy sessions. A refresh outage can therefore affect new physical connections while +existing connections continue working. Requests can wait for healthy busy +sessions to return within their normal acquisition deadline, even when refresh +fails or stalls. Check Entra egress, the federated credential, and PostgreSQL role mapping; do not suggest CLI login, node identity, or a password fallback. Sandbox ServiceAccounts must not carry gateway database federation. See the published gateway configuration documentation for setup. From f8a5a2a2d31fbf2375b874189d1522857a0b5fbf Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Tue, 17 Nov 2026 08:47:20 -0800 Subject: [PATCH 4/5] docs(server): clarify idle-session polling Signed-off-by: Taylor Mutch --- .../src/persistence/azure_workload_identity.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/openshell-server/src/persistence/azure_workload_identity.rs b/crates/openshell-server/src/persistence/azure_workload_identity.rs index d74771ed9d..df941c6ef1 100644 --- a/crates/openshell-server/src/persistence/azure_workload_identity.rs +++ b/crates/openshell-server/src/persistence/azure_workload_identity.rs @@ -28,8 +28,8 @@ use tokio::time::Instant; const POSTGRES_SCOPE: &str = "https://ossrdbms-aad.database.windows.net/.default"; const MAX_CREDENTIAL_BYTES: usize = 64 * 1024; const CLOCK_SKEW: Duration = Duration::from_secs(30); -// SQLx exposes no notification for returned idle connections. Poll only while -// refresh is unavailable/in flight; a valid cached token keeps SQLx's native wait. +// SQLx exposes no notification for returned idle connections. Poll alongside +// credential acquisition so returned sessions can win without waiting for Entra. const REUSE_POLL_INTERVAL: Duration = Duration::from_millis(10); fn config_error(message: &'static str) -> sqlx::Error { From 15e434befba3c772b5f142c72d6174267ce004f3 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Tue, 17 Nov 2026 10:45:56 -0800 Subject: [PATCH 5/5] fix(postgres): preserve transport timeout error classification Signed-off-by: Taylor Mutch --- .../persistence/azure_workload_identity.rs | 11 +++++++- .../azure_workload_identity_tests.rs | 25 +++++++++++++++++++ .../openshell-server/src/persistence/mod.rs | 13 +++++++--- 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/crates/openshell-server/src/persistence/azure_workload_identity.rs b/crates/openshell-server/src/persistence/azure_workload_identity.rs index df941c6ef1..10046fad1f 100644 --- a/crates/openshell-server/src/persistence/azure_workload_identity.rs +++ b/crates/openshell-server/src/persistence/azure_workload_identity.rs @@ -36,8 +36,17 @@ fn config_error(message: &'static str) -> sqlx::Error { sqlx::Error::Configuration(message.into()) } +/// Credential acquisition failed before the caller could send a statement. +/// Keep this distinct from a database transport timeout with an unknown outcome. +#[derive(Debug, thiserror::Error)] +#[error("{0}")] +pub(super) struct CredentialUnavailable(&'static str); + fn unavailable(message: &'static str) -> sqlx::Error { - sqlx::Error::Io(std::io::Error::new(std::io::ErrorKind::TimedOut, message)) + sqlx::Error::Io(std::io::Error::new( + std::io::ErrorKind::TimedOut, + CredentialUnavailable(message), + )) } #[derive(Deserialize)] diff --git a/crates/openshell-server/src/persistence/azure_workload_identity_tests.rs b/crates/openshell-server/src/persistence/azure_workload_identity_tests.rs index 3633b30b6e..11c56cc706 100644 --- a/crates/openshell-server/src/persistence/azure_workload_identity_tests.rs +++ b/crates/openshell-server/src/persistence/azure_workload_identity_tests.rs @@ -449,3 +449,28 @@ async fn azure_postgres_busy_pools_reuse_sessions_during_identity_outage() { // token. Access tokens are used as database passwords and are never printed. assert_busy_pool_reuse(&options, &token).await; } + +#[test] +fn credential_unavailability_preserves_database_io_error_classification() { + use super::super::{PersistenceError, map_db_error}; + + // A transport timeout may happen after a statement was sent. It must not + // gain the pre-acquisition failure classification used by credentials. + let transport_timeout = sqlx::Error::Io(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "database transport timed out", + )); + assert!(matches!( + map_db_error(&transport_timeout), + PersistenceError::Database(_) + )); + let credential_error = unavailable("Azure credential acquisition failed"); + assert!(matches!( + map_db_error(&credential_error), + PersistenceError::Unavailable(message) if message.contains("Azure credential acquisition failed") + )); + assert!(matches!( + map_db_error(&sqlx::Error::PoolTimedOut), + PersistenceError::Unavailable(_) + )); +} diff --git a/crates/openshell-server/src/persistence/mod.rs b/crates/openshell-server/src/persistence/mod.rs index d7b7530c3c..2f9fc8e7ec 100644 --- a/crates/openshell-server/src/persistence/mod.rs +++ b/crates/openshell-server/src/persistence/mod.rs @@ -1586,9 +1586,16 @@ pub fn current_time_ms() -> i64 { } fn map_db_error(error: &sqlx::Error) -> PersistenceError { - if matches!(error, sqlx::Error::PoolTimedOut) - || matches!(error, sqlx::Error::Io(error) if error.kind() == std::io::ErrorKind::TimedOut) - { + let unavailable = match error { + sqlx::Error::PoolTimedOut => true, + sqlx::Error::Io(error) => error.get_ref().is_some_and( + ::is::< + azure_workload_identity::CredentialUnavailable, + >, + ), + _ => false, + }; + if unavailable { return PersistenceError::Unavailable(error.to_string()); } if let sqlx::Error::Database(db) = error