From be8916385dfa0bedc140ccbd8619addf5b2d5519 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Mon, 16 Nov 2026 14:31:32 -0800 Subject: [PATCH 1/7] feat(kubernetes): configure workload and supervisor placement Signed-off-by: Taylor Mutch --- .agents/skills/helm-dev-environment/SKILL.md | 41 +- crates/openshell-driver-kubernetes/README.md | 25 + .../openshell-driver-kubernetes/src/config.rs | 28 + .../openshell-driver-kubernetes/src/driver.rs | 125 +++- crates/openshell-driver-kubernetes/src/lib.rs | 2 + .../openshell-driver-kubernetes/src/main.rs | 27 + .../src/sandbox_runtime.rs | 4 + .../src/scheduling.rs | 571 ++++++++++++++++++ deploy/helm/openshell/README.md | 6 +- .../openshell/ci/values-scheduling-pools.yaml | 29 + .../ci/values-scheduling-required.yaml | 5 + .../openshell/templates/_gateway-workload.tpl | 3 +- deploy/helm/openshell/templates/_helpers.tpl | 48 ++ deploy/helm/openshell/templates/certgen.yaml | 16 + .../openshell/templates/gateway-config.yaml | 2 +- .../helm/openshell/templates/supervisor.yaml | 13 +- .../tests/gateway_config_serializer_test.yaml | 4 +- .../tests/runtime_scheduling_test.yaml | 153 +++++ .../tests/supervisor_affinity_test.yaml | 2 + .../helm/openshell/tests/supervisor_test.yaml | 5 +- deploy/helm/openshell/values.yaml | 13 +- .../helm/test-gateway-resource-coherence.sh | 2 +- docs/how-it-works/gateways/configuration.mdx | 91 +++ e2e/rust/tests/kubernetes_scheduling.rs | 150 +++++ skills/debug-openshell-cluster/SKILL.md | 13 +- tasks/scripts/helm-k3s-local.sh | 9 + tasks/test.toml | 5 + 27 files changed, 1376 insertions(+), 16 deletions(-) create mode 100644 crates/openshell-driver-kubernetes/src/scheduling.rs create mode 100644 deploy/helm/openshell/ci/values-scheduling-pools.yaml create mode 100644 deploy/helm/openshell/ci/values-scheduling-required.yaml create mode 100644 deploy/helm/openshell/tests/runtime_scheduling_test.yaml create mode 100644 e2e/rust/tests/kubernetes_scheduling.rs diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 59a7d15b1d..ee9c8d7854 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -50,6 +50,7 @@ Port mappings created at cluster time (cannot be changed without recreating): | `8080` | Port `80` via k3d load balancer | Envoy Gateway LoadBalancer service (`values-gateway.yaml`) | Override with env vars before running `helm:k3s:create`: +- `HELM_K3S_NODES` (default: `1`; accepts `1`, `2`, or `3` total nodes for a new cluster) - `HELM_K3S_LB_HOST_PORT` (default: `8080`) - `HELM_K3S_PRELOAD_SANDBOX_IMAGE` (default: `nvcr.io/nvidia/base/ubuntu:24.04`; set to an empty value to skip) @@ -125,8 +126,8 @@ chart renders the gateway's `recovery_timeout_secs` from `supervisor.recoveryTimeoutSeconds`, default 60; the driver's `shared_supervisor` table stays unchanged), a RollingUpdate Deployment (`maxSurge: 1`, `maxUnavailable: 0`) with a 60-second -grace period, no affinity unless `supervisor.affinity` is set (top-level -`affinity` applies only to the gateway), the pool identity env vars +grace period, preferred role anti-affinity merged with `supervisor.affinity` +(top-level `affinity` applies only to the gateway), the pool identity env vars (`OPENSHELL_SUPERVISOR_POOL_MEMBER_ID` and `_ADVERTISE_IP` from the Pod name and IP), and a `maxUnavailable: 1` PodDisruptionBudget unless `supervisor.podDisruptionBudget.enabled=false` (default `true`; toggling it @@ -273,6 +274,42 @@ registers the gateway under the worktree-specific k3d cluster name and selects it as the active gateway. Keep the forwarding task running while using those endpoints. +### Role scheduling validation + +Workloads, supervisors, and gateways use `openshell.ai/runtime-role` with values +`workload`, `supervisor`, and `gateway`. Workload/supervisor anti-affinity defaults +to preferred and matches across all namespaces. Set +`scheduling.workloadSupervisorAntiAffinity=required` for hard separation or +`off` to opt out; `scheduling.namespaceSelector` narrows the namespace scope. +Node selectors and tolerations remain opt-in. Gateways use the top-level chart +fields; supervisors use `supervisor.nodeSelector` and `supervisor.tolerations`; +workloads use gateway TOML `sandbox_runtime.workload_placement`. + +Create an isolated cluster with up to three nodes: + +```bash +HELM_K3S_CLUSTER_NAME=openshell-scheduling HELM_K3S_NODES=3 \ +HELM_K3S_LB_HOST_PORT=18080 mise run helm:k3s:create +``` + +`HELM_K3S_NODES` affects new clusters only; it does not resize an existing one. +Use `mise run e2e:kubernetes:scheduling` for the shared-supervisor create/restart +checks. Against the isolated cluster, set `OPENSHELL_E2E_KUBE_CONTEXT` and +`OPENSHELL_E2E_KUBE_BUILD_IMAGES=1` to build matching branch images. +Layer `ci/values-scheduling-required.yaml` through +`OPENSHELL_E2E_KUBE_EXTRA_VALUES` and set `OPENSHELL_E2E_SCHEDULING_MODE=required` +for hard separation. The `ci/values-scheduling-pools.yaml` fixture additionally +requires one node labeled `openshell.ai/node-pool=` for each of +`gateway`, `supervisor`, and `workload`. Taint the two worker pools with +`openshell.ai/node-pool=:NoSchedule` to exercise tolerations; leave the +gateway node untainted for the local-path storage provisioner +(or configure that provisioner to tolerate the gateway taint). Set +`OPENSHELL_E2E_SCHEDULING_POOLS=1` to verify their nodes. Add the managed-workspace +overlay to check cross-namespace placement. Verify a one-eligible-node preferred +case as well. Do not cordon, relabel, or taint nodes in another development +cluster. A pending required workload gets fresh bounded startup grace when +capacity returns; it must not be reaped immediately after becoming scheduled. + ### Viewing local traces The gateway exports OTLP/gRPC to diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index d1db98a0d1..b43a208268 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -88,6 +88,31 @@ The supervisor Pod has a direct, non-controller owner reference to the Sandbox resource. This links its garbage-collection lifecycle to the sandbox without competing with the Agent Sandbox controller for workload-Pod ownership. +### Scheduling + +The operator-owned `sandbox_runtime.pod_anti_affinity` defaults to `preferred`; +`required` enforces workload/supervisor node separation and `off` removes the +generated rule. The driver uses the reserved `openshell.ai/runtime-role` label +(`workload` or `supervisor`), hostname topology, and a native +`pod_anti_affinity_namespace_selector` (empty means all namespaces). Helm also +labels gateways and shared supervisors and renders reciprocal supervisor rules. + +Optional `sandbox_runtime.workload_placement.node_selector` and `tolerations` +apply independently of caller driver-config opt-in. Conflicting caller selectors +fail; tolerations append without duplicates. A reserved template annotation +records operator additions so stop/start can replace them while preserving +caller placement and unrelated admission rules. Restart reconciles only the +reserved role anti-affinity term, using a merge patch that clears removed keys. +Admission checks validate required workload placement before releasing the +bootstrap scheduling gate. A verified unschedulable generation may wait for +capacity; once bound, its scheduling transition starts a bounded startup grace. + +Standalone drivers expose `--pod-anti-affinity`, +`--pod-anti-affinity-namespace-selector` (JSON), and `--workload-placement` (JSON), +with corresponding `OPENSHELL_K8S_*` environment variables. See the +[gateway configuration reference](../../docs/how-it-works/gateways/configuration.mdx#kubernetes-pod-placement) +for deployment examples and cross-namespace scope. + When the gateway exports OTLP traces, the driver sets `OPENSHELL_OTLP_ENDPOINT` on the supervisor Pod to the gateway's endpoint and `TRACEPARENT` to the trace context of the operation that created the Pod. The diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index a18b51e1de..d7217c43bf 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -119,6 +119,12 @@ impl FromStr for KubernetesControlConnection { #[serde(default, deny_unknown_fields)] pub struct KubernetesSandboxRuntimeConfig { pub app_listener: KubernetesAppListenerConfig, + /// Separation of workload pods from supervisor pods (preferred by default). + pub pod_anti_affinity: crate::scheduling::KubernetesPodAntiAffinity, + /// Namespaces whose opposite-role pods participate; empty selects all. + pub pod_anti_affinity_namespace_selector: + k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector, + pub workload_placement: crate::scheduling::KubernetesWorkloadPlacement, /// TCP port of the Sandbox Protocol listener: the workload boundary for /// supervisor-initiated sandboxes, the supervisor for runtime-initiated ones. pub boundary_port: u16, @@ -129,6 +135,10 @@ impl Default for KubernetesSandboxRuntimeConfig { Self { boundary_port: 5500, app_listener: KubernetesAppListenerConfig::default(), + pod_anti_affinity: crate::KubernetesPodAntiAffinity::default(), + pod_anti_affinity_namespace_selector: + k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector::default(), + workload_placement: crate::KubernetesWorkloadPlacement::default(), } } } @@ -146,6 +156,24 @@ pub struct KubernetesAppListenerConfig { impl KubernetesSandboxRuntimeConfig { pub fn validate(&self) -> Result<(), String> { self.app_listener.common.validate()?; + for toleration in &self.workload_placement.tolerations { + let operator = toleration.operator.as_deref().unwrap_or("Equal"); + if !matches!(operator, "Equal" | "Exists") + || (operator == "Exists" + && toleration + .value + .as_ref() + .is_some_and(|value| !value.is_empty())) + || (operator == "Equal" && toleration.key.as_ref().is_none_or(String::is_empty)) + || toleration.effect.as_deref().is_some_and(|effect| { + !matches!(effect, "" | "NoSchedule" | "PreferNoSchedule" | "NoExecute") + }) + || (toleration.toleration_seconds.is_some() + && toleration.effect.as_deref() != Some("NoExecute")) + { + return Err("invalid sandbox_runtime.workload_placement.tolerations: use Kubernetes toleration operators and effects".into()); + } + } if self.app_listener.common.enabled { if !is_dns_subdomain(&self.app_listener.tls_secret_name) { return Err("sandbox_runtime.app_listener.tls_secret_name is required when the app listener is enabled".into()); diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index e1a4226187..0274eaeb15 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -2151,6 +2151,7 @@ impl KubernetesComputeDriver { .namespace_for_workspace(&sandbox.workspace, self.operator_allowlist.as_ref()) .map_err(tonic::Status::failed_precondition)?; let params = SandboxPodParams { + scheduling: Some(&self.config.sandbox_runtime), default_image: &self.config.default_image, image_pull_secrets: &self.config.image_pull_secrets, default_runtime_class_name: &self.config.default_runtime_class_name, @@ -2642,6 +2643,7 @@ impl KubernetesComputeDriver { })?; let log_level = openshell_core::driver_utils::sandbox_log_level(sandbox, "info"); let params = SandboxPodParams { + scheduling: Some(&self.config.sandbox_runtime), default_image: &self.config.default_image, image_pull_policy: self.config.image_pull_policy, image_pull_secrets: &image_pull_secrets, @@ -3585,9 +3587,27 @@ impl KubernetesComputeDriver { ), &self.config.supervisor_runtime_class_name, ); - pods.create(&PostParams::default(), &pod) + let mut value = serde_json::to_value(pod) + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + value["metadata"]["labels"][crate::scheduling::ROLE_LABEL] = "supervisor".into(); + crate::scheduling::configure_affinity( + &mut value["spec"], + self.config.sandbox_runtime.pod_anti_affinity, + &self + .config + .sandbox_runtime + .pod_anti_affinity_namespace_selector, + "workload", + ); + let pod: Pod = serde_json::from_value(value) + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let admitted = pods + .create(&PostParams::default(), &pod) .await - .map_err(KubernetesDriverError::from_kube)? + .map_err(KubernetesDriverError::from_kube)?; + crate::scheduling::validate_supervisor(&admitted, &self.config.sandbox_runtime) + .map_err(KubernetesDriverError::Precondition)?; + admitted .metadata .uid .ok_or_else(|| KubernetesDriverError::Message("supervisor Pod has no UID".to_string())) @@ -3842,6 +3862,8 @@ impl KubernetesComputeDriver { agent_gid, &names.sandbox_secret, )?; + crate::scheduling::validate_workload(&workload_pod, &self.config.sandbox_runtime) + .map_err(KubernetesDriverError::Precondition)?; let workload_pod_uid = workload_pod.metadata.uid.clone().ok_or_else(|| { KubernetesDriverError::Message("workload Pod has no UID".to_string()) @@ -4362,6 +4384,11 @@ impl KubernetesComputeDriver { )); } + let old_template = &object.data["spec"]["podTemplate"]; + let mut template = old_template.clone(); + crate::scheduling::configure_workload(&mut template, &self.config.sandbox_runtime) + .map_err(KubernetesDriverError::Precondition)?; + let scheduling_patch = crate::scheduling::merge_patch(old_template, &template); let control_connection = KubernetesControlConnection::RuntimeInitiated; let placement = self.new_generation_placement(); let names = SandboxRuntimeNames::for_generation(sandbox_id, generation.as_str()); @@ -4539,6 +4566,12 @@ impl KubernetesComputeDriver { &init_containers, &image_pull_secrets, ); + if let Some(fields) = scheduling_patch["spec"].as_object() { + running_patch["spec"]["podTemplate"]["spec"].as_object_mut().expect("restart spec").extend(fields.clone()); + } + if let Some(metadata) = scheduling_patch.get("metadata") { + running_patch["spec"]["podTemplate"]["metadata"] = metadata.clone(); + } running_patch }, ) @@ -5534,6 +5567,74 @@ impl KubernetesComputeDriver { } } + async fn bootstrap_waiting_for_scheduling( + &self, + object: &DynamicObject, + ) -> Result { + if sandbox_runtime_bootstrap_phase(object) != Some(SandboxRuntimeBootstrapPhase::Released) { + return Ok(false); + } + let namespace = object + .metadata + .namespace + .as_deref() + .unwrap_or(&self.config.namespace); + let Some(name) = object.metadata.name.as_deref() else { + return Ok(false); + }; + let Some(annotations) = object.metadata.annotations.as_ref() else { + return Ok(false); + }; + let Some(uid) = annotations.get(ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID) else { + return Ok(false); + }; + let pods = Api::::namespaced(self.client.clone(), namespace); + let Some(workload) = tokio::time::timeout(KUBE_API_TIMEOUT, pods.get_opt(name)) + .await + .map_err(|_| { + KubernetesDriverError::Message("timed out checking workload scheduling".into()) + })? + .map_err(KubernetesDriverError::from_kube)? + else { + return Ok(false); + }; + if workload.metadata.uid.as_ref() != Some(uid) { + return Ok(false); + } + let mut generation_pods = vec![workload]; + if let Some(uid) = annotations.get(ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID) { + let sandbox_id = + sandbox_id_from_object(object).map_err(KubernetesDriverError::Precondition)?; + let name = SandboxRuntimeNames::new(&sandbox_id).supervisor_pod; + let Some(supervisor) = tokio::time::timeout(KUBE_API_TIMEOUT, pods.get_opt(&name)) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out checking supervisor scheduling".into(), + ) + })? + .map_err(KubernetesDriverError::from_kube)? + else { + return Ok(false); + }; + if supervisor.metadata.uid.as_ref() != Some(uid) { + return Ok(false); + } + generation_pods.push(supervisor); + } + if generation_pods.iter().any(|pod| { + pod.metadata.deletion_timestamp.is_some() + || pod.status.as_ref().is_some_and(|status| { + matches!(status.phase.as_deref(), Some("Failed" | "Succeeded")) + }) + }) { + return Ok(false); + } + Ok(generation_pods.iter().any(|pod| { + crate::scheduling::scheduling_wait(pod, openshell_core::time::now_ms(), 300_000) + })) + } + async fn reap_stale_sandbox_runtime_bootstrap( &self, lookup_api: &AgentSandboxApi, @@ -5546,6 +5647,14 @@ impl KubernetesComputeDriver { ) { return; } + match self.bootstrap_waiting_for_scheduling(object).await { + Ok(true) => return, + Ok(false) => {} + Err(error) => { + debug!(%error, "could not verify scheduling wait; bootstrap cleanup will retry"); + return; + } + } if sandbox_runtime_bootstrap_operation(object) != Some("create") { warn!( sandbox = object.metadata.name.as_deref().unwrap_or(""), @@ -7461,6 +7570,10 @@ fn apply_supervisor_sandbox_runtime_boundary( BOUNDARY_PAIR_LABEL.to_string(), serde_json::json!(crate::sandbox_runtime::pair_label_value(params.sandbox_id)), ); + labels.insert( + crate::scheduling::ROLE_LABEL.to_string(), + serde_json::json!("workload"), + ); labels.insert( BOUNDARY_ROLE_LABEL.to_string(), serde_json::json!("workload"), @@ -7820,6 +7933,7 @@ fn default_workspace_volume_claim_templates( /// Parameters shared by `sandbox_to_k8s_spec` and `sandbox_template_to_k8s`. #[allow(clippy::struct_excessive_bools)] struct SandboxPodParams<'a> { + scheduling: Option<&'a crate::KubernetesSandboxRuntimeConfig>, default_image: &'a str, image_pull_policy: Option, image_pull_secrets: &'a [String], @@ -7844,6 +7958,7 @@ struct SandboxPodParams<'a> { impl Default for SandboxPodParams<'_> { fn default() -> Self { Self { + scheduling: None, default_image: "", image_pull_policy: None, image_pull_secrets: &[], @@ -7953,6 +8068,12 @@ fn sandbox_to_k8s_spec( ); } + let config = params.scheduling.cloned().unwrap_or_default(); + crate::scheduling::configure_workload( + root.get_mut("podTemplate") + .expect("pod template is required"), + &config, + )?; Ok(serde_json::Value::Object( std::iter::once(("spec".to_string(), serde_json::Value::Object(root))).collect(), )) diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index f3a1e419ec..59b899ccfc 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -10,6 +10,8 @@ mod lifecycle; pub mod otel_tracing; mod resource_admission; mod sandbox_runtime; +mod scheduling; +pub use scheduling::{KubernetesPodAntiAffinity, KubernetesWorkloadPlacement}; mod shared_supervisor; pub use config::{ diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index b67accbaca..8afbe15c3f 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -154,6 +154,26 @@ struct Args { )] sandbox_runtime_boundary_port: u16, + /// Workload/supervisor node separation: off, preferred, or required. + #[arg( + long, + env = "OPENSHELL_K8S_POD_ANTI_AFFINITY", + default_value = "preferred" + )] + pod_anti_affinity: openshell_driver_kubernetes::KubernetesPodAntiAffinity, + + /// Native Kubernetes namespace selector JSON (default: all namespaces). + #[arg( + long, + env = "OPENSHELL_K8S_POD_ANTI_AFFINITY_NAMESPACE_SELECTOR", + default_value = "{}" + )] + pod_anti_affinity_namespace_selector: String, + + /// Operator workload placement JSON, with `node_selector` and tolerations. + #[arg(long, env = "OPENSHELL_K8S_WORKLOAD_PLACEMENT", default_value = "{}")] + workload_placement: String, + /// Corporate HTTP forward proxy for policy-approved TLS CONNECT egress. #[arg(long, env = "OPENSHELL_UPSTREAM_PROXY")] https_proxy: Option, @@ -294,6 +314,13 @@ async fn main() -> Result<()> { app_listener: openshell_driver_kubernetes::config::KubernetesAppListenerConfig::default(), boundary_port: args.sandbox_runtime_boundary_port, + pod_anti_affinity: args.pod_anti_affinity, + pod_anti_affinity_namespace_selector: serde_json::from_str( + &args.pod_anti_affinity_namespace_selector, + ) + .into_diagnostic()?, + workload_placement: serde_json::from_str(&args.workload_placement) + .into_diagnostic()?, }, runtime_bootstrap_grace_seconds: args.runtime_bootstrap_grace_seconds, sandbox_attach_request_timeout_seconds: args.sandbox_attach_request_timeout_seconds, diff --git a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs index 9ed4ce71e9..0c6c97fd0c 100644 --- a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs +++ b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs @@ -834,6 +834,10 @@ fn common_labels(sandbox_id: &str, component: &str) -> BTreeMap fn control_labels(sandbox_id: &str, gateway_id: &str) -> BTreeMap { let mut labels = common_labels(sandbox_id, "supervisor"); + labels.insert( + crate::scheduling::ROLE_LABEL.to_string(), + "supervisor".to_string(), + ); labels.extend(pair_labels(sandbox_id, "supervisor")); labels.insert( "openshell.ai/gateway-id".to_string(), diff --git a/crates/openshell-driver-kubernetes/src/scheduling.rs b/crates/openshell-driver-kubernetes/src/scheduling.rs new file mode 100644 index 0000000000..ec2dad9673 --- /dev/null +++ b/crates/openshell-driver-kubernetes/src/scheduling.rs @@ -0,0 +1,571 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Operator-owned placement of workload and supervisor pods, across namespaces. + +use std::collections::BTreeMap; +use std::str::FromStr; + +use k8s_openapi::api::core::v1::{Pod, Toleration}; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +pub const ROLE_LABEL: &str = "openshell.ai/runtime-role"; +const PLACEMENT_ANNOTATION: &str = "openshell.ai/workload-placement"; +const BOOTSTRAP_GATE: &str = "openshell.ai/bootstrap"; + +/// Strength of the scheduling rule between workloads and supervisors. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, clap::ValueEnum)] +#[serde(rename_all = "snake_case")] +pub enum KubernetesPodAntiAffinity { + Off, + #[default] + Preferred, + Required, +} + +impl FromStr for KubernetesPodAntiAffinity { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "off" => Ok(Self::Off), + "preferred" => Ok(Self::Preferred), + "required" => Ok(Self::Required), + _ => Err("pod_anti_affinity must be off, preferred, or required".into()), + } + } +} + +/// Optional operator restrictions, independent of caller driver-config opt-in. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct KubernetesWorkloadPlacement { + pub node_selector: BTreeMap, + pub tolerations: Vec, +} + +// Remember only the operator's changes, so changing/removing operator placement +// on restart restores caller settings and preserves unrelated admission rules. +#[derive(Default, Serialize, Deserialize)] +struct AppliedPlacement { + node_selector: BTreeMap, + original_selector: BTreeMap, + added_tolerations: Vec, +} + +fn term(peer: &str, namespaces: &LabelSelector) -> Value { + json!({ + "topologyKey": "kubernetes.io/hostname", + "namespaceSelector": namespaces, + "labelSelector": {"matchLabels": {ROLE_LABEL: peer}}, + }) +} + +fn owned_term(value: &Value, peer: &str) -> bool { + value["topologyKey"] == "kubernetes.io/hostname" + && value["labelSelector"] == json!({"matchLabels": {ROLE_LABEL: peer}}) + && value.as_object().is_some_and(|map| { + map.keys().all(|key| { + matches!( + key.as_str(), + "topologyKey" | "labelSelector" | "namespaceSelector" + ) + }) + }) +} + +/// Reconcile only the reserved role rule. Other affinity is left intact. +pub fn configure_affinity( + spec: &mut Value, + mode: KubernetesPodAntiAffinity, + namespaces: &LabelSelector, + peer: &str, +) { + if spec["affinity"].is_null() && mode == KubernetesPodAntiAffinity::Off { + return; + } + if spec["affinity"].is_null() { + spec["affinity"] = json!({}); + } + let affinity = &mut spec["affinity"]; + if affinity["podAntiAffinity"].is_null() { + affinity["podAntiAffinity"] = json!({}); + } + let anti = &mut affinity["podAntiAffinity"]; + let required = "requiredDuringSchedulingIgnoredDuringExecution"; + let preferred = "preferredDuringSchedulingIgnoredDuringExecution"; + if let Some(terms) = anti.get_mut(required).and_then(Value::as_array_mut) { + terms.retain(|value| !owned_term(value, peer)); + } + if let Some(terms) = anti.get_mut(preferred).and_then(Value::as_array_mut) { + terms + .retain(|value| value["weight"] != 100 || !owned_term(&value["podAffinityTerm"], peer)); + } + let generated = term(peer, namespaces); + let (key, value) = match mode { + KubernetesPodAntiAffinity::Off => (required, None), + KubernetesPodAntiAffinity::Required => (required, Some(generated)), + KubernetesPodAntiAffinity::Preferred => ( + preferred, + Some(json!({"weight": 100, "podAffinityTerm": generated})), + ), + }; + if let Some(value) = value { + if anti[key].is_null() { + anti[key] = json!([]); + } + anti[key] + .as_array_mut() + .expect("affinity terms are arrays") + .push(value); + } + for key in [required, preferred] { + if anti[key].as_array().is_some_and(Vec::is_empty) { + anti.as_object_mut().unwrap().remove(key); + } + } + if anti.as_object().is_some_and(serde_json::Map::is_empty) { + affinity.as_object_mut().unwrap().remove("podAntiAffinity"); + } + if affinity.as_object().is_some_and(serde_json::Map::is_empty) { + spec.as_object_mut().unwrap().remove("affinity"); + } +} + +pub fn configure_workload( + template: &mut Value, + config: &crate::KubernetesSandboxRuntimeConfig, +) -> Result<(), String> { + let previous: AppliedPlacement = template["metadata"]["annotations"][PLACEMENT_ANNOTATION] + .as_str() + .map(serde_json::from_str) + .transpose() + .map_err(|error| format!("invalid stored workload placement: {error}"))? + .unwrap_or_default(); + let spec = &mut template["spec"]; + if !spec["affinity"].is_null() { + serde_json::from_value::(spec["affinity"].clone()) + .map_err(|error| format!("invalid stored affinity: {error}"))?; + } + let mut selector: BTreeMap = + serde_json::from_value(spec["nodeSelector"].clone()) + .or_else(|_| { + if spec["nodeSelector"].is_null() { + Ok(BTreeMap::new()) + } else { + Err("invalid nodeSelector") + } + }) + .map_err(str::to_string)?; + for (key, value) in &previous.node_selector { + if selector.get(key) == Some(value) { + selector.remove(key); + if let Some(original) = previous.original_selector.get(key) { + selector.insert(key.clone(), original.clone()); + } + } + } + let mut applied = AppliedPlacement::default(); + for (key, value) in &config.workload_placement.node_selector { + if let Some(existing) = selector.get(key) { + if existing != value { + return Err(format!( + "workload node selector '{key}' conflicts with operator workload_placement" + )); + } + applied + .original_selector + .insert(key.clone(), existing.clone()); + } + selector.insert(key.clone(), value.clone()); + } + applied + .node_selector + .clone_from(&config.workload_placement.node_selector); + if selector.is_empty() { + spec.as_object_mut().unwrap().remove("nodeSelector"); + } else { + spec["nodeSelector"] = json!(selector); + } + let mut tolerations = spec["tolerations"].as_array().cloned().unwrap_or_default(); + for previous in previous.added_tolerations { + if let Some(index) = tolerations.iter().position(|value| *value == previous) { + tolerations.remove(index); + } + } + for toleration in &config.workload_placement.tolerations { + let value = json!(toleration); + if !tolerations.contains(&value) { + tolerations.push(value.clone()); + applied.added_tolerations.push(value); + } + } + if tolerations.is_empty() { + spec.as_object_mut().unwrap().remove("tolerations"); + } else { + spec["tolerations"] = json!(tolerations); + } + configure_affinity( + spec, + config.pod_anti_affinity, + &config.pod_anti_affinity_namespace_selector, + "supervisor", + ); + if template["metadata"].is_null() { + template["metadata"] = json!({}); + } + if template["metadata"]["labels"].is_null() { + template["metadata"]["labels"] = json!({}); + } + template["metadata"]["labels"][ROLE_LABEL] = "workload".into(); + if template["metadata"]["annotations"].is_null() { + template["metadata"]["annotations"] = json!({}); + } + template["metadata"]["annotations"][PLACEMENT_ANNOTATION] = serde_json::to_string(&applied) + .map_err(|error| error.to_string())? + .into(); + Ok(()) +} + +/// A merge patch must explicitly null removed fields (including selector keys). +pub fn merge_patch(old: &Value, new: &Value) -> Value { + if let (Some(old), Some(new)) = (old.as_object(), new.as_object()) { + let mut patch = serde_json::Map::new(); + for key in old.keys().chain(new.keys()) { + if old.get(key) != new.get(key) { + patch.insert( + key.clone(), + merge_patch( + old.get(key).unwrap_or(&Value::Null), + new.get(key).unwrap_or(&Value::Null), + ), + ); + } + } + Value::Object(patch) + } else { + new.clone() + } +} + +pub fn validate_workload( + pod: &Pod, + config: &crate::KubernetesSandboxRuntimeConfig, +) -> Result<(), String> { + validate_role(pod, config, "workload", "supervisor") +} + +pub fn validate_supervisor( + pod: &Pod, + config: &crate::KubernetesSandboxRuntimeConfig, +) -> Result<(), String> { + let config = crate::KubernetesSandboxRuntimeConfig { + workload_placement: KubernetesWorkloadPlacement::default(), + ..config.clone() + }; + validate_role(pod, &config, "supervisor", "workload") +} + +fn validate_role( + pod: &Pod, + config: &crate::KubernetesSandboxRuntimeConfig, + role: &str, + peer: &str, +) -> Result<(), String> { + let fail = |reason| format!("admitted {role} does not preserve operator scheduling: {reason}"); + if pod + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get(ROLE_LABEL)) + .map(String::as_str) + != Some(role) + { + return Err(fail("runtime role label changed")); + } + let spec = pod.spec.as_ref().ok_or_else(|| fail("missing spec"))?; + for (key, value) in &config.workload_placement.node_selector { + if spec + .node_selector + .as_ref() + .and_then(|selector| selector.get(key)) + != Some(value) + { + return Err(fail("node selector changed")); + } + } + for value in &config.workload_placement.tolerations { + if !spec + .tolerations + .as_ref() + .is_some_and(|values| values.contains(value)) + { + return Err(fail("operator toleration removed")); + } + } + if config.pod_anti_affinity != KubernetesPodAntiAffinity::Required + && config.workload_placement.node_selector.is_empty() + { + return Ok(()); + } + if spec.node_name.as_ref().is_some_and(|name| !name.is_empty()) + || spec + .scheduler_name + .as_deref() + .is_some_and(|name| name != "default-scheduler") + || !spec + .scheduling_gates + .as_ref() + .is_some_and(|gates| gates.iter().any(|gate| gate.name == BOOTSTRAP_GATE)) + { + return Err(fail( + "default scheduler and bootstrap scheduling gate required; nodeName must be empty", + )); + } + if config.pod_anti_affinity == KubernetesPodAntiAffinity::Required { + let expected = term(peer, &config.pod_anti_affinity_namespace_selector); + let value = json!(spec); + if !value["affinity"]["podAntiAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"] + .as_array() + .is_some_and(|terms| terms.contains(&expected)) + { + return Err(fail("required anti-affinity changed")); + } + } + Ok(()) +} + +/// Scheduling waits do not consume startup grace; once bound, startup gets a +/// bounded grace from `PodScheduled`, including after a gateway outage. +pub fn scheduling_wait(pod: &Pod, now_ms: i64, grace_ms: i64) -> bool { + if pod.metadata.deletion_timestamp.is_some() { + return false; + } + let Some(status) = &pod.status else { + return false; + }; + if matches!(status.phase.as_deref(), Some("Failed" | "Succeeded")) { + return false; + } + let Some(spec) = &pod.spec else { + return false; + }; + status.conditions.as_ref().is_some_and(|conditions| { + conditions.iter().any(|condition| { + if condition.type_ != "PodScheduled" { + return false; + } + if condition.status == "False" && condition.reason.as_deref() == Some("Unschedulable") { + return spec.node_name.as_ref().is_none_or(String::is_empty) + && spec.scheduling_gates.as_ref().is_none_or(Vec::is_empty); + } + condition.status == "True" + && condition + .last_transition_time + .as_ref() + .is_some_and(|time| now_ms.saturating_sub(time.0.timestamp_millis()) < grace_ms) + }) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::KubernetesSandboxRuntimeConfig as Config; + + fn template() -> Value { + json!({"metadata": {}, "spec": {"containers": []}}) + } + + fn merge(target: &mut Value, patch: &Value) { + if let Some(fields) = patch.as_object() { + if !target.is_object() { + *target = json!({}); + } + for (key, value) in fields { + if value.is_null() { + target.as_object_mut().unwrap().remove(key); + } else { + merge(&mut target[key], value); + } + } + } else { + *target = patch.clone(); + } + } + + #[test] + fn defaults_prefer_cross_namespace_separation_without_node_placement() { + let config: Config = serde_json::from_value(json!({})).unwrap(); + let mut pod = template(); + configure_workload(&mut pod, &config).unwrap(); + assert_eq!(pod["metadata"]["labels"][ROLE_LABEL], "workload"); + assert!(pod["spec"].get("nodeSelector").is_none()); + assert!(pod["spec"].get("tolerations").is_none()); + let preference = &pod["spec"]["affinity"]["podAntiAffinity"]["preferredDuringSchedulingIgnoredDuringExecution"] + [0]; + assert_eq!(preference["weight"], 100); + assert_eq!( + preference["podAffinityTerm"]["namespaceSelector"], + json!({}) + ); + assert_eq!( + preference["podAffinityTerm"]["labelSelector"]["matchLabels"][ROLE_LABEL], + "supervisor" + ); + assert!(serde_json::from_value::(json!({"pod_anti_affinity": "require"})).is_err()); + assert!( + serde_json::from_value::(json!({"workload_placement": {"nodeSelector": {}}})) + .is_err() + ); + } + + #[test] + fn restart_policy_and_scope_transitions_preserve_operator_rules() { + let mut pod = template(); + pod["spec"]["affinity"] = json!({ + "nodeAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": []}, + "podAntiAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": [ + {"topologyKey": "rack", "labelSelector": {"matchLabels": {"app": "database"}}} + ]} + }); + let original = pod["spec"]["affinity"].clone(); + let mut config = Config::default(); + for (mode, namespaces) in [ + (KubernetesPodAntiAffinity::Required, json!({})), + ( + KubernetesPodAntiAffinity::Preferred, + json!({"matchLabels": {"pool": "a"}}), + ), + (KubernetesPodAntiAffinity::Off, json!({})), + ] { + config.pod_anti_affinity = mode; + config.pod_anti_affinity_namespace_selector = + serde_json::from_value(namespaces).unwrap(); + let old = pod.clone(); + configure_workload(&mut pod, &config).unwrap(); + let mut admitted = old.clone(); + merge(&mut admitted, &merge_patch(&old, &pod)); + assert_eq!(admitted, pod, "merge patch must remove obsolete terms"); + let once = pod.clone(); + configure_workload(&mut pod, &config).unwrap(); + assert_eq!(pod, once); + } + assert_eq!(pod["spec"]["affinity"], original); + } + + #[test] + fn node_placement_restores_caller_values_and_preserves_admission_additions() { + let mut pod = template(); + let caller_toleration = json!({"key": "gpu", "operator": "Exists"}); + pod["spec"]["nodeSelector"] = json!({"pool": "sandbox", "gpu": "yes"}); + pod["spec"]["tolerations"] = json!([caller_toleration]); + let config: Config = serde_json::from_value(json!({"workload_placement": { + "node_selector": {"pool": "sandbox", "operator": "yes"}, + "tolerations": [{"key": "gpu", "operator": "Exists"}, {"key": "sandbox", "operator": "Exists", "effect": "NoSchedule"}] + }})).unwrap(); + configure_workload(&mut pod, &config).unwrap(); + let once = pod.clone(); + configure_workload(&mut pod, &config).unwrap(); + assert_eq!(pod, once); + assert_eq!(pod["spec"]["tolerations"].as_array().unwrap().len(), 2); + pod["spec"]["nodeSelector"]["admission"] = "yes".into(); + configure_workload(&mut pod, &Config::default()).unwrap(); + assert_eq!( + pod["spec"]["nodeSelector"], + json!({"pool": "sandbox", "gpu": "yes", "admission": "yes"}) + ); + assert_eq!(pod["spec"]["tolerations"], json!([caller_toleration])); + let conflict: Config = serde_json::from_value( + json!({"workload_placement": {"node_selector": {"pool": "infrastructure"}}}), + ) + .unwrap(); + assert!( + configure_workload(&mut pod, &conflict) + .unwrap_err() + .contains("conflicts with operator") + ); + } + + #[test] + fn required_admission_rejects_scheduler_bypasses_and_changed_policy() { + let config = Config { + pod_anti_affinity: KubernetesPodAntiAffinity::Required, + ..Config::default() + }; + let mut good = template(); + good["spec"]["schedulingGates"] = json!([{"name": BOOTSTRAP_GATE}]); + configure_workload(&mut good, &config).unwrap(); + validate_workload(&serde_json::from_value(good.clone()).unwrap(), &config).unwrap(); + for (path, value) in [ + ( + "/metadata/labels/openshell.ai~1runtime-role", + json!("supervisor"), + ), + ("/spec/affinity", json!({})), + ("/spec/schedulingGates", json!([])), + ] { + let mut mutated = good.clone(); + *mutated.pointer_mut(path).unwrap() = value; + assert!(validate_workload(&serde_json::from_value(mutated).unwrap(), &config).is_err()); + } + for (key, value) in [("schedulerName", "custom"), ("nodeName", "node-1")] { + let mut mutated = good.clone(); + mutated["spec"][key] = value.into(); + assert!(validate_workload(&serde_json::from_value(mutated).unwrap(), &config).is_err()); + } + } + + #[test] + fn operator_node_placement_is_validated_after_admission() { + let config: Config = serde_json::from_value(json!({"workload_placement": { + "node_selector": {"pool": "sandbox"}, + "tolerations": [{"key": "pool", "operator": "Exists"}] + }})) + .unwrap(); + config.validate().unwrap(); + let mut pod = template(); + pod["spec"]["schedulingGates"] = json!([{"name": BOOTSTRAP_GATE}]); + configure_workload(&mut pod, &config).unwrap(); + validate_workload(&serde_json::from_value(pod.clone()).unwrap(), &config).unwrap(); + let original = pod.clone(); + pod["spec"]["nodeSelector"]["pool"] = "infrastructure".into(); + assert!(validate_workload(&serde_json::from_value(pod).unwrap(), &config).is_err()); + let mut pod = original; + pod["spec"]["tolerations"] = json!([]); + assert!(validate_workload(&serde_json::from_value(pod).unwrap(), &config).is_err()); + let invalid: Config = serde_json::from_value(json!({"workload_placement": { + "tolerations": [{"operator": "Exists", "value": "not-empty"}] + }})) + .unwrap(); + assert!(invalid.validate().is_err()); + } + + #[test] + fn restored_capacity_gets_bounded_startup_grace_from_scheduling_transition() { + let mut pod: Pod = serde_json::from_value(json!({ + "metadata": {}, "spec": {"containers": []}, + "status": {"phase": "Pending", "conditions": [{ + "type": "PodScheduled", "status": "False", "reason": "Unschedulable" + }]} + })) + .unwrap(); + assert!(scheduling_wait(&pod, 10_000_000, 300_000)); + pod.spec.as_mut().unwrap().node_name = Some("new-node".into()); + pod.status.as_mut().unwrap().conditions = Some(serde_json::from_value(json!([{ + "type": "PodScheduled", "status": "True", "lastTransitionTime": "1970-01-01T02:46:40Z" + }])).unwrap()); + assert!( + scheduling_wait(&pod, 10_100_000, 300_000), + "ContainerCreating after a long capacity wait must survive" + ); + assert!( + !scheduling_wait(&pod, 10_300_000, 300_000), + "startup failures must eventually be cleaned up" + ); + pod.status.as_mut().unwrap().phase = Some("Failed".into()); + assert!(!scheduling_wait(&pod, 10_100_000, 300_000)); + } +} diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index c2940a0784..5656a6a990 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -653,6 +653,8 @@ discovery endpoint or its TLS CA. | sandboxServiceAccount.annotations | object | `{}` | Annotations to add to the generated sandbox service account. | | sandboxServiceAccount.create | bool | `true` | Create a service account for sandbox pods. | | sandboxServiceAccount.name | string | `""` | Existing service account name for sandbox pods when sandboxServiceAccount.create is false. | +| scheduling.namespaceSelector | object | `nil` | Native namespace label selector for role anti-affinity. Null inherits gatewayConfig, then {} (all namespaces). | +| scheduling.workloadSupervisorAntiAffinity | string | `nil` | Workload/supervisor anti-affinity: preferred (default), required, or off. Null inherits gatewayConfig, then preferred. Sets both driver and shared-supervisor policy. | | securityContext.allowPrivilegeEscalation | bool | `false` | Whether the gateway container can gain additional privileges. | | securityContext.capabilities.drop | list | `["ALL"]` | Linux capabilities dropped from the gateway container. | | securityContext.runAsNonRoot | bool | `true` | Require the gateway container to run as a non-root user. | @@ -745,7 +747,7 @@ discovery endpoint or its TLS CA. | serviceAccount.annotations | object | `{}` | Annotations to add to the generated service account. | | serviceAccount.create | bool | `true` | Create a service account for the gateway. | | serviceAccount.name | string | `""` | Existing service account name to use when serviceAccount.create is false. | -| supervisor.affinity | object | `{}` | Affinity rules for the shared supervisor pod, rendered only when gatewayConfig selects supervisor_topology = "gateway". Top-level affinity applies only to the gateway pod. | +| supervisor.affinity | object | `{}` | Additional affinity rules merged with the shared supervisor role policy, rendered only when gatewayConfig selects supervisor_topology = "gateway". Top-level affinity applies only to the gateway pod. | | supervisor.autoscaling.behavior | object | `{"scaleDown":{"policies":[{"periodSeconds":120,"type":"Pods","value":1}],"stabilizationWindowSeconds":300}}` | HPA scaling behavior. Remove at most one member every two minutes, after a five-minute stabilization window, to allow ownership handoff. | | supervisor.autoscaling.enabled | bool | `false` | Enable memory-based autoscaling of the shared supervisor Deployment. Requires supervisor_topology = "gateway" and the Kubernetes resource metrics API (usually Metrics Server). When enabled, the HPA owns the replica count and supervisor.replicas is ignored. | | supervisor.autoscaling.maxReplicas | int | `4` | Maximum supervisor replicas. | @@ -757,6 +759,7 @@ discovery endpoint or its TLS CA. | supervisor.image.repository | string | `"openshell/supervisor"` | Supervisor image repository. | | supervisor.image.tag | string | `""` | Supervisor image tag. Defaults to the chart appVersion when empty. | | supervisor.network.additionalCaConfigMapName | string | `""` | Name of an operator-owned ConfigMap with a `ca.crt` key containing additional destination CA certificates for sandbox supervisors. Empty disables additional destination trust. | +| supervisor.nodeSelector | object | `{}` | Optional node selector for shared supervisors. Empty imposes no pool restriction. | | supervisor.podDisruptionBudget.enabled | bool | `true` | Render a PodDisruptionBudget with maxUnavailable 1 for the shared supervisor, used only when gatewayConfig selects supervisor_topology = "gateway". Disable it only when maintenance must evict more than one supervisor Pod at a time, or evict a member while another is unavailable. | | supervisor.podSecurityContext.fsGroup | int | `65534` | fsGroup assigned to the shared supervisor pod. | | supervisor.podSecurityContext.runAsGroup | int | `65534` | Primary GID assigned to the shared supervisor pod. | @@ -766,6 +769,7 @@ discovery endpoint or its TLS CA. | supervisor.recoveryTimeoutSeconds | int | `60` | Seconds a sandbox workload stays frozen after losing its pool member's supervision before the sandbox runtime terminates it, rendered as [openshell.gateway.supervisor_pool] recovery_timeout_secs. Must be an integer from 30 to 300. Null leaves the gateway's pool default of 60 seconds. | | supervisor.replicas | int | `1` | Shared supervisor pool members, used only when gatewayConfig selects supervisor_topology = "gateway". The gateway always coordinates the shared supervisor as a high-availability pool: when a member fails or leaves, another member takes over its sandboxes without restarting their workloads. 1 has no failover; use 2 or more to survive losing a member. Only the supervisor Deployment's replica count depends on this value, so changing it neither restarts the gateway nor replaces running members. Ignored when supervisor.autoscaling.enabled is true. | | supervisor.sandboxRuntime.boundaryPort | int | `5500` | Workload boundary TLS listener port. | +| supervisor.tolerations | list | `[]` | Optional tolerations for shared supervisors. Empty adds no tolerations. | | tolerations | list | `[]` | Tolerations for the gateway pod. | | topologySpreadConstraints | list | `[]` | Topology spread constraints for the gateway pod, used to spread replicas across zones or nodes. Empty = omit the field. | | upstreamProxy | object | `{"authAllowInsecure":false,"authSecret":{"key":"","name":""},"caBundle":{"configMapName":"","key":"ca.crt"},"connectByHostname":false,"noProxy":"","url":""}` | Operator-owned corporate forward proxy for policy-approved TLS egress from Kubernetes sandboxes. The workload cannot select or override it. | diff --git a/deploy/helm/openshell/ci/values-scheduling-pools.yaml b/deploy/helm/openshell/ci/values-scheduling-pools.yaml new file mode 100644 index 0000000000..d0f593e599 --- /dev/null +++ b/deploy/helm/openshell/ci/values-scheduling-pools.yaml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Opt-in placement fixture: label/taint one node for each role before installing. +nodeSelector: + openshell.ai/node-pool: gateway +tolerations: + - key: openshell.ai/node-pool + operator: Equal + value: gateway + effect: NoSchedule +supervisor: + nodeSelector: + openshell.ai/node-pool: supervisor + tolerations: + - key: openshell.ai/node-pool + operator: Equal + value: supervisor + effect: NoSchedule +gatewayConfig: + openshell.drivers.kubernetes: + sandbox_runtime: + workload_placement: + node_selector: + openshell.ai/node-pool: workload + tolerations: + - key: openshell.ai/node-pool + operator: Equal + value: workload + effect: NoSchedule diff --git a/deploy/helm/openshell/ci/values-scheduling-required.yaml b/deploy/helm/openshell/ci/values-scheduling-required.yaml new file mode 100644 index 0000000000..e91b8484db --- /dev/null +++ b/deploy/helm/openshell/ci/values-scheduling-required.yaml @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Layer with values-supervisor.yaml and values-tls-port-forward.yaml for E2E. +scheduling: + workloadSupervisorAntiAffinity: required diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index fed49a498c..9ebf06ae7f 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -31,7 +31,8 @@ metadata: {{- end }} labels: {{- include "openshell.labels" . | nindent 4 }} - {{- with .Values.podLabels }} + openshell.ai/runtime-role: gateway + {{- with (omit .Values.podLabels "openshell.ai/runtime-role") }} {{- toYaml . | nindent 4 }} {{- end }} spec: diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 5177348ae2..3eaccf6b72 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -655,6 +655,29 @@ absent fields so every chart consumer observes the same configuration. {{- if hasKey $gatewayConfig "openshell.drivers.kubernetes.sandbox_runtime" -}} {{- $runtimeConfig = deepCopy (get $gatewayConfig "openshell.drivers.kubernetes.sandbox_runtime") -}} {{- end -}} +{{/* Resolve the chart convenience setting once for the gateway and supervisor. */}} +{{- $scheduling := .Values.scheduling | default dict -}} +{{- $mode := "preferred" -}} +{{- if hasKey $runtimeConfig "pod_anti_affinity" -}}{{- $mode = get $runtimeConfig "pod_anti_affinity" -}}{{- end -}} +{{- if and (hasKey $scheduling "workloadSupervisorAntiAffinity") (ne (get $scheduling "workloadSupervisorAntiAffinity") nil) -}} +{{- $requested := get $scheduling "workloadSupervisorAntiAffinity" -}} +{{- if and (hasKey $runtimeConfig "pod_anti_affinity") (ne $mode $requested) -}} +{{- fail "scheduling.workloadSupervisorAntiAffinity conflicts with gatewayConfig sandbox_runtime.pod_anti_affinity" -}} +{{- end -}} +{{- $mode = $requested -}} +{{- end -}} +{{- if not (has $mode (list "off" "preferred" "required")) -}}{{- fail "pod anti-affinity must be off, preferred, or required" -}}{{- end -}} +{{- $_ := set $runtimeConfig "pod_anti_affinity" $mode -}} +{{- $namespaces := get $runtimeConfig "pod_anti_affinity_namespace_selector" | default dict -}} +{{- if and (hasKey $scheduling "namespaceSelector") (ne (get $scheduling "namespaceSelector") nil) -}} +{{- $requested := get $scheduling "namespaceSelector" -}} +{{- if and (hasKey $runtimeConfig "pod_anti_affinity_namespace_selector") (ne (toJson $namespaces) (toJson $requested)) -}} +{{- fail "scheduling.namespaceSelector conflicts with gatewayConfig pod_anti_affinity_namespace_selector" -}} +{{- end -}} +{{- $namespaces = $requested -}} +{{- end -}} +{{- if not (kindIs "map" $namespaces) -}}{{- fail "pod anti-affinity namespace selector must be a map" -}}{{- end -}} +{{- $_ := set $runtimeConfig "pod_anti_affinity_namespace_selector" $namespaces -}} {{- $appDeployment := include "openshell.appListenerDeployment" . | fromYaml -}} {{- if $appDeployment -}} {{- $_ := set $runtimeConfig "app_listener" $appDeployment -}} @@ -739,6 +762,9 @@ like user-supplied schema-v2 fields during compatibility resolution. */}} Validate chart values that Helm would otherwise accept silently. */}} {{- define "openshell.validateValues" -}} +{{- if and (hasKey .Values.podLabels "openshell.ai/runtime-role") (ne (get .Values.podLabels "openshell.ai/runtime-role") "gateway") -}} +{{- fail "podLabels cannot override openshell.ai/runtime-role=gateway" -}} +{{- end -}} {{- $metricsEnabled := include "openshell.metricsEnabled" . -}} {{- $kubernetes := include "openshell.effectiveKubernetesConfig" . | fromYaml -}} {{- if hasKey $kubernetes "supervisor_topology" -}} @@ -888,3 +914,25 @@ Validate chart values that Helm would otherwise accept silently. {{- end -}} {{- toYaml $app -}} {{- end -}} + +{{/* Preserve operator affinity and add the reciprocal role rule for shared supervisors. */}} +{{- define "openshell.supervisorAffinity" -}} +{{- $kubernetes := include "openshell.effectiveKubernetesConfig" . | fromYaml -}} +{{- $runtime := get $kubernetes "sandbox_runtime" -}} +{{- $affinity := deepCopy (.Values.supervisor.affinity | default dict) -}} +{{- $mode := get $runtime "pod_anti_affinity" -}} +{{- if ne $mode "off" -}} +{{- $term := dict "topologyKey" "kubernetes.io/hostname" "namespaceSelector" (get $runtime "pod_anti_affinity_namespace_selector") "labelSelector" (dict "matchLabels" (dict "openshell.ai/runtime-role" "workload")) -}} +{{- $anti := get $affinity "podAntiAffinity" | default dict -}} +{{- $key := "requiredDuringSchedulingIgnoredDuringExecution" -}} +{{- $value := $term -}} +{{- if eq $mode "preferred" -}} +{{- $key = "preferredDuringSchedulingIgnoredDuringExecution" -}} +{{- $value = dict "weight" 100 "podAffinityTerm" $term -}} +{{- end -}} +{{- $terms := get $anti $key | default list -}} +{{- if not (has $value $terms) -}}{{- $_ := set $anti $key (append $terms $value) -}}{{- end -}} +{{- $_ := set $affinity "podAntiAffinity" $anti -}} +{{- end -}} +{{- if $affinity -}}{{- toYaml $affinity -}}{{- end -}} +{{- end -}} diff --git a/deploy/helm/openshell/templates/certgen.yaml b/deploy/helm/openshell/templates/certgen.yaml index c378916ecc..a435390b66 100644 --- a/deploy/helm/openshell/templates/certgen.yaml +++ b/deploy/helm/openshell/templates/certgen.yaml @@ -83,6 +83,14 @@ spec: {{- end }} spec: restartPolicy: OnFailure + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} serviceAccountName: {{ $hookName }} {{- with .Values.imagePullSecrets }} imagePullSecrets: @@ -155,6 +163,14 @@ spec: {{- end }} spec: restartPolicy: OnFailure + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} serviceAccountName: {{ $hookName }} {{- with .Values.imagePullSecrets }} imagePullSecrets: diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index bd7fefdc07..91499df489 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -19,4 +19,4 @@ metadata: {{- include "openshell.labels" . | nindent 4 }} data: gateway.toml: | -{{ include "openshell.gatewayConfigToml" . | nindent 4 }} +{{ include "openshell.gatewayConfigToml" . | indent 4 }} diff --git a/deploy/helm/openshell/templates/supervisor.yaml b/deploy/helm/openshell/templates/supervisor.yaml index 513f49d283..be167ef535 100644 --- a/deploy/helm/openshell/templates/supervisor.yaml +++ b/deploy/helm/openshell/templates/supervisor.yaml @@ -59,6 +59,7 @@ spec: metadata: labels: {{- include "openshell.supervisorSelectorLabels" . | nindent 8 }} + openshell.ai/runtime-role: supervisor spec: automountServiceAccountToken: false enableServiceLinks: false @@ -75,10 +76,18 @@ spec: securityContext: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.supervisor.affinity }} - affinity: + {{- with .Values.supervisor.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.supervisor.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{- with (include "openshell.supervisorAffinity" .) }} + affinity: + {{- . | nindent 8 }} + {{- end }} containers: - name: supervisor image: {{ get $kubernetes "supervisor_image" | quote }} diff --git a/deploy/helm/openshell/tests/gateway_config_serializer_test.yaml b/deploy/helm/openshell/tests/gateway_config_serializer_test.yaml index 8d0afc30bc..cb56680b69 100644 --- a/deploy/helm/openshell/tests/gateway_config_serializer_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_serializer_test.yaml @@ -88,7 +88,7 @@ tests: pattern: '(?m)^proxy_ca_bundle = "/schema-v2/ca\.crt"$' - matchRegex: path: data["gateway.toml"] - pattern: '(?m)^sandbox_runtime = \{ boundary_port = 7700 \}$' + pattern: '(?m)^sandbox_runtime = \{ boundary_port = 7700, pod_anti_affinity = "preferred", pod_anti_affinity_namespace_selector = \{ \} \}$' - notMatchRegex: path: data["gateway.toml"] pattern: legacy\.example|6600|proxy-ca @@ -100,7 +100,7 @@ tests: asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?m)^sandbox_runtime = \{ boundary_port = 6600 \}$' + pattern: '(?m)^sandbox_runtime = \{ boundary_port = 6600, pod_anti_affinity = "preferred", pod_anti_affinity_namespace_selector = \{ \} \}$' - notMatchRegex: path: data["gateway.toml"] pattern: network_policy_enforced diff --git a/deploy/helm/openshell/tests/runtime_scheduling_test.yaml b/deploy/helm/openshell/tests/runtime_scheduling_test.yaml new file mode 100644 index 0000000000..cbaac4f09a --- /dev/null +++ b/deploy/helm/openshell/tests/runtime_scheduling_test.yaml @@ -0,0 +1,153 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +suite: runtime role scheduling +templates: + - templates/statefulset.yaml + - templates/deployment.yaml + - templates/supervisor.yaml + - templates/gateway-config.yaml + - templates/certgen.yaml +set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway +tests: + - it: labels gateway pods without modifying controller selectors + template: templates/statefulset.yaml + asserts: + - equal: + path: spec.template.metadata.labels["openshell.ai/runtime-role"] + value: gateway + - notExists: + path: spec.selector.matchLabels["openshell.ai/runtime-role"] + - notExists: + path: spec.template.spec.nodeSelector + - notExists: + path: spec.template.spec.tolerations + - it: labels Deployment gateway pods too + template: templates/deployment.yaml + set: + workload.kind: Deployment + server.externalDbSecret: test-postgres + asserts: + - equal: + path: spec.template.metadata.labels["openshell.ai/runtime-role"] + value: gateway + - it: defaults supervisors to soft cross namespace separation without node restrictions + template: templates/supervisor.yaml + asserts: + - equal: + path: spec.template.metadata.labels["openshell.ai/runtime-role"] + value: supervisor + - equal: + path: spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0] + value: + weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + openshell.ai/runtime-role: workload + namespaceSelector: {} + topologyKey: kubernetes.io/hostname + - notExists: + path: spec.template.spec.nodeSelector + - notExists: + path: spec.template.spec.tolerations + - it: merges required role separation with custom affinity and namespace scope + template: templates/supervisor.yaml + set: + scheduling: + workloadSupervisorAntiAffinity: required + namespaceSelector: + matchLabels: + scheduling.example.com/group: a + supervisor: + nodeSelector: + pool: supervisor + tolerations: + - key: pool + operator: Equal + value: supervisor + effect: NoSchedule + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: rack + labelSelector: + matchLabels: + openshell.ai/runtime-role: gateway + asserts: + - lengthEqual: + path: spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution + count: 2 + - equal: + path: spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[1].namespaceSelector.matchLabels + value: + scheduling.example.com/group: a + - equal: + path: spec.template.spec.nodeSelector + value: + pool: supervisor + - equal: + path: spec.template.spec.tolerations[0].value + value: supervisor + - it: allows opting out while retaining role labels + template: templates/supervisor.yaml + set: + scheduling.workloadSupervisorAntiAffinity: "off" + asserts: + - notExists: + path: spec.template.spec.affinity + - equal: + path: spec.template.metadata.labels["openshell.ai/runtime-role"] + value: supervisor + - it: honors direct gateway configuration on supervisors + template: templates/supervisor.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + sandbox_runtime: + pod_anti_affinity: required + asserts: + - exists: + path: spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution + - it: rejects contradictory chart and gateway policies + template: templates/supervisor.yaml + set: + scheduling.workloadSupervisorAntiAffinity: required + gatewayConfig: + openshell.drivers.kubernetes: + sandbox_runtime: + pod_anti_affinity: preferred + asserts: + - failedTemplate: + errorMessage: scheduling.workloadSupervisorAntiAffinity conflicts with gatewayConfig sandbox_runtime.pod_anti_affinity + - it: rejects gateway role overrides + template: templates/supervisor.yaml + set: + podLabels: + openshell.ai/runtime-role: supervisor + asserts: + - failedTemplate: + errorMessage: podLabels cannot override openshell.ai/runtime-role=gateway + - it: places PKI hooks with the gateway on opt in infrastructure nodes + template: templates/certgen.yaml + documentIndex: 3 + set: + nodeSelector: + pool: infrastructure + tolerations: + - key: pool + operator: Equal + value: infrastructure + effect: NoSchedule + asserts: + - equal: + path: spec.template.spec.nodeSelector + value: + pool: infrastructure + - equal: + path: spec.template.spec.tolerations[0].value + value: infrastructure + - notExists: + path: spec.template.metadata.labels["openshell.ai/runtime-role"] diff --git a/deploy/helm/openshell/tests/supervisor_affinity_test.yaml b/deploy/helm/openshell/tests/supervisor_affinity_test.yaml index 6b426a2dc5..fc0927b5e5 100644 --- a/deploy/helm/openshell/tests/supervisor_affinity_test.yaml +++ b/deploy/helm/openshell/tests/supervisor_affinity_test.yaml @@ -9,6 +9,8 @@ release: name: openshell namespace: gateway set: + scheduling: + workloadSupervisorAntiAffinity: off gatewayConfig: openshell.drivers.kubernetes: supervisor_topology: gateway diff --git a/deploy/helm/openshell/tests/supervisor_test.yaml b/deploy/helm/openshell/tests/supervisor_test.yaml index 49a9916cbc..2dd3b8d055 100644 --- a/deploy/helm/openshell/tests/supervisor_test.yaml +++ b/deploy/helm/openshell/tests/supervisor_test.yaml @@ -344,8 +344,9 @@ tests: path: spec.template.spec.terminationGracePeriodSeconds value: 60 documentIndex: 0 - - notExists: - path: spec.template.spec.affinity + - equal: + path: spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].weight + value: 100 documentIndex: 0 - contains: path: spec.template.spec.containers[0].env diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index c8995f7926..c427633ead 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -3,6 +3,13 @@ # Default values for OpenShell +# Scheduling policy for workload/supervisor roles. Does not constrain gateway placement. +scheduling: + # -- Workload/supervisor anti-affinity: preferred (default), required, or off. Null inherits gatewayConfig, then preferred. Sets both driver and shared-supervisor policy. + workloadSupervisorAntiAffinity: null + # -- (object) Native namespace label selector for role anti-affinity. Null inherits gatewayConfig, then {} (all namespaces). + namespaceSelector: null + # -- Number of OpenShell gateway replicas. Values greater than 1 require # server.externalDbSecret because the default SQLite backend is per pod. replicaCount: 1 @@ -130,7 +137,11 @@ supervisor: seccompProfile: # -- Seccomp profile applied to the shared supervisor pod. type: RuntimeDefault - # -- Affinity rules for the shared supervisor pod, rendered only when + # -- Optional node selector for shared supervisors. Empty imposes no pool restriction. + nodeSelector: {} + # -- Optional tolerations for shared supervisors. Empty adds no tolerations. + tolerations: [] + # -- Additional affinity rules merged with the shared supervisor role policy, rendered only when # gatewayConfig selects supervisor_topology = "gateway". Top-level affinity # applies only to the gateway pod. affinity: {} diff --git a/deploy/helm/test-gateway-resource-coherence.sh b/deploy/helm/test-gateway-resource-coherence.sh index df68025831..06766cf00b 100755 --- a/deploy/helm/test-gateway-resource-coherence.sh +++ b/deploy/helm/test-gateway-resource-coherence.sh @@ -18,7 +18,7 @@ render() { --set agentSandbox.preflight.enabled=false \ "$@" >"${work_dir}/${name}.yaml" - if invalid_documents="$(yq ea -r 'select(. != null and ((has("apiVersion") | not) or (has("kind") | not))) | .kind // ""' "${work_dir}/${name}.yaml")" && [[ -n "${invalid_documents}" ]]; then + if invalid_documents="$(yq ea -o=json 'select(. != null and ((has("apiVersion") | not) or (has("kind") | not)))' "${work_dir}/${name}.yaml")" && [[ -n "${invalid_documents}" ]]; then printf 'rendered Kubernetes documents without apiVersion or kind:\n%s\n' "${invalid_documents}" >&2 echo "${name}: rendered an invalid Kubernetes document" >&2 exit 1 diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index b486eaf165..b39c602ea3 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -1459,8 +1459,99 @@ gateway_pod_selector = { "app.kubernetes.io/name" = "openshell", "app.kubernetes [openshell.drivers.kubernetes.sandbox_runtime] # TLS-protected boundary listener reached only by the paired control pod. boundary_port = 5500 +# Workload/supervisor node separation: preferred (default), required, or off. +pod_anti_affinity = "preferred" ``` +### Kubernetes pod placement + +OpenShell labels pods with `openshell.ai/runtime-role`: `workload`, `supervisor`, +or `gateway`. The Kubernetes driver labels workloads and dedicated supervisors; +Helm labels gateways and shared supervisors. These labels do not change Service +or controller selectors and can be used in native pod affinity rules. To target +gateways, select `openshell.ai/runtime-role: gateway` in your own rules under +`supervisor.affinity` or the gateway's top-level Helm `affinity`. These rules +coexist with the generated workload/supervisor policy. + +Workloads and supervisors prefer separate nodes by default. Set +`sandbox_runtime.pod_anti_affinity = "required"` to require separation, or `"off"` +to omit the generated rule. Required mode leaves pods Pending when separation +cannot be satisfied. Preferred mode permits single-node deployments. Gateway +placement is independent; the default rule does not select gateways. + +Rules use `kubernetes.io/hostname` and select the opposite role across **all +namespaces**, including other OpenShell installations. To limit the scope, label +the participating namespaces and configure a native namespace label selector: + +```toml +[openshell.drivers.kubernetes.sandbox_runtime.pod_anti_affinity_namespace_selector.matchLabels] +"example.com/openshell-group" = "production" +``` + +In Helm, `scheduling.workloadSupervisorAntiAffinity` sets the policy for both the +driver and shared-supervisor Deployment. It inherits gateway TOML when unset, +then defaults to `preferred`. `scheduling.namespaceSelector` similarly sets both +sides' namespace selector. Contradictory chart and gateway settings are rejected. +For example: + +```yaml +scheduling: + workloadSupervisorAntiAffinity: required + namespaceSelector: + matchLabels: + example.com/openshell-group: production +``` + +For independently deployed shared supervisors, apply the supervisor role label +and configure native anti-affinity against workloads in the supervisor Pod +template. Use the same namespace scope and policy strength as the gateway: + +```yaml +metadata: + labels: + openshell.ai/runtime-role: supervisor +spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + openshell.ai/runtime-role: workload + namespaceSelector: {} + topologyKey: kubernetes.io/hostname +``` + +This example selects all namespaces. For soft separation, put the term under +`preferredDuringSchedulingIgnoredDuringExecution` as a `podAffinityTerm` with +`weight: 100`. `off` only removes OpenShell-generated rules; operator-authored +rules remain effective. The gateway never changes independently deployed +supervisor pods. + +Node selectors and tolerations are opt-in and empty by default. To select a +sandbox node pool without enabling caller driver config: + +```toml +[openshell.drivers.kubernetes.sandbox_runtime.workload_placement] +node_selector = { "openshell.ai/node-pool" = "sandbox" } +tolerations = [{ key = "openshell.ai/node-pool", operator = "Equal", value = "sandbox", effect = "NoSchedule" }] +``` + +For gateway placement, use Helm's `nodeSelector` and `tolerations`; shared +supervisors use `supervisor.nodeSelector` and `supervisor.tolerations`. PKI hook +Jobs inherit gateway node selectors and tolerations. For +example, set both selectors to `openshell.ai/node-pool: infrastructure` to run +infrastructure together and workloads elsewhere. Selectors constrain eligible +nodes. Tolerations permit use of tainted nodes and do not select a pool. +Anti-affinity alone does not reserve nodes for future supervisors. + +Operator workload selectors merge with caller selectors; conflicting values +are rejected. Operator tolerations append without duplicates and retain caller +tolerations. Placement is applied on creation and stop/start, preserving +unrelated affinity. Running pods are not moved or continuously evicted. Recreate +existing gateway/supervisor pods to acquire the role labels during rollout; +older unlabeled pods are not selected. A capacity wait does not consume the +workload's bounded startup grace after scheduling. + The Kubernetes driver always creates the workload `NetworkPolicy` before releasing a sandbox Pod. Your cluster CNI must enforce ingress and egress `NetworkPolicy` in every sandbox namespace; the Kubernetes API does not verify diff --git a/e2e/rust/tests/kubernetes_scheduling.rs b/e2e/rust/tests/kubernetes_scheduling.rs new file mode 100644 index 0000000000..06fc4cdb54 --- /dev/null +++ b/e2e/rust/tests/kubernetes_scheduling.rs @@ -0,0 +1,150 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e-kubernetes")] + +//! Run via e2e:kubernetes:scheduling with a shared-supervisor fixture. The same +//! test covers one eligible node, required separation, and three node pools. + +use openshell_e2e::harness::cli::run_cli; +use openshell_e2e::harness::sandbox::SandboxGuard; +use serde_json::Value; + +async fn kubectl(args: &[&str]) -> Value { + let mut command = tokio::process::Command::new("kubectl"); + if let Ok(context) = std::env::var("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE") { + command.arg("--context").arg(context); + } + let output = command.args(args).output().await.expect("spawn kubectl"); + assert!( + output.status.success(), + "kubectl {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + serde_json::from_slice(&output.stdout).expect("kubectl JSON") +} + +async fn assert_pool(node: &str, role: &str) { + let node = kubectl(&["get", "node", node, "-o", "json"]).await; + assert_eq!(node["metadata"]["labels"]["openshell.ai/node-pool"], role); +} + +async fn assert_placement(name: &str) -> Value { + let mode = + std::env::var("OPENSHELL_E2E_SCHEDULING_MODE").unwrap_or_else(|_| "preferred".into()); + let pools = std::env::var("OPENSHELL_E2E_SCHEDULING_POOLS").is_ok_and(|value| value == "1"); + let resources = kubectl(&[ + "get", + "sandboxes.agents.x-k8s.io", + "-A", + "-l", + &format!("openshell.ai/sandbox-name={name}"), + "-o", + "json", + ]) + .await; + let resources = resources["items"].as_array().unwrap(); + assert_eq!(resources.len(), 1); + let namespace = resources[0]["metadata"]["namespace"].as_str().unwrap(); + let pod_name = resources[0]["metadata"]["name"].as_str().unwrap(); + let workload = kubectl(&["get", "pod", pod_name, "-n", namespace, "-o", "json"]).await; + assert_eq!( + workload["metadata"]["labels"]["openshell.ai/runtime-role"], + "workload" + ); + let namespace = + std::env::var("OPENSHELL_E2E_KUBE_NAMESPACE").unwrap_or_else(|_| "openshell".into()); + let infrastructure = kubectl(&[ + "get", + "pods", + "-n", + &namespace, + "-l", + "openshell.ai/runtime-role", + "-o", + "json", + ]) + .await; + let key = if mode == "required" { + "requiredDuringSchedulingIgnoredDuringExecution" + } else { + "preferredDuringSchedulingIgnoredDuringExecution" + }; + if mode != "off" { + let rule = &workload["spec"]["affinity"]["podAntiAffinity"][key][0]; + let term = if mode == "preferred" { + &rule["podAffinityTerm"] + } else { + rule + }; + assert_eq!( + term["labelSelector"]["matchLabels"]["openshell.ai/runtime-role"], + "supervisor" + ); + assert_eq!(term["namespaceSelector"], serde_json::json!({})); + } + let mut found_supervisor = false; + let mut gateway_node = None; + for pod in infrastructure["items"].as_array().unwrap() { + if !pod["metadata"]["deletionTimestamp"].is_null() { + continue; + } + let role = pod["metadata"]["labels"]["openshell.ai/runtime-role"] + .as_str() + .unwrap(); + let node = pod["spec"]["nodeName"] + .as_str() + .expect("infrastructure scheduled"); + if role == "supervisor" { + found_supervisor = true; + if mode == "required" { + assert_ne!(pod["spec"]["nodeName"], workload["spec"]["nodeName"]); + } + if mode != "off" { + assert!(pod["spec"]["affinity"]["podAntiAffinity"][key].is_array()); + } + } + if role == "gateway" { + gateway_node = Some(node.to_string()); + } + if pools { + let node = kubectl(&["get", "node", node, "-o", "json"]).await; + assert_eq!(node["metadata"]["labels"]["openshell.ai/node-pool"], role); + } + } + assert!( + found_supervisor, + "shared supervisor must carry the stable role label" + ); + assert!( + gateway_node.is_some(), + "gateway must carry the stable role label" + ); + if pools { + assert_pool(workload["spec"]["nodeName"].as_str().unwrap(), "workload").await; + } + workload +} + +#[tokio::test] +async fn role_placement_survives_workload_restart() { + let mut sandbox = SandboxGuard::create_keep_with_args( + &["--from", "docker.io/library/ubuntu:24.04"], + &["sh", "-c", "echo scheduling-ready; exec sleep infinity"], + "scheduling-ready", + ) + .await + .expect("create scheduled sandbox"); + let before = assert_placement(&sandbox.name).await; + for verb in ["stop", "start"] { + let (output, code) = run_cli(&["sandbox", verb, &sandbox.name]).await; + assert_eq!(code, 0, "{verb}: {output}"); + } + sandbox + .exec(&["true"]) + .await + .expect("restarted sandbox usable"); + let after = assert_placement(&sandbox.name).await; + assert_ne!(before["metadata"]["uid"], after["metadata"]["uid"]); + sandbox.cleanup().await; +} diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 1e8995fd8b..1b83a0d202 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1064,6 +1064,18 @@ kubectl -n openshell get "${GATEWAY_DEPLOYMENT}" -o jsonpath="{.spec.template.sp helm -n openshell get values openshell | grep -E 'repository|tag|supervisorImage|workload' ``` +Workload/supervisor node separation is preferred by default. Check +`sandbox_runtime.pod_anti_affinity` in gateway configuration and +`scheduling.workloadSupervisorAntiAffinity` in Helm values. Required mode needs +separate eligible nodes; inspect both roles' `FailedScheduling` events. The +reserved pod label `openshell.ai/runtime-role` has values `workload`, +`supervisor`, and `gateway`; role anti-affinity selects across all namespaces +unless `pod_anti_affinity_namespace_selector` / `scheduling.namespaceSelector` +restricts scope. Node selectors/tolerations are opt-in: check gateway chart +fields, `supervisor.nodeSelector` / `supervisor.tolerations`, and +`sandbox_runtime.workload_placement`. Existing unlabeled pods need recreation. +The published gateway configuration reference documents the supported fields. + The gateway, sandbox, and supervisor images should use the same release tag. A stale runtime image can make sandbox behavior lag behind gateway policy or protocol changes. Kubernetes sandboxes take the configured supervisor and runtime images each time they start, so a sandbox that kept running through an upgrade uses its previous images until its next stop and start. For vulnerability reports, record the running image digest and scan that exact @@ -1750,7 +1762,6 @@ Shared serving leases last at most fifteen seconds from issuance, independently of configuration renewal; include membership failure detection when measuring a partition's stale-owner interval. Storage failures prevent renewal. - Application ingress consumers call the separate `openshell.v1.ServiceDiscovery` service, rather than `OpenShell`. Discovery snapshots share a one-second cache per workspace (16 cached workspaces, four concurrent collections). The gateway diff --git a/tasks/scripts/helm-k3s-local.sh b/tasks/scripts/helm-k3s-local.sh index 2a5aaf272d..cc04cb0f31 100755 --- a/tasks/scripts/helm-k3s-local.sh +++ b/tasks/scripts/helm-k3s-local.sh @@ -24,6 +24,8 @@ CLUSTER_NAME="${HELM_K3S_CLUSTER_NAME:-openshell-dev${_suffix:+-${_suffix}}}" # k3d caps cluster names at 32 chars; validated in cmd_create so the operator # gets an actionable hint instead of a deep-stack k3d validation error. K3D_CLUSTER_NAME_MAX=32 +# One server plus optional agents for isolated scheduling validation. +CLUSTER_NODES="${HELM_K3S_NODES:-1}" # Host port forwarded to port 80 via the k3d load balancer. # Used by Envoy Gateway's LoadBalancer service (values-gateway.yaml). HOST_LB_PORT="${HELM_K3S_LB_HOST_PORT:-8080}" @@ -73,6 +75,7 @@ Environment: HELM_K3S_CLUSTER_NAME k3d cluster name (default: openshell-dev-) Each git worktree gets its own cluster derived from its branch name. Override to share a single cluster across worktrees. + HELM_K3S_NODES Total nodes for a new cluster: 1 (default), 2, or 3 HELM_K3S_KUBECONFIG kubeconfig file to write/merge (default: repo kubeconfig or \$KUBECONFIG) HELM_K3S_LB_HOST_PORT Host port mapped to load balancer port 80 (default: 8080) HELM_K3S_PRELOAD_SANDBOX_IMAGE @@ -394,6 +397,11 @@ EOF exit 1 fi + if [[ ! "${CLUSTER_NODES}" =~ ^[123]$ ]]; then + echo "error: HELM_K3S_NODES must be 1, 2, or 3." >&2 + exit 1 + fi + local lb_port_map="${HOST_LB_PORT}:80@loadbalancer" if k3d_cluster_exists; then @@ -403,6 +411,7 @@ EOF echo "Creating k3d cluster '${CLUSTER_NAME}'..." k3d cluster create "${CLUSTER_NAME}" \ --wait \ + --agents "$((CLUSTER_NODES - 1))" \ --kubeconfig-update-default=false \ --kubeconfig-switch-context=false \ --port "${lb_port_map}" \ diff --git a/tasks/test.toml b/tasks/test.toml index 26c2ddd6a5..e3a57d34f4 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -423,3 +423,8 @@ description = "Test generated local gateway TOML without starting a runtime" run = "bash tasks/scripts/test-gateway-config.sh" run_windows = "echo Skipping test:gateway-config: Unix gateway scripts do not apply on Windows." hide = true + +["e2e:kubernetes:scheduling"] +description = "Validate shared-supervisor role labels and placement, including workload stop/start" +env = { OPENSHELL_E2E_KUBE_TEST = "kubernetes_scheduling", OPENSHELL_E2E_KUBE_TLS = "1", OPENSHELL_E2E_KUBE_CONFORMANCE = "0" } +run = 'OPENSHELL_E2E_KUBE_EXTRA_VALUES="deploy/helm/openshell/ci/values-supervisor.yaml:deploy/helm/openshell/ci/values-tls-port-forward.yaml${OPENSHELL_E2E_KUBE_EXTRA_VALUES:+:$OPENSHELL_E2E_KUBE_EXTRA_VALUES}" e2e/rust/e2e-kubernetes.sh' From c9a0fe71a756481f1c7286b6977175950ccc4fa1 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Tue, 17 Nov 2026 11:50:20 -0800 Subject: [PATCH 2/7] refactor(helm): derive scheduling policy from gateway config Signed-off-by: Taylor Mutch --- .agents/skills/helm-dev-environment/SKILL.md | 6 ++-- deploy/helm/openshell/README.md | 2 -- .../ci/values-scheduling-required.yaml | 6 ++-- deploy/helm/openshell/templates/_helpers.tpl | 16 ----------- .../tests/runtime_scheduling_test.yaml | 28 ++++++++----------- .../tests/supervisor_affinity_test.yaml | 4 +-- deploy/helm/openshell/values.yaml | 7 ----- docs/how-it-works/gateways/configuration.mdx | 20 ++++++------- skills/debug-openshell-cluster/SKILL.md | 7 ++--- 9 files changed, 34 insertions(+), 62 deletions(-) diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index ee9c8d7854..237501fa02 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -279,8 +279,10 @@ task running while using those endpoints. Workloads, supervisors, and gateways use `openshell.ai/runtime-role` with values `workload`, `supervisor`, and `gateway`. Workload/supervisor anti-affinity defaults to preferred and matches across all namespaces. Set -`scheduling.workloadSupervisorAntiAffinity=required` for hard separation or -`off` to opt out; `scheduling.namespaceSelector` narrows the namespace scope. +`gatewayConfig.openshell.drivers.kubernetes.sandbox_runtime.pod_anti_affinity` +to `required` for hard separation or `off` to opt out. The driver's +`pod_anti_affinity_namespace_selector` narrows namespace scope. The chart derives +shared-supervisor rules from the same gateway configuration. Node selectors and tolerations remain opt-in. Gateways use the top-level chart fields; supervisors use `supervisor.nodeSelector` and `supervisor.tolerations`; workloads use gateway TOML `sandbox_runtime.workload_placement`. diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 5656a6a990..eb3286a13d 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -653,8 +653,6 @@ discovery endpoint or its TLS CA. | sandboxServiceAccount.annotations | object | `{}` | Annotations to add to the generated sandbox service account. | | sandboxServiceAccount.create | bool | `true` | Create a service account for sandbox pods. | | sandboxServiceAccount.name | string | `""` | Existing service account name for sandbox pods when sandboxServiceAccount.create is false. | -| scheduling.namespaceSelector | object | `nil` | Native namespace label selector for role anti-affinity. Null inherits gatewayConfig, then {} (all namespaces). | -| scheduling.workloadSupervisorAntiAffinity | string | `nil` | Workload/supervisor anti-affinity: preferred (default), required, or off. Null inherits gatewayConfig, then preferred. Sets both driver and shared-supervisor policy. | | securityContext.allowPrivilegeEscalation | bool | `false` | Whether the gateway container can gain additional privileges. | | securityContext.capabilities.drop | list | `["ALL"]` | Linux capabilities dropped from the gateway container. | | securityContext.runAsNonRoot | bool | `true` | Require the gateway container to run as a non-root user. | diff --git a/deploy/helm/openshell/ci/values-scheduling-required.yaml b/deploy/helm/openshell/ci/values-scheduling-required.yaml index e91b8484db..d526a920e7 100644 --- a/deploy/helm/openshell/ci/values-scheduling-required.yaml +++ b/deploy/helm/openshell/ci/values-scheduling-required.yaml @@ -1,5 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # Layer with values-supervisor.yaml and values-tls-port-forward.yaml for E2E. -scheduling: - workloadSupervisorAntiAffinity: required +gatewayConfig: + openshell.drivers.kubernetes: + sandbox_runtime: + pod_anti_affinity: required diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 3eaccf6b72..321f8a9f54 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -655,27 +655,11 @@ absent fields so every chart consumer observes the same configuration. {{- if hasKey $gatewayConfig "openshell.drivers.kubernetes.sandbox_runtime" -}} {{- $runtimeConfig = deepCopy (get $gatewayConfig "openshell.drivers.kubernetes.sandbox_runtime") -}} {{- end -}} -{{/* Resolve the chart convenience setting once for the gateway and supervisor. */}} -{{- $scheduling := .Values.scheduling | default dict -}} {{- $mode := "preferred" -}} {{- if hasKey $runtimeConfig "pod_anti_affinity" -}}{{- $mode = get $runtimeConfig "pod_anti_affinity" -}}{{- end -}} -{{- if and (hasKey $scheduling "workloadSupervisorAntiAffinity") (ne (get $scheduling "workloadSupervisorAntiAffinity") nil) -}} -{{- $requested := get $scheduling "workloadSupervisorAntiAffinity" -}} -{{- if and (hasKey $runtimeConfig "pod_anti_affinity") (ne $mode $requested) -}} -{{- fail "scheduling.workloadSupervisorAntiAffinity conflicts with gatewayConfig sandbox_runtime.pod_anti_affinity" -}} -{{- end -}} -{{- $mode = $requested -}} -{{- end -}} {{- if not (has $mode (list "off" "preferred" "required")) -}}{{- fail "pod anti-affinity must be off, preferred, or required" -}}{{- end -}} {{- $_ := set $runtimeConfig "pod_anti_affinity" $mode -}} {{- $namespaces := get $runtimeConfig "pod_anti_affinity_namespace_selector" | default dict -}} -{{- if and (hasKey $scheduling "namespaceSelector") (ne (get $scheduling "namespaceSelector") nil) -}} -{{- $requested := get $scheduling "namespaceSelector" -}} -{{- if and (hasKey $runtimeConfig "pod_anti_affinity_namespace_selector") (ne (toJson $namespaces) (toJson $requested)) -}} -{{- fail "scheduling.namespaceSelector conflicts with gatewayConfig pod_anti_affinity_namespace_selector" -}} -{{- end -}} -{{- $namespaces = $requested -}} -{{- end -}} {{- if not (kindIs "map" $namespaces) -}}{{- fail "pod anti-affinity namespace selector must be a map" -}}{{- end -}} {{- $_ := set $runtimeConfig "pod_anti_affinity_namespace_selector" $namespaces -}} {{- $appDeployment := include "openshell.appListenerDeployment" . | fromYaml -}} diff --git a/deploy/helm/openshell/tests/runtime_scheduling_test.yaml b/deploy/helm/openshell/tests/runtime_scheduling_test.yaml index cbaac4f09a..c813bdca29 100644 --- a/deploy/helm/openshell/tests/runtime_scheduling_test.yaml +++ b/deploy/helm/openshell/tests/runtime_scheduling_test.yaml @@ -56,11 +56,13 @@ tests: - it: merges required role separation with custom affinity and namespace scope template: templates/supervisor.yaml set: - scheduling: - workloadSupervisorAntiAffinity: required - namespaceSelector: - matchLabels: - scheduling.example.com/group: a + gatewayConfig: + openshell.drivers.kubernetes: + sandbox_runtime: + pod_anti_affinity: required + pod_anti_affinity_namespace_selector: + matchLabels: + scheduling.example.com/group: a supervisor: nodeSelector: pool: supervisor @@ -94,7 +96,10 @@ tests: - it: allows opting out while retaining role labels template: templates/supervisor.yaml set: - scheduling.workloadSupervisorAntiAffinity: "off" + gatewayConfig: + openshell.drivers.kubernetes: + sandbox_runtime: + pod_anti_affinity: "off" asserts: - notExists: path: spec.template.spec.affinity @@ -111,17 +116,6 @@ tests: asserts: - exists: path: spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution - - it: rejects contradictory chart and gateway policies - template: templates/supervisor.yaml - set: - scheduling.workloadSupervisorAntiAffinity: required - gatewayConfig: - openshell.drivers.kubernetes: - sandbox_runtime: - pod_anti_affinity: preferred - asserts: - - failedTemplate: - errorMessage: scheduling.workloadSupervisorAntiAffinity conflicts with gatewayConfig sandbox_runtime.pod_anti_affinity - it: rejects gateway role overrides template: templates/supervisor.yaml set: diff --git a/deploy/helm/openshell/tests/supervisor_affinity_test.yaml b/deploy/helm/openshell/tests/supervisor_affinity_test.yaml index fc0927b5e5..67be869a2e 100644 --- a/deploy/helm/openshell/tests/supervisor_affinity_test.yaml +++ b/deploy/helm/openshell/tests/supervisor_affinity_test.yaml @@ -9,11 +9,11 @@ release: name: openshell namespace: gateway set: - scheduling: - workloadSupervisorAntiAffinity: off gatewayConfig: openshell.drivers.kubernetes: supervisor_topology: gateway + sandbox_runtime: + pod_anti_affinity: "off" tests: - it: renders supervisor.affinity unchanged on the supervisor Pod template: templates/supervisor.yaml diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index c427633ead..693367b8df 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -3,13 +3,6 @@ # Default values for OpenShell -# Scheduling policy for workload/supervisor roles. Does not constrain gateway placement. -scheduling: - # -- Workload/supervisor anti-affinity: preferred (default), required, or off. Null inherits gatewayConfig, then preferred. Sets both driver and shared-supervisor policy. - workloadSupervisorAntiAffinity: null - # -- (object) Native namespace label selector for role anti-affinity. Null inherits gatewayConfig, then {} (all namespaces). - namespaceSelector: null - # -- Number of OpenShell gateway replicas. Values greater than 1 require # server.externalDbSecret because the default SQLite backend is per pod. replicaCount: 1 diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index b39c602ea3..9f68e03a08 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -1488,18 +1488,18 @@ the participating namespaces and configure a native namespace label selector: "example.com/openshell-group" = "production" ``` -In Helm, `scheduling.workloadSupervisorAntiAffinity` sets the policy for both the -driver and shared-supervisor Deployment. It inherits gateway TOML when unset, -then defaults to `preferred`. `scheduling.namespaceSelector` similarly sets both -sides' namespace selector. Contradictory chart and gateway settings are rejected. -For example: +In Helm, configure the driver through `gatewayConfig`. The chart derives the +shared-supervisor Deployment's reciprocal policy and namespace scope from the +same settings: ```yaml -scheduling: - workloadSupervisorAntiAffinity: required - namespaceSelector: - matchLabels: - example.com/openshell-group: production +gatewayConfig: + openshell.drivers.kubernetes: + sandbox_runtime: + pod_anti_affinity: required + pod_anti_affinity_namespace_selector: + matchLabels: + example.com/openshell-group: production ``` For independently deployed shared supervisors, apply the supervisor role label diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 1b83a0d202..a3068bcf43 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1065,13 +1065,12 @@ helm -n openshell get values openshell | grep -E 'repository|tag|supervisorImage ``` Workload/supervisor node separation is preferred by default. Check -`sandbox_runtime.pod_anti_affinity` in gateway configuration and -`scheduling.workloadSupervisorAntiAffinity` in Helm values. Required mode needs +`sandbox_runtime.pod_anti_affinity` in gateway configuration (under Helm's +`gatewayConfig`); the chart derives shared-supervisor rules from it. Required mode needs separate eligible nodes; inspect both roles' `FailedScheduling` events. The reserved pod label `openshell.ai/runtime-role` has values `workload`, `supervisor`, and `gateway`; role anti-affinity selects across all namespaces -unless `pod_anti_affinity_namespace_selector` / `scheduling.namespaceSelector` -restricts scope. Node selectors/tolerations are opt-in: check gateway chart +unless `sandbox_runtime.pod_anti_affinity_namespace_selector` restricts scope. Node selectors/tolerations are opt-in: check gateway chart fields, `supervisor.nodeSelector` / `supervisor.tolerations`, and `sandbox_runtime.workload_placement`. Existing unlabeled pods need recreation. The published gateway configuration reference documents the supported fields. From 65cb0de222f7e86ee3eacc0bd72477498c584c9c Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Tue, 17 Nov 2026 12:06:54 -0800 Subject: [PATCH 3/7] docs(kubernetes): add pod placement guide and diagrams Signed-off-by: Taylor Mutch --- .agents/skills/helm-dev-environment/SKILL.md | 2 + crates/openshell-driver-kubernetes/README.md | 2 +- docs/how-it-works/gateways/configuration.mdx | 108 ++------ docs/kubernetes/pod-placement.mdx | 255 +++++++++++++++++++ skills/debug-openshell-cluster/SKILL.md | 2 +- 5 files changed, 277 insertions(+), 92 deletions(-) create mode 100644 docs/kubernetes/pod-placement.mdx diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 237501fa02..dcd7f976de 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -276,6 +276,8 @@ task running while using those endpoints. ### Role scheduling validation +See the published `/kubernetes/pod-placement` guide for deployment examples and role diagrams. + Workloads, supervisors, and gateways use `openshell.ai/runtime-role` with values `workload`, `supervisor`, and `gateway`. Workload/supervisor anti-affinity defaults to preferred and matches across all namespaces. Set diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index b43a208268..674f772b3a 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -110,7 +110,7 @@ capacity; once bound, its scheduling transition starts a bounded startup grace. Standalone drivers expose `--pod-anti-affinity`, `--pod-anti-affinity-namespace-selector` (JSON), and `--workload-placement` (JSON), with corresponding `OPENSHELL_K8S_*` environment variables. See the -[gateway configuration reference](../../docs/how-it-works/gateways/configuration.mdx#kubernetes-pod-placement) +[pod placement guide](../../docs/kubernetes/pod-placement.mdx) for deployment examples and cross-namespace scope. When the gateway exports OTLP traces, the driver sets diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 9f68e03a08..c8a7afd1a0 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -1463,95 +1463,6 @@ boundary_port = 5500 pod_anti_affinity = "preferred" ``` -### Kubernetes pod placement - -OpenShell labels pods with `openshell.ai/runtime-role`: `workload`, `supervisor`, -or `gateway`. The Kubernetes driver labels workloads and dedicated supervisors; -Helm labels gateways and shared supervisors. These labels do not change Service -or controller selectors and can be used in native pod affinity rules. To target -gateways, select `openshell.ai/runtime-role: gateway` in your own rules under -`supervisor.affinity` or the gateway's top-level Helm `affinity`. These rules -coexist with the generated workload/supervisor policy. - -Workloads and supervisors prefer separate nodes by default. Set -`sandbox_runtime.pod_anti_affinity = "required"` to require separation, or `"off"` -to omit the generated rule. Required mode leaves pods Pending when separation -cannot be satisfied. Preferred mode permits single-node deployments. Gateway -placement is independent; the default rule does not select gateways. - -Rules use `kubernetes.io/hostname` and select the opposite role across **all -namespaces**, including other OpenShell installations. To limit the scope, label -the participating namespaces and configure a native namespace label selector: - -```toml -[openshell.drivers.kubernetes.sandbox_runtime.pod_anti_affinity_namespace_selector.matchLabels] -"example.com/openshell-group" = "production" -``` - -In Helm, configure the driver through `gatewayConfig`. The chart derives the -shared-supervisor Deployment's reciprocal policy and namespace scope from the -same settings: - -```yaml -gatewayConfig: - openshell.drivers.kubernetes: - sandbox_runtime: - pod_anti_affinity: required - pod_anti_affinity_namespace_selector: - matchLabels: - example.com/openshell-group: production -``` - -For independently deployed shared supervisors, apply the supervisor role label -and configure native anti-affinity against workloads in the supervisor Pod -template. Use the same namespace scope and policy strength as the gateway: - -```yaml -metadata: - labels: - openshell.ai/runtime-role: supervisor -spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - openshell.ai/runtime-role: workload - namespaceSelector: {} - topologyKey: kubernetes.io/hostname -``` - -This example selects all namespaces. For soft separation, put the term under -`preferredDuringSchedulingIgnoredDuringExecution` as a `podAffinityTerm` with -`weight: 100`. `off` only removes OpenShell-generated rules; operator-authored -rules remain effective. The gateway never changes independently deployed -supervisor pods. - -Node selectors and tolerations are opt-in and empty by default. To select a -sandbox node pool without enabling caller driver config: - -```toml -[openshell.drivers.kubernetes.sandbox_runtime.workload_placement] -node_selector = { "openshell.ai/node-pool" = "sandbox" } -tolerations = [{ key = "openshell.ai/node-pool", operator = "Equal", value = "sandbox", effect = "NoSchedule" }] -``` - -For gateway placement, use Helm's `nodeSelector` and `tolerations`; shared -supervisors use `supervisor.nodeSelector` and `supervisor.tolerations`. PKI hook -Jobs inherit gateway node selectors and tolerations. For -example, set both selectors to `openshell.ai/node-pool: infrastructure` to run -infrastructure together and workloads elsewhere. Selectors constrain eligible -nodes. Tolerations permit use of tainted nodes and do not select a pool. -Anti-affinity alone does not reserve nodes for future supervisors. - -Operator workload selectors merge with caller selectors; conflicting values -are rejected. Operator tolerations append without duplicates and retain caller -tolerations. Placement is applied on creation and stop/start, preserving -unrelated affinity. Running pods are not moved or continuously evicted. Recreate -existing gateway/supervisor pods to acquire the role labels during rollout; -older unlabeled pods are not selected. A capacity wait does not consume the -workload's bounded startup grace after scheduling. - The Kubernetes driver always creates the workload `NetworkPolicy` before releasing a sandbox Pod. Your cluster CNI must enforce ingress and egress `NetworkPolicy` in every sandbox namespace; the Kubernetes API does not verify @@ -1570,6 +1481,21 @@ The gateway verifies supervisor JWT-SVIDs with JWT bundles fetched from the SPIFFE Workload API, so this validation path does not require gateway access to the SPIRE OIDC discovery endpoint or its TLS CA. +### Kubernetes pod placement + +The following fields are under `[openshell.drivers.kubernetes.sandbox_runtime]`: + +| Field | Default | Purpose | +| --- | --- | --- | +| `pod_anti_affinity` | `"preferred"` | Workload/supervisor node separation: `preferred`, `required`, or `off`. | +| `pod_anti_affinity_namespace_selector` | `{}` | Native Kubernetes label selector for peer namespaces. Empty matches all namespaces. | +| `workload_placement.node_selector` | `{}` | Operator-owned workload node selector. Conflicting caller values are rejected. | +| `workload_placement.tolerations` | `[]` | Operator tolerations appended without duplicates to caller tolerations. | + +Helm derives the shared-supervisor policy from these gateway settings. See +[Pod Placement](/kubernetes/pod-placement) for role labels, separation modes, +node pools, independently deployed supervisors, and Pending-pod diagnostics. + ### MXC The MXC driver runs native Windows workloads through `wxc-exec`. Use this @@ -1993,7 +1919,9 @@ compute_driver = "kyma" socket_path = "/run/openshell/kyma-compute-driver.sock" ``` -## Preflight package configuration {#gateway-config-preflight} + + +## Preflight package configuration Before starting a package-managed gateway, validate the selected file without changing it: diff --git a/docs/kubernetes/pod-placement.mdx b/docs/kubernetes/pod-placement.mdx new file mode 100644 index 0000000000..f43e2cfa14 --- /dev/null +++ b/docs/kubernetes/pod-placement.mdx @@ -0,0 +1,255 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Pod Placement" +sidebar-title: "Pod Placement" +description: "Configure Kubernetes placement for OpenShell gateways, supervisors, and sandbox workloads." +position: 8 +--- + +Workloads and supervisors prefer separate nodes by default. Node selectors and +tolerations are opt-in: the default deployment needs no node-pool labels, +taints, or dedicated nodes. + +## Choose the separation policy + +Set `pod_anti_affinity` in the gateway configuration to `preferred`, `required`, +or `off`. Preferred mode permits colocation when capacity is limited, including +a single-node cluster: + +```mermaid +flowchart LR + subgraph single["One eligible node: preferred mode"] + gateway["Gateway"] + supervisor["Shared supervisor"] + workload["Sandbox workloads"] + end +``` + +To require workload/supervisor separation, configure gateway TOML: + +```toml +[openshell.drivers.kubernetes.sandbox_runtime] +pod_anti_affinity = "required" +``` + +In Helm values, use the same setting through `gatewayConfig`: + +```yaml +gatewayConfig: + openshell.drivers.kubernetes: + sandbox_runtime: + pod_anti_affinity: required +``` + +The chart derives the shared-supervisor Deployment's reciprocal rule from this +setting. The driver configures workloads and dedicated supervisors. Required +mode leaves pods Pending when no separate eligible node exists; it does not +fall back to colocation. `off` omits generated anti-affinity rules while keeping +operator-authored rules effective. Gateway placement is independent of this +policy. + +## Select namespace scope + +Rules use `kubernetes.io/hostname` and select the opposite pod role across all +namespaces, including other OpenShell installations. This covers shared +supervisors deployed in a separate namespace from their workloads. + +To scope separation to one installation, label the participating supervisor and +workload namespaces with a common label, then select that label: + +```yaml +gatewayConfig: + openshell.drivers.kubernetes: + sandbox_runtime: + pod_anti_affinity_namespace_selector: + matchLabels: + example.com/openshell-group: production +``` + +The chart applies the same namespace scope to shared-supervisor rules. An empty +selector, `{}`, matches all namespaces. This selector identifies peer +namespaces; it does not choose nodes. + +## Place pods in separate node pools + +Use node selectors to constrain eligible nodes. Gateways use Helm's +`nodeSelector`; shared supervisors use `supervisor.nodeSelector`; workloads use +the operator-owned `sandbox_runtime.workload_placement.node_selector` in gateway +configuration. Caller driver-config opt-in is not required. + +For example, gateways and supervisors can share an infrastructure pool while +sandbox workloads use another pool: + +```mermaid +flowchart LR + subgraph infrastructure["Infrastructure pool"] + gateway["Gateway"] + supervisor["Shared supervisors"] + end + subgraph sandboxes["Sandbox pool"] + workloads["Sandbox workloads"] + end +``` + +Label the nodes in those pools with `openshell.ai/node-pool=infrastructure` or +`openshell.ai/node-pool=sandbox`, then configure: + +```yaml +nodeSelector: + openshell.ai/node-pool: infrastructure +supervisor: + nodeSelector: + openshell.ai/node-pool: infrastructure +gatewayConfig: + openshell.drivers.kubernetes: + sandbox_runtime: + workload_placement: + node_selector: + openshell.ai/node-pool: sandbox +``` + +To give every role a separate pool, label nodes with the corresponding pool +value and change the selectors: + +```mermaid +flowchart LR + subgraph gatewayPool["Gateway pool: node A"] + gateway["Gateway"] + end + subgraph supervisorPool["Supervisor pool: node B"] + supervisor["Shared supervisors"] + end + subgraph sandboxPool["Sandbox pool: node C"] + workloads["Sandbox workloads"] + end +``` + +```yaml +nodeSelector: + openshell.ai/node-pool: gateway +supervisor: + nodeSelector: + openshell.ai/node-pool: supervisor +gatewayConfig: + openshell.drivers.kubernetes: + sandbox_runtime: + workload_placement: + node_selector: + openshell.ai/node-pool: sandbox +``` + +Explicit node selectors constrain placement even when anti-affinity remains +preferred. Operator workload selectors merge with caller selectors; conflicting +values are rejected. Anti-affinity alone does not reserve capacity for future +supervisors. + +### Tolerate pool taints + +If a pool has a `NoSchedule` taint, give its pods the matching toleration. +Gateways use Helm's `tolerations`, shared supervisors use +`supervisor.tolerations`, and workloads use +`sandbox_runtime.workload_placement.tolerations`. For a sandbox pool tainted +`openshell.ai/node-pool=sandbox:NoSchedule`, add: + +```yaml +gatewayConfig: + openshell.drivers.kubernetes: + sandbox_runtime: + workload_placement: + tolerations: + - key: openshell.ai/node-pool + operator: Equal + value: sandbox + effect: NoSchedule +``` + +Tolerations permit use of tainted nodes; they do not select a pool. Operator +workload tolerations append without duplicates and retain caller tolerations. +PKI hook Jobs inherit the gateway's node selectors and tolerations. Cluster +services, including storage provisioners, need their own placement configuration +when nodes are tainted. + +## Use role labels in custom affinity rules + +OpenShell uses the common label key `openshell.ai/runtime-role`: + +| Pod | Label value | Applied by | +| --- | --- | --- | +| Sandbox workload | `workload` | Kubernetes driver | +| Shared supervisor | `supervisor` | Helm or its deployment owner | +| Dedicated supervisor | `supervisor` | Kubernetes driver | +| Gateway | `gateway` | Helm or its deployment owner | + +These pod labels preserve existing Service and controller selectors. For +example, to prefer placing shared supervisors away from gateways, add: + +```yaml +supervisor: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + openshell.ai/runtime-role: gateway + namespaceSelector: {} + topologyKey: kubernetes.io/hostname +``` + +This example selects gateways across all namespaces. Set its namespace selector +to narrow the scope. Custom `supervisor.affinity` rules coexist with generated +workload/supervisor rules. Top-level Helm `affinity` configures gateway pods. + +## Configure independently deployed supervisors + +The gateway configures workload pods; the supervisor deployment owner configures +shared supervisors. Apply the supervisor role label and a reciprocal native +anti-affinity rule in the supervisor Pod template. For required separation: + +```yaml +metadata: + labels: + openshell.ai/runtime-role: supervisor +spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + openshell.ai/runtime-role: workload + namespaceSelector: {} + topologyKey: kubernetes.io/hostname +``` + +Use the same namespace scope and policy strength as the gateway. For soft +separation, put the term under +`preferredDuringSchedulingIgnoredDuringExecution` as a `podAffinityTerm` with +`weight: 100`. Changing the gateway setting does not change independently +deployed supervisor pods. + +## Apply placement changes + +Workload placement applies on creation and stop/start, preserving unrelated +affinity. Running pods are not moved or continuously evicted. Roll out gateway +and supervisor Pod templates to apply their changed placement or acquire role +labels. Older unlabeled pods are not selected by role anti-affinity. + +## Diagnose Pending pods + +Inspect placement, node-pool labels, and the affected pod's scheduling events: + +```shell +kubectl get pods -A -l openshell.ai/runtime-role -o wide +kubectl get nodes -L openshell.ai/node-pool +kubectl describe pod POD --namespace NAMESPACE +``` + +Check for insufficient eligible nodes, conflicting node selectors or affinity, +and missing tolerations. Required separation needs enough eligible nodes for +both roles. A verified capacity wait does not consume the workload's bounded +startup grace after scheduling. + +See the [gateway configuration reference](/how-it-works/gateways/configuration#kubernetes-pod-placement) +for field names and defaults. diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index a3068bcf43..b0e67a8e2e 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1073,7 +1073,7 @@ reserved pod label `openshell.ai/runtime-role` has values `workload`, unless `sandbox_runtime.pod_anti_affinity_namespace_selector` restricts scope. Node selectors/tolerations are opt-in: check gateway chart fields, `supervisor.nodeSelector` / `supervisor.tolerations`, and `sandbox_runtime.workload_placement`. Existing unlabeled pods need recreation. -The published gateway configuration reference documents the supported fields. +The published `/kubernetes/pod-placement` guide covers placement configuration and diagnostics. The gateway, sandbox, and supervisor images should use the same release tag. A stale runtime image can make sandbox behavior lag behind gateway policy or protocol changes. Kubernetes sandboxes take the configured supervisor and runtime images each time they start, so a sandbox that kept running through an upgrade uses its previous images until its next stop and start. From d626300642c817638b65ee40b4b36ddb57a1488b Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Tue, 17 Nov 2026 13:15:18 -0800 Subject: [PATCH 4/7] docs(kubernetes): clarify example node pool labels Signed-off-by: Taylor Mutch --- docs/kubernetes/pod-placement.mdx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/kubernetes/pod-placement.mdx b/docs/kubernetes/pod-placement.mdx index f43e2cfa14..0d3c3f1ef0 100644 --- a/docs/kubernetes/pod-placement.mdx +++ b/docs/kubernetes/pod-placement.mdx @@ -78,6 +78,12 @@ Use node selectors to constrain eligible nodes. Gateways use Helm's the operator-owned `sandbox_runtime.workload_placement.node_selector` in gateway configuration. Caller driver-config opt-in is not required. +You might use, for example, `openshell.ai/node-pool` as a node label to organize +where pods run within your cluster. This label is only an example; OpenShell +does not create or require it. Use your own node label keys and values in +selectors and tolerations. The built-in `openshell.ai/runtime-role` label +identifies pod roles for automatic anti-affinity. + For example, gateways and supervisors can share an infrastructure pool while sandbox workloads use another pool: From 8427475ffccc49e03fda405cdda9e9ff6671ea0d Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Tue, 17 Nov 2026 13:24:58 -0800 Subject: [PATCH 5/7] docs(kubernetes): explain placement across deployment methods Signed-off-by: Taylor Mutch --- docs/kubernetes/pod-placement.mdx | 101 +++++++++++++++++++----------- 1 file changed, 66 insertions(+), 35 deletions(-) diff --git a/docs/kubernetes/pod-placement.mdx b/docs/kubernetes/pod-placement.mdx index 0d3c3f1ef0..f1294c880b 100644 --- a/docs/kubernetes/pod-placement.mdx +++ b/docs/kubernetes/pod-placement.mdx @@ -7,15 +7,30 @@ description: "Configure Kubernetes placement for OpenShell gateways, supervisors position: 8 --- -Workloads and supervisors prefer separate nodes by default. Node selectors and -tolerations are opt-in: the default deployment needs no node-pool labels, -taints, or dedicated nodes. +Kubernetes places OpenShell pods using standard labels, node selectors, +tolerations, and affinity rules. These concepts apply whether you use the +OpenShell Helm chart, an operator, or deployment manifests you maintain. + +Placement configuration follows the owner of each pod: + +- The Kubernetes driver configures workload placement and workload/supervisor + anti-affinity through gateway configuration, including dedicated supervisors. +- Configure gateways and shared supervisors in their deployment's Pod templates. + The OpenShell Helm chart renders these templates and derives shared-supervisor + anti-affinity from the gateway configuration. With another deployment method, + apply the equivalent labels and scheduling rules in your own templates. + +Examples identified as Helm values below are specific to the OpenShell chart. +[Custom deployment manifests](#use-your-own-deployment-manifests) use native +Kubernetes Pod fields. ## Choose the separation policy -Set `pod_anti_affinity` in the gateway configuration to `preferred`, `required`, -or `off`. Preferred mode permits colocation when capacity is limited, including -a single-node cluster: +The Kubernetes driver defaults to `preferred` workload/supervisor anti-affinity. +Set `pod_anti_affinity` in the gateway configuration to `required` or `off` to +change it. Preferred mode permits colocation when capacity is limited, including +a single-node cluster. Node selectors and tolerations are opt-in; the default +policy needs no node-pool labels, taints, or dedicated nodes: ```mermaid flowchart LR @@ -26,7 +41,7 @@ flowchart LR end ``` -To require workload/supervisor separation, configure gateway TOML: +To require workload/supervisor node separation, configure gateway TOML: ```toml [openshell.drivers.kubernetes.sandbox_runtime] @@ -42,9 +57,9 @@ gatewayConfig: pod_anti_affinity: required ``` -The chart derives the shared-supervisor Deployment's reciprocal rule from this -setting. The driver configures workloads and dedicated supervisors. Required -mode leaves pods Pending when no separate eligible node exists; it does not +When using the chart, it also derives the shared-supervisor Deployment's +reciprocal rule from this setting. Required mode leaves pods Pending when no +separate eligible node exists; it does not fall back to colocation. `off` omits generated anti-affinity rules while keeping operator-authored rules effective. Gateway placement is independent of this policy. @@ -56,7 +71,8 @@ namespaces, including other OpenShell installations. This covers shared supervisors deployed in a separate namespace from their workloads. To scope separation to one installation, label the participating supervisor and -workload namespaces with a common label, then select that label: +workload namespaces with a common label, then select that label. For example, +use these Helm values: ```yaml gatewayConfig: @@ -67,16 +83,20 @@ gatewayConfig: example.com/openshell-group: production ``` -The chart applies the same namespace scope to shared-supervisor rules. An empty -selector, `{}`, matches all namespaces. This selector identifies peer -namespaces; it does not choose nodes. +The chart applies the same namespace scope to shared-supervisor rules. In custom +supervisor manifests, set the native anti-affinity term's `namespaceSelector` to +the same selector. An empty selector, `{}`, matches all namespaces. This +selector identifies peer namespaces; it does not choose nodes. ## Place pods in separate node pools -Use node selectors to constrain eligible nodes. Gateways use Helm's -`nodeSelector`; shared supervisors use `supervisor.nodeSelector`; workloads use -the operator-owned `sandbox_runtime.workload_placement.node_selector` in gateway -configuration. Caller driver-config opt-in is not required. +Use node selectors to constrain eligible nodes. In your own gateway or +shared-supervisor Pod templates, set `spec.nodeSelector`. The OpenShell chart +exposes these fields as `nodeSelector` and `supervisor.nodeSelector` respectively. +For driver-created workloads, use the operator-owned +`sandbox_runtime.workload_placement.node_selector` in gateway configuration, +regardless of how you deploy the gateway. Caller driver-config opt-in is not +required. You might use, for example, `openshell.ai/node-pool` as a node label to organize where pods run within your cluster. This label is only an example; OpenShell @@ -99,7 +119,7 @@ flowchart LR ``` Label the nodes in those pools with `openshell.ai/node-pool=infrastructure` or -`openshell.ai/node-pool=sandbox`, then configure: +`openshell.ai/node-pool=sandbox`. With the OpenShell Helm chart, configure: ```yaml nodeSelector: @@ -131,6 +151,8 @@ flowchart LR end ``` +For the OpenShell Helm chart, the values are: + ```yaml nodeSelector: openshell.ai/node-pool: gateway @@ -152,11 +174,12 @@ supervisors. ### Tolerate pool taints -If a pool has a `NoSchedule` taint, give its pods the matching toleration. -Gateways use Helm's `tolerations`, shared supervisors use -`supervisor.tolerations`, and workloads use -`sandbox_runtime.workload_placement.tolerations`. For a sandbox pool tainted -`openshell.ai/node-pool=sandbox:NoSchedule`, add: +If a pool has a `NoSchedule` taint, give its pods the matching toleration. Set +`spec.tolerations` in custom gateway or shared-supervisor Pod templates. The +OpenShell chart exposes these fields as `tolerations` and +`supervisor.tolerations` respectively. For driver-created workloads, use +`sandbox_runtime.workload_placement.tolerations`. These Helm values configure a +workload toleration for `openshell.ai/node-pool=sandbox:NoSchedule`: ```yaml gatewayConfig: @@ -172,9 +195,10 @@ gatewayConfig: Tolerations permit use of tainted nodes; they do not select a pool. Operator workload tolerations append without duplicates and retain caller tolerations. -PKI hook Jobs inherit the gateway's node selectors and tolerations. Cluster -services, including storage provisioners, need their own placement configuration -when nodes are tainted. +With the OpenShell chart, PKI hook Jobs inherit the gateway's node selectors and +tolerations. Other deployment methods must configure their supporting Jobs as +needed. Cluster services, including storage provisioners, need their own +placement configuration when nodes are tainted. ## Use role labels in custom affinity rules @@ -187,8 +211,9 @@ OpenShell uses the common label key `openshell.ai/runtime-role`: | Dedicated supervisor | `supervisor` | Kubernetes driver | | Gateway | `gateway` | Helm or its deployment owner | -These pod labels preserve existing Service and controller selectors. For -example, to prefer placing shared supervisors away from gateways, add: +These pod labels preserve existing Service and controller selectors. Use them +in native `spec.affinity` rules in your own Pod templates. For example, these +OpenShell Helm values prefer placing shared supervisors away from gateways: ```yaml supervisor: @@ -208,11 +233,15 @@ This example selects gateways across all namespaces. Set its namespace selector to narrow the scope. Custom `supervisor.affinity` rules coexist with generated workload/supervisor rules. Top-level Helm `affinity` configures gateway pods. -## Configure independently deployed supervisors +## Use your own deployment manifests + +Apply `openshell.ai/runtime-role: gateway` to your gateway pods and +`openshell.ai/runtime-role: supervisor` to shared-supervisor pods. Configure +their node selectors, tolerations, and affinity using native Pod fields. -The gateway configures workload pods; the supervisor deployment owner configures -shared supervisors. Apply the supervisor role label and a reciprocal native -anti-affinity rule in the supervisor Pod template. For required separation: +For example, this shared-supervisor Pod template requires separation from +workloads. In a Deployment or StatefulSet manifest, place this fragment under +`spec.template`: ```yaml metadata: @@ -229,11 +258,13 @@ spec: topologyKey: kubernetes.io/hostname ``` -Use the same namespace scope and policy strength as the gateway. For soft +Continue to configure driver-created pods through gateway TOML, as shown above. +Give shared supervisors the same namespace scope and policy strength. For soft separation, put the term under `preferredDuringSchedulingIgnoredDuringExecution` as a `podAffinityTerm` with `weight: 100`. Changing the gateway setting does not change independently -deployed supervisor pods. +deployed supervisor pods. Your deployment tooling must maintain those labels +and rules across rollouts, replacements, and scaling. ## Apply placement changes From f63a57c7c81ffb88143c57faea36f89131213b16 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Tue, 17 Nov 2026 13:29:23 -0800 Subject: [PATCH 6/7] docs(kubernetes): trim generic placement guidance Signed-off-by: Taylor Mutch --- docs/kubernetes/pod-placement.mdx | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/docs/kubernetes/pod-placement.mdx b/docs/kubernetes/pod-placement.mdx index f1294c880b..f37d76b88d 100644 --- a/docs/kubernetes/pod-placement.mdx +++ b/docs/kubernetes/pod-placement.mdx @@ -59,11 +59,13 @@ gatewayConfig: When using the chart, it also derives the shared-supervisor Deployment's reciprocal rule from this setting. Required mode leaves pods Pending when no -separate eligible node exists; it does not -fall back to colocation. `off` omits generated anti-affinity rules while keeping +separate eligible node exists; it does not fall back to colocation. `off` omits generated anti-affinity rules while keeping operator-authored rules effective. Gateway placement is independent of this policy. +Workload placement takes effect on creation or stop/start. Existing pods are not +moved automatically. + ## Select namespace scope Rules use `kubernetes.io/hostname` and select the opposite pod role across all @@ -266,27 +268,5 @@ separation, put the term under deployed supervisor pods. Your deployment tooling must maintain those labels and rules across rollouts, replacements, and scaling. -## Apply placement changes - -Workload placement applies on creation and stop/start, preserving unrelated -affinity. Running pods are not moved or continuously evicted. Roll out gateway -and supervisor Pod templates to apply their changed placement or acquire role -labels. Older unlabeled pods are not selected by role anti-affinity. - -## Diagnose Pending pods - -Inspect placement, node-pool labels, and the affected pod's scheduling events: - -```shell -kubectl get pods -A -l openshell.ai/runtime-role -o wide -kubectl get nodes -L openshell.ai/node-pool -kubectl describe pod POD --namespace NAMESPACE -``` - -Check for insufficient eligible nodes, conflicting node selectors or affinity, -and missing tolerations. Required separation needs enough eligible nodes for -both roles. A verified capacity wait does not consume the workload's bounded -startup grace after scheduling. - See the [gateway configuration reference](/how-it-works/gateways/configuration#kubernetes-pod-placement) for field names and defaults. From 1d61d785823d6f3686cf4ee80a600e746f8b1d03 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Tue, 17 Nov 2026 14:59:57 -0800 Subject: [PATCH 7/7] fix(kubernetes): preserve placement policies across admission and restart Signed-off-by: Taylor Mutch --- .agents/skills/helm-dev-environment/SKILL.md | 4 +- crates/openshell-driver-kubernetes/README.md | 4 +- .../openshell-driver-kubernetes/src/driver.rs | 51 +++- .../src/scheduling.rs | 243 +++++++++++++++--- e2e/rust/Cargo.toml | 7 + e2e/rust/tests/kubernetes_scheduling.rs | 4 +- tasks/test.toml | 2 +- 7 files changed, 268 insertions(+), 47 deletions(-) diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index dcd7f976de..2126f12af7 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -298,7 +298,9 @@ HELM_K3S_LB_HOST_PORT=18080 mise run helm:k3s:create `HELM_K3S_NODES` affects new clusters only; it does not resize an existing one. Use `mise run e2e:kubernetes:scheduling` for the shared-supervisor create/restart -checks. Against the isolated cluster, set `OPENSHELL_E2E_KUBE_CONTEXT` and +checks. This task enables the dedicated `e2e-kubernetes-scheduling` feature; +the scheduling test is excluded from the general Kubernetes suite. Against the +isolated cluster, set `OPENSHELL_E2E_KUBE_CONTEXT` and `OPENSHELL_E2E_KUBE_BUILD_IMAGES=1` to build matching branch images. Layer `ci/values-scheduling-required.yaml` through `OPENSHELL_E2E_KUBE_EXTRA_VALUES` and set `OPENSHELL_E2E_SCHEDULING_MODE=required` diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 674f772b3a..a17e535925 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -102,7 +102,9 @@ apply independently of caller driver-config opt-in. Conflicting caller selectors fail; tolerations append without duplicates. A reserved template annotation records operator additions so stop/start can replace them while preserving caller placement and unrelated admission rules. Restart reconciles only the -reserved role anti-affinity term, using a merge patch that clears removed keys. +exact anti-affinity term recorded as inserted by the driver, using a merge patch +that clears removed keys. Independently authored role rules retain their scope +and strength. Admission checks validate required workload placement before releasing the bootstrap scheduling gate. A verified unschedulable generation may wait for capacity; once bound, its scheduling transition starts a bounded startup grace. diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 0274eaeb15..0eff82d052 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -5631,7 +5631,11 @@ impl KubernetesComputeDriver { return Ok(false); } Ok(generation_pods.iter().any(|pod| { - crate::scheduling::scheduling_wait(pod, openshell_core::time::now_ms(), 300_000) + crate::scheduling::scheduling_wait( + pod, + openshell_core::time::now_ms(), + Duration::from_secs(self.config.runtime_bootstrap_grace_seconds), + ) })) } @@ -13674,6 +13678,51 @@ mod tests { Arc>>, ); + #[tokio::test] + async fn capacity_recovery_uses_configured_bootstrap_grace() { + let scheduled = k8s_openapi::apimachinery::pkg::apis::meta::v1::Time( + (SystemTime::now() - Duration::from_mins(1)).into(), + ); + for grace in [30, 900] { + let config = KubernetesComputeConfig { + runtime_bootstrap_grace_seconds: grace, + ..KubernetesComputeConfig::default() + }; + let pod = serde_json::json!({ + "metadata": {"name": "demo", "uid": "current-uid"}, + "spec": {"containers": [], "nodeName": "new-node"}, + "status": {"phase": "Pending", "conditions": [{ + "type": "PodScheduled", "status": "True", "lastTransitionTime": scheduled + }]} + }); + let (driver, remaining, _) = scripted_driver( + config, + vec![( + http::Method::GET, + "/api/v1/namespaces/openshell/pods/demo", + kube_test_response(http::StatusCode::OK, pod), + )], + ); + let object: DynamicObject = serde_json::from_value(serde_json::json!({ + "apiVersion": "agents.x-k8s.io/v1beta1", "kind": "Sandbox", + "metadata": {"name": "demo", "namespace": "openshell", "annotations": { + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE: "released", + ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID: "current-uid", + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT: "1", + }} + })) + .unwrap(); + assert_eq!( + driver + .bootstrap_waiting_for_scheduling(&object) + .await + .unwrap(), + grace == 900 + ); + assert!(remaining.lock().unwrap().is_empty()); + } + } + /// Build a driver whose client answers `steps` in order and records each /// request body. fn scripted_driver( diff --git a/crates/openshell-driver-kubernetes/src/scheduling.rs b/crates/openshell-driver-kubernetes/src/scheduling.rs index ec2dad9673..94c695a70a 100644 --- a/crates/openshell-driver-kubernetes/src/scheduling.rs +++ b/crates/openshell-driver-kubernetes/src/scheduling.rs @@ -5,6 +5,7 @@ use std::collections::BTreeMap; use std::str::FromStr; +use std::time::Duration; use k8s_openapi::api::core::v1::{Pod, Toleration}; use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; @@ -49,10 +50,36 @@ pub struct KubernetesWorkloadPlacement { // Remember only the operator's changes, so changing/removing operator placement // on restart restores caller settings and preserves unrelated admission rules. #[derive(Default, Serialize, Deserialize)] +#[serde(default)] struct AppliedPlacement { node_selector: BTreeMap, original_selector: BTreeMap, - added_tolerations: Vec, + added_tolerations: Vec, + anti_affinity: Option, +} + +#[derive(Serialize, Deserialize)] +struct AppliedAntiAffinity { + key: String, + value: Value, +} + +fn normalized_toleration(value: &Toleration) -> Toleration { + let mut value = value.clone(); + if value.operator.as_ref().is_none_or(String::is_empty) { + value.operator = Some("Equal".into()); + } + for field in [&mut value.key, &mut value.value, &mut value.effect] { + if field.as_ref().is_some_and(String::is_empty) { + *field = None; + } + } + value +} + +fn toleration_matches(value: &Value, expected: &Toleration) -> bool { + serde_json::from_value::(value.clone()) + .is_ok_and(|value| normalized_toleration(&value) == normalized_toleration(expected)) } fn term(peer: &str, namespaces: &LabelSelector) -> Value { @@ -63,28 +90,27 @@ fn term(peer: &str, namespaces: &LabelSelector) -> Value { }) } -fn owned_term(value: &Value, peer: &str) -> bool { - value["topologyKey"] == "kubernetes.io/hostname" - && value["labelSelector"] == json!({"matchLabels": {ROLE_LABEL: peer}}) - && value.as_object().is_some_and(|map| { - map.keys().all(|key| { - matches!( - key.as_str(), - "topologyKey" | "labelSelector" | "namespaceSelector" - ) - }) - }) -} - -/// Reconcile only the reserved role rule. Other affinity is left intact. +/// Add the role rule to a newly constructed supervisor spec. pub fn configure_affinity( spec: &mut Value, mode: KubernetesPodAntiAffinity, namespaces: &LabelSelector, peer: &str, ) { + reconcile_affinity(spec, mode, namespaces, peer, None); +} + +// Stored ownership includes the strength and namespace scope. Similar rules +// independently authored by operators must survive policy changes. +fn reconcile_affinity( + spec: &mut Value, + mode: KubernetesPodAntiAffinity, + namespaces: &LabelSelector, + peer: &str, + previous: Option<&AppliedAntiAffinity>, +) -> Option { if spec["affinity"].is_null() && mode == KubernetesPodAntiAffinity::Off { - return; + return None; } if spec["affinity"].is_null() { spec["affinity"] = json!({}); @@ -96,12 +122,11 @@ pub fn configure_affinity( let anti = &mut affinity["podAntiAffinity"]; let required = "requiredDuringSchedulingIgnoredDuringExecution"; let preferred = "preferredDuringSchedulingIgnoredDuringExecution"; - if let Some(terms) = anti.get_mut(required).and_then(Value::as_array_mut) { - terms.retain(|value| !owned_term(value, peer)); - } - if let Some(terms) = anti.get_mut(preferred).and_then(Value::as_array_mut) { - terms - .retain(|value| value["weight"] != 100 || !owned_term(&value["podAffinityTerm"], peer)); + if let Some(previous) = previous + && let Some(terms) = anti.get_mut(&previous.key).and_then(Value::as_array_mut) + && let Some(index) = terms.iter().position(|value| value == &previous.value) + { + terms.remove(index); } let generated = term(peer, namespaces); let (key, value) = match mode { @@ -112,14 +137,19 @@ pub fn configure_affinity( Some(json!({"weight": 100, "podAffinityTerm": generated})), ), }; + let mut applied = None; if let Some(value) = value { if anti[key].is_null() { anti[key] = json!([]); } - anti[key] - .as_array_mut() - .expect("affinity terms are arrays") - .push(value); + let terms = anti[key].as_array_mut().expect("affinity terms are arrays"); + if !terms.contains(&value) { + terms.push(value.clone()); + applied = Some(AppliedAntiAffinity { + key: key.into(), + value, + }); + } } for key in [required, preferred] { if anti[key].as_array().is_some_and(Vec::is_empty) { @@ -132,6 +162,7 @@ pub fn configure_affinity( if affinity.as_object().is_some_and(serde_json::Map::is_empty) { spec.as_object_mut().unwrap().remove("affinity"); } + applied } pub fn configure_workload( @@ -191,14 +222,20 @@ pub fn configure_workload( } let mut tolerations = spec["tolerations"].as_array().cloned().unwrap_or_default(); for previous in previous.added_tolerations { - if let Some(index) = tolerations.iter().position(|value| *value == previous) { + if let Some(index) = tolerations + .iter() + .position(|value| toleration_matches(value, &previous)) + { tolerations.remove(index); } } for toleration in &config.workload_placement.tolerations { - let value = json!(toleration); - if !tolerations.contains(&value) { - tolerations.push(value.clone()); + if !tolerations + .iter() + .any(|value| toleration_matches(value, toleration)) + { + let value = normalized_toleration(toleration); + tolerations.push(json!(&value)); applied.added_tolerations.push(value); } } @@ -207,11 +244,12 @@ pub fn configure_workload( } else { spec["tolerations"] = json!(tolerations); } - configure_affinity( + applied.anti_affinity = reconcile_affinity( spec, config.pod_anti_affinity, &config.pod_anti_affinity_namespace_selector, "supervisor", + previous.anti_affinity.as_ref(), ); if template["metadata"].is_null() { template["metadata"] = json!({}); @@ -297,11 +335,11 @@ fn validate_role( } } for value in &config.workload_placement.tolerations { - if !spec - .tolerations - .as_ref() - .is_some_and(|values| values.contains(value)) - { + if !spec.tolerations.as_ref().is_some_and(|values| { + values + .iter() + .any(|actual| normalized_toleration(actual) == normalized_toleration(value)) + }) { return Err(fail("operator toleration removed")); } } @@ -339,7 +377,8 @@ fn validate_role( /// Scheduling waits do not consume startup grace; once bound, startup gets a /// bounded grace from `PodScheduled`, including after a gateway outage. -pub fn scheduling_wait(pod: &Pod, now_ms: i64, grace_ms: i64) -> bool { +pub fn scheduling_wait(pod: &Pod, now_ms: i64, grace: Duration) -> bool { + let grace_ms = i64::try_from(grace.as_millis()).unwrap_or(i64::MAX); if pod.metadata.deletion_timestamp.is_some() { return false; } @@ -489,6 +528,128 @@ mod tests { ); } + #[test] + fn tolerations_accept_native_defaults_without_duplicates() { + for configured in [ + json!({"key": "pool", "value": "sandbox", "effect": "NoSchedule"}), + json!({"key": "", "operator": "Exists", "value": "", "effect": ""}), + ] { + let config: Config = serde_json::from_value(json!({ + "workload_placement": {"tolerations": [configured]} + })) + .unwrap(); + config.validate().unwrap(); + let mut pod = template(); + configure_workload(&mut pod, &config).unwrap(); + let mut admitted = serde_json::from_value::(pod.clone()).unwrap(); + for actual in admitted + .spec + .as_mut() + .unwrap() + .tolerations + .as_mut() + .unwrap() + { + *actual = normalized_toleration(actual); + } + validate_workload(&admitted, &config).unwrap(); + pod["spec"]["tolerations"] = json!(admitted.spec.unwrap().tolerations.unwrap()); + configure_workload(&mut pod, &config).unwrap(); + assert_eq!(pod["spec"]["tolerations"].as_array().unwrap().len(), 1); + configure_workload(&mut pod, &Config::default()).unwrap(); + assert!(pod["spec"].get("tolerations").is_none()); + } + + let configured = json!({"key": "pool", "value": "sandbox", "effect": "NoSchedule"}); + let config: Config = serde_json::from_value(json!({ + "workload_placement": {"tolerations": [configured]} + })) + .unwrap(); + let mut pod = template(); + let caller = + json!({"key": "pool", "operator": "Equal", "value": "sandbox", "effect": "NoSchedule"}); + pod["spec"]["tolerations"] = json!([caller]); + configure_workload(&mut pod, &config).unwrap(); + configure_workload(&mut pod, &Config::default()).unwrap(); + assert_eq!(pod["spec"]["tolerations"], json!([caller])); + } + + #[test] + fn restart_preserves_role_rules_with_independent_scope_and_strength() { + let mut pod = template(); + let mut config = Config::default(); + configure_workload(&mut pod, &config).unwrap(); + let required = term( + "supervisor", + &serde_json::from_value(json!({"matchLabels": {"group": "critical"}})).unwrap(), + ); + let preferred = json!({"weight": 100, "podAffinityTerm": term("supervisor", &serde_json::from_value(json!({"matchLabels": {"group": "other"}})).unwrap())}); + let anti = &mut pod["spec"]["affinity"]["podAntiAffinity"]; + anti["requiredDuringSchedulingIgnoredDuringExecution"] = json!([required]); + anti["preferredDuringSchedulingIgnoredDuringExecution"] + .as_array_mut() + .unwrap() + .push(preferred.clone()); + for mode in [ + KubernetesPodAntiAffinity::Required, + KubernetesPodAntiAffinity::Preferred, + KubernetesPodAntiAffinity::Off, + ] { + config.pod_anti_affinity = mode; + config.pod_anti_affinity_namespace_selector = + serde_json::from_value(json!({"matchLabels": {"group": "updated"}})).unwrap(); + configure_workload(&mut pod, &config).unwrap(); + let anti = &pod["spec"]["affinity"]["podAntiAffinity"]; + assert!( + anti["requiredDuringSchedulingIgnoredDuringExecution"] + .as_array() + .unwrap() + .contains(&required) + ); + assert!( + anti["preferredDuringSchedulingIgnoredDuringExecution"] + .as_array() + .unwrap() + .contains(&preferred) + ); + } + let anti = &pod["spec"]["affinity"]["podAntiAffinity"]; + assert_eq!( + anti["requiredDuringSchedulingIgnoredDuringExecution"], + json!([required]) + ); + assert_eq!( + anti["preferredDuringSchedulingIgnoredDuringExecution"], + json!([preferred]) + ); + } + + #[test] + fn existing_identical_rule_is_not_claimed_by_driver() { + let mut pod = template(); + let required = term("supervisor", &LabelSelector::default()); + pod["spec"]["affinity"] = json!({"podAntiAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": [required]}}); + // Missing ownership must not claim existing rules. + pod["metadata"] = json!({"annotations": {PLACEMENT_ANNOTATION: r#"{"node_selector":{},"original_selector":{},"added_tolerations":[]}"#}}); + let config = Config { + pod_anti_affinity: KubernetesPodAntiAffinity::Required, + ..Config::default() + }; + configure_workload(&mut pod, &config).unwrap(); + configure_workload( + &mut pod, + &Config { + pod_anti_affinity: KubernetesPodAntiAffinity::Off, + ..Config::default() + }, + ) + .unwrap(); + assert_eq!( + pod["spec"]["affinity"]["podAntiAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"], + json!([required]) + ); + } + #[test] fn required_admission_rejects_scheduler_bypasses_and_changed_policy() { let config = Config { @@ -552,20 +713,20 @@ mod tests { }]} })) .unwrap(); - assert!(scheduling_wait(&pod, 10_000_000, 300_000)); + assert!(scheduling_wait(&pod, 10_000_000, Duration::from_mins(5))); pod.spec.as_mut().unwrap().node_name = Some("new-node".into()); pod.status.as_mut().unwrap().conditions = Some(serde_json::from_value(json!([{ "type": "PodScheduled", "status": "True", "lastTransitionTime": "1970-01-01T02:46:40Z" }])).unwrap()); assert!( - scheduling_wait(&pod, 10_100_000, 300_000), + scheduling_wait(&pod, 10_100_000, Duration::from_mins(5)), "ContainerCreating after a long capacity wait must survive" ); assert!( - !scheduling_wait(&pod, 10_300_000, 300_000), + !scheduling_wait(&pod, 10_300_000, Duration::from_mins(5)), "startup failures must eventually be cleaned up" ); pod.status.as_mut().unwrap().phase = Some("Failed".into()); - assert!(!scheduling_wait(&pod, 10_100_000, 300_000)); + assert!(!scheduling_wait(&pod, 10_100_000, Duration::from_mins(5))); } } diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 5e557abcdf..d0ed40c805 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -33,6 +33,8 @@ e2e-config-push = ["e2e-docker"] e2e-gpu = ["e2e"] e2e-docker-gpu = ["e2e-docker", "e2e-gpu"] e2e-kubernetes = ["e2e"] +# The dedicated shared-supervisor scheduling fixture. +e2e-kubernetes-scheduling = ["e2e-kubernetes"] e2e-kubernetes-ha = ["e2e-kubernetes"] e2e-kubernetes-credential-drivers = ["e2e-kubernetes"] e2e-kubernetes-workspace-managed = ["e2e-kubernetes"] @@ -308,3 +310,8 @@ module_name_repetitions = "allow" must_use_candidate = "allow" missing_errors_doc = "allow" missing_panics_doc = "allow" + +[[test]] +name = "kubernetes_scheduling" +path = "tests/kubernetes_scheduling.rs" +required-features = ["e2e-kubernetes-scheduling"] diff --git a/e2e/rust/tests/kubernetes_scheduling.rs b/e2e/rust/tests/kubernetes_scheduling.rs index 06fc4cdb54..584c29f46f 100644 --- a/e2e/rust/tests/kubernetes_scheduling.rs +++ b/e2e/rust/tests/kubernetes_scheduling.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#![cfg(feature = "e2e-kubernetes")] +#![cfg(feature = "e2e-kubernetes-scheduling")] //! Run via e2e:kubernetes:scheduling with a shared-supervisor fixture. The same //! test covers one eligible node, required separation, and three node pools. @@ -60,7 +60,7 @@ async fn assert_placement(name: &str) -> Value { "-n", &namespace, "-l", - "openshell.ai/runtime-role", + "openshell.ai/runtime-role in (gateway,supervisor),app.kubernetes.io/instance=openshell", "-o", "json", ]) diff --git a/tasks/test.toml b/tasks/test.toml index e3a57d34f4..d67689729d 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -426,5 +426,5 @@ hide = true ["e2e:kubernetes:scheduling"] description = "Validate shared-supervisor role labels and placement, including workload stop/start" -env = { OPENSHELL_E2E_KUBE_TEST = "kubernetes_scheduling", OPENSHELL_E2E_KUBE_TLS = "1", OPENSHELL_E2E_KUBE_CONFORMANCE = "0" } +env = { OPENSHELL_E2E_KUBE_TEST = "kubernetes_scheduling", OPENSHELL_E2E_KUBE_TLS = "1", OPENSHELL_E2E_KUBE_CONFORMANCE = "0", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e-kubernetes-scheduling,e2e-host-gateway" } run = 'OPENSHELL_E2E_KUBE_EXTRA_VALUES="deploy/helm/openshell/ci/values-supervisor.yaml:deploy/helm/openshell/ci/values-tls-port-forward.yaml${OPENSHELL_E2E_KUBE_EXTRA_VALUES:+:$OPENSHELL_E2E_KUBE_EXTRA_VALUES}" e2e/rust/e2e-kubernetes.sh'