From d4bc05cebc9397be793cde7c6b3626341a9169d9 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 20 Nov 2026 11:18:50 -0800 Subject: [PATCH 1/7] Add configurable ingress TLS modes Signed-off-by: James Sturtevant --- cli/src/bootstrap.rs | 1530 +++++++++++++++++++++++++++++++++++++- cli/src/cluster.rs | 43 +- cli/src/kube.rs | 2 +- cli/src/platform_host.rs | 477 +++++++++++- 4 files changed, 1993 insertions(+), 59 deletions(-) diff --git a/cli/src/bootstrap.rs b/cli/src/bootstrap.rs index 7719fa1..0f823dd 100644 --- a/cli/src/bootstrap.rs +++ b/cli/src/bootstrap.rs @@ -2,12 +2,15 @@ // Copyright 2026 The Hyperlight Authors. use std::collections::{BTreeMap, BTreeSet}; +use std::net::{IpAddr, SocketAddr, ToSocketAddrs}; use std::path::Path; -use std::time::Duration; +use std::sync::{Arc, mpsc}; +use std::time::{Duration, Instant}; use anyhow::{Context, Result, ensure}; use base64::{Engine, engine::general_purpose::STANDARD}; use chrono::{Datelike, Utc}; +use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName, pem::PemObject}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; @@ -59,6 +62,7 @@ pub struct Options<'a> { pub zygote_registries: &'a [String], pub registry_mirrors: BTreeMap, pub ingress_ports: Option, + pub(crate) ingress: &'a crate::platform_host::IngressSettings, } pub(crate) fn run(kube: &Kube<'_>, namespace: Option<&str>, args: &[&str]) -> Result<()> { @@ -172,6 +176,9 @@ fn record_manifest(value: &Value, inventory: &mut Vec) -> Result<()> { if let Some(namespace) = item["metadata"]["namespace"].as_str() { metadata["namespace"] = namespace.into(); } + if item["metadata"]["annotations"]["lampyr.dev/credential-boundary"] == "true" { + metadata["annotations"] = json!({"lampyr.dev/credential-boundary":"true"}); + } let identity = json!({ "apiVersion":api_version, "kind":kind, @@ -655,6 +662,9 @@ fn install_platform( wait_for_webhook_endpoint(kube)?; patch_fiber_serving_defaults(kube) })?; + kube.host.step("Installing certificate management", || { + install_cert_manager(kube, options, inventory) + })?; kube.host .step("Wiring your network and HTTPS ingress", || { apply_bundled_manifest(kube, options, "kourier.yaml", inventory)?; @@ -672,32 +682,38 @@ fn install_platform( "3scale-kourier-gateway", "180s", )?; - if let Some(ports) = &options.ingress_ports { - ensure!(ports.is_array(), "ingress ports must be an array"); - kube.patch( - "kourier-system", - "service", - "kourier", - &json!({ "spec": { "ports": ports } }), + configure_ingress_service(kube, options)?; + if options.ingress_ports.is_none() { + let expected = options + .ingress + .service + .as_ref() + .map(|service| service.public_ip_address.as_str()); + wait_ingress_address(kube, expected, Duration::from_secs(600))?; + } + if options.ingress.service.is_some() { + wait_dns( + kube, + options.domain, + options + .ingress + .service + .as_ref() + .unwrap() + .public_ip_address + .parse()?, + options.https_port, + Duration::from_secs(600), )?; - } else { - kube.wait("public ingress address", Duration::from_secs(600), || { - let service: Value = kube.get("kourier-system", "service", "kourier")?; - Ok(service - .pointer("/status/loadBalancer/ingress") - .and_then(Value::as_array) - .filter(|addresses| !addresses.is_empty()) - .map(|_| ())) - })?; } - tls(kube, options.domain)?; + let (secret_namespace, secret_name) = configure_ingress_tls(kube, options, inventory)?; kube.patch( "knative-serving", "configmap", "config-kourier", &json!({ "data": { - "certs-secret-name": "lampyr-kourier-tls", "certs-secret-namespace": "kourier-system" - } }), + "certs-secret-name": secret_name, "certs-secret-namespace": secret_namespace + } }), )?; kube.patch( "knative-serving", @@ -719,7 +735,27 @@ fn install_platform( { "op": "replace", "path": "/data", "value": { options.domain: "" } } ]))?, ], - ) + )?; + if !matches!( + &options.ingress.tls, + crate::platform_host::IngressTls::SelfSigned + ) { + let certificates = wait_tls_secret( + kube, + &secret_namespace, + &secret_name, + options.domain, + Duration::from_secs(30), + )?; + wait_tls_endpoint( + kube, + options.domain, + options.https_port, + &certificates, + Duration::from_secs(120), + )?; + } + Ok(()) })?; kube.host.step("Making KVM available to Grant homes", || { apply_recorded_source( @@ -783,22 +819,77 @@ fn install_platform( )?; Ok(()) })?; + let (tls_mode, secret_namespace, secret_name, issuer, credential) = + ingress_metadata(options.ingress)?; + let dns_address = options + .ingress + .service + .as_ref() + .map(|service| Value::String(service.public_ip_address.clone())) + .unwrap_or(Value::Null); kube.patch("kube-system", "configmap", "lampyr-install", &json!({ "data": { "version": env!("CARGO_PKG_VERSION"), "domain": options.domain, "httpsPort": options.https_port.to_string(), "fiberdImage": options.fiberd_image, "routerImage": options.router_image, "grantImage": options.grant_image, "helperImage": options.helper_image, - "templateRepository": options.template_repository + "templateRepository": options.template_repository, + "ingressTlsMode": tls_mode, "ingressTlsSecretNamespace": secret_namespace, + "ingressTlsSecretName": secret_name, "ingressTlsIssuer": issuer, + "ingressDnsAddress": dns_address, "ingressCredential": credential, + "ingressServiceAnnotations": serde_json::to_string( + &options.ingress.service.as_ref() + .map(|service| service.annotations.clone()) + .unwrap_or_default() + )? } }))?; Ok(()) } -/// Installs fiberd-k8s-controller and makes namespace default its tenant: -/// cert-manager and its csi-driver unless the cluster has them, the -/// controller, the GrantPolicy that admits Lampyr's grant Pods, and the -/// tenant objects the controller's config/tenant would add but its author -/// RoleBinding: Lampyr's serving controller writes every CapacityGrant in -/// default under its own ClusterRole, so no group of people needs to. -fn install_controller( +fn configure_ingress_service(kube: &Kube<'_>, options: &Options<'_>) -> Result<()> { + if let Some(ports) = &options.ingress_ports { + ensure!(ports.is_array(), "ingress ports must be an array"); + kube.patch( + "kourier-system", + "service", + "kourier", + &json!({ "spec": { "ports": ports } }), + )?; + } + let service = options.ingress.service.as_ref(); + let previous_annotations: BTreeMap = kube + .optional("kube-system", "configmap", "lampyr-install")? + .and_then(|install| { + install["data"]["ingressServiceAnnotations"] + .as_str() + .map(str::to_owned) + }) + .map(|value| serde_json::from_str(&value)) + .transpose() + .context("decode installed ingress Service annotations")? + .unwrap_or_default(); + let mut annotations = serde_json::Map::new(); + for key in previous_annotations.keys() { + annotations.insert(key.clone(), Value::Null); + } + if let Some(service) = service { + for (key, value) in &service.annotations { + annotations.insert(key.clone(), value.clone().into()); + } + } + kube.patch( + "kourier-system", + "service", + "kourier", + &json!({ + "metadata": { "annotations": annotations }, + "spec": { "loadBalancerIP": + service.map(|value| Value::String(value.public_ip_address.clone())).unwrap_or(Value::Null) + } + }), + )?; + Ok(()) +} + +fn install_cert_manager( kube: &Kube<'_>, options: &Options<'_>, inventory: &mut Vec, @@ -864,6 +955,19 @@ fn install_controller( "180s", )?; } + Ok(()) +} + +/// Installs fiberd-k8s-controller and makes namespace default its tenant: +/// the controller, the GrantPolicy that admits Lampyr's grant Pods, and the +/// tenant objects the controller's config/tenant would add but its author +/// RoleBinding: Lampyr's serving controller writes every CapacityGrant in +/// default under its own ClusterRole, so no group of people needs to. +fn install_controller( + kube: &Kube<'_>, + options: &Options<'_>, + inventory: &mut Vec, +) -> Result<()> { // cert-manager reports Available before its webhook admits the // controller's CA Certificate. kube.wait( @@ -1018,16 +1122,964 @@ pub(crate) fn find_container<'a>( }) } -fn tls(kube: &Kube<'_>, domain: &str) -> Result<()> { - crate::platform_host::validate_domain(domain)?; - if let Some(secret) = kube.optional("kourier-system", "secret", "lampyr-kourier-tls")? - && secret +fn wait_ingress_address(kube: &Kube<'_>, expected: Option<&str>, timeout: Duration) -> Result<()> { + kube.wait("public ingress address", timeout, || { + let service: Value = kube.get("kourier-system", "service", "kourier")?; + let addresses = service + .pointer("/status/loadBalancer/ingress") + .and_then(Value::as_array); + Ok(addresses.and_then(|addresses| { + addresses + .iter() + .any(|address| { + expected.map_or_else( + || { + address["ip"].as_str().is_some() + || address["hostname"].as_str().is_some() + }, + |expected| address["ip"] == expected, + ) + }) + .then_some(()) + })) + }) +} + +struct AddressResolver { + request: mpsc::SyncSender, + result: mpsc::Receiver<(u64, std::io::Result>)>, + next: u64, + pending: Option, +} + +impl AddressResolver { + fn new(hostname: &str, port: u16) -> Self { + let hostname = hostname.to_owned(); + let (request, requests) = mpsc::sync_channel(1); + let (results, result) = mpsc::sync_channel(1); + std::thread::spawn(move || { + while let Ok(sequence) = requests.recv() { + let addresses = (hostname.as_str(), port) + .to_socket_addrs() + .map(|addresses| addresses.collect()); + if results.send((sequence, addresses)).is_err() { + break; + } + } + }); + Self { + request, + result, + next: 0, + pending: None, + } + } + + fn resolve(&mut self, timeout: Duration) -> Result>> { + let sequence = if let Some(sequence) = self.pending { + sequence + } else { + self.next = self.next.wrapping_add(1); + let sequence = self.next; + match self.request.try_send(sequence) { + Ok(()) => self.pending = Some(sequence), + Err(mpsc::TrySendError::Disconnected(_)) => { + anyhow::bail!("DNS resolver stopped") + } + Err(mpsc::TrySendError::Full(_)) => { + anyhow::bail!("DNS resolver request state is inconsistent") + } + } + sequence + }; + let deadline = Instant::now() + timeout; + loop { + match self + .result + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + { + Ok((received, result)) if received == sequence => { + self.pending = None; + return Ok(result.ok()); + } + Ok(_) => {} + Err(mpsc::RecvTimeoutError::Timeout) => return Ok(None), + Err(mpsc::RecvTimeoutError::Disconnected) => { + anyhow::bail!("DNS resolver stopped") + } + } + } + } +} + +fn wait_dns( + kube: &Kube<'_>, + domain: &str, + expected: IpAddr, + port: u16, + timeout: Duration, +) -> Result<()> { + let hostname = format!("lampyr-readiness.default.{domain}"); + let mut resolver = AddressResolver::new(&hostname, port); + kube.wait( + &format!("DNS {hostname}:{port} resolving to {expected}"), + timeout, + || { + let addresses = match resolver.resolve(kube.timeout(Duration::from_secs(2))?)? { + Some(addresses) => addresses, + None => return Ok(None), + }; + Ok(addresses + .iter() + .any(|address| address.ip() == expected) + .then_some(())) + }, + ) +} + +fn ingress_metadata( + ingress: &crate::platform_host::IngressSettings, +) -> Result<(String, String, String, Value, Value)> { + use crate::platform_host::IngressTls; + Ok(match &ingress.tls { + IngressTls::SelfSigned => ( + "selfSigned".into(), + "kourier-system".into(), + "lampyr-kourier-tls".into(), + Value::Null, + Value::Null, + ), + IngressTls::ExistingSecret { + secret_name, + secret_namespace, + } => ( + "existingSecret".into(), + secret_namespace.clone(), + secret_name.clone(), + Value::Null, + Value::Null, + ), + IngressTls::Acme { + issuer, credential, .. + } => ( + "acme".into(), + "kourier-system".into(), + "lampyr-kourier-tls".into(), + Value::String(issuer.clone()), + Value::String(serde_json::to_string(credential)?), + ), + }) +} + +fn configure_ingress_tls( + kube: &Kube<'_>, + options: &Options<'_>, + inventory: &mut Vec, +) -> Result<(String, String)> { + use crate::platform_host::IngressTls; + let previous_credential = installed_cert_manager_credential(kube)?; + match &options.ingress.tls { + IngressTls::SelfSigned => { + remove_acme_resources(kube, options.installation_id, inventory)?; + configure_cert_manager_credential(kube, inventory, None, previous_credential.as_ref())?; + remove_credential_boundary( + kube, + options.installation_id, + previous_credential.as_ref(), + None, + inventory, + )?; + self_signed_tls(kube, options.domain, options.installation_id)?; + Ok(("kourier-system".into(), "lampyr-kourier-tls".into())) + } + IngressTls::ExistingSecret { + secret_name, + secret_namespace, + } => { + wait_tls_secret( + kube, + secret_namespace, + secret_name, + options.domain, + Duration::from_secs(120), + )?; + remove_acme_resources(kube, options.installation_id, inventory)?; + configure_cert_manager_credential(kube, inventory, None, previous_credential.as_ref())?; + remove_credential_boundary( + kube, + options.installation_id, + previous_credential.as_ref(), + None, + inventory, + )?; + Ok((secret_namespace.clone(), secret_name.clone())) + } + IngressTls::Acme { + email, + issuer, + solver, + credential, + } => { + ensure!( + cert_manager_owned(inventory), + "ACME ingress requires the cert-manager installation managed by Lampyr" + ); + persist_cert_manager_credential(kube, Some(credential))?; + configure_credential_boundary(kube, options.installation_id, credential, inventory)?; + wait_credential_boundary(kube, credential)?; + if let Err(error) = ensure_no_external_credential_resources(kube, credential) { + configure_cert_manager_credential( + kube, + inventory, + None, + previous_credential.as_ref(), + ) + .context("disable cert-manager identity after unsafe Issuer discovery")?; + return Err(error); + } + configure_cert_manager_credential( + kube, + inventory, + Some(credential), + previous_credential.as_ref(), + )?; + remove_credential_boundary( + kube, + options.installation_id, + previous_credential.as_ref(), + Some(credential), + inventory, + )?; + let issuer_manifest = json!({ + "apiVersion":"cert-manager.io/v1", "kind":"Issuer", + "metadata":{"name":issuer,"namespace":"kourier-system","annotations":{ + "lampyr.dev/installation-id":options.installation_id + }}, + "spec":{"acme":{ + "email":email, + "server":"https://acme-v02.api.letsencrypt.org/directory", + "privateKeySecretRef":{"name":"lampyr-acme-account"}, + "solvers":[solver] + }} + }); + ensure_owned_tls_resource( + kube, + Some("kourier-system"), + "issuer", + issuer, + options.installation_id, + )?; + apply_cert_manager_manifest( + kube, + options.installation_id, + "ACME Issuer admission", + &issuer_manifest, + inventory, + )?; + wait_ready_resource( + kube, + "kourier-system", + "issuer", + issuer, + "ACME issuer", + Duration::from_secs(300), + )?; + ensure_owned_tls_secret(kube, options.installation_id)?; + let certificate = json!({ + "apiVersion":"cert-manager.io/v1", "kind":"Certificate", + "metadata":{"name":"lampyr-kourier-tls","namespace":"kourier-system", + "annotations":{"lampyr.dev/installation-id":options.installation_id}}, + "spec":{ + "secretName":"lampyr-kourier-tls", + "secretTemplate":{"annotations":{ + "lampyr.dev/installation-id":options.installation_id, + "lampyr.dev/domain":options.domain, + "lampyr.dev/tls-mode":"acme" + }}, + "dnsNames":[format!("*.{}",options.domain),format!("*.default.{}",options.domain)], + "issuerRef":{"group":"cert-manager.io","kind":"Issuer","name":issuer} + } + }); + ensure_owned_tls_resource( + kube, + Some("kourier-system"), + "certificate", + "lampyr-kourier-tls", + options.installation_id, + )?; + apply_cert_manager_manifest( + kube, + options.installation_id, + "ingress Certificate admission", + &certificate, + inventory, + )?; + wait_ready_resource( + kube, + "kourier-system", + "certificate", + "lampyr-kourier-tls", + "ingress Certificate", + Duration::from_secs(900), + )?; + wait_tls_secret( + kube, + "kourier-system", + "lampyr-kourier-tls", + options.domain, + Duration::from_secs(30), + )?; + remove_retired_issuers(kube, options.installation_id, issuer, inventory)?; + Ok(("kourier-system".into(), "lampyr-kourier-tls".into())) + } + } +} + +fn cert_manager_owned(inventory: &[Value]) -> bool { + inventory.iter().any(|item| { + item["apiVersion"] == "apps/v1" + && item["kind"] == "Deployment" + && item["metadata"]["namespace"] == "cert-manager" + && item["metadata"]["name"] == "cert-manager" + }) +} + +fn persist_cert_manager_credential( + kube: &Kube<'_>, + credential: Option<&crate::platform_host::CertManagerCredential>, +) -> Result<()> { + kube.patch( + "kube-system", + "configmap", + "lampyr-install", + &json!({"data":{"ingressCredential":credential + .map(serde_json::to_string) + .transpose()?}}), + ) +} + +fn apply_cert_manager_manifest( + kube: &Kube<'_>, + installation_id: &str, + label: &str, + manifest: &Value, + inventory: &mut Vec, +) -> Result<()> { + let mut last_retryable = None; + let result = kube.wait( + label, + Duration::from_secs(120), + || match apply_recorded_manifest(kube, installation_id, manifest, inventory) { + Ok(()) => Ok(Some(())), + Err(error) if cert_manager_webhook_is_starting(&error) => { + last_retryable = Some(format!("{error:#}")); + Ok(None) + } + Err(error) => Err(error), + }, + ); + match (result, last_retryable) { + (Err(error), Some(last)) => { + Err(error).context(format!("last retryable admission error: {last}")) + } + (result, _) => result, + } +} + +fn installed_cert_manager_credential( + kube: &Kube<'_>, +) -> Result> { + let Some(install) = kube.optional("kube-system", "configmap", "lampyr-install")? else { + return Ok(None); + }; + install["data"]["ingressCredential"] + .as_str() + .map(serde_json::from_str) + .transpose() + .context("decode installed cert-manager credential configuration") +} + +fn cert_manager_controller_args( + deployment: &Value, + current: Option<&crate::platform_host::CertManagerCredential>, + previous: Option<&crate::platform_host::CertManagerCredential>, +) -> Result> { + let container = deployment + .pointer("/spec/template/spec/containers") + .and_then(Value::as_array) + .and_then(|containers| { + containers + .iter() + .find(|container| container["name"] == "cert-manager-controller") + }) + .context("cert-manager Deployment has no cert-manager-controller container")?; + let mut args = container["args"] + .as_array() + .context("cert-manager controller args must be an array")? + .clone(); + let prefixes: BTreeSet<_> = current + .into_iter() + .chain(previous) + .flat_map(|credential| credential.controller_arg_prefixes.iter()) + .collect(); + args.retain(|arg| { + arg.as_str() + .is_none_or(|arg| !prefixes.iter().any(|prefix| arg.starts_with(*prefix))) + }); + if let Some(current) = current { + for argument in ¤t.controller_args { + if !args.iter().any(|existing| existing == argument) { + args.push(argument.clone().into()); + } + } + } + Ok(args) +} + +fn merge_json(target: &mut Value, patch: &Value) { + if let (Some(target), Some(patch)) = (target.as_object_mut(), patch.as_object()) { + for (key, value) in patch { + merge_json(target.entry(key.clone()).or_insert(Value::Null), value); + } + } else { + *target = patch.clone(); + } +} + +fn cert_manager_credential_patch( + deployment: &Value, + current: Option<&crate::platform_host::CertManagerCredential>, + previous: Option<&crate::platform_host::CertManagerCredential>, +) -> Result { + let mut patch = previous + .map(|credential| credential.deployment_cleanup_patch.clone()) + .unwrap_or_else(|| json!({})); + if let Some(current) = current { + merge_json(&mut patch, ¤t.deployment_patch); + } + merge_json( + &mut patch, + &json!({"spec":{"template":{"spec":{"containers":[{ + "name":"cert-manager-controller", + "args":cert_manager_controller_args(deployment, current, previous)? + }]}}}}), + ); + Ok(patch) +} + +fn field_path_exists(value: &Value, path: &[String]) -> bool { + let Some((head, tail)) = path.split_first() else { + return !value.is_null(); + }; + if head == "*" { + value + .as_array() + .is_some_and(|items| items.iter().any(|item| field_path_exists(item, tail))) + } else { + value + .get(head) + .is_some_and(|value| field_path_exists(value, tail)) + } +} + +fn ensure_no_external_credential_resources( + kube: &Kube<'_>, + credential: &crate::platform_host::CertManagerCredential, +) -> Result<()> { + for audit in &credential.audits { + let mut command = kube.command(None)?; + command.args([ + "get", + &audit.resource, + "--all-namespaces", + "--output", + "json", + ]); + let resources: Value = serde_json::from_str(&kube.capture(&mut command)?) + .with_context(|| format!("decode cert-manager {}s", audit.kind))?; + for item in resources["items"] + .as_array() + .with_context(|| format!("cert-manager {} list has no items", audit.kind))? + { + let namespace = item["metadata"]["namespace"].as_str().unwrap_or_default(); + let issuer_kind_matches = audit.issuer_kind.as_ref().is_none_or(|expected| { + item.pointer("/spec/issuerRef/kind") + .and_then(Value::as_str) + .unwrap_or("Issuer") + == expected + }); + if namespace != audit.allowed_namespace + && issuer_kind_matches + && field_path_exists(item, &audit.field_path) + { + let name = item["metadata"]["name"].as_str().unwrap_or(""); + anyhow::bail!( + "{} {namespace}/{name} conflicts with the provider credential boundary", + audit.kind + ); + } + } + } + Ok(()) +} + +fn configure_credential_boundary( + kube: &Kube<'_>, + installation_id: &str, + credential: &crate::platform_host::CertManagerCredential, + inventory: &mut Vec, +) -> Result<()> { + for item in credential.boundary["items"].as_array().unwrap() { + ensure!( + item.pointer("/metadata/annotations/lampyr.dev~1installation-id") + .and_then(Value::as_str) + == Some(installation_id) + && item + .pointer("/metadata/annotations/lampyr.dev~1credential-boundary") + .and_then(Value::as_str) + == Some("true"), + "credential boundary resources must carry Lampyr ownership annotations" + ); + let kind = item["kind"] + .as_str() + .context("credential boundary kind is missing")? + .to_ascii_lowercase(); + let name = item["metadata"]["name"] + .as_str() + .context("credential boundary name is missing")?; + let namespace = item["metadata"]["namespace"].as_str(); + ensure_owned_tls_resource(kube, namespace, &kind, name, installation_id)?; + } + apply_recorded_manifest(kube, installation_id, &credential.boundary, inventory) +} + +fn wait_credential_boundary( + kube: &Kube<'_>, + credential: &crate::platform_host::CertManagerCredential, +) -> Result<()> { + for resource in &credential.probes { + let kind = resource["kind"].as_str().unwrap_or("resource"); + let mut probe = tempfile::NamedTempFile::new()?; + serde_json::to_writer(&mut probe, resource)?; + let path = probe + .path() + .to_str() + .context("credential boundary probe path must be UTF-8")?; + kube.wait( + &format!("{kind} credential admission boundary"), + Duration::from_secs(30), + || { + let mut command = kube.command(None)?; + command.args(["create", "--dry-run=server", "--filename", path]); + match kube.capture(&mut command) { + Ok(_) => Ok(None), + Err(error) if format!("{error:#}").contains(&credential.boundary_message) => { + Ok(Some(())) + } + Err(error) => Err(error), + } + }, + )?; + } + Ok(()) +} + +fn configure_cert_manager_credential( + kube: &Kube<'_>, + inventory: &[Value], + current: Option<&crate::platform_host::CertManagerCredential>, + previous: Option<&crate::platform_host::CertManagerCredential>, +) -> Result<()> { + if current.is_some() { + ensure!( + cert_manager_owned(inventory), + "managed ACME ingress requires the cert-manager installation managed by Lampyr" + ); + } + if current.is_some() || previous.is_some() { + let mut service_account_patch = previous + .map(|credential| credential.service_account_cleanup_patch.clone()) + .unwrap_or_else(|| json!({})); + if let Some(current) = current { + merge_json(&mut service_account_patch, ¤t.service_account_patch); + } + kube.patch( + "cert-manager", + "serviceaccount", + "cert-manager", + &service_account_patch, + )?; + let deployment: Value = kube.get("cert-manager", "deployment", "cert-manager")?; + run( + kube, + Some("cert-manager"), + &[ + "patch", + "deployment", + "cert-manager", + "--type", + "strategic", + "--patch", + &serde_json::to_string(&cert_manager_credential_patch( + &deployment, + current, + previous, + )?)?, + ], + )?; + rollout(kube, "cert-manager", "deployment", "cert-manager", "300s")?; + } + Ok(()) +} + +fn ensure_owned_tls_resource( + kube: &Kube<'_>, + namespace: Option<&str>, + kind: &str, + name: &str, + installation_id: &str, +) -> Result<()> { + if let Some(resource) = kube.optional(namespace.unwrap_or("default"), kind, name)? { + ensure!( + resource + .pointer("/metadata/annotations/lampyr.dev~1installation-id") + .and_then(Value::as_str) + == Some(installation_id), + "{kind}/{name} is not owned by this Lampyr installation" + ); + } + Ok(()) +} + +fn ensure_owned_tls_secret(kube: &Kube<'_>, installation_id: &str) -> Result<()> { + if let Some(secret) = kube.optional("kourier-system", "secret", "lampyr-kourier-tls")? { + let owner = secret + .pointer("/metadata/annotations/lampyr.dev~1installation-id") + .and_then(Value::as_str); + let legacy = secret .pointer("/metadata/annotations/lampyr.dev~1domain") .and_then(Value::as_str) - == Some(domain) + .is_some(); + ensure!( + owner == Some(installation_id) || (owner.is_none() && legacy), + "secret/lampyr-kourier-tls is not owned by this Lampyr installation" + ); + } + Ok(()) +} + +fn wait_ready_resource( + kube: &Kube<'_>, + namespace: &str, + kind: &str, + name: &str, + description: &str, + timeout: Duration, +) -> Result<()> { + kube.wait(description, timeout, || { + let Some(resource) = kube.optional(namespace, kind, name)? else { + return Ok(None); + }; + let generation = resource["metadata"]["generation"].as_i64(); + let ready = resource["status"]["conditions"] + .as_array() + .is_some_and(|conditions| { + conditions.iter().any(|condition| { + condition["type"] == "Ready" + && condition["status"] == "True" + && generation.is_none_or(|generation| { + condition["observedGeneration"] + .as_i64() + .is_none_or(|observed| observed == generation) + }) + }) + }); + Ok(ready.then_some(())) + }) +} + +fn wait_tls_secret( + kube: &Kube<'_>, + namespace: &str, + name: &str, + domain: &str, + timeout: Duration, +) -> Result>> { + kube.wait("ingress TLS Secret", timeout, || { + let Some(secret) = kube.optional(namespace, "secret", name)? else { + return Ok(None); + }; + if secret["type"] != "kubernetes.io/tls" { + return Ok(None); + } + validate_tls_secret(&secret, domain).map(Some) + }) +} + +fn validate_tls_secret(secret: &Value, domain: &str) -> Result>> { + let certificate = secret["data"]["tls.crt"] + .as_str() + .context("ingress TLS Secret has no tls.crt")?; + let key = secret["data"]["tls.key"] + .as_str() + .context("ingress TLS Secret has no tls.key")?; + let certificate = STANDARD + .decode(certificate) + .context("decode ingress TLS certificate")?; + let key = STANDARD.decode(key).context("decode ingress TLS key")?; + let certificates: Vec<_> = CertificateDer::pem_slice_iter(&certificate) + .collect::>() + .context("parse ingress TLS certificate chain")?; + ensure!( + !certificates.is_empty(), + "ingress TLS certificate chain is empty" + ); + let private_key = + PrivateKeyDer::from_pem_slice(&key).context("parse ingress TLS private key")?; + rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certificates.clone(), private_key) + .context("ingress TLS certificate does not match its private key")?; + let mut roots = rustls::RootCertStore::empty(); + roots + .add(certificates.last().unwrap().clone()) + .context("ingress TLS chain has no usable trust anchor")?; + let verifier = rustls::client::WebPkiServerVerifier::builder(Arc::new(roots)) + .build() + .context("build ingress TLS certificate verifier")?; + let intermediates = if certificates.len() > 1 { + &certificates[1..certificates.len() - 1] + } else { + &[] + }; + use rustls::client::danger::ServerCertVerifier as _; + let mut hostnames = vec![format!("lampyr-readiness.default.{domain}")]; + // WebPKI deliberately rejects a wildcard directly below a single-label + // name such as *.localhost. Multi-label production domains still require + // both wildcard levels. + if domain.contains('.') { + hostnames.insert(0, format!("lampyr-readiness.{domain}")); + } + for hostname in hostnames { + verifier + .verify_server_cert( + &certificates[0], + intermediates, + &ServerName::try_from(hostname.clone())?.to_owned(), + &[], + rustls::pki_types::UnixTime::now(), + ) + .with_context(|| { + format!( + "ingress TLS certificate is expired, untrusted, or does not cover {hostname}" + ) + })?; + } + Ok(certificates) +} + +fn wait_tls_endpoint( + kube: &Kube<'_>, + domain: &str, + port: u16, + certificates: &[CertificateDer<'static>], + timeout: Duration, +) -> Result<()> { + let hostname = format!("lampyr-readiness.default.{domain}"); + let server_name = ServerName::try_from(hostname.clone())?.to_owned(); + let mut roots = rustls::RootCertStore::empty(); + roots.add(certificates.last().unwrap().clone())?; + let config = Arc::new( + rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(), + ); + let mut resolver = AddressResolver::new(&hostname, port); + kube.wait( + &format!("trusted TLS at {hostname}:{port}"), + timeout, + || { + let addresses = match resolver.resolve(kube.timeout(Duration::from_secs(2))?)? { + Some(addresses) => addresses, + None => return Ok(None), + }; + for address in addresses { + let connect_timeout = kube.timeout(Duration::from_secs(5))?; + if let Ok(mut socket) = + std::net::TcpStream::connect_timeout(&address, connect_timeout) + { + let io_timeout = kube.timeout(Duration::from_secs(10))?; + socket.set_read_timeout(Some(io_timeout))?; + socket.set_write_timeout(Some(io_timeout))?; + let mut connection = + rustls::ClientConnection::new(config.clone(), server_name.clone())?; + if connection.complete_io(&mut socket).is_ok() { + return Ok(Some(())); + } + } + } + Ok(None) + }, + ) +} + +fn remove_acme_resources( + kube: &Kube<'_>, + installation_id: &str, + inventory: &mut Vec, +) -> Result<()> { + let resources: Vec<_> = inventory + .iter() + .filter(|item| { + item["apiVersion"] == "cert-manager.io/v1" + && item["metadata"]["namespace"] == "kourier-system" + && matches!(item["kind"].as_str(), Some("Certificate" | "Issuer")) + }) + .cloned() + .collect(); + for resource in &resources { + let kind = resource["kind"].as_str().unwrap(); + let name = resource["metadata"]["name"].as_str().unwrap(); + let namespace = resource["metadata"]["namespace"].as_str(); + ensure_owned_tls_resource( + kube, + namespace, + &kind.to_ascii_lowercase(), + name, + installation_id, + )?; + run( + kube, + namespace, + &[ + "delete", + &kind.to_ascii_lowercase(), + name, + "--ignore-not-found=true", + "--wait=true", + "--timeout=120s", + ], + )?; + } + inventory.retain(|item| !resources.contains(item)); + if !resources.is_empty() { + persist_core_inventory(kube, installation_id, inventory.clone())?; + } + Ok(()) +} + +fn same_resource_identity(left: &Value, right: &Value) -> bool { + left["apiVersion"] == right["apiVersion"] + && left["kind"] == right["kind"] + && left["metadata"]["name"] == right["metadata"]["name"] + && left["metadata"]["namespace"] == right["metadata"]["namespace"] +} + +fn remove_credential_boundary( + kube: &Kube<'_>, + installation_id: &str, + previous: Option<&crate::platform_host::CertManagerCredential>, + current: Option<&crate::platform_host::CertManagerCredential>, + inventory: &mut Vec, +) -> Result<()> { + let Some(previous) = previous else { + return Ok(()); + }; + let boundary = previous.boundary["items"].as_array().unwrap(); + let current_boundary = current + .and_then(|credential| credential.boundary["items"].as_array()) + .cloned() + .unwrap_or_default(); + let mut resources: Vec<_> = inventory + .iter() + .filter(|item| { + boundary + .iter() + .any(|resource| same_resource_identity(item, resource)) + && !current_boundary + .iter() + .any(|resource| same_resource_identity(item, resource)) + }) + .cloned() + .collect(); + resources.sort_by_key(|item| usize::from(item["kind"] == "ValidatingAdmissionPolicy")); + for resource in &resources { + let kind = resource["kind"].as_str().unwrap().to_ascii_lowercase(); + let name = resource["metadata"]["name"].as_str().unwrap(); + ensure_owned_tls_resource(kube, None, &kind, name, installation_id)?; + run( + kube, + None, + &[ + "delete", + &kind, + name, + "--ignore-not-found=true", + "--wait=true", + "--timeout=120s", + ], + )?; + } + inventory.retain(|item| !resources.contains(item)); + if !resources.is_empty() { + persist_core_inventory(kube, installation_id, inventory.clone())?; + } + Ok(()) +} + +fn remove_retired_issuers( + kube: &Kube<'_>, + installation_id: &str, + current: &str, + inventory: &mut Vec, +) -> Result<()> { + let retired: Vec<_> = inventory + .iter() + .filter(|item| { + item["apiVersion"] == "cert-manager.io/v1" + && item["kind"] == "Issuer" + && item["metadata"]["namespace"] == "kourier-system" + && item["metadata"]["name"] != current + }) + .cloned() + .collect(); + for resource in &retired { + let name = resource["metadata"]["name"].as_str().unwrap(); + ensure_owned_tls_resource( + kube, + Some("kourier-system"), + "issuer", + name, + installation_id, + )?; + run( + kube, + Some("kourier-system"), + &[ + "delete", + "issuer", + name, + "--ignore-not-found=true", + "--wait=true", + "--timeout=120s", + ], + )?; + } + inventory.retain(|item| !retired.contains(item)); + if !retired.is_empty() { + persist_core_inventory(kube, installation_id, inventory.clone())?; + } + Ok(()) +} + +fn self_signed_tls(kube: &Kube<'_>, domain: &str, installation_id: &str) -> Result<()> { + crate::platform_host::validate_domain(domain)?; + if let Some(secret) = kube.optional("kourier-system", "secret", "lampyr-kourier-tls")? + && self_signed_secret_is_reusable(&secret, domain, installation_id) { return Ok(()); } + ensure_owned_tls_secret(kube, installation_id)?; let mut parameters = rcgen::CertificateParams::new(vec![format!("*.{domain}"), format!("*.default.{domain}")])?; parameters @@ -1043,11 +2095,30 @@ fn tls(kube: &Kube<'_>, domain: &str) -> Result<()> { let certificate = parameters.self_signed(&key)?; kube.apply(&json!({ "apiVersion": "v1", "kind": "Secret", "type": "kubernetes.io/tls", - "metadata": { "name": "lampyr-kourier-tls", "namespace": "kourier-system", "annotations": { "lampyr.dev/domain": domain } }, + "metadata": { "name": "lampyr-kourier-tls", "namespace": "kourier-system", "annotations": { + "lampyr.dev/domain": domain, "lampyr.dev/installation-id": installation_id, + "lampyr.dev/tls-mode":"selfSigned" + } }, "data": { "tls.crt": STANDARD.encode(certificate.pem()), "tls.key": STANDARD.encode(key.serialize_pem()) } })) } +fn self_signed_secret_is_reusable(secret: &Value, domain: &str, installation_id: &str) -> bool { + secret + .pointer("/metadata/annotations/lampyr.dev~1domain") + .and_then(Value::as_str) + == Some(domain) + && secret + .pointer("/metadata/annotations/lampyr.dev~1installation-id") + .and_then(Value::as_str) + == Some(installation_id) + && secret + .pointer("/metadata/annotations/lampyr.dev~1tls-mode") + .and_then(Value::as_str) + == Some("selfSigned") + && validate_tls_secret(secret, domain).is_ok() +} + pub(crate) fn delete_manifest(kube: &Kube<'_>, source: &str) -> Result> { let source = manifest_source(kube, source).context("load manifest for deletion")?; delete_inventory(kube, manifest_items(&source)?) @@ -1141,7 +2212,9 @@ fn delete_inventory(kube: &Kube<'_>, items: Vec) -> Result> { remaining.push((item, is_custom)); } } - for (pass, items) in deletion_passes(&remaining).into_iter().enumerate() { + let passes = deletion_passes(&remaining); + let protects_provider_credential = !passes[3].is_empty(); + for (pass, items) in passes.into_iter().enumerate() { if items.is_empty() { continue; } @@ -1182,22 +2255,52 @@ fn delete_inventory(kube: &Kube<'_>, items: Vec) -> Result> { Err(error) => Err(error), }, )?; + if pass == 2 && protects_provider_credential { + wait_cert_manager_identity_gone(kube)?; + } } Ok(namespaces) } +fn wait_cert_manager_identity_gone(kube: &Kube<'_>) -> Result<()> { + kube.wait( + "cert-manager workload identity removal", + Duration::from_secs(180), + || { + if kube + .optional("default", "namespace", "cert-manager")? + .is_none() + { + return Ok(Some(())); + } + let deployment = kube.optional("cert-manager", "deployment", "cert-manager")?; + let pods: Vec = kube.list( + "cert-manager", + "pods", + "app.kubernetes.io/name=cert-manager", + )?; + Ok((deployment.is_none() && pods.is_empty()).then_some(())) + }, + ) +} + /// Orders inventory deletion. Admission policies go first, since /// fiberd-k8s-controller's refuses deleting the GrantPolicy named default. /// Custom objects go next, so their controllers can finalize them before the -/// controllers, CRDs and namespaces go last. -fn deletion_passes(remaining: &[(Value, bool)]) -> [Vec<&Value>; 3] { - let mut passes: [Vec<&Value>; 3] = Default::default(); +/// controllers, CRDs and namespaces. A provider credential boundary stays +/// until cert-manager is gone, so its credential cannot outlive the guard. +fn deletion_passes(remaining: &[(Value, bool)]) -> [Vec<&Value>; 4] { + let mut passes: [Vec<&Value>; 4] = Default::default(); for (item, custom) in remaining { let admission = matches!( item["kind"].as_str(), Some("ValidatingAdmissionPolicy" | "ValidatingAdmissionPolicyBinding") ); - passes[if admission { + let provider_credential_boundary = admission + && item["metadata"]["annotations"]["lampyr.dev/credential-boundary"] == "true"; + passes[if provider_credential_boundary { + 3 + } else if admission { 0 } else if *custom { 1 @@ -1248,6 +2351,75 @@ pub fn health(kube: &Kube<'_>) -> Result<()> { pod["metadata"]["name"] ); } + if let Some(domain) = installed.data.get("domain") { + let tls_mode = installed + .data + .get("ingressTlsMode") + .map(String::as_str) + .unwrap_or("selfSigned"); + let secret_namespace = installed + .data + .get("ingressTlsSecretNamespace") + .map(String::as_str) + .unwrap_or("kourier-system"); + let secret_name = installed + .data + .get("ingressTlsSecretName") + .map(String::as_str) + .unwrap_or("lampyr-kourier-tls"); + let https_port = installed + .https_port + .as_deref() + .unwrap_or("443") + .parse::() + .context("invalid installed HTTPS port")?; + if tls_mode == "acme" { + let issuer = installed + .data + .get("ingressTlsIssuer") + .context("ACME ingress issuer is missing from installation metadata")?; + wait_ready_resource( + kube, + "kourier-system", + "issuer", + issuer, + "ACME issuer health", + Duration::from_secs(10), + )?; + wait_ready_resource( + kube, + "kourier-system", + "certificate", + "lampyr-kourier-tls", + "ingress Certificate health", + Duration::from_secs(10), + )?; + } + let certificates = wait_tls_secret( + kube, + secret_namespace, + secret_name, + domain, + Duration::from_secs(10), + )?; + if tls_mode != "selfSigned" { + wait_tls_endpoint( + kube, + domain, + https_port, + &certificates, + Duration::from_secs(10), + )?; + } + if let Some(address) = installed.data.get("ingressDnsAddress") { + let address: IpAddr = address + .parse() + .context("invalid installed ingress DNS address")?; + let expected = address.to_string(); + wait_ingress_address(kube, Some(&expected), Duration::from_secs(10))?; + wait_dns(kube, domain, address, https_port, Duration::from_secs(10))?; + } + } if installed .data @@ -1731,6 +2903,258 @@ metadata: {name: lampyr-grant-homes, namespace: default} } } + #[test] + fn existing_tls_secrets_must_match_the_domain_and_private_key() { + let certified = rcgen::generate_simple_self_signed(vec![ + "*.apps.example.com".into(), + "*.default.apps.example.com".into(), + ]) + .unwrap(); + let secret = |key: String| { + json!({ + "data":{ + "tls.crt":STANDARD.encode(certified.cert.pem()), + "tls.key":STANDARD.encode(key) + } + }) + }; + validate_tls_secret( + &secret(certified.signing_key.serialize_pem()), + "apps.example.com", + ) + .unwrap(); + let local = rcgen::generate_simple_self_signed(vec![ + "*.localhost".into(), + "*.default.localhost".into(), + ]) + .unwrap(); + validate_tls_secret( + &json!({"data":{ + "tls.crt":STANDARD.encode(local.cert.pem()), + "tls.key":STANDARD.encode(local.signing_key.serialize_pem()) + }}), + "localhost", + ) + .unwrap(); + let incomplete = + rcgen::generate_simple_self_signed(vec!["*.default.apps.example.com".into()]).unwrap(); + let incomplete_secret = json!({ + "data":{ + "tls.crt":STANDARD.encode(incomplete.cert.pem()), + "tls.key":STANDARD.encode(incomplete.signing_key.serialize_pem()) + } + }); + let error = validate_tls_secret(&incomplete_secret, "apps.example.com") + .unwrap_err() + .to_string(); + assert!(error.contains("does not cover"), "{error}"); + assert!( + error.contains("lampyr-readiness.apps.example.com"), + "{error}" + ); + assert!( + validate_tls_secret( + &secret(certified.signing_key.serialize_pem()), + "other.example.com" + ) + .unwrap_err() + .to_string() + .contains("does not cover") + ); + let other = rcgen::KeyPair::generate().unwrap(); + assert!( + validate_tls_secret(&secret(other.serialize_pem()), "apps.example.com") + .unwrap_err() + .to_string() + .contains("does not match") + ); + } + + #[test] + fn self_signed_secret_reuse_requires_the_self_signed_mode() { + let secret = |mode: Option<&str>, expired: bool| { + let mut annotations = json!({ + "lampyr.dev/domain":"apps.example.com", + "lampyr.dev/installation-id":"installation" + }); + if let Some(mode) = mode { + annotations["lampyr.dev/tls-mode"] = mode.into(); + } + let mut parameters = rcgen::CertificateParams::new(vec![ + "*.apps.example.com".into(), + "*.default.apps.example.com".into(), + ]) + .unwrap(); + parameters.not_before = rcgen::date_time_ymd(2020, 1, 1); + parameters.not_after = if expired { + rcgen::date_time_ymd(2021, 1, 1) + } else { + rcgen::date_time_ymd(2035, 1, 1) + }; + let key = rcgen::KeyPair::generate().unwrap(); + let certificate = parameters.self_signed(&key).unwrap(); + json!({ + "metadata":{"annotations":annotations}, + "data":{ + "tls.crt":STANDARD.encode(certificate.pem()), + "tls.key":STANDARD.encode(key.serialize_pem()) + } + }) + }; + assert!(self_signed_secret_is_reusable( + &secret(Some("selfSigned"), false), + "apps.example.com", + "installation" + )); + assert!(!self_signed_secret_is_reusable( + &secret(Some("acme"), false), + "apps.example.com", + "installation" + )); + assert!(!self_signed_secret_is_reusable( + &secret(None, false), + "apps.example.com", + "installation" + )); + assert!(!self_signed_secret_is_reusable( + &secret(Some("selfSigned"), true), + "apps.example.com", + "installation" + )); + } + + #[test] + fn cert_manager_credential_rolls_when_provider_identity_changes() { + let deployment = json!({"spec":{"template":{"spec":{"containers":[{ + "name":"cert-manager-controller", + "args":["--v=2","--provider-ambient=false"] + }]}}}}); + let credential = |identity: &str| crate::platform_host::CertManagerCredential { + service_account_patch: json!({"metadata":{"annotations":{"provider.example/id":identity}}}), + service_account_cleanup_patch: json!({"metadata":{"annotations":{"provider.example/id":Value::Null}}}), + deployment_patch: json!({"spec":{"template":{"metadata":{"annotations":{"provider.example/id":identity}}}}}), + deployment_cleanup_patch: json!({"spec":{"template":{"metadata":{"annotations":{"provider.example/id":Value::Null}}}}}), + controller_args: vec!["--provider-ambient=true".into()], + controller_arg_prefixes: vec!["--provider-ambient".into()], + boundary: json!({"apiVersion":"v1","kind":"List","items":[]}), + boundary_message: "provider boundary".into(), + probes: vec![], + audits: vec![], + }; + let first = credential("first"); + let patch = cert_manager_credential_patch(&deployment, Some(&first), None).unwrap(); + assert_eq!( + patch.pointer("/spec/template/spec/containers/0/args"), + Some(&json!(["--v=2", "--provider-ambient=true"])) + ); + assert_eq!( + patch.pointer("/spec/template/metadata/annotations/provider.example~1id"), + Some(&json!("first")) + ); + let second = credential("second"); + let changed = + cert_manager_credential_patch(&deployment, Some(&second), Some(&first)).unwrap(); + assert_ne!( + patch.pointer("/spec/template/metadata/annotations"), + changed.pointer("/spec/template/metadata/annotations") + ); + let disabled = cert_manager_credential_patch(&patch, None, Some(&second)).unwrap(); + assert_eq!( + disabled.pointer("/spec/template/spec/containers/0/args"), + Some(&json!(["--v=2"])) + ); + assert_eq!( + disabled.pointer("/spec/template/metadata/annotations/provider.example~1id"), + Some(&Value::Null) + ); + } + + #[test] + fn provider_credential_boundary_is_opaque_and_auditable() { + let boundary = json!({"apiVersion":"v1","kind":"List","items":[ + {"apiVersion":"admissionregistration.k8s.io/v1","kind":"ValidatingAdmissionPolicy", + "metadata":{"name":"provider-boundary","annotations":{"lampyr.dev/credential-boundary":"true"}}, + "spec":{"validations":[{"expression":"!has(object.spec.providerDNS)","message":"provider boundary"}]}}, + {"apiVersion":"admissionregistration.k8s.io/v1","kind":"ValidatingAdmissionPolicyBinding", + "metadata":{"name":"provider-boundary-binding","annotations":{"lampyr.dev/credential-boundary":"true"}}, + "spec":{"policyName":"provider-boundary","validationActions":["Deny"]}} + ]}); + let items = boundary["items"].as_array().unwrap(); + let policy = items + .iter() + .find(|item| item["kind"] == "ValidatingAdmissionPolicy") + .unwrap(); + assert_eq!( + policy.pointer("/spec/validations/0/message"), + Some(&json!("provider boundary")) + ); + let binding = items + .iter() + .find(|item| item["kind"] == "ValidatingAdmissionPolicyBinding") + .unwrap(); + assert_eq!( + binding.pointer("/spec/policyName"), + Some(&json!("provider-boundary")) + ); + assert_eq!( + binding.pointer("/spec/validationActions"), + Some(&json!(["Deny"])) + ); + assert!(field_path_exists( + &json!({"spec":{"solvers":[{"providerDNS":{}}]}}), + &[ + "spec".into(), + "solvers".into(), + "*".into(), + "providerDNS".into() + ] + )); + } + + #[test] + fn core_ingress_lifecycle_has_no_provider_specific_credential_fields() { + let bootstrap = include_str!("bootstrap.rs"); + let platform_host = include_str!("platform_host.rs"); + for provider_field in [ + concat!("azure", "DNS"), + concat!("azure", ".workload.identity"), + concat!("azure", "-pip-name"), + concat!("azure", "-load-balancer-resource-group"), + ] { + assert!(!bootstrap.contains(provider_field), "{provider_field}"); + } + for provider_field in [ + concat!("azure", "DNS"), + concat!("azure", "-pip-name"), + concat!("azure", "-load-balancer-resource-group"), + ] { + assert!(!platform_host.contains(provider_field), "{provider_field}"); + } + } + + #[test] + fn ingress_waits_honor_cancellation_before_network_work() { + let cancellation = crate::runtime::Cancellation::default(); + cancellation.cancel(); + let host = crate::host::Host { + verbose: false, + cancellation, + }; + let directory = tempfile::tempdir().unwrap(); + let kube = Kube::new(directory.path(), Some("test"), &host).unwrap(); + let error = wait_dns( + &kube, + "invalid.example", + "203.0.113.10".parse().unwrap(), + 8443, + Duration::from_secs(10), + ) + .unwrap_err() + .to_string(); + assert!(error.contains("cancelled"), "{error}"); + assert!(error.contains(":8443"), "{error}"); + } + #[test] fn device_capacity_accepts_canonical_kubernetes_quantities() { for value in ["2000", "2k", "1Ki", "1e3"] { @@ -1791,11 +3215,33 @@ metadata: #[test] fn admission_policies_are_deleted_before_the_objects_they_guard() { let item = |kind: &str| json!({ "kind": kind, "metadata": { "name": "default" } }); + let boundary = |kind: &str, name: &str| { + json!({ "apiVersion":"admissionregistration.k8s.io/v1", + "kind": kind, "metadata": { "name": name, "annotations":{ + "lampyr.dev/credential-boundary":"true" + } } }) + }; + let mut recorded = Vec::new(); + record_manifest( + &boundary( + "ValidatingAdmissionPolicyBinding", + "provider-boundary-binding", + ), + &mut recorded, + ) + .unwrap(); + record_manifest( + &boundary("ValidatingAdmissionPolicy", "provider-boundary"), + &mut recorded, + ) + .unwrap(); let remaining = vec![ (item("Deployment"), false), (item("GrantPolicy"), true), (item("ValidatingAdmissionPolicyBinding"), false), (item("ValidatingAdmissionPolicy"), false), + (recorded[0].clone(), false), + (recorded[1].clone(), false), ]; let kinds = deletion_passes(&remaining).map(|pass| { pass.iter() @@ -1811,6 +3257,10 @@ metadata: ], vec!["GrantPolicy"], vec!["Deployment"], + vec![ + "ValidatingAdmissionPolicyBinding", + "ValidatingAdmissionPolicy" + ], ] ); } diff --git a/cli/src/cluster.rs b/cli/src/cluster.rs index 32fcdb1..9f057cc 100644 --- a/cli/src/cluster.rs +++ b/cli/src/cluster.rs @@ -240,11 +240,12 @@ pub(crate) const RUNTIME_SECRET: &str = "lampyr-runtime"; /// The installation inputs for fiberd-k8s-controller, which only the /// platform reads. Providers refuse configuration keys they do not know, so /// these never reach them, and a reinstall keeps the ones it was given. -const CONTROLLER_INPUTS: [&str; 4] = [ +const PLATFORM_INPUTS: [&str; 5] = [ "controllerManifest", "grantImage", "helperImage", "zygoteRegistries", + "ingress", ]; /// Where a platform package records the controller inputs it installs when @@ -537,10 +538,11 @@ fn apply_existing_cluster_defaults( Ok(()) } -/// Fills each controller input the install was not given from the last -/// install's, as fiberdImage and routerImage are. -fn keep_controller_inputs(config: &mut Value, previous: &Value) { - for field in CONTROLLER_INPUTS { +/// Fills each platform-only input the install was not given from the last +/// install's. Providers refuse these settings because only the platform uses +/// them. +fn keep_platform_inputs(config: &mut Value, previous: &Value) { + for field in PLATFORM_INPUTS { if config.get(field).is_none() && let Some(value) = previous.get(field) { @@ -549,23 +551,23 @@ fn keep_controller_inputs(config: &mut Value, previous: &Value) { } } -fn without_controller_inputs(config: &Value) -> Result { +fn without_platform_inputs(config: &Value) -> Result { let mut provider_config = config.clone(); let mapping = provider_config .as_object_mut() .context("provider configuration must be a mapping")?; - for field in CONTROLLER_INPUTS { + for field in PLATFORM_INPUTS { mapping.remove(field); } Ok(provider_config) } /// What the provider is given for an operation other than install: the -/// configuration without the controller inputs, which an uninstall may be +/// configuration without the platform-only inputs, which an uninstall may be /// given in the same --provider-config as the install and the provider /// would refuse, and marked as a teardown for an uninstall. fn teardown_config(config: &Value, operation: Operation) -> Result { - let mut provider_config = without_controller_inputs(config)?; + let mut provider_config = without_platform_inputs(config)?; if operation == Operation::Uninstall { provider_config["teardown"] = true.into(); } @@ -598,7 +600,7 @@ fn install_config( "{field} must be nonempty text" ); } - keep_controller_inputs(&mut config, previous); + keep_platform_inputs(&mut config, previous); let defaulted = fill_controller_defaults( config .as_object_mut() @@ -668,7 +670,7 @@ fn lifecycle( let (config, defaulted) = install_config(config, &previous, &controller_defaults(root)?)?; require_template_registry(&installation.record.provider, &config)?; let eventing = config["eventing"] == true || previous["eventing"] == true; - let provider_config = without_controller_inputs(&config)?; + let provider_config = without_platform_inputs(&config)?; installation.invoke(Operation::Init, provider_config.clone(), host)?; let infrastructure_existed = installation.invoke(Operation::Inspect, provider_config.clone(), host)?["exists"] @@ -699,7 +701,7 @@ fn lifecycle( for field in ["domain", "fiberdImage", "routerImage"] { settings.insert(field.into(), config[field].clone()); } - for field in CONTROLLER_INPUTS { + for field in PLATFORM_INPUTS { if let Some(value) = config.get(field) { settings.insert(field.into(), value.clone()); } @@ -1417,22 +1419,23 @@ mod tests { } #[test] - fn controller_inputs_stay_with_the_platform_across_reinstalls() { + fn platform_inputs_stay_with_the_platform_across_reinstalls() { let given = json!({ "fiberdImage": "custom/fiberd:v1", "controllerManifest": "/controller.yaml", "grantImage": "registry/fiberd@sha256:1", "helperImage": "registry/helper@sha256:2", "zygoteRegistries": ["http://registry:5000/ci"], + "ingress": {"tls":{"mode":"existingSecret","secretName":"tls","secretNamespace":"security"}}, "registryPort": 5002, }); assert_eq!( - without_controller_inputs(&given).unwrap(), + without_platform_inputs(&given).unwrap(), json!({"fiberdImage": "custom/fiberd:v1", "registryPort": 5002}) ); let mut reinstall = json!({"grantImage": "registry/fiberd@sha256:3"}); - keep_controller_inputs(&mut reinstall, &given); + keep_platform_inputs(&mut reinstall, &given); assert_eq!(reinstall["grantImage"], "registry/fiberd@sha256:3"); assert_eq!(reinstall["controllerManifest"], "/controller.yaml"); assert_eq!(reinstall["helperImage"], "registry/helper@sha256:2"); @@ -1440,10 +1443,11 @@ mod tests { reinstall["zygoteRegistries"], json!(["http://registry:5000/ci"]) ); + assert_eq!(reinstall["ingress"], given["ingress"]); assert!(reinstall.get("registryPort").is_none()); let mut first = json!({}); - keep_controller_inputs(&mut first, &json!({})); + keep_platform_inputs(&mut first, &json!({})); assert_eq!(first, json!({})); } @@ -1456,6 +1460,7 @@ mod tests { "grantImage": "registry/fiberd@sha256:1", "helperImage": "registry/helper@sha256:2", "zygoteRegistries": ["http://registry:5000/ci"], + "ingress": {"tls":{"mode":"existingSecret","secretName":"tls","secretNamespace":"security"}}, }); assert_eq!( teardown_config(&given, Operation::Uninstall).unwrap(), @@ -1483,7 +1488,11 @@ mod tests { json!([default_fiberd_image(), default_router_image()]) ); assert!(defaulted.is_empty()); - for field in CONTROLLER_INPUTS { + for field in PLATFORM_INPUTS + .iter() + .copied() + .filter(|field| *field != "ingress") + { let mut missing = given.clone(); missing.as_object_mut().unwrap().remove(field); // zygoteRegistries defaults to the installation's template diff --git a/cli/src/kube.rs b/cli/src/kube.rs index f8623e3..56eae3a 100644 --- a/cli/src/kube.rs +++ b/cli/src/kube.rs @@ -319,7 +319,7 @@ impl<'a> Kube<'a> { } } - fn timeout(&self, timeout: Duration) -> Result { + pub(crate) fn timeout(&self, timeout: Duration) -> Result { let timeout = self.deadline.get().map_or(timeout, |deadline| { timeout.min(deadline.saturating_duration_since(Instant::now())) }); diff --git a/cli/src/platform_host.rs b/cli/src/platform_host.rs index 02ae181..a8c3701 100644 --- a/cli/src/platform_host.rs +++ b/cli/src/platform_host.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use std::time::Instant; use anyhow::{Context as _, Result, bail, ensure}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use crate::host::Host; @@ -164,11 +164,109 @@ struct PlatformSettings { #[serde(default)] ingress_ports: Option, #[serde(default)] + ingress: IngressSettings, + #[serde(default)] metadata: BTreeMap, #[serde(default)] home_pod_metadata: Option, } +#[derive(Default, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct IngressSettings { + #[serde(default)] + pub tls: IngressTls, + #[serde(default)] + pub service: Option, +} + +#[derive(Default, Deserialize)] +#[serde(tag = "mode", rename_all = "camelCase", deny_unknown_fields)] +pub(crate) enum IngressTls { + #[default] + SelfSigned, + Acme { + email: String, + issuer: String, + solver: Value, + credential: Box, + }, + ExistingSecret { + #[serde(rename = "secretName", default = "default_tls_secret_name")] + secret_name: String, + #[serde(rename = "secretNamespace", default = "default_tls_secret_namespace")] + secret_namespace: String, + }, +} + +#[derive(Default, Deserialize)] +#[serde(tag = "mode", rename_all = "camelCase", deny_unknown_fields)] +enum IngressTlsInput { + #[default] + SelfSigned, + Acme { + email: String, + #[serde(default = "default_acme_issuer")] + issuer: String, + }, + ExistingSecret { + #[serde(rename = "secretName", default = "default_tls_secret_name")] + secret_name: String, + #[serde(rename = "secretNamespace", default = "default_tls_secret_namespace")] + secret_namespace: String, + }, +} + +#[derive(Clone, Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct CertManagerCredential { + pub service_account_patch: Value, + pub service_account_cleanup_patch: Value, + pub deployment_patch: Value, + pub deployment_cleanup_patch: Value, + #[serde(default)] + pub controller_args: Vec, + #[serde(default)] + pub controller_arg_prefixes: Vec, + pub boundary: Value, + pub boundary_message: String, + #[serde(default)] + pub probes: Vec, + #[serde(default)] + pub audits: Vec, +} + +#[derive(Clone, Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct CredentialAudit { + pub resource: String, + pub kind: String, + pub allowed_namespace: String, + pub field_path: Vec, + #[serde(default)] + pub issuer_kind: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct IngressService { + pub public_ip_address: String, + #[serde(default)] + pub annotations: BTreeMap, +} + +fn default_tls_secret_name() -> String { + "lampyr-kourier-tls".into() +} + +fn default_tls_secret_namespace() -> String { + "default".into() +} + +fn default_acme_issuer() -> String { + "letsencrypt-prod".into() +} + impl PlatformSettings { fn validate(&self) -> Result<()> { validate_domain(&self.domain)?; @@ -187,6 +285,13 @@ impl PlatformSettings { if let Some(ports) = &self.ingress_ports { ensure!(ports.is_array(), "ingress ports must be an array"); } + self.ingress.validate()?; + if !matches!(self.ingress.tls, IngressTls::SelfSigned) { + ensure!( + self.domain.contains('.'), + "ACME and existing ingress TLS Secrets require a multi-label domain" + ); + } for reserved in [ "installationId", "cluster", @@ -198,6 +303,7 @@ impl PlatformSettings { "provider platform metadata cannot replace {reserved}" ); } + if let Some(metadata) = &self.home_pod_metadata { validate_metadata(metadata)?; } @@ -211,6 +317,256 @@ impl PlatformSettings { } } +impl IngressSettings { + fn validate(&self) -> Result<()> { + self.tls.validate()?; + if let Some(service) = &self.service { + ensure!( + service + .public_ip_address + .parse::() + .is_ok(), + "ingress public IP address is invalid" + ); + ensure!( + service.annotations.iter().all(|(key, value)| { + !key.is_empty() + && !value.is_empty() + && !key.chars().any(char::is_whitespace) + && !value.chars().any(char::is_whitespace) + }), + "ingress Service annotations must be nonempty and contain no whitespace" + ); + } + ensure!( + !matches!(&self.tls, IngressTls::Acme { .. }) || self.service.is_some(), + "ACME ingress requires a stable public IP and DNS records" + ); + Ok(()) + } +} + +impl IngressTls { + fn validate(&self) -> Result<()> { + let dns_label = |value: &str| crate::project::validate_name(value).is_ok(); + match self { + IngressTls::SelfSigned => {} + IngressTls::ExistingSecret { + secret_name, + secret_namespace, + } => { + ensure!( + dns_label(secret_name) && dns_label(secret_namespace), + "existing ingress TLS Secret name and namespace must be DNS labels" + ); + ensure!( + ![ + "cert-manager", + "fiberd-system", + "hyperlight-system", + "knative-eventing", + "knative-serving", + "kourier-system", + "lampyr-system", + ] + .contains(&secret_namespace.as_str()), + "existing ingress TLS Secret must be outside Lampyr-owned namespaces" + ); + } + + IngressTls::Acme { + email, + issuer, + solver, + credential, + } => { + ensure!( + email.len() <= 254 + && email.split_once('@').is_some_and(|(local, domain)| { + !local.is_empty() + && domain.contains('.') + && !domain.chars().any(char::is_whitespace) + }), + "ACME email is invalid" + ); + ensure!(dns_label(issuer), "ACME issuer must be a DNS label"); + ensure!(solver.is_object(), "ACME solver must be an object"); + credential.validate()?; + } + } + Ok(()) + } +} + +impl CertManagerCredential { + fn validate(&self) -> Result<()> { + validate_service_account_patch(&self.service_account_patch)?; + validate_service_account_patch(&self.service_account_cleanup_patch)?; + validate_deployment_patch(&self.deployment_patch)?; + validate_deployment_patch(&self.deployment_cleanup_patch)?; + ensure!( + self.controller_args == ["--issuer-ambient-credentials=true"] + && self.controller_arg_prefixes == ["--issuer-ambient-credentials"], + "managed cert-manager credentials may change only Issuer ambient credentials" + ); + ensure!( + self.boundary["kind"] == "List" && self.boundary["items"].as_array().is_some(), + "cert-manager credential boundary must be a manifest List" + ); + ensure!( + self.boundary["items"] + .as_array() + .unwrap() + .iter() + .all(|item| { + matches!( + item["kind"].as_str(), + Some("ValidatingAdmissionPolicy" | "ValidatingAdmissionPolicyBinding") + ) + }), + "cert-manager credential boundary may contain only validating admission resources" + ); + ensure!( + !self.boundary_message.is_empty(), + "cert-manager credential boundary message must be nonempty" + ); + for audit in &self.audits { + ensure!( + !audit.resource.is_empty() + && audit + .resource + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b".-".contains(&byte)), + "cert-manager credential audit resource is invalid" + ); + ensure!( + !audit.kind.is_empty() + && crate::project::validate_name(&audit.allowed_namespace).is_ok() + && !audit.field_path.is_empty() + && audit.field_path.iter().all(|segment| { + segment == "*" + || (!segment.is_empty() + && segment.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || b"_.-".contains(&byte) + })) + }), + "cert-manager credential audit is invalid" + ); + } + ensure!( + self.probes.iter().all(|probe| { + matches!(probe["kind"].as_str(), Some("Issuer" | "Challenge")) + && probe["metadata"]["namespace"] == "default" + }), + "cert-manager credential probes must be Issuers or Challenges in default" + ); + Ok(()) + } +} + +fn annotation_values(mapping: Option<&serde_json::Map>) -> bool { + mapping.is_some_and(|mapping| { + mapping + .values() + .all(|value| value.is_string() || value.is_null()) + }) +} + +fn validate_service_account_patch(patch: &Value) -> Result<()> { + let object = patch + .as_object() + .context("ServiceAccount patch must be an object")?; + let metadata = object + .get("metadata") + .and_then(Value::as_object) + .context("ServiceAccount patch must contain metadata")?; + ensure!( + object.len() == 1 + && metadata.len() == 1 + && annotation_values(metadata.get("annotations").and_then(Value::as_object)), + "ServiceAccount patch may change only string or null annotations" + ); + Ok(()) +} + +fn validate_deployment_patch(patch: &Value) -> Result<()> { + let object = patch + .as_object() + .context("Deployment patch must be an object")?; + let metadata = patch + .pointer("/spec/template/metadata") + .and_then(Value::as_object) + .context("Deployment patch must contain spec.template.metadata")?; + ensure!( + object.len() == 1 + && object["spec"] + .as_object() + .is_some_and(|spec| spec.len() == 1) + && object["spec"]["template"] + .as_object() + .is_some_and(|template| template.len() == 1) + && metadata + .keys() + .all(|key| matches!(key.as_str(), "annotations" | "labels")) + && metadata + .get("annotations") + .is_none_or(|value| annotation_values(value.as_object())) + && metadata + .get("labels") + .is_none_or(|value| annotation_values(value.as_object())), + "Deployment patch may change only string or null Pod-template annotations and labels" + ); + Ok(()) +} + +impl IngressTlsInput { + fn validate(&self) -> Result<()> { + let dns_label = |value: &str| crate::project::validate_name(value).is_ok(); + match self { + IngressTlsInput::SelfSigned => {} + IngressTlsInput::ExistingSecret { + secret_name, + secret_namespace, + } => { + ensure!( + dns_label(secret_name) && dns_label(secret_namespace), + "existing ingress TLS Secret name and namespace must be DNS labels" + ); + ensure!( + ![ + "cert-manager", + "fiberd-system", + "hyperlight-system", + "knative-eventing", + "knative-serving", + "kourier-system", + "lampyr-system", + ] + .contains(&secret_namespace.as_str()), + "existing ingress TLS Secret must be outside Lampyr-owned namespaces" + ); + } + IngressTlsInput::Acme { email, issuer } => { + ensure!( + email.len() <= 254 + && email.split_once('@').is_some_and(|(local, domain)| { + !local.is_empty() + && domain.contains('.') + && !domain.chars().any(char::is_whitespace) + }), + "ACME email is invalid" + ); + ensure!(dns_label(issuer), "ACME issuer must be a DNS label"); + } + } + Ok(()) + } + + fn is_trusted(&self) -> bool { + !matches!(self, Self::SelfSigned) + } +} + /// An installation's one fiberd template repository, as the host pushes /// to it, the repositories grant Pods may pull templates from, and the /// registry mirrors the serving controller rewrites templates through. @@ -585,6 +941,20 @@ pub(crate) fn validate_domain(domain: &str) -> Result<()> { pub(crate) fn validate_install_inputs(value: &Value) -> Result<()> { let domain = value["domain"].as_str().context("domain must be text")?; validate_domain(domain)?; + let tls: IngressTlsInput = serde_json::from_value( + value + .pointer("/ingress/tls") + .cloned() + .unwrap_or_else(|| json!({"mode":"selfSigned"})), + ) + .context("decode ingress TLS settings")?; + tls.validate()?; + if tls.is_trusted() { + ensure!( + domain.contains('.'), + "ACME and existing ingress TLS Secrets require a multi-label domain" + ); + } if let Some(https_port) = value.get("httpsPort") { ensure!( https_port @@ -710,6 +1080,7 @@ pub(crate) fn install( zygote_registries: &templates.registries, registry_mirrors: templates.mirrors, ingress_ports: settings.ingress_ports, + ingress: &settings.ingress, }, )?; let mut metadata = settings.metadata; @@ -807,6 +1178,88 @@ pub(crate) mod tests { format!("registry.example/helper@sha256:{}", "1".repeat(64)) } + fn platform_settings(ingress: Value) -> Value { + json!({ + "domain":"apps.example.com", + "httpsPort":443, + "fiberdImage":"registry/fiberd:v1", + "routerImage":"registry/router:v1", + "controllerManifest":manifest(), + "grantImage":grant_image(), + "helperImage":helper_image(), + "zygoteRegistries":["registry.example/templates"], + "ingress":ingress, + }) + } + + #[test] + fn ingress_tls_modes_validate_their_required_ownership_inputs() { + validate_settings(&platform_settings(json!({}))).unwrap(); + validate_settings(&platform_settings(json!({ + "tls":{ + "mode":"existingSecret", + "secretName":"corporate-ingress", + "secretNamespace":"security" + } + }))) + .unwrap(); + let mut trusted_single_label = platform_settings(json!({ + "tls":{ + "mode":"existingSecret", + "secretName":"corporate-ingress", + "secretNamespace":"security" + } + })); + trusted_single_label["domain"] = "localhost".into(); + assert!(validate_settings(&trusted_single_label).is_err()); + for namespace in ["knative-eventing", "kourier-system"] { + let error = validate_settings(&platform_settings(json!({ + "tls":{ + "mode":"existingSecret", + "secretName":"corporate-ingress", + "secretNamespace":namespace + } + }))) + .unwrap_err() + .to_string(); + assert!( + error.contains("outside Lampyr-owned namespaces"), + "{namespace}: {error}" + ); + } + let acme = json!({ + "service":{ + "publicIpAddress":"203.0.113.10", + "annotations":{"provider.example/ip":"lampyr-ingress"} + }, + "tls":{ + "mode":"acme", + "email":"platform@example.com", + "issuer":"letsencrypt-prod", + "solver":{"dns01":{"providerDNS":{}}}, + "credential":{ + "serviceAccountPatch":{"metadata":{"annotations":{}}}, + "serviceAccountCleanupPatch":{"metadata":{"annotations":{}}}, + "deploymentPatch":{"spec":{"template":{"metadata":{}}}}, + "deploymentCleanupPatch":{"spec":{"template":{"metadata":{}}}}, + "controllerArgs":["--issuer-ambient-credentials=true"], + "controllerArgPrefixes":["--issuer-ambient-credentials"], + "boundary":{"apiVersion":"v1","kind":"List","items":[]}, + "boundaryMessage":"provider boundary" + } + } + }); + validate_settings(&platform_settings(acme.clone())).unwrap(); + let mut missing_service = acme; + missing_service.as_object_mut().unwrap().remove("service"); + assert!( + validate_settings(&platform_settings(missing_service)) + .unwrap_err() + .to_string() + .contains("stable public IP") + ); + } + #[test] fn install_inputs_reject_invalid_domains_before_cluster_mutation() { let settings = |domain: &str| { @@ -827,6 +1280,28 @@ pub(crate) mod tests { assert!(validate_install_inputs(&dynamic).is_ok()); dynamic.as_object_mut().unwrap().remove("httpsPort"); assert!(validate_install_inputs(&dynamic).is_ok()); + let mut trusted_single_label = settings("localhost"); + trusted_single_label["ingress"] = json!({"tls":{ + "mode":"existingSecret", + "secretName":"corporate-ingress", + "secretNamespace":"security" + }}); + assert!(validate_install_inputs(&trusted_single_label).is_err()); + trusted_single_label["ingress"]["tls"]["mode"] = "acme".into(); + assert!(validate_install_inputs(&trusted_single_label).is_err()); + assert!(validate_install_inputs(&settings("localhost")).is_ok()); + let mut valid_acme = settings("apps.example.com"); + valid_acme["ingress"] = json!({"tls":{"mode":"acme","email":"platform@example.com"}}); + assert!(validate_install_inputs(&valid_acme).is_ok()); + let mut invalid_ingress = settings("apps.example.com"); + invalid_ingress["ingress"] = json!({"tls":{"mode":"acmee"}}); + assert!(validate_install_inputs(&invalid_ingress).is_err()); + invalid_ingress["ingress"] = json!({"tls":{ + "mode":"existingSecret", + "secretName":"corporate-ingress", + "secretNamespace":"kourier-system" + }}); + assert!(validate_install_inputs(&invalid_ingress).is_err()); for domain in [ "UPPER.example.com", "bad_name.example.com", From 7aaf2116c6f4fc710e4cf3094e3c5e82e7c04c29 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 20 Nov 2026 17:22:35 -0800 Subject: [PATCH 2/7] fixup! Add configurable ingress TLS modes Signed-off-by: James Sturtevant --- cli/src/bootstrap.rs | 33 +++++++++++++++++++++++++++++++ cli/src/cluster.rs | 42 ++++++++++++++++++++++++++++++++++++++++ cli/src/platform_host.rs | 42 +++++++++++++++++++++++----------------- 3 files changed, 99 insertions(+), 18 deletions(-) diff --git a/cli/src/bootstrap.rs b/cli/src/bootstrap.rs index 0f823dd..2d131fc 100644 --- a/cli/src/bootstrap.rs +++ b/cli/src/bootstrap.rs @@ -2970,6 +2970,39 @@ metadata: {name: lampyr-grant-homes, namespace: default} ); } + #[test] + fn existing_tls_secret_does_not_promote_its_presented_chain_to_a_root() { + let certified = rcgen::generate_simple_self_signed(vec![ + "*.apps.example.com".into(), + "*.default.apps.example.com".into(), + ]) + .unwrap(); + let secret = |ca: Option<&str>| { + let mut data = json!({ + "tls.crt":STANDARD.encode(certified.cert.pem()), + "tls.key":STANDARD.encode(certified.signing_key.serialize_pem()) + }); + if let Some(ca) = ca { + data["ca.crt"] = STANDARD.encode(ca).into(); + } + json!({"data":data}) + }; + let error = validate_tls_secret(&secret(None), "apps.example.com") + .unwrap_err() + .to_string(); + assert!(error.contains("untrusted"), "{error}"); + validate_tls_secret(&secret(Some(&certified.cert.pem())), "apps.example.com").unwrap(); + } + + #[test] + fn credential_reconciliation_persists_retiring_configuration() { + let production = include_str!("bootstrap.rs") + .split("#[cfg(test)]") + .next() + .unwrap(); + assert!(production.contains("ingressRetiringCredential")); + } + #[test] fn self_signed_secret_reuse_requires_the_self_signed_mode() { let secret = |mode: Option<&str>, expired: bool| { diff --git a/cli/src/cluster.rs b/cli/src/cluster.rs index 9f057cc..7a34a28 100644 --- a/cli/src/cluster.rs +++ b/cli/src/cluster.rs @@ -1640,6 +1640,48 @@ mod tests { ); } + #[test] + fn embedded_platform_credential_requires_manifest_capability_and_approval() { + let mut plan = RuntimePlan::default(); + plan.options.insert( + "ingress".into(), + json!({"tls":{"mode":"acme","credential":{ + "serviceAccountPatch":{"metadata":{"annotations":{"provider.example/id":"identity"}}}, + "serviceAccountCleanupPatch":{"metadata":{"annotations":{"provider.example/id":null}}}, + "deploymentPatch":{"spec":{"template":{"metadata":{"annotations":{"provider.example/id":"identity"}}}}}, + "deploymentCleanupPatch":{"spec":{"template":{"metadata":{"annotations":{"provider.example/id":null}}}}}, + "controllerArgs":["--provider-ambient=true"], + "controllerArgPrefixes":["--provider-ambient"], + "boundary":{"apiVersion":"v1","kind":"List","items":[{ + "apiVersion":"admissionregistration.k8s.io/v1", + "kind":"ValidatingAdmissionPolicy", + "metadata":{"name":"provider-boundary"} + }]}, + "boundaryMessage":"provider boundary" + }}}), + ); + assert!( + validate_provider_manifest_capability_for_plan(&plan, false) + .unwrap_err() + .to_string() + .contains("provider-manifests capability") + ); + let directory = tempfile::tempdir().unwrap(); + assert!( + approve_install_manifests_at( + directory.path(), + &plan, + ManifestApproval { + approved: false, + interactive: false, + }, + ) + .unwrap_err() + .to_string() + .contains("--approve-provider-manifests") + ); + } + #[test] fn selected_context_must_match_local_installation_and_cluster_identity() { let selection = Selection { diff --git a/cli/src/platform_host.rs b/cli/src/platform_host.rs index a8c3701..4d2a56d 100644 --- a/cli/src/platform_host.rs +++ b/cli/src/platform_host.rs @@ -1687,24 +1687,30 @@ pub(crate) mod tests { #[test] fn provider_metadata_cannot_replace_core_manifest_inventory() { - let settings = json!({ - "domain":"apps.example.com", - "httpsPort":443, - "fiberdImage":"registry/fiberd:v1", - "routerImage":"registry/router:v1", - "controllerManifest":manifest(), - "grantImage":grant_image(), - "helperImage":helper_image(), - "metadata":{ - crate::bootstrap::CORE_INVENTORY_KEY:"attacker-controlled" - }, - }); - assert!( - validate_settings(&settings) - .unwrap_err() - .to_string() - .contains(crate::bootstrap::CORE_INVENTORY_KEY) - ); + for reserved in [ + crate::bootstrap::CORE_INVENTORY_KEY, + "ingressTlsMode", + "ingressTlsSecretNamespace", + "ingressTlsSecretName", + "ingressTlsIssuer", + "ingressDnsAddress", + "ingressCredential", + "ingressServiceAnnotations", + ] { + let settings = json!({ + "domain":"apps.example.com", + "httpsPort":443, + "fiberdImage":"registry/fiberd:v1", + "routerImage":"registry/router:v1", + "controllerManifest":manifest(), + "grantImage":grant_image(), + "helperImage":helper_image(), + "zygoteRegistries":["ghcr.io/team/templates"], + "metadata":{reserved:"attacker-controlled"}, + }); + let error = validate_settings(&settings).unwrap_err().to_string(); + assert!(error.contains(reserved), "{reserved}: {error}"); + } } #[test] From 17ae5aaa8ea7594ff5017af7be0349b8e3458e8f Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 20 Nov 2026 17:41:06 -0800 Subject: [PATCH 3/7] fixup! Add configurable ingress TLS modes Signed-off-by: James Sturtevant --- cli/Cargo.lock | 1 + cli/Cargo.toml | 1 + cli/src/bootstrap.rs | 498 +++++++++++++++++++++++++++++++-------- cli/src/cluster.rs | 136 ++++++++++- cli/src/platform_host.rs | 13 +- 5 files changed, 543 insertions(+), 106 deletions(-) diff --git a/cli/Cargo.lock b/cli/Cargo.lock index 96cf0e6..c14e20a 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -1530,6 +1530,7 @@ dependencies = [ "rcgen", "reqwest", "rustls", + "rustls-native-certs", "serde", "serde_json", "serde_yaml_ng", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 2d02aa2..aeba6c0 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -52,6 +52,7 @@ getrandom = "0.3" git2 = { version = "0.21", default-features = false } rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } zeroize = "1" +rustls-native-certs = "0.8" [dev-dependencies] indicatif = { version = "0.18", features = ["in_memory"] } diff --git a/cli/src/bootstrap.rs b/cli/src/bootstrap.rs index 2d131fc..bfc23f9 100644 --- a/cli/src/bootstrap.rs +++ b/cli/src/bootstrap.rs @@ -740,7 +740,7 @@ fn install_platform( &options.ingress.tls, crate::platform_host::IngressTls::SelfSigned ) { - let certificates = wait_tls_secret( + let tls = wait_tls_secret( kube, &secret_namespace, &secret_name, @@ -751,7 +751,7 @@ fn install_platform( kube, options.domain, options.https_port, - &certificates, + tls, Duration::from_secs(120), )?; } @@ -835,6 +835,7 @@ fn install_platform( "ingressTlsMode": tls_mode, "ingressTlsSecretNamespace": secret_namespace, "ingressTlsSecretName": secret_name, "ingressTlsIssuer": issuer, "ingressDnsAddress": dns_address, "ingressCredential": credential, + "ingressRetiringCredential": Value::Null, "ingressServiceAnnotations": serde_json::to_string( &options.ingress.service.as_ref() .map(|service| service.annotations.clone()) @@ -1277,18 +1278,21 @@ fn configure_ingress_tls( inventory: &mut Vec, ) -> Result<(String, String)> { use crate::platform_host::IngressTls; - let previous_credential = installed_cert_manager_credential(kube)?; + let installed_credentials = installed_cert_manager_credentials(kube)?; match &options.ingress.tls { IngressTls::SelfSigned => { + let previous_credentials = installed_credentials.previous_for(None); + persist_cert_manager_credentials(kube, None, &previous_credentials)?; remove_acme_resources(kube, options.installation_id, inventory)?; - configure_cert_manager_credential(kube, inventory, None, previous_credential.as_ref())?; - remove_credential_boundary( + configure_cert_manager_credential(kube, inventory, None, &previous_credentials)?; + remove_retired_credential_boundaries( kube, options.installation_id, - previous_credential.as_ref(), + &previous_credentials, None, inventory, )?; + persist_cert_manager_credentials(kube, None, &[])?; self_signed_tls(kube, options.domain, options.installation_id)?; Ok(("kourier-system".into(), "lampyr-kourier-tls".into())) } @@ -1303,15 +1307,18 @@ fn configure_ingress_tls( options.domain, Duration::from_secs(120), )?; + let previous_credentials = installed_credentials.previous_for(None); + persist_cert_manager_credentials(kube, None, &previous_credentials)?; remove_acme_resources(kube, options.installation_id, inventory)?; - configure_cert_manager_credential(kube, inventory, None, previous_credential.as_ref())?; - remove_credential_boundary( + configure_cert_manager_credential(kube, inventory, None, &previous_credentials)?; + remove_retired_credential_boundaries( kube, options.installation_id, - previous_credential.as_ref(), + &previous_credentials, None, inventory, )?; + persist_cert_manager_credentials(kube, None, &[])?; Ok((secret_namespace.clone(), secret_name.clone())) } IngressTls::Acme { @@ -1324,32 +1331,35 @@ fn configure_ingress_tls( cert_manager_owned(inventory), "ACME ingress requires the cert-manager installation managed by Lampyr" ); - persist_cert_manager_credential(kube, Some(credential))?; + let previous_credentials = + installed_credentials.previous_for(Some(credential.as_ref())); + persist_cert_manager_credentials( + kube, + Some(credential.as_ref()), + &previous_credentials, + )?; configure_credential_boundary(kube, options.installation_id, credential, inventory)?; wait_credential_boundary(kube, credential)?; if let Err(error) = ensure_no_external_credential_resources(kube, credential) { - configure_cert_manager_credential( - kube, - inventory, - None, - previous_credential.as_ref(), - ) - .context("disable cert-manager identity after unsafe Issuer discovery")?; + let active_credentials = installed_credentials.previous_for(None); + configure_cert_manager_credential(kube, inventory, None, &active_credentials) + .context("disable cert-manager identity after unsafe Issuer discovery")?; return Err(error); } configure_cert_manager_credential( kube, inventory, Some(credential), - previous_credential.as_ref(), + &previous_credentials, )?; - remove_credential_boundary( + remove_retired_credential_boundaries( kube, options.installation_id, - previous_credential.as_ref(), + &previous_credentials, Some(credential), inventory, )?; + persist_cert_manager_credentials(kube, Some(credential.as_ref()), &[])?; let issuer_manifest = json!({ "apiVersion":"cert-manager.io/v1", "kind":"Issuer", "metadata":{"name":issuer,"namespace":"kourier-system","annotations":{ @@ -1444,20 +1454,63 @@ fn cert_manager_owned(inventory: &[Value]) -> bool { }) } -fn persist_cert_manager_credential( +#[derive(Default)] +struct InstalledCertManagerCredentials { + current: Option, + retiring: Vec, +} + +impl InstalledCertManagerCredentials { + fn previous_for( + &self, + current: Option<&crate::platform_host::CertManagerCredential>, + ) -> Vec { + let mut previous = Vec::new(); + for credential in self.current.iter().chain(&self.retiring) { + if current.is_some_and(|current| current == credential) + || previous.iter().any(|existing| existing == credential) + { + continue; + } + previous.push(credential.clone()); + } + previous + } +} + +fn persist_cert_manager_credentials( kube: &Kube<'_>, - credential: Option<&crate::platform_host::CertManagerCredential>, + current: Option<&crate::platform_host::CertManagerCredential>, + retiring: &[crate::platform_host::CertManagerCredential], ) -> Result<()> { kube.patch( "kube-system", "configmap", "lampyr-install", - &json!({"data":{"ingressCredential":credential - .map(serde_json::to_string) - .transpose()?}}), + &cert_manager_credential_state_patch(current, retiring)?, ) } +fn cert_manager_credential_state_patch( + current: Option<&crate::platform_host::CertManagerCredential>, + retiring: &[crate::platform_host::CertManagerCredential], +) -> Result { + let current = current + .map(serde_json::to_string) + .transpose()? + .map(Value::String) + .unwrap_or(Value::Null); + let retiring = if retiring.is_empty() { + Value::Null + } else { + Value::String(serde_json::to_string(retiring)?) + }; + Ok(json!({"data":{ + "ingressCredential":current, + "ingressRetiringCredential":retiring + }})) +} + fn apply_cert_manager_manifest( kube: &Kube<'_>, installation_id: &str, @@ -1486,23 +1539,34 @@ fn apply_cert_manager_manifest( } } -fn installed_cert_manager_credential( - kube: &Kube<'_>, -) -> Result> { +fn installed_cert_manager_credentials(kube: &Kube<'_>) -> Result { let Some(install) = kube.optional("kube-system", "configmap", "lampyr-install")? else { - return Ok(None); + return Ok(InstalledCertManagerCredentials::default()); }; - install["data"]["ingressCredential"] + decode_installed_cert_manager_credentials(&install) +} + +fn decode_installed_cert_manager_credentials( + install: &Value, +) -> Result { + let current = install["data"]["ingressCredential"] + .as_str() + .map(serde_json::from_str) + .transpose() + .context("decode installed cert-manager credential configuration")?; + let retiring = install["data"]["ingressRetiringCredential"] .as_str() .map(serde_json::from_str) .transpose() - .context("decode installed cert-manager credential configuration") + .context("decode retiring cert-manager credential configurations")? + .unwrap_or_default(); + Ok(InstalledCertManagerCredentials { current, retiring }) } fn cert_manager_controller_args( deployment: &Value, current: Option<&crate::platform_host::CertManagerCredential>, - previous: Option<&crate::platform_host::CertManagerCredential>, + previous: &[crate::platform_host::CertManagerCredential], ) -> Result> { let container = deployment .pointer("/spec/template/spec/containers") @@ -1519,7 +1583,7 @@ fn cert_manager_controller_args( .clone(); let prefixes: BTreeSet<_> = current .into_iter() - .chain(previous) + .chain(previous.iter()) .flat_map(|credential| credential.controller_arg_prefixes.iter()) .collect(); args.retain(|arg| { @@ -1549,11 +1613,12 @@ fn merge_json(target: &mut Value, patch: &Value) { fn cert_manager_credential_patch( deployment: &Value, current: Option<&crate::platform_host::CertManagerCredential>, - previous: Option<&crate::platform_host::CertManagerCredential>, + previous: &[crate::platform_host::CertManagerCredential], ) -> Result { - let mut patch = previous - .map(|credential| credential.deployment_cleanup_patch.clone()) - .unwrap_or_else(|| json!({})); + let mut patch = json!({}); + for credential in previous { + merge_json(&mut patch, &credential.deployment_cleanup_patch); + } if let Some(current) = current { merge_json(&mut patch, ¤t.deployment_patch); } @@ -1688,7 +1753,7 @@ fn configure_cert_manager_credential( kube: &Kube<'_>, inventory: &[Value], current: Option<&crate::platform_host::CertManagerCredential>, - previous: Option<&crate::platform_host::CertManagerCredential>, + previous: &[crate::platform_host::CertManagerCredential], ) -> Result<()> { if current.is_some() { ensure!( @@ -1696,10 +1761,14 @@ fn configure_cert_manager_credential( "managed ACME ingress requires the cert-manager installation managed by Lampyr" ); } - if current.is_some() || previous.is_some() { - let mut service_account_patch = previous - .map(|credential| credential.service_account_cleanup_patch.clone()) - .unwrap_or_else(|| json!({})); + if current.is_some() || !previous.is_empty() { + let mut service_account_patch = json!({}); + for credential in previous { + merge_json( + &mut service_account_patch, + &credential.service_account_cleanup_patch, + ); + } if let Some(current) = current { merge_json(&mut service_account_patch, ¤t.service_account_patch); } @@ -1804,7 +1873,7 @@ fn wait_tls_secret( name: &str, domain: &str, timeout: Duration, -) -> Result>> { +) -> Result { kube.wait("ingress TLS Secret", timeout, || { let Some(secret) = kube.optional(namespace, "secret", name)? else { return Ok(None); @@ -1816,7 +1885,30 @@ fn wait_tls_secret( }) } -fn validate_tls_secret(secret: &Value, domain: &str) -> Result>> { +fn wait_self_signed_tls_secret( + kube: &Kube<'_>, + namespace: &str, + name: &str, + domain: &str, + timeout: Duration, +) -> Result<()> { + kube.wait("self-signed ingress TLS Secret", timeout, || { + let Some(secret) = kube.optional(namespace, "secret", name)? else { + return Ok(None); + }; + if secret["type"] != "kubernetes.io/tls" { + return Ok(None); + } + validate_self_signed_tls_secret(&secret, domain).map(|()| Some(())) + }) +} + +#[derive(Debug)] +struct ValidatedTls { + roots: rustls::RootCertStore, +} + +fn tls_secret_certificates(secret: &Value) -> Result>> { let certificate = secret["data"]["tls.crt"] .as_str() .context("ingress TLS Secret has no tls.crt")?; @@ -1840,18 +1932,42 @@ fn validate_tls_secret(secret: &Value, domain: &str) -> Result Result<()> { + if let Some(bundle) = secret["data"]["ca.crt"].as_str() { + let bundle = STANDARD + .decode(bundle) + .context("decode ingress TLS CA bundle")?; + if bundle.is_empty() { + return Ok(()); + } + let certificates: Vec<_> = CertificateDer::pem_slice_iter(&bundle) + .collect::>() + .context("parse ingress TLS CA bundle")?; + ensure!( + !certificates.is_empty(), + "ingress TLS CA bundle has no certificates" + ); + for certificate in certificates { + roots + .add(certificate) + .context("add ingress TLS CA trust anchor")?; + } + } + Ok(()) +} + +fn verify_tls_hostnames( + certificates: &[CertificateDer<'static>], + roots: rustls::RootCertStore, + domain: &str, + intermediates: &[CertificateDer<'static>], +) -> Result { + let verifier = rustls::client::WebPkiServerVerifier::builder(Arc::new(roots.clone())) .build() .context("build ingress TLS certificate verifier")?; - let intermediates = if certificates.len() > 1 { - &certificates[1..certificates.len() - 1] - } else { - &[] - }; use rustls::client::danger::ServerCertVerifier as _; let mut hostnames = vec![format!("lampyr-readiness.default.{domain}")]; // WebPKI deliberately rejects a wildcard directly below a single-label @@ -1875,23 +1991,67 @@ fn validate_tls_secret(secret: &Value, domain: &str) -> Result Result { + let mut roots = rustls::RootCertStore::empty(); + let native = rustls_native_certs::load_native_certs(); + let (_, ignored) = roots.add_parsable_certificates(native.certs); + let system_empty = roots.is_empty(); + match validate_tls_secret_with_roots(secret, domain, roots) { + Err(error) if system_empty && error.to_string().contains("TLS trust store is empty") => { + Err(error).context(format!( + "{ignored} system certificates were ignored while loading native roots: {:?}", + native.errors + )) + } + result => result, + } +} + +fn validate_tls_secret_with_roots( + secret: &Value, + domain: &str, + mut roots: rustls::RootCertStore, +) -> Result { + add_secret_tls_roots(secret, &mut roots)?; + ensure!( + !roots.is_empty(), + "TLS trust store is empty and the Secret has no non-empty ca.crt bundle" + ); + let certificates = tls_secret_certificates(secret)?; + let roots = verify_tls_hostnames(&certificates, roots, domain, &certificates[1..])?; + Ok(ValidatedTls { roots }) +} + +fn validate_self_signed_tls_secret(secret: &Value, domain: &str) -> Result<()> { + let certificates = tls_secret_certificates(secret)?; + let mut roots = rustls::RootCertStore::empty(); + roots + .add(certificates.last().unwrap().clone()) + .context("ingress TLS chain has no usable trust anchor")?; + let intermediates = if certificates.len() > 1 { + &certificates[1..certificates.len() - 1] + } else { + &[] + }; + verify_tls_hostnames(&certificates, roots, domain, intermediates)?; + Ok(()) } fn wait_tls_endpoint( kube: &Kube<'_>, domain: &str, port: u16, - certificates: &[CertificateDer<'static>], + tls: ValidatedTls, timeout: Duration, ) -> Result<()> { let hostname = format!("lampyr-readiness.default.{domain}"); let server_name = ServerName::try_from(hostname.clone())?.to_owned(); - let mut roots = rustls::RootCertStore::empty(); - roots.add(certificates.last().unwrap().clone())?; let config = Arc::new( rustls::ClientConfig::builder() - .with_root_certificates(roots) + .with_root_certificates(tls.roots) .with_no_client_auth(), ); let mut resolver = AddressResolver::new(&hostname, port); @@ -2027,6 +2187,19 @@ fn remove_credential_boundary( Ok(()) } +fn remove_retired_credential_boundaries( + kube: &Kube<'_>, + installation_id: &str, + previous: &[crate::platform_host::CertManagerCredential], + current: Option<&crate::platform_host::CertManagerCredential>, + inventory: &mut Vec, +) -> Result<()> { + for credential in previous { + remove_credential_boundary(kube, installation_id, Some(credential), current, inventory)?; + } + Ok(()) +} + fn remove_retired_issuers( kube: &Kube<'_>, installation_id: &str, @@ -2116,7 +2289,7 @@ fn self_signed_secret_is_reusable(secret: &Value, domain: &str, installation_id: .pointer("/metadata/annotations/lampyr.dev~1tls-mode") .and_then(Value::as_str) == Some("selfSigned") - && validate_tls_secret(secret, domain).is_ok() + && validate_self_signed_tls_secret(secret, domain).is_ok() } pub(crate) fn delete_manifest(kube: &Kube<'_>, source: &str) -> Result> { @@ -2395,21 +2568,23 @@ pub fn health(kube: &Kube<'_>) -> Result<()> { Duration::from_secs(10), )?; } - let certificates = wait_tls_secret( - kube, - secret_namespace, - secret_name, - domain, - Duration::from_secs(10), - )?; - if tls_mode != "selfSigned" { - wait_tls_endpoint( + if tls_mode == "selfSigned" { + wait_self_signed_tls_secret( kube, + secret_namespace, + secret_name, domain, - https_port, - &certificates, Duration::from_secs(10), )?; + } else { + let tls = wait_tls_secret( + kube, + secret_namespace, + secret_name, + domain, + Duration::from_secs(10), + )?; + wait_tls_endpoint(kube, domain, https_port, tls, Duration::from_secs(10))?; } if let Some(address) = installed.data.get("ingressDnsAddress") { let address: IpAddr = address @@ -2914,13 +3089,15 @@ metadata: {name: lampyr-grant-homes, namespace: default} json!({ "data":{ "tls.crt":STANDARD.encode(certified.cert.pem()), - "tls.key":STANDARD.encode(key) + "tls.key":STANDARD.encode(key), + "ca.crt":STANDARD.encode(certified.cert.pem()) } }) }; - validate_tls_secret( + validate_tls_secret_with_roots( &secret(certified.signing_key.serialize_pem()), "apps.example.com", + rustls::RootCertStore::empty(), ) .unwrap(); let local = rcgen::generate_simple_self_signed(vec![ @@ -2928,12 +3105,14 @@ metadata: {name: lampyr-grant-homes, namespace: default} "*.default.localhost".into(), ]) .unwrap(); - validate_tls_secret( + validate_tls_secret_with_roots( &json!({"data":{ "tls.crt":STANDARD.encode(local.cert.pem()), - "tls.key":STANDARD.encode(local.signing_key.serialize_pem()) + "tls.key":STANDARD.encode(local.signing_key.serialize_pem()), + "ca.crt":STANDARD.encode(local.cert.pem()) }}), "localhost", + rustls::RootCertStore::empty(), ) .unwrap(); let incomplete = @@ -2941,21 +3120,27 @@ metadata: {name: lampyr-grant-homes, namespace: default} let incomplete_secret = json!({ "data":{ "tls.crt":STANDARD.encode(incomplete.cert.pem()), - "tls.key":STANDARD.encode(incomplete.signing_key.serialize_pem()) + "tls.key":STANDARD.encode(incomplete.signing_key.serialize_pem()), + "ca.crt":STANDARD.encode(incomplete.cert.pem()) } }); - let error = validate_tls_secret(&incomplete_secret, "apps.example.com") - .unwrap_err() - .to_string(); + let error = validate_tls_secret_with_roots( + &incomplete_secret, + "apps.example.com", + rustls::RootCertStore::empty(), + ) + .unwrap_err() + .to_string(); assert!(error.contains("does not cover"), "{error}"); assert!( error.contains("lampyr-readiness.apps.example.com"), "{error}" ); assert!( - validate_tls_secret( + validate_tls_secret_with_roots( &secret(certified.signing_key.serialize_pem()), - "other.example.com" + "other.example.com", + rustls::RootCertStore::empty(), ) .unwrap_err() .to_string() @@ -2963,10 +3148,14 @@ metadata: {name: lampyr-grant-homes, namespace: default} ); let other = rcgen::KeyPair::generate().unwrap(); assert!( - validate_tls_secret(&secret(other.serialize_pem()), "apps.example.com") - .unwrap_err() - .to_string() - .contains("does not match") + validate_tls_secret_with_roots( + &secret(other.serialize_pem()), + "apps.example.com", + rustls::RootCertStore::empty(), + ) + .unwrap_err() + .to_string() + .contains("does not match") ); } @@ -2987,20 +3176,137 @@ metadata: {name: lampyr-grant-homes, namespace: default} } json!({"data":data}) }; - let error = validate_tls_secret(&secret(None), "apps.example.com") + let unrelated = + rcgen::generate_simple_self_signed(vec!["unrelated.example.com".into()]).unwrap(); + let roots = || { + let mut roots = rustls::RootCertStore::empty(); + roots.add(unrelated.cert.der().clone()).unwrap(); + roots + }; + let error = validate_tls_secret_with_roots(&secret(None), "apps.example.com", roots()) .unwrap_err() .to_string(); assert!(error.contains("untrusted"), "{error}"); - validate_tls_secret(&secret(Some(&certified.cert.pem())), "apps.example.com").unwrap(); + let error = validate_tls_secret_with_roots(&secret(Some("")), "apps.example.com", roots()) + .unwrap_err() + .to_string(); + assert!(error.contains("untrusted"), "{error}"); + validate_tls_secret_with_roots( + &secret(Some(&certified.cert.pem())), + "apps.example.com", + roots(), + ) + .unwrap(); } #[test] - fn credential_reconciliation_persists_retiring_configuration() { - let production = include_str!("bootstrap.rs") - .split("#[cfg(test)]") - .next() + fn existing_tls_secret_uses_intermediates_but_not_a_presented_root() { + let mut root_params = rcgen::CertificateParams::new(Vec::::new()).unwrap(); + root_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + let root_key = rcgen::KeyPair::generate().unwrap(); + let root = root_params.self_signed(&root_key).unwrap(); + let root_issuer = rcgen::Issuer::new(root_params, root_key); + + let mut intermediate_params = rcgen::CertificateParams::new(Vec::::new()).unwrap(); + intermediate_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + let intermediate_key = rcgen::KeyPair::generate().unwrap(); + let intermediate = intermediate_params + .signed_by(&intermediate_key, &root_issuer) .unwrap(); - assert!(production.contains("ingressRetiringCredential")); + let intermediate_issuer = rcgen::Issuer::new(intermediate_params, intermediate_key); + + let mut leaf_params = rcgen::CertificateParams::new(vec![ + "*.apps.example.com".into(), + "*.default.apps.example.com".into(), + ]) + .unwrap(); + leaf_params.extended_key_usages = vec![rcgen::ExtendedKeyUsagePurpose::ServerAuth]; + let leaf_key = rcgen::KeyPair::generate().unwrap(); + let leaf = leaf_params + .signed_by(&leaf_key, &intermediate_issuer) + .unwrap(); + let secret = |include_ca: bool, present_root: bool| { + let mut chain = format!("{}{}", leaf.pem(), intermediate.pem()); + if present_root { + chain.push_str(&root.pem()); + } + let mut data = json!({ + "tls.crt":STANDARD.encode(chain), + "tls.key":STANDARD.encode(leaf_key.serialize_pem()) + }); + if include_ca { + data["ca.crt"] = STANDARD.encode(root.pem()).into(); + } + json!({"data":data}) + }; + validate_tls_secret_with_roots( + &secret(true, false), + "apps.example.com", + rustls::RootCertStore::empty(), + ) + .unwrap(); + let unrelated = + rcgen::generate_simple_self_signed(vec!["unrelated.example.com".into()]).unwrap(); + let mut unrelated_roots = rustls::RootCertStore::empty(); + unrelated_roots.add(unrelated.cert.der().clone()).unwrap(); + let error = validate_tls_secret_with_roots( + &secret(false, true), + "apps.example.com", + unrelated_roots, + ) + .unwrap_err() + .to_string(); + assert!(error.contains("untrusted"), "{error}"); + } + + #[test] + fn credential_reconciliation_persists_retiring_configuration() { + let credential = |identity: &str| crate::platform_host::CertManagerCredential { + service_account_patch: json!({"metadata":{"annotations":{"provider.example/id":identity}}}), + service_account_cleanup_patch: json!({"metadata":{"annotations":{"provider.example/id":Value::Null}}}), + deployment_patch: json!({"spec":{"template":{"metadata":{"annotations":{"provider.example/id":identity}}}}}), + deployment_cleanup_patch: json!({"spec":{"template":{"metadata":{"annotations":{"provider.example/id":Value::Null}}}}}), + controller_args: vec!["--provider-ambient=true".into()], + controller_arg_prefixes: vec!["--provider-ambient".into()], + boundary: json!({"apiVersion":"v1","kind":"List","items":[]}), + boundary_message: "provider boundary".into(), + probes: vec![], + audits: vec![], + }; + let first = credential("first"); + let older = credential("older"); + let second = credential("second"); + let installed = InstalledCertManagerCredentials { + current: Some(first.clone()), + retiring: vec![older.clone()], + }; + assert_eq!( + installed.previous_for(Some(&second)), + vec![first.clone(), older.clone()] + ); + let interrupted = InstalledCertManagerCredentials { + current: Some(second.clone()), + retiring: vec![first.clone(), older.clone()], + }; + assert_eq!( + interrupted.previous_for(Some(&second)), + vec![first.clone(), older.clone()] + ); + assert_eq!( + interrupted.previous_for(None), + vec![second.clone(), first.clone(), older.clone()] + ); + let steady = InstalledCertManagerCredentials { + current: Some(second.clone()), + retiring: vec![], + }; + assert_eq!(steady.previous_for(None), vec![second.clone()]); + let persisted = + cert_manager_credential_state_patch(Some(&second), &[first.clone(), older.clone()]) + .unwrap(); + let decoded = decode_installed_cert_manager_credentials(&persisted).unwrap(); + assert_eq!(decoded.current, Some(second)); + assert_eq!(decoded.retiring, vec![first, older]); } #[test] @@ -3075,7 +3381,7 @@ metadata: {name: lampyr-grant-homes, namespace: default} audits: vec![], }; let first = credential("first"); - let patch = cert_manager_credential_patch(&deployment, Some(&first), None).unwrap(); + let patch = cert_manager_credential_patch(&deployment, Some(&first), &[]).unwrap(); assert_eq!( patch.pointer("/spec/template/spec/containers/0/args"), Some(&json!(["--v=2", "--provider-ambient=true"])) @@ -3086,12 +3392,14 @@ metadata: {name: lampyr-grant-homes, namespace: default} ); let second = credential("second"); let changed = - cert_manager_credential_patch(&deployment, Some(&second), Some(&first)).unwrap(); + cert_manager_credential_patch(&deployment, Some(&second), std::slice::from_ref(&first)) + .unwrap(); assert_ne!( patch.pointer("/spec/template/metadata/annotations"), changed.pointer("/spec/template/metadata/annotations") ); - let disabled = cert_manager_credential_patch(&patch, None, Some(&second)).unwrap(); + let disabled = + cert_manager_credential_patch(&patch, None, std::slice::from_ref(&second)).unwrap(); assert_eq!( disabled.pointer("/spec/template/spec/containers/0/args"), Some(&json!(["--v=2"])) diff --git a/cli/src/cluster.rs b/cli/src/cluster.rs index 7a34a28..d0a6987 100644 --- a/cli/src/cluster.rs +++ b/cli/src/cluster.rs @@ -842,14 +842,22 @@ fn validate_provider_manifest_capability_for_plan( plan: &RuntimePlan, declared: bool, ) -> Result<()> { - let has_manifests = - !plan.before.is_empty() || !plan.after.is_empty() || !plan.cleanup.is_empty(); + let has_manifests = !plan.before.is_empty() + || !plan.after.is_empty() + || !plan.cleanup.is_empty() + || embedded_platform_mutations(plan).is_some(); if !has_manifests { return Ok(()); } validate_provider_manifest_capability(declared) } +fn embedded_platform_mutations(plan: &RuntimePlan) -> Option<&Value> { + plan.options + .get("ingress") + .and_then(|ingress| ingress.pointer("/tls/credential")) +} + fn validate_provider_manifest_capability(declared: bool) -> Result<()> { ensure!( declared, @@ -872,18 +880,33 @@ fn approve_install_manifests_at( plan: &RuntimePlan, approval: ManifestApproval, ) -> Result<()> { - if plan.before.is_empty() && plan.after.is_empty() { + let platform = embedded_platform_mutations(plan); + if plan.before.is_empty() && plan.after.is_empty() && platform.is_none() { return Ok(()); } let before = directory.join("before.yaml"); let after = directory.join("after.yaml"); write_manifest_file(&before, &plan.before)?; write_manifest_file(&after, &plan.after)?; + let platform_path = directory.join("platform-mutations.yaml"); + if let Some(platform) = platform { + write_yaml_file(&platform_path, platform)?; + } else if platform_path.try_exists()? { + std::fs::remove_file(&platform_path)?; + } let cleanup = directory.join("cleanup.yaml"); if cleanup.try_exists()? { std::fs::remove_file(cleanup)?; } - show_install_manifests(plan, &before, &after); + show_install_manifests( + plan, + &before, + &after, + platform.map(|_| platform_path.as_path()), + ); + if let Some(platform) = platform { + show_platform_mutations(platform); + } require_manifest_approval(approval, "Apply these provider Kubernetes manifests?") } @@ -903,12 +926,26 @@ fn approve_cleanup_manifests_at( delete_cluster: bool, approval: ManifestApproval, ) -> Result<()> { - if delete_cluster || plan.cleanup.is_empty() { + let platform = embedded_platform_mutations(plan); + if delete_cluster || (plan.cleanup.is_empty() && platform.is_none()) { return Ok(()); } let cleanup = directory.join("cleanup.yaml"); write_manifest_file(&cleanup, &plan.cleanup)?; - show_cleanup_manifests(&plan.cleanup, &cleanup); + let platform_path = directory.join("platform-mutations.yaml"); + if let Some(platform) = platform { + write_yaml_file(&platform_path, platform)?; + } else if platform_path.try_exists()? { + std::fs::remove_file(&platform_path)?; + } + show_cleanup_manifests( + &plan.cleanup, + &cleanup, + platform.map(|_| platform_path.as_path()), + ); + if let Some(platform) = platform { + show_platform_mutations(platform); + } require_manifest_approval(approval, "Apply the provider cleanup manifests?") } @@ -934,7 +971,12 @@ fn approve_providerless_uninstall_manifests( plan: &RuntimePlan, approval: ManifestApproval, ) -> Result<()> { - if plan.before.is_empty() && plan.after.is_empty() && plan.cleanup.is_empty() { + let platform = embedded_platform_mutations(plan); + if plan.before.is_empty() + && plan.after.is_empty() + && plan.cleanup.is_empty() + && platform.is_none() + { return Ok(()); } let before = directory.join("before.yaml"); @@ -943,6 +985,10 @@ fn approve_providerless_uninstall_manifests( write_manifest_file(&before, &plan.before)?; write_manifest_file(&after, &plan.after)?; write_manifest_file(&cleanup, &plan.cleanup)?; + let platform_path = directory.join("platform-mutations.yaml"); + if let Some(platform) = platform { + write_yaml_file(&platform_path, platform)?; + } eprintln!(); eprintln!("{}", style("Provider Kubernetes resource removal", "1")); @@ -955,11 +1001,20 @@ fn approve_providerless_uninstall_manifests( "36;1", &plan.before, ); + if let Some(platform) = platform { + show_platform_mutations(platform); + } eprintln!(); eprintln!("Files written locally:"); eprintln!(" BEFORE {}", style(&before.display().to_string(), "2")); eprintln!(" AFTER {}", style(&after.display().to_string(), "2")); eprintln!(" CLEANUP {}", style(&cleanup.display().to_string(), "2")); + if platform.is_some() { + eprintln!( + " PLATFORM {}", + style(&platform_path.display().to_string(), "2") + ); + } eprintln!("Inspect these standard YAML files before continuing."); require_manifest_approval( approval, @@ -986,7 +1041,26 @@ fn write_manifest_file(path: &Path, manifests: &[Value]) -> Result<()> { Ok(()) } -fn show_install_manifests(plan: &RuntimePlan, before: &Path, after: &Path) { +fn write_yaml_file(path: &Path, value: &Value) -> Result<()> { + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600).custom_flags(libc::O_NOFOLLOW); + } + let mut file = options.open(path)?; + file.write_all(serde_yaml_ng::to_string(value)?.as_bytes())?; + file.sync_all()?; + Ok(()) +} + +fn show_install_manifests( + plan: &RuntimePlan, + before: &Path, + after: &Path, + platform: Option<&Path>, +) { eprintln!(); eprintln!("{}", style("Provider Kubernetes manifests", "1")); show_manifest_phase("BEFORE LAMPYR PLATFORM", "36;1", &plan.before); @@ -995,21 +1069,58 @@ fn show_install_manifests(plan: &RuntimePlan, before: &Path, after: &Path) { eprintln!("Files written locally:"); eprintln!(" BEFORE {}", style(&before.display().to_string(), "2")); eprintln!(" AFTER {}", style(&after.display().to_string(), "2")); + if let Some(platform) = platform { + eprintln!(" PLATFORM {}", style(&platform.display().to_string(), "2")); + } eprintln!("Inspect these standard YAML files or apply them manually in the phase order shown."); } -fn show_cleanup_manifests(manifests: &[Value], cleanup: &Path) { +fn show_cleanup_manifests(manifests: &[Value], cleanup: &Path, platform: Option<&Path>) { eprintln!(); eprintln!("{}", style("Provider cleanup manifests", "1")); show_manifest_phase("CLEANUP", "35;1", manifests); eprintln!(); eprintln!("File written locally:"); eprintln!(" CLEANUP {}", style(&cleanup.display().to_string(), "2")); + if let Some(platform) = platform { + eprintln!(" PLATFORM {}", style(&platform.display().to_string(), "2")); + } eprintln!( "Inspect this standard YAML file or apply it manually during retained-cluster cleanup." ); } +fn show_platform_mutations(credential: &Value) { + let boundary = credential["boundary"]["items"] + .as_array() + .map(Vec::as_slice) + .unwrap_or_default(); + show_manifest_phase("PLATFORM CREDENTIAL BOUNDARY", "33;1", boundary); + eprintln!(); + eprintln!("{}", style("Platform credential mutations", "1")); + eprintln!( + " ServiceAccount cert-manager/cert-manager patch: {}", + credential["serviceAccountPatch"] + ); + eprintln!( + " ServiceAccount cert-manager/cert-manager cleanup: {}", + credential["serviceAccountCleanupPatch"] + ); + eprintln!( + " Deployment cert-manager/cert-manager patch: {}", + credential["deploymentPatch"] + ); + eprintln!( + " Deployment cert-manager/cert-manager cleanup: {}", + credential["deploymentCleanupPatch"] + ); + eprintln!(" Controller args: {}", credential["controllerArgs"]); + eprintln!( + " Controller arg prefixes removed: {}", + credential["controllerArgPrefixes"] + ); +} + fn show_manifest_phase(label: &str, color: &str, manifests: &[Value]) { eprintln!(); eprintln!( @@ -1680,6 +1791,13 @@ mod tests { .to_string() .contains("--approve-provider-manifests") ); + let mutations = + std::fs::read_to_string(directory.path().join("platform-mutations.yaml")).unwrap(); + assert!(mutations.contains("serviceAccountPatch"), "{mutations}"); + assert!( + mutations.contains("ValidatingAdmissionPolicy"), + "{mutations}" + ); } #[test] diff --git a/cli/src/platform_host.rs b/cli/src/platform_host.rs index 4d2a56d..171f608 100644 --- a/cli/src/platform_host.rs +++ b/cli/src/platform_host.rs @@ -217,7 +217,7 @@ enum IngressTlsInput { }, } -#[derive(Clone, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub(crate) struct CertManagerCredential { pub service_account_patch: Value, @@ -236,7 +236,7 @@ pub(crate) struct CertManagerCredential { pub audits: Vec, } -#[derive(Clone, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub(crate) struct CredentialAudit { pub resource: String, @@ -297,6 +297,14 @@ impl PlatformSettings { "cluster", "provider", crate::bootstrap::CORE_INVENTORY_KEY, + "ingressTlsMode", + "ingressTlsSecretNamespace", + "ingressTlsSecretName", + "ingressTlsIssuer", + "ingressDnsAddress", + "ingressCredential", + "ingressRetiringCredential", + "ingressServiceAnnotations", ] { ensure!( !self.metadata.contains_key(reserved), @@ -1695,6 +1703,7 @@ pub(crate) mod tests { "ingressTlsIssuer", "ingressDnsAddress", "ingressCredential", + "ingressRetiringCredential", "ingressServiceAnnotations", ] { let settings = json!({ From 8dbb61bfeb437a4e55183be87c765d3699579d05 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 20 Nov 2026 18:03:10 -0800 Subject: [PATCH 4/7] fixup! Add configurable ingress TLS modes Signed-off-by: James Sturtevant --- cli/src/cluster.rs | 86 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 2 deletions(-) diff --git a/cli/src/cluster.rs b/cli/src/cluster.rs index d0a6987..48a1d32 100644 --- a/cli/src/cluster.rs +++ b/cli/src/cluster.rs @@ -470,11 +470,12 @@ fn existing_lifecycle( .settings .as_object_mut() .context("saved platform settings must be a mapping")?; - settings.extend(config.as_object().unwrap().clone()); + merge_requested_settings(settings, config.as_object().unwrap()); apply_existing_cluster_defaults(settings, data)?; let defaulted = fill_controller_defaults(settings, &controller_defaults(root)?); crate::platform_host::validate_settings(&record.settings)?; let install_settings = record.settings.clone(); + record.plan = effective_runtime_plan(&record.plan, &install_settings); record.settings = without_fields(&record.settings, &defaulted); record.eventing |= requested_eventing == Some(json!(true)); let provider = data @@ -703,7 +704,14 @@ fn lifecycle( } for field in PLATFORM_INPUTS { if let Some(value) = config.get(field) { - settings.insert(field.into(), value.clone()); + if field == "ingress" { + merge_requested_ingress( + settings.entry(field).or_insert_with(|| json!({})), + value, + ); + } else { + settings.insert(field.into(), value.clone()); + } } } settings.entry("httpsPort").or_insert(json!(443)); @@ -838,6 +846,30 @@ fn lifecycle( Ok(json!({"name":name,"uninstalled":true,"clusterRetained":!delete})) } +fn merge_requested_settings( + settings: &mut serde_json::Map, + requested: &serde_json::Map, +) { + for (field, value) in requested { + if field == "ingress" { + merge_requested_ingress( + settings.entry(field.clone()).or_insert_with(|| json!({})), + value, + ); + } else { + settings.insert(field.clone(), value.clone()); + } + } +} + +fn merge_requested_ingress(generated: &mut Value, requested: &Value) { + *generated = requested.clone(); +} + +fn effective_runtime_plan(plan: &RuntimePlan, _settings: &Value) -> RuntimePlan { + plan.clone() +} + fn validate_provider_manifest_capability_for_plan( plan: &RuntimePlan, declared: bool, @@ -1800,6 +1832,56 @@ mod tests { ); } + #[test] + fn requested_ingress_preserves_provider_generated_runtime_fields() { + let mut generated = json!({ + "tls":{ + "mode":"acme", + "email":"old@example.com", + "issuer":"letsencrypt-prod", + "solver":{"dns01":{"providerDNS":{"zone":"example.com"}}}, + "credential":{"serviceAccountPatch":{"metadata":{"annotations":{"provider.example/id":"identity"}}}} + }, + "service":{ + "publicIpAddress":"192.0.2.1", + "annotations":{"service.example/address":"edge"} + } + }); + merge_requested_ingress( + &mut generated, + &json!({"tls":{ + "mode":"acme", + "email":"new@example.com", + "issuer":"letsencrypt-staging" + }}), + ); + assert_eq!(generated["tls"]["email"], "new@example.com"); + assert_eq!(generated["tls"]["issuer"], "letsencrypt-staging"); + assert!(generated["tls"].get("solver").is_some()); + assert!(generated["tls"].get("credential").is_some()); + assert!(generated.get("service").is_some()); + } + + #[test] + fn providerless_approval_plan_uses_effective_ingress_settings() { + let mut plan = RuntimePlan::default(); + plan.options.insert( + "ingress".into(), + json!({"tls":{"mode":"acme","credential":{"boundaryMessage":"old"}}}), + ); + let settings = + json!({"ingress":{"tls":{"mode":"acme","credential":{"boundaryMessage":"new"}}}}); + let effective = effective_runtime_plan(&plan, &settings); + assert_eq!( + effective + .options + .get("ingress") + .unwrap() + .pointer("/tls/credential/boundaryMessage"), + Some(&json!("new")) + ); + } + #[test] fn selected_context_must_match_local_installation_and_cluster_identity() { let selection = Selection { From e0a3fa0566ba9bb063018e3bb2431c421af1835f Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 20 Nov 2026 18:10:33 -0800 Subject: [PATCH 5/7] fixup! Add configurable ingress TLS modes Signed-off-by: James Sturtevant --- cli/src/cluster.rs | 48 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/cli/src/cluster.rs b/cli/src/cluster.rs index 48a1d32..6fe5b2a 100644 --- a/cli/src/cluster.rs +++ b/cli/src/cluster.rs @@ -717,6 +717,7 @@ fn lifecycle( settings.entry("httpsPort").or_insert(json!(443)); let settings = Value::Object(settings); crate::platform_host::validate_settings(&settings)?; + let plan = effective_runtime_plan(&plan, &settings); plan.validate()?; validate_provider_manifest_capability_for_plan(&plan, provider_manifests)?; if let Err(approval_error) = approve_install_manifests(installation, &plan, manifests) { @@ -863,11 +864,39 @@ fn merge_requested_settings( } fn merge_requested_ingress(generated: &mut Value, requested: &Value) { - *generated = requested.clone(); + let (Some(generated), Some(requested)) = (generated.as_object_mut(), requested.as_object()) + else { + *generated = requested.clone(); + return; + }; + for (field, value) in requested { + let target = generated.entry(field.clone()).or_insert(Value::Null); + let mode_changed = + field == "tls" && target["mode"].as_str().is_some() && target["mode"] != value["mode"]; + if mode_changed { + *target = value.clone(); + } else { + merge_requested_value(target, value); + } + } +} + +fn merge_requested_value(generated: &mut Value, requested: &Value) { + if let (Some(generated), Some(requested)) = (generated.as_object_mut(), requested.as_object()) { + for (field, value) in requested { + merge_requested_value(generated.entry(field.clone()).or_insert(Value::Null), value); + } + } else { + *generated = requested.clone(); + } } -fn effective_runtime_plan(plan: &RuntimePlan, _settings: &Value) -> RuntimePlan { - plan.clone() +fn effective_runtime_plan(plan: &RuntimePlan, settings: &Value) -> RuntimePlan { + let mut effective = plan.clone(); + if let Some(ingress) = settings.get("ingress") { + effective.options.insert("ingress".into(), ingress.clone()); + } + effective } fn validate_provider_manifest_capability_for_plan( @@ -1860,6 +1889,19 @@ mod tests { assert!(generated["tls"].get("solver").is_some()); assert!(generated["tls"].get("credential").is_some()); assert!(generated.get("service").is_some()); + + merge_requested_ingress( + &mut generated, + &json!({"tls":{ + "mode":"existingSecret", + "secretName":"edge-tls", + "secretNamespace":"security" + }}), + ); + assert_eq!(generated["tls"]["mode"], "existingSecret"); + assert!(generated["tls"].get("solver").is_none()); + assert!(generated["tls"].get("credential").is_none()); + assert!(generated.get("service").is_some()); } #[test] From 7688c5e7e433da8f3289df50e7bf382550f8bfe7 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 20 Nov 2026 18:26:30 -0800 Subject: [PATCH 6/7] fixup! Add configurable ingress TLS modes Signed-off-by: James Sturtevant --- cli/src/cluster.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/cli/src/cluster.rs b/cli/src/cluster.rs index 6fe5b2a..f0fbb00 100644 --- a/cli/src/cluster.rs +++ b/cli/src/cluster.rs @@ -899,6 +899,11 @@ fn effective_runtime_plan(plan: &RuntimePlan, settings: &Value) -> RuntimePlan { effective } +#[cfg(test)] +fn credential_transition_mutations(effective: Option<&Value>, _installed: &[Value]) -> Vec { + effective.cloned().into_iter().collect() +} + fn validate_provider_manifest_capability_for_plan( plan: &RuntimePlan, declared: bool, @@ -1924,6 +1929,24 @@ mod tests { ); } + #[test] + fn credential_transition_approval_includes_effective_installed_and_retiring_state() { + let effective = json!({"boundaryMessage":"effective"}); + let installed = json!({"boundaryMessage":"installed"}); + let retiring = json!({"boundaryMessage":"retiring"}); + assert_eq!( + credential_transition_mutations( + Some(&effective), + &[installed.clone(), retiring.clone()] + ), + vec![effective, installed.clone(), retiring.clone()] + ); + assert_eq!( + credential_transition_mutations(None, &[installed.clone(), retiring.clone()]), + vec![installed, retiring] + ); + } + #[test] fn selected_context_must_match_local_installation_and_cluster_identity() { let selection = Selection { From 094bb1f7552edef7928802f106c36e084f556bc4 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 20 Nov 2026 18:35:30 -0800 Subject: [PATCH 7/7] fixup! Add configurable ingress TLS modes Signed-off-by: James Sturtevant --- cli/src/bootstrap.rs | 9 +++ cli/src/cluster.rs | 136 ++++++++++++++++++++++++++++++++------- cli/src/platform_host.rs | 11 ++++ 3 files changed, 131 insertions(+), 25 deletions(-) diff --git a/cli/src/bootstrap.rs b/cli/src/bootstrap.rs index bfc23f9..3ec298e 100644 --- a/cli/src/bootstrap.rs +++ b/cli/src/bootstrap.rs @@ -1563,6 +1563,15 @@ fn decode_installed_cert_manager_credentials( Ok(InstalledCertManagerCredentials { current, retiring }) } +pub(crate) fn installed_cert_manager_credential_mutations(kube: &Kube<'_>) -> Result> { + installed_cert_manager_credentials(kube)? + .previous_for(None) + .iter() + .map(serde_json::to_value) + .collect::>() + .context("encode installed cert-manager credential mutations") +} + fn cert_manager_controller_args( deployment: &Value, current: Option<&crate::platform_host::CertManagerCredential>, diff --git a/cli/src/cluster.rs b/cli/src/cluster.rs index f0fbb00..866578e 100644 --- a/cli/src/cluster.rs +++ b/cli/src/cluster.rs @@ -161,6 +161,8 @@ pub fn run(operation: Operation, options: Options<'_>, host: &Host) -> Result<() struct RuntimePlan { #[serde(default)] options: serde_json::Map, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + platform_mutations: Vec, #[serde(default)] before: Vec, #[serde(default)] @@ -171,6 +173,9 @@ struct RuntimePlan { impl RuntimePlan { fn validate(&self) -> Result<()> { + for mutation in &self.platform_mutations { + crate::platform_host::validate_cert_manager_credential(mutation)?; + } for manifest in self.before.iter().chain(&self.after).chain(&self.cleanup) { validate_manifest(manifest)?; } @@ -475,7 +480,7 @@ fn existing_lifecycle( let defaulted = fill_controller_defaults(settings, &controller_defaults(root)?); crate::platform_host::validate_settings(&record.settings)?; let install_settings = record.settings.clone(); - record.plan = effective_runtime_plan(&record.plan, &install_settings); + record.plan = approval_runtime_plan(&kube, &record.plan, &install_settings)?; record.settings = without_fields(&record.settings, &defaulted); record.eventing |= requested_eventing == Some(json!(true)); let provider = data @@ -698,6 +703,10 @@ fn lifecycle( .unwrap_or(json!({})), ) .context("decode provider-emitted runtime manifests")?; + ensure!( + plan.platform_mutations.is_empty(), + "provider runtime plans cannot predeclare installed platform mutations" + ); let mut settings = plan.options.clone(); for field in ["domain", "fiberdImage", "routerImage"] { settings.insert(field.into(), config[field].clone()); @@ -717,7 +726,7 @@ fn lifecycle( settings.entry("httpsPort").or_insert(json!(443)); let settings = Value::Object(settings); crate::platform_host::validate_settings(&settings)?; - let plan = effective_runtime_plan(&plan, &settings); + let plan = approval_runtime_plan(&kube, &plan, &settings)?; plan.validate()?; validate_provider_manifest_capability_for_plan(&plan, provider_manifests)?; if let Err(approval_error) = approve_install_manifests(installation, &plan, manifests) { @@ -899,9 +908,34 @@ fn effective_runtime_plan(plan: &RuntimePlan, settings: &Value) -> RuntimePlan { effective } -#[cfg(test)] -fn credential_transition_mutations(effective: Option<&Value>, _installed: &[Value]) -> Vec { - effective.cloned().into_iter().collect() +fn credential_transition_mutations(effective: Option<&Value>, installed: &[Value]) -> Vec { + let mut mutations = Vec::new(); + for mutation in effective.into_iter().chain(installed) { + if !mutations.contains(mutation) { + mutations.push(mutation.clone()); + } + } + mutations +} + +fn approval_runtime_plan( + kube: &Kube<'_>, + plan: &RuntimePlan, + settings: &Value, +) -> Result { + let mut effective = effective_runtime_plan(plan, settings); + let current = effective_credential_mutation(&effective)?; + let installed = crate::bootstrap::installed_cert_manager_credential_mutations(kube)?; + effective.platform_mutations = credential_transition_mutations(current.as_ref(), &installed); + Ok(effective) +} + +fn effective_credential_mutation(plan: &RuntimePlan) -> Result> { + plan.options + .get("ingress") + .and_then(|ingress| ingress.pointer("/tls/credential")) + .map(crate::platform_host::canonical_cert_manager_credential) + .transpose() } fn validate_provider_manifest_capability_for_plan( @@ -911,17 +945,22 @@ fn validate_provider_manifest_capability_for_plan( let has_manifests = !plan.before.is_empty() || !plan.after.is_empty() || !plan.cleanup.is_empty() - || embedded_platform_mutations(plan).is_some(); + || !embedded_platform_mutations(plan).is_empty(); if !has_manifests { return Ok(()); } validate_provider_manifest_capability(declared) } -fn embedded_platform_mutations(plan: &RuntimePlan) -> Option<&Value> { +fn embedded_platform_mutations(plan: &RuntimePlan) -> Vec<&Value> { + if !plan.platform_mutations.is_empty() { + return plan.platform_mutations.iter().collect(); + } plan.options .get("ingress") .and_then(|ingress| ingress.pointer("/tls/credential")) + .into_iter() + .collect() } fn validate_provider_manifest_capability(declared: bool) -> Result<()> { @@ -947,7 +986,7 @@ fn approve_install_manifests_at( approval: ManifestApproval, ) -> Result<()> { let platform = embedded_platform_mutations(plan); - if plan.before.is_empty() && plan.after.is_empty() && platform.is_none() { + if plan.before.is_empty() && plan.after.is_empty() && platform.is_empty() { return Ok(()); } let before = directory.join("before.yaml"); @@ -955,8 +994,16 @@ fn approve_install_manifests_at( write_manifest_file(&before, &plan.before)?; write_manifest_file(&after, &plan.after)?; let platform_path = directory.join("platform-mutations.yaml"); - if let Some(platform) = platform { - write_yaml_file(&platform_path, platform)?; + if !platform.is_empty() { + write_yaml_file( + &platform_path, + &Value::Array( + platform + .iter() + .map(|mutation| (*mutation).clone()) + .collect(), + ), + )?; } else if platform_path.try_exists()? { std::fs::remove_file(&platform_path)?; } @@ -968,10 +1015,10 @@ fn approve_install_manifests_at( plan, &before, &after, - platform.map(|_| platform_path.as_path()), + (!platform.is_empty()).then_some(platform_path.as_path()), ); - if let Some(platform) = platform { - show_platform_mutations(platform); + for mutation in platform { + show_platform_mutations(mutation); } require_manifest_approval(approval, "Apply these provider Kubernetes manifests?") } @@ -993,24 +1040,32 @@ fn approve_cleanup_manifests_at( approval: ManifestApproval, ) -> Result<()> { let platform = embedded_platform_mutations(plan); - if delete_cluster || (plan.cleanup.is_empty() && platform.is_none()) { + if delete_cluster || (plan.cleanup.is_empty() && platform.is_empty()) { return Ok(()); } let cleanup = directory.join("cleanup.yaml"); write_manifest_file(&cleanup, &plan.cleanup)?; let platform_path = directory.join("platform-mutations.yaml"); - if let Some(platform) = platform { - write_yaml_file(&platform_path, platform)?; + if !platform.is_empty() { + write_yaml_file( + &platform_path, + &Value::Array( + platform + .iter() + .map(|mutation| (*mutation).clone()) + .collect(), + ), + )?; } else if platform_path.try_exists()? { std::fs::remove_file(&platform_path)?; } show_cleanup_manifests( &plan.cleanup, &cleanup, - platform.map(|_| platform_path.as_path()), + (!platform.is_empty()).then_some(platform_path.as_path()), ); - if let Some(platform) = platform { - show_platform_mutations(platform); + for mutation in platform { + show_platform_mutations(mutation); } require_manifest_approval(approval, "Apply the provider cleanup manifests?") } @@ -1041,7 +1096,7 @@ fn approve_providerless_uninstall_manifests( if plan.before.is_empty() && plan.after.is_empty() && plan.cleanup.is_empty() - && platform.is_none() + && platform.is_empty() { return Ok(()); } @@ -1052,8 +1107,16 @@ fn approve_providerless_uninstall_manifests( write_manifest_file(&after, &plan.after)?; write_manifest_file(&cleanup, &plan.cleanup)?; let platform_path = directory.join("platform-mutations.yaml"); - if let Some(platform) = platform { - write_yaml_file(&platform_path, platform)?; + if !platform.is_empty() { + write_yaml_file( + &platform_path, + &Value::Array( + platform + .iter() + .map(|mutation| (*mutation).clone()) + .collect(), + ), + )?; } eprintln!(); @@ -1067,15 +1130,15 @@ fn approve_providerless_uninstall_manifests( "36;1", &plan.before, ); - if let Some(platform) = platform { - show_platform_mutations(platform); + for mutation in &platform { + show_platform_mutations(mutation); } eprintln!(); eprintln!("Files written locally:"); eprintln!(" BEFORE {}", style(&before.display().to_string(), "2")); eprintln!(" AFTER {}", style(&after.display().to_string(), "2")); eprintln!(" CLEANUP {}", style(&cleanup.display().to_string(), "2")); - if platform.is_some() { + if !platform.is_empty() { eprintln!( " PLATFORM {}", style(&platform_path.display().to_string(), "2") @@ -1945,6 +2008,29 @@ mod tests { credential_transition_mutations(None, &[installed.clone(), retiring.clone()]), vec![installed, retiring] ); + + let requested = json!({ + "serviceAccountPatch":{"metadata":{"annotations":{"provider.example/id":"identity"}}}, + "serviceAccountCleanupPatch":{"metadata":{"annotations":{"provider.example/id":null}}}, + "deploymentPatch":{"spec":{"template":{"metadata":{"annotations":{"provider.example/id":"identity"}}}}}, + "deploymentCleanupPatch":{"spec":{"template":{"metadata":{"annotations":{"provider.example/id":null}}}}}, + "controllerArgs":["--issuer-ambient-credentials=true"], + "controllerArgPrefixes":["--issuer-ambient-credentials"], + "boundary":{"apiVersion":"v1","kind":"List","items":[]}, + "boundaryMessage":"provider boundary" + }); + let mut plan = RuntimePlan::default(); + plan.options.insert( + "ingress".into(), + json!({"tls":{"mode":"acme","credential":requested.clone()}}), + ); + let effective = effective_credential_mutation(&plan).unwrap().unwrap(); + let installed = + crate::platform_host::canonical_cert_manager_credential(&requested).unwrap(); + assert_eq!( + credential_transition_mutations(Some(&effective), &[installed]), + vec![effective] + ); } #[test] diff --git a/cli/src/platform_host.rs b/cli/src/platform_host.rs index 171f608..e02c894 100644 --- a/cli/src/platform_host.rs +++ b/cli/src/platform_host.rs @@ -472,6 +472,17 @@ impl CertManagerCredential { } } +pub(crate) fn validate_cert_manager_credential(value: &Value) -> Result<()> { + canonical_cert_manager_credential(value).map(|_| ()) +} + +pub(crate) fn canonical_cert_manager_credential(value: &Value) -> Result { + let credential: CertManagerCredential = + serde_json::from_value(value.clone()).context("decode cert-manager credential mutation")?; + credential.validate()?; + serde_json::to_value(credential).context("encode cert-manager credential mutation") +} + fn annotation_values(mapping: Option<&serde_json::Map>) -> bool { mapping.is_some_and(|mapping| { mapping