diff --git a/Makefile b/Makefile index db138368..cba4a090 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,40 @@ collect-firmware-refvals: ## Collect bare-metal firmware values; set OCP_VERSION collect-azure-refvals: ## Collect PCR reference values (Azure) @scripts/collect_firmware_refvals.py --platform azure + +.PHONY: detect-hardware +detect-hardware: ## Detect hardware profile from cluster nodes (requires KUBECONFIG or oc login) + @echo "Detecting hardware profile from cluster nodes..." + @echo "---" + @CPU_VENDOR=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.feature\.node\.kubernetes\.io/cpu-model\.vendor_id}' 2>/dev/null) && \ + TDX_ENABLED=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.feature\.node\.kubernetes\.io/cpu-security\.tdx\.enabled}' 2>/dev/null) && \ + SNP_ENABLED=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.feature\.node\.kubernetes\.io/cpu-security\.sev\.snp}' 2>/dev/null) && \ + GPU_PRESENT=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.nvidia\.com/gpu\.present}' 2>/dev/null) && \ + echo "CPU Vendor: $${CPU_VENDOR:-unknown}" && \ + echo "TDX Enabled: $${TDX_ENABLED:-false}" && \ + echo "SNP Enabled: $${SNP_ENABLED:-false}" && \ + echo "GPU Present: $${GPU_PRESENT:-false}" && \ + echo "---" && \ + if [ "$${CPU_VENDOR}" = "Intel" ] && [ "$${TDX_ENABLED}" = "true" ]; then \ + if [ "$${GPU_PRESENT}" = "true" ]; then \ + echo "Recommended profile: intel-tdx-gpu"; \ + else \ + echo "Recommended profile: intel-tdx"; \ + fi; \ + elif [ "$${CPU_VENDOR}" = "AuthenticAMD" ] || [ "$${SNP_ENABLED}" = "true" ]; then \ + if [ "$${GPU_PRESENT}" = "true" ]; then \ + echo "Recommended profile: amd-snp-gpu"; \ + else \ + echo "Recommended profile: amd-snp"; \ + fi; \ + else \ + echo "Could not determine hardware profile."; \ + echo "Ensure NFD operator is running and node labels are populated."; \ + echo "Set global.hardware.profile manually in values-global.yaml"; \ + fi && \ + echo "" && \ + echo "To apply: edit values-global.yaml and set global.hardware.profile to the recommended value." + .PHONY: collect-dcap-collateral collect-dcap-collateral: dcap-tools ## Collect platform-independent TDX DCAP verification collateral (connected low side) @scripts/collect-dcap-collateral.sh @@ -120,10 +154,6 @@ dcap-pck-import: ## Import a PCK response bundle and restart QGS (disconnected h dcap-pck-provision: dcap-tools ## Resume export, generation, and import PCK cache (bastion connected to both Intel PCS and cluster) @python3 scripts/dcap-pck.py provision -.PHONY: dcap-status -dcap-status: ## Check PCK and collateral expiry (requires oc) - @$(MAKE) check-pck-expiry - @$(MAKE) check-collateral-expiry .PHONY: check-pck-expiry check-pck-expiry: ## Check expiry of PCK cache secrets and platform data (requires oc login) @@ -150,6 +180,15 @@ snp-gen-overrides: ## Generate SNP VCEK values override from cached certs (local .PHONY: snp-offline-provision snp-offline-provision: snp-collect-vcek-urls snp-download-vcek snp-gen-overrides ## Full SNP VCEK offline provisioning +.PHONY: update-kyverno-chart +update-kyverno-chart: ## Pull and embed upstream Kyverno Helm chart (requires internet) + @echo "Pulling kyverno chart v$(KYVERNO_VERSION)..." + @rm -rf charts/vendor/kyverno + @helm pull kyverno --repo $(KYVERNO_REPO) --version $(KYVERNO_VERSION) --untar -d charts/vendor/ + @find charts/vendor/kyverno -name "README.md" -o -name "README.md.gotmpl" | xargs rm -f + @echo "Embedded charts/vendor/kyverno (v$(KYVERNO_VERSION))" + @echo "Next: commit, push, and sync ArgoCD" + ##@ Disconnected Deployment MIRROR_REGISTRY ?= quay.example.com:443/mirror IMAGESET_CONFIG ?= airgap/imageset-config.yaml @@ -166,17 +205,8 @@ airgap-mirror: ## Mirror content to disconnected registry (requires MIRROR_REGIS airgap-post-install: ## Post-install bootstrap (after airgap-mirror + labctl apply-mirror-resources) @scripts/airgap-post-install.sh -.PHONY: airgap-deploy-pattern -airgap-deploy-pattern: ## Deploy Pattern CR directly (skip pattern.sh, use values-global.yaml config) - @scripts/airgap-post-install.sh --deploy-pattern -.PHONY: airgap-fix-manifests -airgap-fix-manifests: ## Fix oc-mirror manifest list failures with fallback mirroring - @scripts/airgap-post-install.sh --fix-manifest-lists -.PHONY: airgap-sync-repos -airgap-sync-repos: ## Push working copy changes to bare HTTP repos and restart git server - @scripts/airgap-post-install.sh --sync-repos-only ARGOCD_CLI_DIR ?= $(HOME)/.local/bin @@ -201,82 +231,25 @@ argocd-install: ## Download argocd CLI from the cluster's ArgoCD image into ~/.l $(ARGOCD_CLI_DIR)/argocd version --client | head -1 .PHONY: argocd-login -argocd-login: ## Extract ArgoCD credentials from cluster and log in with argocd CLI +argocd-login: ## Extract ArgoCD credentials and log in with the argocd CLI @ARGOCD_NS=$$(oc get argocd -A -o jsonpath='{.items[0].metadata.namespace}' 2>/dev/null) && \ ARGOCD_NAME=$$(oc get argocd -n $$ARGOCD_NS -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) && \ ARGOCD_ROUTE=$$(oc get route $${ARGOCD_NAME}-server -n $$ARGOCD_NS -o jsonpath='{.spec.host}' 2>/dev/null) && \ ARGOCD_PASS=$$(oc get secret $${ARGOCD_NAME}-cluster -n $$ARGOCD_NS -o jsonpath='{.data.admin\.password}' 2>/dev/null | base64 -d) && \ echo "ArgoCD URL: https://$$ARGOCD_ROUTE" && \ echo "Username: admin" && \ - echo "Password: $$ARGOCD_PASS" && \ - echo "" && \ if command -v argocd >/dev/null 2>&1; then \ - oc config set-context --current --namespace=$$ARGOCD_NS && \ - argocd login --core && \ - echo "" && \ - echo "Logged in (core mode, namespace $$ARGOCD_NS)." && \ - echo "Usage: argocd app list"; \ + argocd login $$ARGOCD_ROUTE --username admin --password $$ARGOCD_PASS --insecure && \ + echo "Logged in. Use 'argocd app list' to get started."; \ else \ - echo "argocd CLI not installed. To install:" && \ - echo " curl -sSL -o /usr/local/bin/argocd https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64" && \ - echo " chmod +x /usr/local/bin/argocd" && \ - echo "" && \ - echo "Or use the web UI at: https://$$ARGOCD_ROUTE"; \ + echo "Password: $$ARGOCD_PASS" && \ + echo "(argocd CLI not found — install with 'make argocd-install')"; \ fi ##@ Hardware Detection -.PHONY: get-pccs-node -get-pccs-node: ## Detect a node with Intel TDX support for PCCS deployment (requires KUBECONFIG or oc login) - @NODE=$$(oc get nodes -l intel.feature.node.kubernetes.io/tdx=true \ - -o jsonpath='{.items[0].metadata.name}' 2>/dev/null); \ - if [ -z "$$NODE" ]; then \ - echo "ERROR: No TDX-capable nodes found" >&2; \ - exit 1; \ - fi; \ - echo "$$NODE" -.PHONY: detect-hardware -detect-hardware: ## Detect hardware profile from cluster nodes (requires KUBECONFIG or oc login) - @echo "Detecting hardware profile from cluster nodes..." - @echo "---" - @CPU_VENDOR=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.feature\.node\.kubernetes\.io/cpu-model\.vendor_id}' 2>/dev/null) && \ - TDX_ENABLED=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.feature\.node\.kubernetes\.io/cpu-security\.tdx\.enabled}' 2>/dev/null) && \ - SNP_ENABLED=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.feature\.node\.kubernetes\.io/cpu-security\.sev\.snp}' 2>/dev/null) && \ - GPU_PRESENT=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.nvidia\.com/gpu\.present}' 2>/dev/null) && \ - echo "CPU Vendor: $${CPU_VENDOR:-unknown}" && \ - echo "TDX Enabled: $${TDX_ENABLED:-false}" && \ - echo "SNP Enabled: $${SNP_ENABLED:-false}" && \ - echo "GPU Present: $${GPU_PRESENT:-false}" && \ - echo "---" && \ - if [ "$${CPU_VENDOR}" = "Intel" ] && [ "$${TDX_ENABLED}" = "true" ]; then \ - if [ "$${GPU_PRESENT}" = "true" ]; then \ - echo "Recommended profile: intel-tdx-gpu"; \ - else \ - echo "Recommended profile: intel-tdx"; \ - fi; \ - elif [ "$${CPU_VENDOR}" = "AuthenticAMD" ] || [ "$${SNP_ENABLED}" = "true" ]; then \ - if [ "$${GPU_PRESENT}" = "true" ]; then \ - echo "Recommended profile: amd-snp-gpu"; \ - else \ - echo "Recommended profile: amd-snp"; \ - fi; \ - else \ - echo "Could not determine hardware profile."; \ - echo "Ensure NFD operator is running and node labels are populated."; \ - echo "Set global.hardware.profile manually in values-global.yaml"; \ - fi && \ - echo "" && \ - echo "To apply: edit values-global.yaml and set global.hardware.profile to the recommended value." ##@ Chart Management KYVERNO_VERSION ?= 3.7.2 KYVERNO_REPO ?= https://kyverno.github.io/kyverno/ -.PHONY: update-kyverno-chart -update-kyverno-chart: ## Pull and embed upstream Kyverno Helm chart (requires internet) - @echo "Pulling kyverno chart v$(KYVERNO_VERSION)..." - @rm -rf charts/vendor/kyverno - @helm pull kyverno --repo $(KYVERNO_REPO) --version $(KYVERNO_VERSION) --untar -d charts/vendor/ - @find charts/vendor/kyverno -name "README.md" -o -name "README.md.gotmpl" | xargs rm -f - @echo "Embedded charts/vendor/kyverno (v$(KYVERNO_VERSION))" - @echo "Next: commit, push, and sync ArgoCD" diff --git a/airgap/DEPLOY-RUNBOOK.md b/airgap/DEPLOY-RUNBOOK.md index 4eb40e87..fb2b11e0 100644 --- a/airgap/DEPLOY-RUNBOOK.md +++ b/airgap/DEPLOY-RUNBOOK.md @@ -1,1587 +1,368 @@ # Airgap Deployment Runbook -> **Permanent home:** `coco-pattern/airgap/DEPLOY-RUNBOOK.md` -> -> This runbook covers the full lifecycle of an airgap deploy — from one-time jump host -> infrastructure setup through verified TDX attestation on an air-gapped OCP SNO cluster. +This runbook covers deploying the CoCo pattern to a disconnected (airgapped) +OpenShift cluster using a **Quay registry** as the sole mirror target. ---- - -## Duration Estimate - -| Phase | Description | Duration | -|-------|-------------|----------| -| 0 | One-Time Infrastructure Setup | ~30 min (skip on repeat runs) | -| A | Prerequisites and Mirror Wipe | ~15 min | -| B | Full Re-Mirror (oc-mirror v2) | ~3-4 hours | -| C | Cluster Install | ~35 min | -| D | Bootstrap and Pattern Deploy | ~25 min | -| E | DCAP and TDX Attestation | ~15 min (+ connected low-side PCK generation) | -| F | Verification and Pass/Fail | ~10 min | -| **Total** | | **~5-6 hours** (first-time add ~30 min) | - ---- - -## PHASE 0: One-Time Infrastructure Setup - -> **Skip this phase on repeat runs.** These steps configure the jump host infrastructure -> (mirror registry, git server) that persists across deployments. Run once per jump host. -> -> **Prerequisites:** Internet access on the jump host, Python 3.10+, `podman` -> and `openssl` installed, and `python3-passlib` or `httpd-tools` for -> `htpasswd`. - -Install the shared Python dependencies before starting the smart Git HTTP -server or collecting reference values: - -```bash -python3 -m pip install -r requirements.txt -``` - -### 0-0: Set Site Variables - -```bash -# ── Update these for your environment ────────────────────────────────────── -export MIRROR_REGISTRY=MIRROR_REGISTRY_HOST:8443 # host:port of your mirror registry -export GIT_SERVER_PORT=8080 # port for local git HTTP server -# ─────────────────────────────────────────────────────────────────────────── -export MREG_HOST="${MIRROR_REGISTRY%%:*}" -export MREG_PORT="${MIRROR_REGISTRY##*:}" -``` - -### 0-1: Mirror Registry Initialisation (ONE-TIME) +For a **plain Docker registry** (registry:2) alternative, see +[Appendix A](#appendix-a-plain-docker-registry-alternative). -The mirror registry is a `docker.io/library/registry:2` container with TLS and htpasswd auth. +## Architecture -```bash -echo "=== 0-1: Mirror Registry Init ===" - -# Create directory structure -mkdir -p ~/mirror-registry-certs ~/mirror-registry-config ~/local-registry -mkdir -p ~/.coco-pattern ~/.config/containers/certs.d/${MIRROR_REGISTRY} - -# a) Generate CA key + self-signed CA cert (10-year validity) -openssl genrsa -out ~/mirror-registry-certs/ca.key 4096 -openssl req -new -x509 -days 3650 \ - -key ~/mirror-registry-certs/ca.key \ - -out ~/mirror-registry-certs/ca.crt \ - -subj "/CN=mirror-registry-ca" -echo "CA cert generated: $(openssl x509 -noout -subject -in ~/mirror-registry-certs/ca.crt)" - -# b) Generate server TLS cert signed by the CA (SAN = host IP) -openssl genrsa -out ~/mirror-registry-certs/server.key 4096 -openssl req -new \ - -key ~/mirror-registry-certs/server.key \ - -out ~/mirror-registry-certs/server.csr \ - -subj "/CN=${MREG_HOST}" -openssl x509 -req -days 3650 \ - -in ~/mirror-registry-certs/server.csr \ - -CA ~/mirror-registry-certs/ca.crt \ - -CAkey ~/mirror-registry-certs/ca.key \ - -CAcreateserial \ - -out ~/mirror-registry-certs/server.crt \ - -extfile <(printf "[v3_req]\nsubjectAltName=IP:${MREG_HOST}\n") \ - -extensions v3_req -echo "Server cert signed by CA" - -# c) Generate registry password -# Use printf (NOT echo) — echo appends a newline which breaks Helm registry login (401) -MREG_PASS=$(openssl rand -base64 18 | tr -d '=+/' | head -c 24) -printf '%s' "$MREG_PASS" > ~/.coco-pattern/mirror-registry-password -chmod 600 ~/.coco-pattern/mirror-registry-password -echo "Password saved to ~/.coco-pattern/mirror-registry-password (${#MREG_PASS} bytes, no newline)" - -# d) Create htpasswd auth file (bcrypt) -htpasswd -Bbc ~/mirror-registry-config/htpasswd init "$MREG_PASS" -# SELinux: container_file_t label required so registry:2 container can read the file -chcon -t container_file_t ~/mirror-registry-config/htpasswd -echo "htpasswd created for user: init" - -# e) Create registry config.yml -cat > ~/mirror-registry-config/config.yml << 'EOF' -version: 0.1 -log: - level: warn -storage: - filesystem: - rootdirectory: /var/lib/registry - delete: - enabled: true -http: - addr: :8443 - tls: - certificate: /certs/server.crt - key: /certs/server.key -auth: - htpasswd: - realm: basic-realm - path: /auth/htpasswd -EOF - -# f) Create and start podman container -# config.yml → /etc/docker/registry/config.yml (where registry:2 reads it) -# htpasswd → /auth/htpasswd (where config.yml references it) -# certs → /certs/ (where config.yml references server.crt/server.key) -podman create \ - --name local-registry \ - -p "${MREG_PORT}:8443" \ - -v ~/local-registry:/var/lib/registry:z \ - -v ~/mirror-registry-config/config.yml:/etc/docker/registry/config.yml:z \ - -v ~/mirror-registry-config/htpasswd:/auth/htpasswd:z \ - -v ~/mirror-registry-certs:/certs:z \ - docker.io/library/registry:2 -podman start local-registry -echo "local-registry container started" - -# g) Persist as systemd user service (survives reboots and logout) -mkdir -p ~/.config/systemd/user -podman generate systemd --name local-registry --restart-policy always \ - > ~/.config/systemd/user/local-registry.service -systemctl --user daemon-reload -systemctl --user enable local-registry.service -loginctl enable-linger "$USER" -echo "local-registry systemd user service enabled" - -# h) Trust the CA for container tools (per-user, no sudo required) -# podman/skopeo/oc-mirror read from this directory automatically. -# curl: pass --cacert ~/mirror-registry-certs/ca.crt -# oc image mirror: pass --insecure=true -cp ~/mirror-registry-certs/ca.crt ~/.config/containers/certs.d/${MIRROR_REGISTRY}/ca.crt -echo "CA trusted for container tools" - -# h2) Skip sigstore attachment lookups for certified vendors -# Intel, HashiCorp, and NVIDIA images on registry.connect.redhat.com lack cosign -# .sig manifests. Without this, oc-mirror fails with "name unknown: Image not found". -mkdir -p ~/.config/containers/registries.d -cat > ~/.config/containers/registries.d/no-sigstore-certified.yaml << 'REGCFG' -docker: - registry.connect.redhat.com/intel: - use-sigstore-attachments: false - registry.connect.redhat.com/hashicorp: - use-sigstore-attachments: false - registry.connect.redhat.com/nvidia: - use-sigstore-attachments: false -REGCFG -echo "Sigstore attachment lookups disabled for certified vendors" - -# i) Build combined-ca-bundle.pem (used by labctl --additional-trust-bundle) -# The mirror-registry CA is the only CA required — Quay is not used in this deployment. -cp ~/mirror-registry-certs/ca.crt ~/combined-ca-bundle.pem -echo "combined-ca-bundle.pem created: $(grep -c 'BEGIN CERTIFICATE' ~/combined-ca-bundle.pem) cert(s)" - -# j) Verify -CATALOG=$(curl -s -u "init:$MREG_PASS" \ - --cacert ~/mirror-registry-certs/ca.crt \ - https://${MIRROR_REGISTRY}/v2/_catalog) -echo "Registry catalog check: $CATALOG" -[ "$CATALOG" = '{"repositories":[]}' ] && echo "PASS: registry healthy and empty" || echo "WARN: unexpected response" -``` - -### 0-2: Seed pull-secret.json with Mirror Registry Credentials - -The Red Hat pull-secret must be downloaded from [console.redhat.com](https://console.redhat.com) before running this step. - -```bash -echo "=== 0-2: Seed pull-secret.json ===" - -# Download pull-secret.json from https://console.redhat.com → OpenShift → Downloads -# Place at ~/pull-secret.json before proceeding. - -MREG_PASS=$(cat ~/.coco-pattern/mirror-registry-password) -python3 - << 'EOF' -import json, base64, os - -ps_path = os.path.expanduser("~/pull-secret.json") -mreg = os.environ["MIRROR_REGISTRY"] -mreg_pass = open(os.path.expanduser("~/.coco-pattern/mirror-registry-password")).read() -mreg_auth = base64.b64encode(f"init:{mreg_pass}".encode()).decode() +```text +┌──────────────┐ ┌───────────────────┐ ┌─────────────────┐ +│ Upstream │────▶│ Quay Mirror │────▶│ OpenShift SNO │ +│ Registries │ │ (org: mirror) │ │ (node-02) │ +│ (internet) │ │ │ │ │ +└──────────────┘ └───────────────────┘ └─────────────────┘ + │ │ + │ ┌────────────────┐ │ + └──▶│ Git HTTP │◀──┘ + │ Server (:8080)│ + └────────────────┘ -with open(ps_path) as f: - ps = json.load(f) +```text -ps["auths"][mreg] = { - "auth": mreg_auth, - "username": "init", - "password": mreg_pass -} +All image content is mirrored to a single Quay org. The cluster pulls +exclusively from the mirror (default catalogs disabled, IDMS/ITMS +redirect all upstream registries). The pattern source code is served +via a local Git HTTP server. -with open(ps_path, "w") as f: - json.dump(ps, f, indent=2) +## Duration -print("Added mirror-registry auth to pull-secret.json") -print("Registries:", list(ps["auths"].keys())) -EOF -``` - -> **When to re-run:** Only if `~/.coco-pattern/mirror-registry-password` is regenerated. -> The htpasswd auth survives mirror storage wipes — only image data is cleared in Phase A-4. - -### 0-3: Local Git Server Initialisation (ONE-TIME per jump host) - -The local Git HTTP server makes pattern repos available to the cluster's patterns-operator -during the deploy. `scripts/airgap-post-install.sh --sync-repos-only` creates bare repos -and starts the service; subsequent runs auto-sync via the D-1 step. - -```bash -echo "=== 0-3: Git Server Init ===" - -# a) Clone the pattern repositories (internet required — run before air-gapping) -git clone -b dev/airgap-testing \ - https://github.com/butler54/coco-pattern.git ~/coco-pattern -git clone -b dev/phase1-testing \ - https://github.com/butler54/trustee-chart.git ~/trustee-chart -git clone -b dev/phase1-testing \ - https://github.com/butler54/sandboxed-containers-chart.git ~/sandboxed-containers-chart -git clone -b dev/phase1-testing \ - https://github.com/butler54/sandboxed-policies-chart.git ~/sandboxed-policies-chart - -echo "Repos cloned:" -for d in coco-pattern trustee-chart sandboxed-containers-chart sandboxed-policies-chart; do - echo " ~/$d: $(git -C ~/$d rev-parse --short HEAD 2>/dev/null)" -done - -# b) Create bare repos in ~/public_html/git/ and start git-http.service -cd ~/coco-pattern -export MIRROR_REGISTRY # must be set from 0-0 -scripts/airgap-post-install.sh --sync-repos-only -# This: -# - Creates ~/public_html/git/*.git bare repos -# - Creates and enables systemd user service: git-http.service -# - Reports Git HTTP URLs on the local network - -# c) Verify -sleep 3 -curl -s "http://localhost:${GIT_SERVER_PORT}/coco-pattern.git/info/refs?service=git-upload-pack" | head -c 80 -echo "" -[ $? -eq 0 ] && echo "PASS: git HTTP server responding" || echo "FAIL: check git-http.service status" -``` - -> **On subsequent runs:** bare repo sync happens automatically in D-1 via -> `scripts/airgap-post-install.sh`. Phase 0-3 is one-time only. +| Phase | Description | Duration | +|---|---|---| +| A | Mirror preparation | ~4h (one-time) | +| B | Cluster install | ~35 min | +| C | Pattern deploy | ~2h | +| D | Secrets + attestation | ~30 min | +| **Total** | From ISO boot to 15/15 healthy | **~2.5-3h** | --- -## PHASE A: Prerequisites and Mirror Wipe (~15 min) - -### A-0: Start tmux Session and Set Up Logging (FIRST STEP) +## Prerequisites -> **This is the only SSH session you need for the entire run.** -> Start tmux immediately after connecting — all steps, including long-running ones, run as -> named windows within the single `deploy` session. To disconnect safely at any point: `Ctrl-b d`. -> To reconnect to the running session: `ssh jump` then `tmux attach -t deploy`. +### Mirror Content -```bash -ssh chbutler@user-jump.int.apac-tech-lab.net -tmux new-session -s deploy -``` - -Everything below runs inside the `deploy` session. Long-running steps open a new named window -(`Ctrl-b c` or via `tmux new-window -n `) and run in the foreground — no nohup needed. -Switch between windows with `Ctrl-b ` or `Ctrl-b n/p`. - -```bash -# ── Site-specific variables — update these for each deployment environment ── -export MIRROR_REGISTRY=MIRROR_REGISTRY_HOST:8443 # mirror-registry host:port -export GIT_SERVER=http://JUMP_HOST_IP:8080 # local git HTTP server -# Persist to all tmux windows (restore in each window with: eval "$(tmux showenv -g)") -tmux set-environment -g MIRROR_REGISTRY "$MIRROR_REGISTRY" -tmux set-environment -g GIT_SERVER "$GIT_SERVER" - -export LOG=~/coco-pattern/logs/run-25-$(date +%Y%m%d).log -mkdir -p ~/coco-pattern/logs -touch "$LOG" -echo "=== Phase 25 Run Start: $(date) ===" | tee -a "$LOG" -echo "LOG=$LOG MIRROR_REGISTRY=$MIRROR_REGISTRY GIT_SERVER=$GIT_SERVER" | tee -a "$LOG" -tmux set-environment -g LOG "$LOG" - -# local-registry credentials (htpasswd auth, username: init) -export MREG_USER=init -export MREG_PASS=$(cat ~/.coco-pattern/mirror-registry-password) -echo "local-registry credentials loaded (user: $MREG_USER)" | tee -a "$LOG" - -# Ensure local-registry CA is trusted by container tools (idempotent) -MREG_HOST="${MIRROR_REGISTRY%%:*}" -MREG_PORT="${MIRROR_REGISTRY##*:}" -mkdir -p ~/.config/containers/certs.d/${MIRROR_REGISTRY} -cp ~/mirror-registry-certs/ca.crt ~/.config/containers/certs.d/${MIRROR_REGISTRY}/ca.crt -echo "local-registry CA trusted for container tools" | tee -a "$LOG" - -# Per-user cert dir handles podman/skopeo/oc-mirror TLS. -# curl: pass --cacert ~/mirror-registry-certs/ca.crt -# oc image mirror: pass --insecure=true - -# Verify sigstore skip config is in place (created in Phase 0-1 step h2) -ls ~/.config/containers/registries.d/no-sigstore-certified.yaml && \ - echo "Sigstore skip config present" | tee -a "$LOG" || \ - echo "WARN: no-sigstore-certified.yaml missing — Intel/HashiCorp/NVIDIA mirrors may fail" | tee -a "$LOG" -``` +The Quay mirror must contain all images the pattern resolves. Verify with +`skopeo list-tags` against each repo (the REST API is unreliable for org +robots). Key content: -> At run end, Claude will SSH in and read this log file for findings analysis. +- OCP release payload (exact installer version, e.g. 4.22.17) +- Operator catalogs (redhat, certified, community — v4.22) +- VP Helm charts (clustergroup, trustee, acm, etc.) +- VP utility images (utility-container, imperative-container, pattern-install) +- Workload images (ubi9/httpd-24, ubi9/ubi) +- Intel DCAP operator + QGS images +- Kyverno images (mirrored from ghcr.io under the `kyverno/` path) +- hashicorp/vault (the version the chart resolves, e.g. 2.1.2-ubi) -### A-0.5: Tool Version Verification (MUST PASS before proceeding) +### Imageset Pin Check (K16) -> **Critical:** PATH order issues can cause the wrong binary to be used silently. -> `~/.local/bin` may contain old versions that override `~/bin/` symlinks. -> Every check below must show the expected version. **Stop and fix any mismatch.** +Pattern values use range pins (`~0.1.10`, `0.2.*`) resolved at deploy time. +The imageset must mirror the **newest version within each range** — not the +range floor. Run a range-check before every deploy: ```bash -echo "=== A-0.5: Tool Version Verification ===" 2>&1 | tee -a "$LOG" -OCP_MINOR="4.22" # Expected OCP minor version for this run - -# 1. openshift-install — version string must contain expected OCP minor version -OI_VERSION=$(openshift-install version 2>/dev/null | head -1) -echo "openshift-install: $OI_VERSION" | tee -a "$LOG" -if ! echo "$OI_VERSION" | grep -q "^openshift-install ${OCP_MINOR}"; then - echo "FAIL: openshift-install reports wrong version (expected ${OCP_MINOR}.x, got: $OI_VERSION)" | tee -a "$LOG" - echo "Fix: run 'labctl generate ...' first to symlink the correct binary, or check PATH order" | tee -a "$LOG" - exit 1 -fi -echo "PASS: openshift-install is ${OCP_MINOR}.x" | tee -a "$LOG" - -# 2. oc-mirror — version string must contain expected OCP minor version -OM_VERSION=$(oc-mirror version 2>&1 | grep -i 'gitversion' | grep -o '[0-9]\+\.[0-9]\+\.[0-9]*' | head -1) -echo "oc-mirror version: $OM_VERSION" | tee -a "$LOG" -if ! echo "$OM_VERSION" | grep -q "^${OCP_MINOR}"; then - echo "FAIL: oc-mirror reports wrong version (expected ${OCP_MINOR}.x, got: $OM_VERSION)" | tee -a "$LOG" - echo "Fix: ln -sf ~/ocp_42208_bin/oc-mirror ~/bin/oc-mirror" | tee -a "$LOG" - exit 1 -fi -echo "PASS: oc-mirror is ${OCP_MINOR}.x" | tee -a "$LOG" - -# 3. labctl — verify installed -LABCTL_VERSION=$(labctl --version 2>/dev/null) -echo "labctl: $LABCTL_VERSION" | tee -a "$LOG" - -# 4. Release image digest alignment (informational — only meaningful after B-2) -OI_DIGEST=$(openshift-install version 2>/dev/null | grep 'release image' | grep -o 'sha256:[0-9a-f]*') -OM_DIGEST=$(grep -o 'sha256-[0-9a-f]*' \ - ~/oc-mirror-workspace/working-dir/cluster-resources/signature-configmap.json 2>/dev/null | \ - head -1 | sed 's/sha256-/sha256:/') -echo "openshift-install release digest: $OI_DIGEST" | tee -a "$LOG" -echo "oc-mirror release digest: ${OM_DIGEST:-not yet run}" | tee -a "$LOG" -[ -n "$OM_DIGEST" ] && [ "$OI_DIGEST" = "$OM_DIGEST" ] && \ - echo "PASS: release image digests match" | tee -a "$LOG" || \ - echo "INFO: digest mismatch or oc-mirror not yet run (check again after B-2)" | tee -a "$LOG" - -echo "A-0.5: Version verification complete at $(date)" | tee -a "$LOG" -``` - -### A-1: State Verification (check before deleting) - -```bash -echo "=== A-1: State Verification ===" 2>&1 | tee -a "$LOG" - -cd ~/coco-pattern -git remote get-url origin 2>&1 | tee -a "$LOG" -# EXPECTED: ${GIT_SERVER}/coco-pattern.git -git branch --show-current 2>&1 | tee -a "$LOG" -# EXPECTED: dev/airgap-testing - -du -sh ~/oc-mirror-workspace 2>&1 | tee -a "$LOG" - -# Check local-registry state (htpasswd auth required) -echo "local-registry catalog:" 2>&1 | tee -a "$LOG" -curl -sk -u "$MREG_USER:$MREG_PASS" https://${MIRROR_REGISTRY}/v2/_catalog 2>&1 | tee -a "$LOG" +# For each chart pin in values-*.yaml, verify the newest in-range tag +# is present in the mirror +skopeo list-tags docker:///validatedpatterns/ | \ + python3 -c "import json,sys; tags=json.load(sys.stdin)['Tags']; print(sorted(tags))" -grep -c "BEGIN CERTIFICATE" ~/combined-ca-bundle.pem 2>&1 | tee -a "$LOG" -# EXPECTED: 3 (Quay 2-chain + mirror-registry 1 cert) - -# Verify pull-secret.json has local-registry credentials (required for cluster node pulls via IDMS) -echo "pull-secret registries:" 2>&1 | tee -a "$LOG" -python3 -c "import json; [print(k) for k in json.load(open('pull-secret.json'))['auths']]" 2>&1 | tee -a "$LOG" -# MUST include: ${MIRROR_REGISTRY} -# If missing, run from appendix A.6 before proceeding - -echo "A-1: State verification complete at $(date)" 2>&1 | tee -a "$LOG" -``` - -### A-3: oc-mirror Workspace Wipe - -> **DESTRUCTIVE — confirm A-1 shows correct git branch and mirror-registry state before proceeding.** - -```bash -echo "=== A-3: oc-mirror Workspace Wipe ===" 2>&1 | tee -a "$LOG" - -ls -la ~/oc-mirror-workspace 2>&1 | tee -a "$LOG" -rm -rf ~/oc-mirror-workspace -echo "oc-mirror-workspace removed" 2>&1 | tee -a "$LOG" - -# Also clear any cached oc-mirror state -rm -rf ~/.oc-mirror 2>/dev/null || true -echo "A-3: oc-mirror wipe complete at $(date)" 2>&1 | tee -a "$LOG" ``` -### A-4: local-registry Decommission (Stop / Wipe / Restart) - -> **DESTRUCTIVE: stop container, wipe storage, restart empty.** -> Container name: `local-registry`. Storage: `~/local-registry` (owned by chbutler — no sudo). -> No auth required for registry API. +### Credentials and Certificates ```bash -echo "=== A-4: local-registry Decommission ===" 2>&1 | tee -a "$LOG" +# Mirror registry credentials (robot account) +export MIRROR_REGISTRY=:443/mirror # INCLUDES the org path +export MIRROR_USER= +export MIRROR_PASS= -# Confirm state before wiping -echo "Container status:" 2>&1 | tee -a "$LOG" -podman ps --format "{{.Names}} {{.Status}}" | grep -i registry 2>&1 | tee -a "$LOG" -echo "Storage size:" 2>&1 | tee -a "$LOG" -du -sh ~/local-registry/ 2>&1 | tee -a "$LOG" +# CA certificate chain +cp ~/.coco-pattern/registry-ca-cert.pem +cat > ~/combined-ca-bundle.pem -# Stop container -podman stop local-registry 2>&1 | tee -a "$LOG" -echo "local-registry stopped" 2>&1 | tee -a "$LOG" +# Intel PCS API key (for PCK provisioning, 32 bytes, no trailing newline) +printf '%s' "" > ~/.coco-pattern/intel-pcs-api-key -# Wipe storage (chbutler owns ~/local-registry — no sudo needed) -rm -rf ~/local-registry/* -echo "Storage wiped" 2>&1 | tee -a "$LOG" +# DCAP collateral (collected on a connected host, ~30 day freshness) +# ~/.coco-pattern/dcap-offline/platform_collaterals.json -# Restart container (reinitializes on empty storage) -podman start local-registry 2>&1 | tee -a "$LOG" -sleep 10 +```text -# Verify empty catalog (htpasswd auth required) -curl -sk -u "$MREG_USER:$MREG_PASS" https://${MIRROR_REGISTRY}/v2/_catalog 2>&1 | tee -a "$LOG" -# EXPECTED: {"repositories":[]} - -echo "A-4: local-registry decommission complete at $(date)" 2>&1 | tee -a "$LOG" -``` +### Values-Secret (Convention Location) -### A-5: Verify Empty State +The values-secret file **must** be at `~/values-secret-.yaml`. +The Ansible secret loader (`find_vp_secrets`) searches home-directory +conventions and falls back to the repo template — it never reads a +repo-local generated file. ```bash -echo "=== A-5: Verify Empty State ===" 2>&1 | tee -a "$LOG" +# Generate (writes to ~/values-secret-.yaml) +make gen-secrets -# local-registry catalog must be empty (htpasswd auth required) -echo "local-registry catalog after wipe:" 2>&1 | tee -a "$LOG" -curl -sk -u "$MREG_USER:$MREG_PASS" https://${MIRROR_REGISTRY}/v2/_catalog 2>&1 | tee -a "$LOG" -# EXPECTED: {"repositories":[]} +# Uncomment airgap blocks (registry CA, bootstrap secrets) +scripts/enable-airgap-secrets.sh -# oc-mirror workspace must not exist -if [ -d ~/oc-mirror-workspace ]; then - echo "WARNING: oc-mirror-workspace still exists" 2>&1 | tee -a "$LOG" -else - echo "PASS: oc-mirror-workspace absent" 2>&1 | tee -a "$LOG" -fi - -echo "A-5: Empty state verification complete at $(date)" 2>&1 | tee -a "$LOG" -``` - ---- - -## PHASE B: Full Re-Mirror (~3-4 hours) - -> **Skip Phase B if mirror-registry already has all required content.** -> Run B-0 to verify. If all checks pass, go directly to Phase C. - -### B-0: Verify Mirror Content and Tool Versions (run before deciding to skip Phase B) - -> **CRITICAL: oc-mirror must be version-matched to the OCP release being mirrored.** -> Cross-version oc-mirror rebuilds manifests with different SHA256 digests than openshift-install -> expects. This causes every component image pull to return 404 from mirror-registry, with the -> node falling back to internet quay.io (blocked by fake gateway) and stalling. -> `labctl generate` symlinks `~/bin/oc-mirror` from the OCP version bin dir automatically. - -```bash -echo "=== B-0: Mirror Content Verification ===" 2>&1 | tee -a "$LOG" -MREG_PASS=$(cat ~/.coco-pattern/mirror-registry-password) - -echo "--- oc-mirror version check (MUST match OCP version) ---" 2>&1 | tee -a "$LOG" -oc-mirror version 2>&1 | grep -i 'version\|GitVersion' | tee -a "$LOG" -# MUST contain "4.22" — if not, update symlink: ln -sf ~/ocp_42208_bin/oc-mirror ~/bin/oc-mirror - -echo "--- VP OCI Helm charts ---" 2>&1 | tee -a "$LOG" -for chart in clustergroup hashicorp-vault acm openshift-external-secrets \ - sandboxed-containers sandboxed-policies trustee; do - result=$(curl -sk -u "init:$MREG_PASS" \ - "https://${MIRROR_REGISTRY}/v2/validatedpatterns/${chart}/tags/list" 2>/dev/null) - tags=$(echo "$result" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('tags','MISSING'))" 2>/dev/null) - echo " validatedpatterns/${chart}: ${tags}" 2>&1 | tee -a "$LOG" -done - -echo "--- Operator catalog indexes ---" 2>&1 | tee -a "$LOG" -for idx in redhat/redhat-operator-index redhat/certified-operator-index redhat/community-operator-index; do - result=$(curl -sk -u "init:$MREG_PASS" \ - "https://${MIRROR_REGISTRY}/v2/${idx}/tags/list" 2>/dev/null) - tags=$(echo "$result" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('tags','MISSING'))" 2>/dev/null) - echo " ${idx}: ${tags}" 2>&1 | tee -a "$LOG" -done - -echo "--- OCP release image digest check ---" 2>&1 | tee -a "$LOG" -# The release image MUST be accessible at the digest openshift-install has hardcoded. -# This digest is set by the version-matched oc-mirror run — do NOT patch manually. -RELEASE_DIGEST=$(~/ocp_42208_bin/openshift-install version 2>/dev/null | grep 'release image' | awk '{print $NF}' | sed 's/.*@//') -if [ -n "$RELEASE_DIGEST" ]; then - HTTP=$(curl -sk -u "init:$MREG_PASS" -o /dev/null -w '%{http_code}' \ - "https://${MIRROR_REGISTRY}/v2/openshift/release-images/manifests/sha256:${RELEASE_DIGEST}") - [ "$HTTP" = "200" ] && echo "PASS: release image digest present" | tee -a "$LOG" || \ - echo "FAIL: release image digest missing (HTTP $HTTP) — must run B-2 with version-matched oc-mirror" | tee -a "$LOG" -fi - -echo "--- oc-mirror cluster-resources ---" 2>&1 | tee -a "$LOG" -ls ~/oc-mirror-workspace/working-dir/cluster-resources/ 2>/dev/null | tee -a "$LOG" - -echo "B-0: Mirror verification complete at $(date)" 2>&1 | tee -a "$LOG" -``` - -> **If all VP charts show tags, catalog indexes show v4.22, oc-mirror version matches, and release digest is present → skip to Phase C.** -> If anything is missing or wrong → run Phase B (full re-mirror). - ---- - -### B-1: Sync Repos to Latest - -```bash -echo "=== B-1: Sync Repos ===" 2>&1 | tee -a "$LOG" - -# Sync git mirror (bare repo) from GitHub -cd ~/public_html/git/coco-pattern.git -git fetch https://github.com/butler54/coco-pattern.git dev/airgap-testing:dev/airgap-testing --force \ - 2>&1 | tee -a "$LOG" +# Edit the bootstrap_secrets block for your mirror: +# url: :443//validatedpatterns +# username: +# password path: ~/.coco-pattern/registry-password +# +# Edit the registryCaCert block: +# pem path: ~/.coco-pattern/registry-ca-cert.pem -# Pull into working copy -cd ~/coco-pattern -git fetch origin 2>&1 | tee -a "$LOG" -git reset --hard origin/dev/airgap-testing 2>&1 | tee -a "$LOG" - -# Verify remote and branch -git remote get-url origin 2>&1 | tee -a "$LOG" -# MUST: ${GIT_SERVER}/coco-pattern.git -git branch --show-current 2>&1 | tee -a "$LOG" -# MUST: dev/airgap-testing - -# Verify clustergroup chart version in imageset-config -echo "Checking imageset-config clustergroup version:" 2>&1 | tee -a "$LOG" -grep "clustergroup" ~/coco-pattern/airgap/imageset-config-4.22.yaml 2>&1 | tee -a "$LOG" -# EXPECTED: clustergroup:0.9.58 - -# Verify values-baremetal-airgap.yaml is 2-line shape -echo "Checking values-baremetal-airgap.yaml shape:" 2>&1 | tee -a "$LOG" -cat ~/coco-pattern/values-baremetal-airgap.yaml 2>&1 | tee -a "$LOG" -# EXPECTED: 2 lines with global.catalogSource and global.catalogSourceNamespace - -# Verify deprecated scripts are quarantined -echo "Checking scripts/deprecated/:" 2>&1 | tee -a "$LOG" -ls ~/coco-pattern/scripts/deprecated/ 2>&1 | tee -a "$LOG" -# EXPECTED: exactly 3 files (fix-patterns-operator-images.sh, rebuild-patterns-operator-bundle.sh, deploy-pattern-without-operator.sh) - -echo "B-1: Repo sync complete at $(date)" 2>&1 | tee -a "$LOG" ``` -### B-2: Run oc-mirror v2 (Full Re-Mirror) +### Git Server -> **Long-running (~3-4 hours). Open a new window in the `deploy` session for this step.** -> -> **oc-mirror version must match OCP version.** `labctl generate` (C-1) automatically symlinks -> `~/bin/oc-mirror` from `~/ocp_42208_bin/oc-mirror`. Verify with B-0 before proceeding. -> A version mismatch causes oc-mirror to rebuild manifests with different SHA256 digests than -> openshift-install expects, resulting in 404s on every component image during node boot. +The pattern source is served via a local Git HTTP server: ```bash -tmux new-window -n "oc-mirror" # opens within the current session; switches you into it -eval "$(tmux showenv -g LOG)"; eval "$(tmux showenv -g MIRROR_REGISTRY)"; eval "$(tmux showenv -g GIT_SERVER)" - -echo "=== B-2: oc-mirror v2 Full Re-Mirror ===" 2>&1 | tee -a "$LOG" -echo "Started at $(date)" 2>&1 | tee -a "$LOG" - -# Verify oc-mirror version before running (must match OCP 4.22) -oc-mirror version 2>&1 | grep -i 'version\|GitVersion' | tee -a "$LOG" -# MUST show 4.22.x — if not, stop and fix: ln -sf ~/ocp_42208_bin/oc-mirror ~/bin/oc-mirror - -# Authenticate oc-mirror against local-registry (CA trusted via certs.d from A-0) -podman login ${MIRROR_REGISTRY} \ - --username "$MREG_USER" \ - --password "$MREG_PASS" \ - 2>&1 | tee -a "$LOG" -# EXPECTED: Login Succeeded! - -# Run oc-mirror v2 — full mirror to mirror-registry (sole airgap target, Phase 25.2) -# Note: --v2 (double dash) is mandatory; -v2 (single dash) is not recognised -# CA trust comes from ~/.config/containers/certs.d/${MIRROR_REGISTRY}/ca.crt (set in A-0) -oc-mirror \ - --config ~/coco-pattern/airgap/imageset-config-4.22.yaml \ - docker://${MIRROR_REGISTRY} \ - --dest-tls-verify=true \ - --workspace file://$HOME/oc-mirror-workspace \ - --v2 \ - 2>&1 | tee -a "$LOG" - -echo "B-2: oc-mirror complete at $(date)" 2>&1 | tee -a "$LOG" -``` +# Bare repos under ~/public_html/git/ +# Smart HTTP on :8080 (started by airgap-post-install.sh step 9) -> **Wait for oc-mirror to complete before proceeding to B-3.** -> Output is visible live in this window. Switch windows with `Ctrl-b ` while it runs. -> Expected completion: 3-4 hours. -> oc-mirror v2 is resumable — re-running the same command skips already-mirrored content. -> -> **Cosign signature failures:** Intel, HashiCorp, and NVIDIA images on -> `registry.connect.redhat.com` lack cosign `.sig` manifests. The -> `no-sigstore-certified.yaml` config (created in Phase 0-1 step h2) disables -> sigstore attachment lookups for these vendors, so oc-mirror handles them normally. -> -> **Transient failures (fixed by re-running):** -> `multicluster-engine/hive-rhel9` is 600MB+ and reliably times out on the first oc-mirror run -> (`context deadline exceeded` during blob upload). When hive-rhel9 times out, oc-mirror also -> skips `mce-operator-bundle` entirely — this cascades to: MCE CSV not installed → MCH stuck -> at Installing → ACM policy controller never starts → `credential` secret never created → -> trustee KbsConfig blocks forever. -> **If B-3 shows hive-rhel9 or mce-operator-bundle MISSING: re-run the B-2 oc-mirror command.** -> oc-mirror resumes from the workspace cache and retries only what failed — do NOT skip to Phase C. - -### B-3: Post-Mirror Verification - -```bash -echo "=== B-3: Post-Mirror Verification ===" 2>&1 | tee -a "$LOG" - -# Count cluster-resources YAML files (expect ~9: IDMS, ITMS, CatalogSources, signatures) -ls ~/oc-mirror-workspace/working-dir/cluster-resources/*.yaml 2>&1 | tee -a "$LOG" -COUNT=$(ls ~/oc-mirror-workspace/working-dir/cluster-resources/*.yaml | wc -l) -echo "cluster-resources count: $COUNT (expected ~10)" 2>&1 | tee -a "$LOG" - -# Paginate through full registry catalog — v2/_catalog returns max 100 repos per page -# CRITICAL: do NOT use a bare curl; with 270+ repos the first page misses everything past repo 100 -echo "Spot-checking key repos in local-registry (paginated):" 2>&1 | tee -a "$LOG" -python3 - << 'PYEOF' 2>&1 | tee -a "$LOG" -import json, urllib.request, urllib.error, base64, os - -creds = base64.b64encode(b"init:" + open(os.path.expanduser("~/.coco-pattern/mirror-registry-password"), "rb").read().strip()).decode() -headers = {"Authorization": f"Basic {creds}"} - -repos = [] -last = "" -while True: - url = f"https://${MIRROR_REGISTRY}/v2/_catalog?n=100{f'&last={last}' if last else ''}" - req = urllib.request.Request(url, headers=headers) - ctx = __import__("ssl").create_default_context() - ctx.load_verify_locations(os.path.expanduser("~/mirror-registry-certs/ca.crt")) - try: - with urllib.request.urlopen(req, context=ctx) as r: - page = json.loads(r.read()).get("repositories", []) - except Exception as e: - print(f"ERROR: {e}"); break - repos.extend(page) - if len(page) < 100: break - last = page[-1] - -print(f"Total repos: {len(repos)}") -checks = [ - ("openshift/release-images", "OCP release images"), - ("openshift/release", "OCP release components"), - ("validatedpatterns/clustergroup", "clustergroup chart"), - ("community-operator-pipeline-prod/patterns-operator", "patterns-operator"), - ("hashicorp/vault", "Vault"), - ("intel/intel-tdx-dcap-operator", "Intel TDX DCAP"), - ("multicluster-engine/hive-rhel9", "MCE hive (large — timeout risk; re-run oc-mirror if missing)"), - ("multicluster-engine/mce-operator-bundle", "MCE bundle (skipped by oc-mirror if hive timed out)"), - ("build-of-trustee/trustee-rhel9", "Trustee"), - ("multicluster-engine/hive-rhel9", "MCE Hive"), - ("openshift-sandboxed-containers/osc-cloud-api-adaptor-rhel9", "CoCo cloud-api-adaptor"), -] -for repo, label in checks: - status = "OK" if repo in repos else "MISSING" - print(f" {status} {label} ({repo})") -PYEOF - -# Verify IDMS files cover expected namespaces -echo "IDMS namespace coverage:" 2>&1 | tee -a "$LOG" -grep "${MIRROR_REGISTRY}" ~/oc-mirror-workspace/working-dir/cluster-resources/idms-oc-mirror.yaml | wc -l | xargs -I{} echo " idms-oc-mirror.yaml: {} mirror entries" | tee -a "$LOG" -# idms-manual-mirrors.yaml no longer needed — oc-mirror generates IDMS for Intel/Hashicorp -# when registries.d/no-sigstore-certified.yaml is in place - -echo "B-3: Post-mirror verification complete at $(date)" 2>&1 | tee -a "$LOG" -``` - -### B-3a: Manual Mirror — ELIMINATED - -> **No longer needed.** The `no-sigstore-certified.yaml` registries.d config -> (Phase 0-1 step h2) disables sigstore attachment lookups for Intel, HashiCorp, -> and NVIDIA on `registry.connect.redhat.com`. oc-mirror now mirrors all images -> successfully and generates IDMS entries for them automatically. -> -> **ITMS (tag-based mirrors)** for `registry.connect.redhat.com/intel` and -> `/hashicorp` are still needed — oc-mirror only generates IDMS (digest-based). -> The supplementary ITMS lives at `airgap/itms-manual-mirrors.yaml` and is applied -> automatically by `airgap-post-install.sh` in D-1. - -```bash -echo "=== B-3a: Verify supplementary ITMS in repo ===" | tee -a "$LOG" -cat ~/coco-pattern/airgap/itms-manual-mirrors.yaml 2>&1 | tee -a "$LOG" -# EXPECTED: ImageTagMirrorSet covering registry.connect.redhat.com/intel and /hashicorp -``` +```text --- -## PHASE C: Cluster Install (~35 min) +## Phase B: Cluster Install -### C-1: Generate Configs and Build ISO - -> labctl now generates the fake gateway directly (172.25.53.254) instead of `route-type: blackhole`, -> and embeds wipe-disks.ign into the ISO via `coreos-installer iso ignition embed`. No manual -> patching or separate openshift-install call needed. +### B-1: Generate the ISO ```bash -echo "=== C-1: Generate Configs and Build ISO ===" 2>&1 | tee -a "$LOG" -source ~/.envrc - -# ── labctl (APAC lab-specific) ─────────────────────────────────────────────── -# labctl is an internal APAC lab tool that reads node configuration from an -# XLSX file and generates OCP agent-based install configs with the correct -# NIC, iDRAC, and network settings for this lab environment. -# -# For other environments, replace labctl generate with: -# openshift-install agent create image --dir -# and mount the ISO via your BMC virtual media interface manually. -# ───────────────────────────────────────────────────────────────────────────── labctl generate \ - --node node-02 \ - --xlsx "$HOME/APAC Technology Lab Details.xlsx" \ - --ocp-version 42208 \ - --output-dir ~/node-02-airgap-output \ + --node \ + --xlsx \ + --ocp-version \ + --output-dir \ --mirror-resources ~/oc-mirror-workspace/working-dir/cluster-resources \ --additional-trust-bundle ~/combined-ca-bundle.pem \ - --http-serve-path $HOME/public_html/ \ - 2>&1 | tee -a "$LOG" - -# Verify: fake gateway present, no blackhole, ISO built and served -grep "172.25.53.254" ~/node-02-airgap-output/agent-config.yaml 2>&1 | tee -a "$LOG" && \ - ! grep "blackhole" ~/node-02-airgap-output/agent-config.yaml && \ - echo "PASS: fake gateway in agent-config, no blackhole" 2>&1 | tee -a "$LOG" -ls -lh ~/public_html/node-02-42208.iso 2>&1 | tee -a "$LOG" && \ - echo "PASS: ISO present at http serve path" 2>&1 | tee -a "$LOG" - -echo "C-1: Config generation and ISO build complete at $(date)" 2>&1 | tee -a "$LOG" -``` - -### C-3: Mount ISO via iDRAC - -```bash -echo "=== C-3: Mount ISO via iDRAC ===" 2>&1 | tee -a "$LOG" -source ~/.envrc + --http-serve-path ~/public_html -# ── labctl (APAC lab-specific) ─────────────────────────────────────────────── -# labctl is an internal APAC lab tool that reads node configuration from an -# XLSX file and generates OCP agent-based install configs with the correct -# NIC, iDRAC, and network settings for this lab environment. -# -# For other environments, replace labctl generate with: -# openshift-install agent create image --dir -# and mount the ISO via your BMC virtual media interface manually. -# ───────────────────────────────────────────────────────────────────────────── -labctl idrac mount-iso \ - --node node-02 \ - --xlsx "$HOME/APAC Technology Lab Details.xlsx" \ - --iso-url "http://user-jump.int.apac-tech-lab.net/~chbutler/node-02-42208.iso" \ - 2>&1 | tee -a "$LOG" - -echo "C-3: ISO mounted at $(date)" 2>&1 | tee -a "$LOG" ``` -### C-4: Wait for Install (~30 min) +### B-2: Boot and Install ```bash -tmux new-window -n "cluster-install" # new window within the deploy session -eval "$(tmux showenv -g LOG)"; eval "$(tmux showenv -g MIRROR_REGISTRY)"; eval "$(tmux showenv -g GIT_SERVER)" - -echo "=== C-4: Wait for Install ===" 2>&1 | tee -a "$LOG" -cd ~/node-02-airgap-output/42208_build +# Mount the ISO via iDRAC/virtual media, one-time boot +# Wait for install (~35 min): +openshift-install agent wait-for install-complete \ + --dir /_build -~/ocp_42208_bin/openshift-install agent wait-for install-complete \ - --dir . \ - --log-level info \ - 2>&1 | tee -a "$LOG" - -echo "C-4: Install complete at $(date)" 2>&1 | tee -a "$LOG" -``` +```text -> **Wait for install to complete before proceeding to C-5.** -> Expected log line: `Install complete!` and `Access the OpenShift web-console here:` -> Switch windows with `Ctrl-b ` while waiting. - -### C-5: Health Check +### B-3: Verify ```bash -echo "=== C-5: Health Check ===" 2>&1 | tee -a "$LOG" - -# Find the build directory (changes per run) -BUILD_DIR=$(ls -td ~/node-02-airgap-output/4*_build* 2>/dev/null | head -1) -echo "Build dir: $BUILD_DIR" 2>&1 | tee -a "$LOG" - -export KUBECONFIG="${BUILD_DIR}/auth/kubeconfig" -echo "KUBECONFIG=$KUBECONFIG" | tee -a "$LOG" - -oc get clusterversion 2>&1 | tee -a "$LOG" -# EXPECTED: VERSION 4.22.8, STATUS Available -oc get nodes 2>&1 | tee -a "$LOG" -# EXPECTED: node-02 Ready - -echo "C-5: Health check complete at $(date)" 2>&1 | tee -a "$LOG" -``` - -### C-6: Set KUBECONFIG - -```bash -echo "=== C-6: Set KUBECONFIG ===" 2>&1 | tee -a "$LOG" - -# The build directory name changes per run — set it from the actual output -BUILD_DIR=$(ls -td ~/node-02-airgap-output/4*_build* 2>/dev/null | head -1) -export KUBECONFIG="${BUILD_DIR}/auth/kubeconfig" - -echo "export KUBECONFIG=${BUILD_DIR}/auth/kubeconfig" >> ~/.envrc -echo "KUBECONFIG set to: $KUBECONFIG" 2>&1 | tee -a "$LOG" - -# Verify -oc whoami 2>&1 | tee -a "$LOG" -# EXPECTED: system:admin +export KUBECONFIG=/_build/auth/kubeconfig +oc get clusterversion # version, Available=True +oc get clusteroperators # all Available +oc get nodes # Ready -echo "C-6: KUBECONFIG configured at $(date)" 2>&1 | tee -a "$LOG" ``` --- -## PHASE D: Bootstrap and Pattern Deploy (~25 min) +## Phase C: Pattern Deploy -### D-1: Run airgap-post-install.sh - -> `airgap-post-install.sh` handles all of the following in one pass: -> -> - Disables OperatorHub default catalogs -> - Applies CatalogSources from oc-mirror cluster-resources (deleting stale ones) -> - Applies ALL oc-mirror cluster-resources: IDMS, ITMS, ClusterCatalog (OLM v1), signature ConfigMap -> - Applies supplementary IDMS (`idms-manual-mirrors.yaml` — intel/hashicorp, created in B-3a) -> - Applies supplementary ITMS (`airgap/itms-manual-mirrors.yaml` — intel/hashicorp, from repo) -> - Patches `mirrorSourcePolicy: NeverContactSource` on all IDMS entries (prevents internet fallback) -> - Configures the git HTTP server and patterns-operator-config ConfigMap +### C-1: Airgap Post-Install ```bash -echo "=== D-1: airgap-post-install.sh ===" 2>&1 | tee -a "$LOG" cd ~/coco-pattern -# MIRROR_REGISTRY was set in A-0; confirm it is still set -echo "MIRROR_REGISTRY=$MIRROR_REGISTRY" | tee -a "$LOG" - -scripts/airgap-post-install.sh 2>&1 | tee -a "$LOG" +export MIRROR_REGISTRY=:443/mirror # INCLUDES the org! +export GIT_SERVER=http://:8080 +export ENABLE_ROUTINGVIAHOST=true # required for pod→lab egress -# Verify catalog pods are Running before proceeding to D-2 -echo "--- CatalogSource pods ---" 2>&1 | tee -a "$LOG" -oc get pods -n openshift-marketplace 2>&1 | tee -a "$LOG" -# EXPECTED: 3 pods Running (redhat, certified, community operator indexes) +scripts/airgap-post-install.sh -echo "D-1: airgap-post-install complete at $(date)" 2>&1 | tee -a "$LOG" -``` +```text -### D-1.5: Wait for MCO if airgap-post-install.sh patched any IDMS/ITMS +This applies: -> **This step is now automated.** `airgap-post-install.sh` (step 3b/3c) patches `NeverContactSource` -> on all IDMS and ITMS objects — including the bootstrap `image-digest-mirror` IDMS created by -> `labctl` at cluster install time. If any objects were patched, MCO re-renders `registries.conf`. -> On SNO this triggers a **node reboot (~5 min)**. Check the script output for -> `"MCO will re-render registries.conf"` and wait for the node to recover before proceeding. +- IDMS/ITMS/CatalogSources from the oc-mirror workspace +- `mirrorSourcePolicy: NeverContactSource` on all mirror rules +- Repoints any default-catalog subscriptions to the mirrored catalogs +- Configures the Git HTTP server +- Creates the patterns-operator-config ConfigMap +- Enables OVN routingViaHost (verified after applying) -```bash -# If airgap-post-install.sh printed a reboot warning, wait here: -echo "=== D-1.5: Wait for node if MCO re-rendered ===" 2>&1 | tee -a "$LOG" -until oc get node -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' \ - 2>/dev/null | grep -q True; do - echo " Node not Ready yet — $(date)" | tee -a "$LOG" - sleep 15 -done -echo "Node Ready at $(date)" 2>&1 | tee -a "$LOG" -oc get nodes 2>&1 | tee -a "$LOG" -``` +**Verify**: catalog pods Running, routingViaHost=true. -### D-2: Enable routingViaHost and Wait for MCO +### C-2: Deploy the Pattern ```bash -echo "=== D-2: Enable routingViaHost ===" 2>&1 | tee -a "$LOG" - -oc patch network.operator.openshift.io cluster --type merge \ - -p '{"spec":{"defaultNetwork":{"ovnKubernetesConfig":{"gatewayConfig":{"routingViaHost":true}}}}}' \ - 2>&1 | tee -a "$LOG" - -echo "Waiting for MCO to complete (expect 5-10 min including reboot)..." 2>&1 | tee -a "$LOG" -while true; do - STATUS=$(oc get mcp master -o jsonpath='{.status.conditions[?(@.type=="Updated")].status}' 2>/dev/null) - UPDATING=$(oc get mcp master -o jsonpath='{.status.conditions[?(@.type=="Updating")].status}' 2>/dev/null) - echo "STATUS: $(date) — MCO Updated=$STATUS Updating=$UPDATING" 2>&1 | tee -a "$LOG" - [ "$STATUS" = "True" ] && [ "$UPDATING" = "False" ] && echo "MCO complete" 2>&1 | tee -a "$LOG" && break - sleep 30 -done - -echo "D-2: routingViaHost enabled at $(date)" 2>&1 | tee -a "$LOG" -``` +export PATTERN_DISCONNECTED_HOME=:443//validatedpatterns -### D-3: Prepare Secrets on Disk - -```bash -echo "=== D-3: Prepare Secrets ===" 2>&1 | tee -a "$LOG" -cd ~/coco-pattern +./pattern.sh make install -make cache-keys 2>&1 | tee -a "$LOG" -make cache-registry-ca 2>&1 | tee -a "$LOG" -./scripts/gen-secrets.sh 2>&1 | tee -a "$LOG" - -# registryCaCert and bootstrap_secrets (mirror-registry-helm-oci) are -# commented out by default in values-secret.yaml.template (they only apply -# to airgap deployments). Uncomment both blocks in the generated -# values-secret file now that cache-registry-ca has populated the CA cert. -make enable-airgap-secrets 2>&1 | tee -a "$LOG" - -# Strip trailing newline from mirror-registry-password. -# load-bootstrap reads this file verbatim via the values-secret.yaml path: field and -# the Ansible role does not strip whitespace. A trailing \n causes Helm to send -# "password\n" to the registry, which never matches the htpasswd hash → 401. -printf '%s' "$(cat ~/.coco-pattern/mirror-registry-password)" \ - > ~/.coco-pattern/mirror-registry-password -echo "mirror-registry-password length: $(wc -c < ~/.coco-pattern/mirror-registry-password) bytes (expected 24, no newline)" \ - 2>&1 | tee -a "$LOG" - -echo "D-3: Secrets prepared at $(date)" 2>&1 | tee -a "$LOG" ``` -### D-4: Launch Pattern Deploy - -```bash -tmux new-window -n "pattern-deploy" # new window within the deploy session -eval "$(tmux showenv -g LOG)"; eval "$(tmux showenv -g MIRROR_REGISTRY)"; eval "$(tmux showenv -g GIT_SERVER)" - -echo "=== D-4: Pattern Deploy ===" 2>&1 | tee -a "$LOG" -cd ~/coco-pattern - -# Verify git remote and branch before deploy -git remote get-url origin 2>&1 | tee -a "$LOG" -# MUST: ${GIT_SERVER}/coco-pattern.git -git branch --show-current 2>&1 | tee -a "$LOG" -# MUST: dev/airgap-testing +This: -./pattern.sh make install \ - 2>&1 | tee -a "$LOG" - -echo "D-4: pattern.sh complete at $(date)" 2>&1 | tee -a "$LOG" -``` +- Creates the patterns-operator Subscription (from the mirrored catalog) +- Creates the Pattern CR (from values-global.yaml) +- Creates the ArgoCD instance (vp-gitops namespace) +- Loads bootstrap secrets (the ArgoCD OCI chart-pull credential) -> **Immediately switch to your main window and start D-5.** `Ctrl-b ` to switch. -> D-5 runs in parallel — it loops waiting for vp-gitops to appear, then fires automatically. -> **Do not wait** — if ArgoCD comes up before D-5 completes it will try to pull charts without credentials. -> MCO reboots during kata deployment are normal — not a failure signal. +**Verify**: `Bootstrap phase: 1 secret(s) created` in the output. +If it says 0, the values-secret is not at the convention location. -### D-5: Inject CA + Load Bootstrap Secrets +### C-3: ArgoCD CA + Bootstrap (parallel with C-2) -> **Start this immediately after launching D-4** from the main tmux window. -> D-5 internally waits for `vp-gitops` to exist — start it now so it fires the moment ArgoCD is ready. -> If you delay, ArgoCD may come up and fail to pull VP OCI Helm charts before credentials are loaded. -> -> Steps D-5 performs once vp-gitops appears: -> -> 1. Inject mirror-registry private CA into ArgoCD's TLS trust store -> 2. `make load-bootstrap` — pre-seeds ArgoCD with mirror-registry OCI Helm credentials -> (from the `bootstrap_secrets` block in values-secret.yaml; no vault required) -> 3. Hard-refresh ArgoCD so it pulls chart manifests from mirror-registry immediately +Run in parallel with the pattern deploy — must fire as soon as ArgoCD +exists: ```bash -echo "=== D-5: ArgoCD CA + Bootstrap Secrets ===" 2>&1 | tee -a "$LOG" -cd ~/coco-pattern - -# Wait for ArgoCD (vp-gitops namespace) -echo "Waiting for vp-gitops namespace..." 2>&1 | tee -a "$LOG" +# Wait for vp-gitops namespace until oc get ns vp-gitops &>/dev/null; do sleep 10; done -echo "vp-gitops active at $(date)" 2>&1 | tee -a "$LOG" - -# Wait for repo-server deployment to exist before patching -until oc get deployment vp-gitops-repo-server -n vp-gitops &>/dev/null; do sleep 10; done - -# Inject mirror-registry CA into ArgoCD TLS trust. -# mirror-registry uses the lab private CA — provide the chain so ArgoCD verifies properly. -# Note: CA cert contains literal newlines which must be escaped for JSON; use Python to build patch. -echo "Injecting mirror-registry CA into argocd-tls-certs-cm..." 2>&1 | tee -a "$LOG" -python3 -c " -import json, subprocess, sys -ca = open('$HOME/mirror-registry-certs/ca.crt').read() -import os; mreg_host = os.environ.get('MIRROR_REGISTRY','MIRROR_REGISTRY_HOST:8443').split(':')[0] -patch = json.dumps({'data': {mreg_host: ca}}) -r = subprocess.run(['oc','patch','configmap','argocd-tls-certs-cm','-n','vp-gitops','--type','merge','-p',patch], capture_output=True, text=True) -print(r.stdout or r.stderr) -sys.exit(r.returncode) -" 2>&1 | tee -a "$LOG" - -# Restart ALL vp-gitops pods to pick up the new CA and any secret changes. -# repo-server uses the CA for Helm registry login; other components may cache certs. -echo "Rolling all vp-gitops deployments..." 2>&1 | tee -a "$LOG" -oc get deployments -n vp-gitops --no-headers -o name 2>/dev/null | \ - while read dep; do oc rollout restart "$dep" -n vp-gitops 2>&1 | tee -a "$LOG"; done - -# Wait for repo-server specifically before loading secrets (it handles Helm OCI auth) -oc rollout status deployment/vp-gitops-repo-server -n vp-gitops --timeout=120s \ - 2>&1 | tee -a "$LOG" - -# Load bootstrap secrets — creates ArgoCD OCI Helm repo secret for mirror-registry. -# This uses values-secret.yaml bootstrap_secrets section; runs before vault init. -# IMPORTANT: mirror-registry-password must have no trailing newline (fixed in D-3). -# A trailing \n causes Helm to send "password\n" → registry rejects with 401. -echo "Loading bootstrap secrets (ArgoCD OCI Helm repo auth)..." 2>&1 | tee -a "$LOG" -make load-bootstrap 2>&1 | tee -a "$LOG" - -# Verify the secret was created and password has correct length (no trailing newline) -oc get secret mirror-registry-helm-oci -n vp-gitops 2>&1 | tee -a "$LOG" -# EXPECTED: mirror-registry-helm-oci Opaque ... -PASS_LEN=$(oc get secret mirror-registry-helm-oci -n vp-gitops \ - -o jsonpath='{.data.password}' 2>/dev/null | base64 -d | wc -c) -echo "mirror-registry-helm-oci password length: ${PASS_LEN} bytes (expected: same as mirror-registry-password file)" \ - 2>&1 | tee -a "$LOG" -if [ "${PASS_LEN}" -gt 24 ]; then - echo "WARN: password may have trailing newline — patch the secret:" 2>&1 | tee -a "$LOG" - echo " MREG_PASS=\$(cat ~/.coco-pattern/mirror-registry-password | tr -d '\\n')" 2>&1 | tee -a "$LOG" - echo " oc create secret generic mirror-registry-helm-oci -n vp-gitops \\" 2>&1 | tee -a "$LOG" - echo " --from-literal=type=helm --from-literal=enableOCI=true \\" 2>&1 | tee -a "$LOG" - echo " --from-literal=url=${MIRROR_REGISTRY}/validatedpatterns \\" 2>&1 | tee -a "$LOG" - echo " --from-literal=name=mirror-registry-charts \\" 2>&1 | tee -a "$LOG" - echo " --from-literal=username=init --from-literal=password=\"\$MREG_PASS\" \\" 2>&1 | tee -a "$LOG" - echo " --dry-run=client -o yaml | oc apply -f -" 2>&1 | tee -a "$LOG" -fi - -# Trigger hard refresh — ArgoCD now has CA trust + credentials for mirror-registry -oc annotate applications.argoproj.io coco-pattern-baremetal -n vp-gitops \ - argocd.argoproj.io/refresh=hard --overwrite 2>&1 | tee -a "$LOG" - -sleep 30 - -# Verify child apps are appearing -echo "--- ArgoCD child apps ---" 2>&1 | tee -a "$LOG" -oc get applications.argoproj.io -A --no-headers 2>/dev/null | \ - awk '{print $2, $4, $5}' | column -t 2>&1 | tee -a "$LOG" -# EXPECTED: vault, acm, kyverno, openshift-external-secrets, storage, etc. are listed - -echo "D-5: ArgoCD configured at $(date)" 2>&1 | tee -a "$LOG" -``` -### D-6: Wait for Vault and Load Secrets +# Inject the mirror CA into ArgoCD's TLS trust +oc patch configmap argocd-tls-certs-cm -n vp-gitops \ + --type merge -p "{\"data\":{\"\":\"$(cat ~/.coco-pattern/registry-ca-cert.pem)\"}}" -```bash -echo "=== D-6: Wait for Vault + Load Secrets ===" 2>&1 | tee -a "$LOG" -cd ~/coco-pattern +# Roll the deployments to pick up the CA +oc get deployments -n vp-gitops --no-headers -o name | \ + while read dep; do oc rollout restart "$dep" -n vp-gitops; done +oc rollout status deployment/vp-gitops-repo-server -n vp-gitops -while true; do - INIT=$(oc exec -n vault vault-0 -- vault status -format=json 2>/dev/null | \ - python3 -c "import sys,json; print(json.loads(sys.stdin.read()).get('initialized',''))" 2>/dev/null || echo "") - SEALED=$(oc exec -n vault vault-0 -- vault status -format=json 2>/dev/null | \ - python3 -c "import sys,json; print(json.loads(sys.stdin.read()).get('sealed',''))" 2>/dev/null || echo "") - echo "STATUS: $(date) — Vault init=$INIT sealed=$SEALED" 2>&1 | tee -a "$LOG" - [ "$INIT" = "True" ] && [ "$SEALED" = "False" ] && echo "Vault ready" 2>&1 | tee -a "$LOG" && break - sleep 15 -done +# Load bootstrap secrets (if not already done by pattern.sh) +make load-bootstrap -./pattern.sh make load-secrets 2>&1 | tee -a "$LOG" -# EXPECTED: 10 secrets injected +# Verify: the secret exists, is labeled, and has the correct URL +oc get secret mirror-registry-helm-oci -n vp-gitops \ + -o jsonpath='{.metadata.labels.argocd\.argoproj\.io/secret-type}' +# Expected: repository -echo "D-6: Secrets loaded at $(date)" 2>&1 | tee -a "$LOG" -``` - ---- +```text -## PHASE E: DCAP and TDX Attestation (~10-15 min) +### C-4: Known Timing Issues -### E-1: Verify DCAP Readiness +Three issues occur consistently in disconnected deployments. Each has a +one-line fix: -```bash -echo "=== E-1: DCAP Readiness ===" 2>&1 | tee -a "$LOG" +#### MCE Subscription (30-60 min after C-1) -oc get pods -n intel-dcap-operator-system -l app=intel-tdx-qgs --no-headers \ - 2>&1 | tee -a "$LOG" -# EXPECTED: NAME 2/2 Running -# NOTE: QGS DaemonSet runs in intel-dcap-operator-system, not openshift-operators - -oc get secrets -n intel-dcap-operator-system -l type=platform-data --no-headers \ - 2>&1 | tee -a "$LOG" -# EXPECTED: 1 secret (name is the QE ID hex string) - -echo "E-1: DCAP readiness verified at $(date)" 2>&1 | tee -a "$LOG" -``` - -### E-2: QGS Socket Port Verification - -> `charts/all/baremetal/templates/vsock-mco.yaml` deploys a MachineConfig drop-in at -> `/etc/kata-containers/kata-tdx/config.d/96-kata-kernel-config` that sets `socket_port=0`. -> Verify that the drop-in exists and the effective port is 0. The base `configuration.toml` -> may still show 4050 — that is overridden by the drop-in, which kata reads last. +The MCH operator creates the `multicluster-engine` subscription with +`source: redhat-operators` (the default catalog, disabled in airgap). +The MCH then blocks in `Installing`. ```bash -echo "=== E-2: QGS Socket Port Verification ===" 2>&1 | tee -a "$LOG" - -NODE=$(oc get nodes -o name | head -1) -echo "Checking node: $NODE" 2>&1 | tee -a "$LOG" - -# Check the drop-in file (authoritative — overrides configuration.toml) -DROP_IN=$(oc debug $NODE -- chroot /host bash -c \ - "cat /etc/kata-containers/kata-tdx/config.d/96-kata-kernel-config 2>/dev/null" \ - 2>/dev/null) -echo "Drop-in content: $DROP_IN" 2>&1 | tee -a "$LOG" - -if echo "$DROP_IN" | grep -q "socket_port=0"; then - echo "PASS: socket_port=0 in drop-in — QGS will use unix socket" 2>&1 | tee -a "$LOG" -else - echo "FAIL: drop-in missing or socket_port not 0 — record as DEV-1 in DEVIATIONS.md" 2>&1 | tee -a "$LOG" - echo " See Appendix A.3 for the fallback sed command (apply ONLY after recording DEV-1)" 2>&1 | tee -a "$LOG" -fi +oc patch subscriptions.operators.coreos.com multicluster-engine \ + -n multicluster-engine --type merge \ + -p '{"spec":{"source":"","sourceNamespace":"openshift-marketplace"}}' -echo "E-2: QGS socket port verification complete at $(date)" 2>&1 | tee -a "$LOG" ``` -> **If verification fails (DEV-1):** The MachineConfig drop-in was not applied — check that -> `oc get mc | grep kata-tdx` shows a MachineConfig with the drop-in content. -> See Appendix Section A.3 for the temporary sed fallback. Record in DEVIATIONS.md. +#### ITMS Org-Prefix (kyverno pods ImagePullBackOff) -### E-3: PCK Lifecycle - -PCK certificates are specific to each QGS platform. The Make targets create checksummed request and response bundles, validate the complete QE-ID/PCE-ID set, and restart QGS only after all matching PCK Secrets are applied. - -The importer discovers the QGS DaemonSet from the `pck-certs-watcher` pod owner reference, avoiding a version-specific DaemonSet label or name. Set `DCAP_QGS_DAEMONSET` only if the namespace has more than one candidate. - -If this bastion can reach **both** the disconnected cluster and Intel PCS, run the combined workflow. The API key is read from `INTEL_PCS_API_KEY` if exported, otherwise Make prompts without echoing it. Never put the key on the Make command line. +The manual ITMS mirrors must include the org prefix for Quay targets. ```bash -echo "=== E-3: Combined PCK provisioning ===" 2>&1 | tee -a "$LOG" -cd ~/coco-pattern -make dcap-pck-provision 2>&1 | tee -a "$LOG" -``` +# Verify: all mirrors should include // +oc get itms itms-manual-mirrors -o json | \ + python3 -c "import json,sys; [print(m['source'],'->',m['mirrors']) for m in json.load(sys.stdin)['spec']['imageTagMirrors']]" -If the high-side cluster cannot reach the connected low-side host, use the split workflow: +```text -```bash -# High side: create ~/.coco-pattern/dcap-pck/platform-request and transfer it high -> low. -make dcap-platform-export - -# Low side: copy platform-request to ~/.coco-pattern/dcap-pck/, then run these commands. -make dcap-tools dcap-pck-generate - -# Transfer ~/.coco-pattern/dcap-pck/pck-response low -> high, then import it on the high side. -make dcap-pck-import -``` +If any mirror is missing the org, patch the ITMS and delete the affected pods. -### E-4: Collect DCAP Collateral and Firmware Reference Values +#### Initdata Staleness (CVM workloads in CreateContainerError) -`collect-dcap-collateral` is independent of PCK provisioning. It collects public Intel TDX verification collateral on the connected low side and does not require an Intel PCS API key. Transfer `~/.coco-pattern/dcap-offline/platform_collaterals.json` to the high side before E-5. +The `cc_init_data` annotation is immutable at pod creation. Workload pods +created before the KBS is ready will retry indefinitely with stale initdata. ```bash -echo "=== E-4: Collect DCAP Collateral + Firmware Refvals ===" 2>&1 | tee -a "$LOG" -cd ~/coco-pattern - -make collect-dcap-collateral 2>&1 | tee -a "$LOG" - -# Verify QeIdentity is populated (must be >100 chars) -python3 -c " -import json, os -c = json.load(open(os.path.expanduser('~/.coco-pattern/dcap-offline/platform_collaterals.json'))) -qi = c.get('collaterals', {}).get('qeidentity_early') or c.get('collaterals', {}).get('qeidentity') -print('qeidentity present:', isinstance(qi, dict), '— PASS' if isinstance(qi, dict) else '— FAIL: recollect collateral') -" 2>&1 | tee -a "$LOG" - -make collect-firmware-refvals 2>&1 | tee -a "$LOG" +# Prerequisite: KBS Running and attestation returning 200 +oc delete pod -n hello-openshift --all +oc delete pod -n kbs-access --all -echo "E-4: DCAP collateral and refvals collected at $(date)" 2>&1 | tee -a "$LOG" -``` - -### E-5: Reload Secrets - -```bash -echo "=== E-5: Reload Secrets ===" 2>&1 | tee -a "$LOG" -cd ~/coco-pattern - -./pattern.sh make load-secrets 2>&1 | tee -a "$LOG" -# EXPECTED: 10 secrets injected (including DCAP collateral and firmware refvals) - -echo "E-5: Secrets reloaded at $(date)" 2>&1 | tee -a "$LOG" ``` --- -## PHASE F: Verification and Pass/Fail (~10 min) +## Phase D: Secrets and Attestation -### F-1: Wait for initdata ConfigMaps +### D-1: Load Vault Secrets ```bash -echo "=== F-1: Wait for initdata ConfigMaps ===" 2>&1 | tee -a "$LOG" +# Waits for vault-0, then loads all secrets from the values-secret file +./pattern.sh make load-secrets -while true; do - COUNT=$(oc get configmap -n imperative -l coco.io/type=initdata --no-headers 2>/dev/null | wc -l) - echo "STATUS: $(date) — initdata ConfigMaps: $COUNT/2" 2>&1 | tee -a "$LOG" - [ "$COUNT" -ge 2 ] && echo "Initdata ready" 2>&1 | tee -a "$LOG" && break - sleep 30 -done - -echo "F-1: initdata ConfigMaps ready at $(date)" 2>&1 | tee -a "$LOG" -``` +```text -### F-2: Bounce Workload Pods +### D-2: PCK Provisioning (parallel with D-1) ```bash -echo "=== F-2: Bounce Workload Pods ===" 2>&1 | tee -a "$LOG" +export INTEL_PCS_API_KEY=$(cat ~/.coco-pattern/intel-pcs-api-key) +make dcap-pck-provision -oc delete pods --all -n hello-openshift --force --grace-period=0 \ - 2>&1 | tee -a "$LOG" -oc delete pods --all -n kbs-access --force --grace-period=0 \ - 2>&1 | tee -a "$LOG" +# Restart the QGS to pick up the provisioned PCK +oc delete pod -n intel-dcap-operator-system -l app=intel-tdx-dcap-qgs -echo "Waiting 90 seconds for pods to restart..." 2>&1 | tee -a "$LOG" -sleep 90 - -echo "F-2: Pods bounced at $(date)" 2>&1 | tee -a "$LOG" ``` -### F-3: Verify Pods Running +### D-3: Firmware Reference Values ```bash -echo "=== F-3: Verify Pods Running ===" 2>&1 | tee -a "$LOG" - -echo "--- hello-openshift ---" 2>&1 | tee -a "$LOG" -oc get pods -n hello-openshift 2>&1 | tee -a "$LOG" -# EXPECTED: 3/3 Running (standard, insecure-policy, secure) - -echo "--- kbs-access ---" 2>&1 | tee -a "$LOG" -oc get pods -n kbs-access 2>&1 | tee -a "$LOG" -# EXPECTED: kbs-access-curl 1/1, kbs-access-sealed 1/1 (or similar) - -echo "F-3: Pod status captured at $(date)" 2>&1 | tee -a "$LOG" -``` - -### F-4: Verify KBS Attestation - -```bash -echo "=== F-4: Verify KBS Attestation ===" 2>&1 | tee -a "$LOG" - -oc logs deployment/trustee-deployment -n trustee-operator-system -c kbs --tail=20 \ - 2>&1 | tee -a "$LOG" -# EXPECTED: POST /attest 200, GET /resource 200 - -echo "F-4: KBS attestation verified at $(date)" 2>&1 | tee -a "$LOG" -``` - -### F-5: Verify Image Sources - -```bash -echo "=== F-5: Verify Image Sources (all from mirror-registry) ===" 2>&1 | tee -a "$LOG" - -for ns in hello-openshift kbs-access trustee-operator-system; do - echo "--- $ns ---" 2>&1 | tee -a "$LOG" - oc get pods -n $ns -o jsonpath='{range .items[*]}{.metadata.name}{": "}{.spec.containers[*].image}{"\n"}{end}' \ - 2>&1 | tee -a "$LOG" -done - -# Flag any image not from mirror-registry or the internal OCP registry -echo "Upstream refs (should be empty):" 2>&1 | tee -a "$LOG" -oc get pods -A -o jsonpath='{range .items[*]}{.spec.containers[*].image}{"\n"}{end}' 2>/dev/null \ - | grep -v "${MIRROR_REGISTRY}\|image-registry.openshift-image-registry" \ - | sort | uniq \ - 2>&1 | tee -a "$LOG" || echo "All images from expected registries" | tee -a "$LOG" - -echo "F-5: Image sources verified at $(date)" 2>&1 | tee -a "$LOG" -``` - -### F-6: Verify ArgoCD Apps - -```bash -echo "=== F-6: Verify ArgoCD Apps ===" 2>&1 | tee -a "$LOG" - -oc get applications.argoproj.io -n vp-gitops \ - 2>&1 | tee -a "$LOG" -# EXPECTED: All apps Synced+Healthy - -APP_COUNT=$(oc get applications.argoproj.io -n vp-gitops --no-headers 2>/dev/null | wc -l) -HEALTHY_COUNT=$(oc get applications.argoproj.io -n vp-gitops --no-headers 2>/dev/null \ - | grep -c "Synced.*Healthy" || echo "0") -echo "ArgoCD: $HEALTHY_COUNT/$APP_COUNT apps Synced+Healthy" 2>&1 | tee -a "$LOG" - -echo "F-6: ArgoCD status captured at $(date)" 2>&1 | tee -a "$LOG" -``` - -### F-7: Phase 24 Validation Checks (D-07) - -> **These checks validate that Phase 24 cleanup decisions did not break the deploy.** -> Record results in DEVIATIONS.md if any check fails. - -```bash -echo "=== F-7: Phase 24 Validation Checks (D-07) ===" 2>&1 | tee -a "$LOG" - -# Check 1: scripts/deprecated/ contains exactly 3 scripts, none were invoked during deploy -echo "--- Check 1: scripts/deprecated/ contents ---" 2>&1 | tee -a "$LOG" -ls -la ~/coco-pattern/scripts/deprecated/ 2>&1 | tee -a "$LOG" -DEPRECATED_COUNT=$(ls ~/coco-pattern/scripts/deprecated/ | wc -l) -echo "Deprecated script count: $DEPRECATED_COUNT (expected: 3)" 2>&1 | tee -a "$LOG" -if [ "$DEPRECATED_COUNT" = "3" ]; then - echo "PASS: Exactly 3 deprecated scripts present (none removed, none invoked)" 2>&1 | tee -a "$LOG" -else - echo "DEVIATION: Record deprecated count mismatch in DEVIATIONS.md" 2>&1 | tee -a "$LOG" -fi - -# Check 2: values-baremetal-airgap.yaml is the 2-line global overlay (D-05 / Phase 24 C-05) -echo "--- Check 2: values-baremetal-airgap.yaml shape ---" 2>&1 | tee -a "$LOG" -cat ~/coco-pattern/values-baremetal-airgap.yaml 2>&1 | tee -a "$LOG" -OVERLAY_LINES=$(grep -c "global\." ~/coco-pattern/values-baremetal-airgap.yaml 2>/dev/null || echo "0") -echo "global.* lines in overlay: $OVERLAY_LINES (expected: 2)" 2>&1 | tee -a "$LOG" -grep -q "global.catalogSource" ~/coco-pattern/values-baremetal-airgap.yaml && \ - grep -q "global.catalogSourceNamespace" ~/coco-pattern/values-baremetal-airgap.yaml && \ - echo "PASS: 2-line global catalogSource overlay active" 2>&1 | tee -a "$LOG" || \ - echo "DEVIATION: values-baremetal-airgap.yaml does not match expected 2-line shape" 2>&1 | tee -a "$LOG" - -# Check 3: mirror-registry is still present and serving -echo "--- Check 3: mirror-registry health ---" 2>&1 | tee -a "$LOG" -curl -sk --cacert ~/mirror-registry-certs/ca.crt \ - https://${MIRROR_REGISTRY}/v2/_catalog \ - 2>&1 | tee -a "$LOG" -MIRROR_STATUS=$? -if [ "$MIRROR_STATUS" = "0" ]; then - echo "PASS: mirror-registry accessible at ${MIRROR_REGISTRY}" 2>&1 | tee -a "$LOG" -else - echo "DEVIATION: mirror-registry not accessible — record in DEVIATIONS.md" 2>&1 | tee -a "$LOG" -fi - -# Check 4: Record whether VP hybrid images required mirror-registry or pulled from Quay alone -echo "--- Check 4: VP hybrid image source ---" 2>&1 | tee -a "$LOG" -echo "Review B-4 log output: did airgap-post-install.sh WARN about VP images?" 2>&1 | tee -a "$LOG" -grep "WARN.*VP\|WARN.*patterns-operator\|mirror-registry" "$LOG" | head -5 2>&1 | tee -a "$LOG" -echo "If WARNs appeared → mirror-registry still needed (record in DEVIATIONS.md as Phase 26 prerequisite confirmed)" 2>&1 | tee -a "$LOG" -echo "If no WARNs → VP images served from Quay alone (record as Phase 26 milestone: mirror-registry removable)" 2>&1 | tee -a "$LOG" - -echo "F-7: Phase 24 validation checks complete at $(date)" 2>&1 | tee -a "$LOG" -``` - -### F-8: Pass/Fail Criteria Table +# Collect from the live node (re-run if the node was reinstalled) +make collect-firmware-refvals +./pattern.sh make load-secrets # reload with fresh values -Fill in the ACTUAL column after running the above steps. Record any failures in DEVIATIONS.md. - -```bash -echo "=== F-8: Pass/Fail Summary ===" 2>&1 | tee -a "$LOG" - -# Collect key status values for the summary table -echo "patterns-operator CSV:" 2>&1 | tee -a "$LOG" -oc get csv -n openshift-operators -l operators.coreos.com/patterns-operator.openshift-operators \ - --no-headers 2>&1 | tee -a "$LOG" - -echo "Vault status:" 2>&1 | tee -a "$LOG" -oc exec -n vault vault-0 -- vault status 2>&1 | tee -a "$LOG" - -echo "MCH status:" 2>&1 | tee -a "$LOG" -oc get mch -A 2>&1 | tee -a "$LOG" - -echo "=== Run Complete: $(date) ===" 2>&1 | tee -a "$LOG" -``` - -**Pass/Fail Criteria Reference Table:** - -| Criterion | Expected | Actual | -|-----------|----------|--------| -| patterns-operator CSV Succeeded | Via OLM from mirror-registry community catalog | | -| Pattern CR reconciles | targetRepo = HTTP Git URL (not GitHub) | | -| Vault init without manual SA creation | Sync-wave fix still working | | -| 15/15 (or current count) ArgoCD apps Synced+Healthy | All green | | -| KBS attestation affirming | POST /attest 200 | | -| KBS resource delivery | GET /resource 200 | | -| hello-openshift 3/3 Running | standard + insecure-policy + secure | | -| kbs-access-curl Running | Uses privileged SCC SA | | -| kbs-access-sealed Running | Uses Secret volume | | -| secret.txt contains KBS resource | Fetched via CDH from KBS | | -| All images from mirror-registry | No upstream refs | | -| MCH Running | ACM + MCE healthy | | -| QGS socket_port=0 WITHOUT manual sed | D-06 — MCO fix working (new row) | | -| No mirrorSourcePolicy conflicts after D-1.5 | PASS: all IDMS/ITMS have NeverContactSource (new row) | | -| scripts/deprecated/ untouched — 3 files, none invoked | D-07 — no deprecated scripts called (new row) | | -| 2-line global catalogSource overlay active in values-baremetal-airgap.yaml | D-07 — Phase 24 C-05 migration active (new row) | | -| mirror-registry serving VP hybrid images | D-07 — confirms two-registry still needed OR single-registry milestone | | - -**Known Warnings (not failures):** - -| Warning | Explanation | -|---------|-------------| -| VP hybrid images WARN in airgap-post-install.sh | Images pre-pushed to mirror-registry. Script cannot re-push (already there). | -| insights operator unavailable | Expected in airgap — cannot reach console.redhat.com | -| clusterversion Error reconciling | Caused by insights operator. Cosmetic. | -| MCO reboots during kata deployment | Normal behavior — not a failure signal | - ---- - -## APPENDIX - -### A.1: DEVIATIONS.md Template (D-04) - -Copy this template to `~/coco-pattern/DEVIATIONS.md` at the start of the run. -Add a new `## DEV-N` section for each deviation discovered during execution. - -```markdown -# Run 25 Deviations - -**Run date:** YYYY-MM-DD -**Operator:** chbutler -**Log file:** ~/coco-pattern/logs/run-25-YYYYMMDD.log - -## Format - -Each deviation entry: -- What happened (observation) -- What was done (action taken) -- Impact on Phase 26 cleanup decisions +```text --- -## DEV-1: [Title — fill in if QGS socket_port fix not automatic] - -**Phase step:** E-2 -**What happened:** [e.g., socket_port was 4050 instead of 0 in kata-tdx/configuration.toml] -**What was done:** [e.g., applied fallback sed — see Appendix A.3] -**Impact on Phase 26:** [e.g., MCO-layer QGS fix not working; Phase 26 must investigate and automate] - ---- - -## DEV-2: [Title — fill in if QE ID mismatch required pcsclient.py] - -**Phase step:** E-3 -**What happened:** [e.g., cluster QE ID c0560e9b019a720a6a09149ec518bbe1 did not match cached ID] -**What was done:** [e.g., ran pcsclient.py interactively to generate new PCK material] -**Impact on Phase 26:** [e.g., hypothesis disproved — SGX platform reset required; investigate trigger] - ---- - -## DEV-N: [Add more entries as needed] - -**Phase step:** -**What happened:** -**What was done:** -**Impact on Phase 26:** -``` - -### A.2: local-registry API Reference - -Container name: `local-registry` (podman). Image: `docker.io/library/registry:2`. -Storage: `~/local-registry` (host bind mount, owned by chbutler). -Config: `~/mirror-registry-config/config.yml`. Certs: `~/mirror-registry-certs/`. -Auth: htpasswd (`~/mirror-registry-config/htpasswd`). Username: `init`. -Password file: `~/.coco-pattern/mirror-registry-password` (600, git-ignored). - -```bash -# List all repos -curl -sk -u "$MREG_USER:$MREG_PASS" https://${MIRROR_REGISTRY}/v2/_catalog - -# List tags for a repo -curl -sk -u "$MREG_USER:$MREG_PASS" https://${MIRROR_REGISTRY}/v2//tags/list -``` - -TLS: self-signed cert — use `-sk` (skip TLS verify) or `--cacert ~/mirror-registry-certs/ca.crt` - -Decommission: `podman stop local-registry && rm -rf ~/local-registry/* && podman start local-registry` - -SELinux note: htpasswd file requires `container_file_t` label. If recreating, run: -`chcon -t container_file_t ~/mirror-registry-config/htpasswd` - -CA trust for container tools (podman, oc-mirror, skopeo): +## Verification ```bash -mkdir -p ~/.config/containers/certs.d/${MIRROR_REGISTRY} -cp ~/mirror-registry-certs/ca.crt ~/.config/containers/certs.d/${MIRROR_REGISTRY}/ca.crt -``` +# All applications Synced+Healthy +oc get applications.argoproj.io -A --no-headers | \ + awk '$3!="Synced" || $4!="Healthy"' +# Expected: no output -This is idempotent and is run automatically in A-0. It enables `--dest-tls-verify=true` in oc-mirror -and removes the need for `--tls-verify=false` in podman login. +# Attestation returning 200 +oc logs -n trustee-operator-system deployment/trustee-deployment \ + --tail=5 | grep "attest.*200" -### A.3: QGS Socket Port Fallback (Deviation Footnote Only) - -> **Apply ONLY if E-2 verification fails and DEV-1 is recorded in DEVIATIONS.md.** -> This is a deviation fallback — not a normal execution step. - -```bash -# FALLBACK ONLY (if E-2 fails) — record as DEV-1 first -echo "Applying QGS socket port fallback sed (DEV-1 deviation)..." 2>&1 | tee -a "$LOG" -NODE=$(oc get nodes -o name | head -1) -oc debug $NODE -- chroot /host bash -c \ - "sed -i 's/tdx_quote_generation_service_socket_port = 4050/tdx_quote_generation_service_socket_port = 0/' \ - /etc/kata-containers/kata-tdx/configuration.toml" \ - 2>&1 | tee -a "$LOG" -echo "Fallback sed applied — record in DEVIATIONS.md as DEV-1" 2>&1 | tee -a "$LOG" -``` +# CVM workloads pulling from the mirror +oc get pods -n hello-openshift -o jsonpath='{.items[0].spec.containers[0].image}' -### A.4: Post-Run Log Retrieval - -After the run completes, Claude SSHes in to analyze the log: - -```bash -# Claude retrieves log from jump host -ssh chbutler@user-jump.int.apac-tech-lab.net "cat ~/coco-pattern/logs/run-25-*.log" -``` - -The operator does not need to gzip, commit, or serve the log file. Claude reads it directly via SSH. -After analysis, commit DEVIATIONS.md to coco-gsd: - -```bash -# On jump host — commit DEVIATIONS.md back -cd ~/coco-pattern -git add DEVIATIONS.md -git commit -m "docs(run-25): record run deviations" -git push origin dev/airgap-testing - -# Then in coco-gsd -cd ~/coco-gsd -cp ../coco-pattern/DEVIATIONS.md .planning/phases/25-manual-logged-jump-host-deploy-runbook/RUN-25-DEVIATIONS.md -git add .planning/phases/25-manual-logged-jump-host-deploy-runbook/RUN-25-DEVIATIONS.md -git commit -m "docs(25): import run-25 deviations log" -git push origin main ``` -### A.5: Environment Variable Quick-Reference - -Set these before starting the run: - -```bash -export LOG=~/coco-pattern/logs/run-25-YYYYMMDD.log -export MREG_USER=init -export MREG_PASS=$(cat ~/.coco-pattern/mirror-registry-password) -``` - -Auto-set during run: - -```bash -export KUBECONFIG= -export MIRROR_REGISTRY=MIRROR_REGISTRY_HOST:8443 -``` - -Persistent in `~/.envrc`: +--- -```bash -export LABCTL_IDRAC_USER=chbutler -export LABCTL_IDRAC_PASSWORD="..." -``` +## Appendix A: Plain Docker Registry Alternative -### A.6: Add local-registry to pull-secret.json +A plain Docker registry (registry:2) can be used instead of Quay. The key +differences: -The cluster's global pull-secret (embedded in `install-config.yaml` by labctl) must include -credentials for MIRROR_REGISTRY_HOST:8443. Without this, cluster nodes cannot pull images redirected -by the IDMS files generated by oc-mirror. +| Aspect | Quay | Docker registry | +|---|---|---| +| Content path | `//` (org prefix) | `/` (flat) | +| MIRROR_REGISTRY | `:443/` | `:8443` (no org) | +| Authentication | Robot account (token) | htpasswd (init user) | +| CA | Quay CA chain | Self-signed CA | +| Setup | Pre-existing Quay | podman run registry:2 | -Run once after initial auth setup, or if the registry password is regenerated: +### Docker Registry Setup ```bash -MREG_PASS=$(cat ~/.coco-pattern/mirror-registry-password) -MREG_AUTH=$(echo -n "init:$MREG_PASS" | base64 -w 0) +# One-time: generate CA, htpasswd, config, and start the container +# (see the Phase 0 section of the historical runbook for the full script) -python3 - <<'EOF' -import json, os +# The MIRROR_REGISTRY does NOT include an org prefix: +export MIRROR_REGISTRY=:8443 -ps_path = os.path.expanduser("~/pull-secret.json") -with open(ps_path) as f: - ps = json.load(f) +# All other configuration is the same, minus the org prefix: +# helmRepoUrl: :8443/validatedpatterns +# image overrides: :8443/ubi9/... +# PATTERN_DISCONNECTED_HOME: :8443/validatedpatterns +# ITMS mirrors: :8443/ (no org) -import base64 -mreg_pass = open(os.path.expanduser("~/.coco-pattern/mirror-registry-password")).read().strip() -mreg_auth = base64.b64encode(f"init:{mreg_pass}".encode()).decode() - -ps["auths"]["${MIRROR_REGISTRY}"] = { - "auth": mreg_auth, - "username": "init", - "password": mreg_pass -} - -with open(ps_path, "w") as f: - json.dump(ps, f, indent=2) - -print("Done. Registries in pull-secret.json:") -for k in ps["auths"]: - print(f" {k}") -EOF -``` +```text -> **When to re-run:** Only if `~/.coco-pattern/mirror-registry-password` changes (password -> regeneration). The htpasswd auth survives `rm -rf ~/local-registry/*` (A-4) — only image -> data is wiped, not auth config. +The Docker registry stores all manifest types (including OCI image indexes +and hybrid manifests that Quay may reject). However, it lacks Quay's +org-based isolation, robot accounts, and API — use Quay where available. diff --git a/airgap/imageset-config-4.22.yaml b/airgap/imageset-config-4.22.yaml index 598af589..9bd9794a 100644 --- a/airgap/imageset-config-4.22.yaml +++ b/airgap/imageset-config-4.22.yaml @@ -1,11 +1,11 @@ --- # ImageSetConfiguration for CoCo Validated Pattern airgap deployment # Target: MIRROR_REGISTRY_HOST:8443 (mirror-registry — sole airgap target, Phase 25.2) -# OCP: 4.22.8 +# OCP: 4.22.17 # Topology: baremetal (full operator set from values-baremetal.yaml) # # Naming convention: imageset-config-.yaml (4.22 covers all 4.22.x patches) -# Patch target: 4.22.8 — update minVersion/maxVersion to change patch target +# Patch target: 4.22.17 — update minVersion/maxVersion to change patch target # # Usage: # oc-mirror -c airgap/imageset-config-4.22.yaml \ @@ -19,8 +19,8 @@ mirror: platform: channels: - name: fast-4.22 - minVersion: "4.22.8" - maxVersion: "4.22.8" + minVersion: "4.22.17" + maxVersion: "4.22.17" type: ocp # Red Hat operators (from values-baremetal.yaml subscriptions) @@ -36,10 +36,15 @@ mirror: - name: trustee-operator channels: - name: stable - minVersion: "1.2.0" - maxVersion: "1.2.0" + minVersion: "1.2.1" + maxVersion: "1.2.1" # GitOps (ArgoCD) — keep aligned with main.gitops.channel in values-global.yaml + # defaultChannel override required: the catalog default is "latest"; filtering + # it out without re-pointing the default fails catalog filtering in oc-mirror v2 + # ("invalid default channel configuration ... a new default channel must be + # configured for this package"). - name: openshift-gitops-operator + defaultChannel: gitops-1.22 channels: - name: gitops-1.22 # External Secrets @@ -71,11 +76,17 @@ mirror: # Certified operators - catalog: registry.redhat.io/redhat/certified-operator-index:v4.22 packages: - - name: gpu-operator-certified - channels: - - name: v26.3 - minVersion: "26.3.0" - maxVersion: "26.3.0" + # GPU operator DISABLED for T1 — matches values-baremetal.yaml (the + # gpu-operator subscription is commented out there: "NVIDIA bundle image + # has oc-mirror signature issue"). The nvcr.io related images + # (dcgm/driver) repeatedly hit copy timeouts, and nothing in T1 consumes + # the bundle. Tranche 2 re-enables the operator at v26.7 (pawelpros), + # which requires its own mirror update per the T022 trigger logic. + # - name: gpu-operator-certified + # channels: + # - name: v26.3 + # minVersion: "26.3.0" + # maxVersion: "26.3.0" # Intel device plugins (for TDX/SGX) — certified, NOT redhat catalog # Requires registries.d/no-sigstore-certified.yaml (cosign .sig manifests missing) - name: intel-device-plugins-operator @@ -114,8 +125,8 @@ mirror: # --- VP Helm OCI charts --- - name: quay.io/validatedpatterns/clustergroup:0.9.58 - name: quay.io/validatedpatterns/acm:0.1.27 - # Minimum selected by the Vault applications (~0.1.10); mirror newer patches before use. - - name: quay.io/validatedpatterns/hashicorp-vault:0.1.10 + # values pin ~0.1.10 resolves the newest 0.1.x — mirror that, not the floor + - name: quay.io/validatedpatterns/hashicorp-vault:0.1.11 - name: quay.io/validatedpatterns/openshift-external-secrets:0.0.4 - name: quay.io/validatedpatterns/trustee:0.10.2 - name: quay.io/validatedpatterns/sandboxed-containers:0.2.1 @@ -128,7 +139,10 @@ mirror: - name: quay.io/validatedpatterns/utility-container:latest - name: quay.io/validatedpatterns/imperative-container:latest - name: quay.io/validatedpatterns/imperative-container:v1 - - name: quay.io/validatedpatterns/pattern-ui-catalog:stable-v1 + # pattern-ui-catalog deliberately NOT mirrored: non-essential (console-tab + # catalog content; nothing in the pattern or the operator CSV deploys it) + # and its OCI-index-wrapping-Docker-children manifest is rejected by Quay. + # See coco-gsd specs/003-quay-sole-retest. # --- UBI minimal (ArgoCD repo-server init container) --- - name: registry.redhat.io/ubi9/ubi-minimal:latest @@ -149,15 +163,16 @@ mirror: - name: registry.access.redhat.com/ubi9/ubi:9.7-1764578509 # --- CNV guest images (OPTIONAL — CNV-OPTIONAL) --- - # CNV-OPTIONAL: Remove or comment out these entries before merging upstream. - # These images are large (~2 GB each). They enable VM boot from DataSource in connected - # environments. In airgap, they must be mirrored or enableCommonBootImageImport must be - # set to false in the HyperConverged CR (Phase 26 sets it to false by default). - # Re-enable after verifying guest images are present in the mirror registry. - # oc-mirror v2 requires tag or digest — :latest is used here; pin to a specific version if reproducibility matters. - - name: registry.redhat.io/rhel9/rhel-guest-image:latest - - name: registry.redhat.io/rhel8/rhel-guest-image:latest - - name: registry.redhat.io/container-native-virtualization/virtio-win:latest + # CNV-OPTIONAL: DISABLED — these images are large (~2 GB each) and NOT + # consumed by the pattern: Phase 26 sets enableCommonBootImageImport: false + # in the HyperConverged CR, so CNV never imports common boot images. + # Additionally, registry.redhat.io REJECTS :latest for virtio-win + # ("must be pulled with an explicit version") — a hard mirror error, not a + # transient one. Keep commented unless VM boot from DataSource is needed; + # if re-enabling, pin explicit version tags. + # - name: registry.redhat.io/rhel9/rhel-guest-image:latest + # - name: registry.redhat.io/rhel8/rhel-guest-image:latest + # - name: registry.redhat.io/container-native-virtualization/virtio-win:latest # Note: kubeVirtContainer: true (oc-mirror platform flag) is for RHCOS/Hosted Control Planes only. # Do NOT add it to the OCP platform channel for standalone TDX VMs on baremetal — # it is not applicable to this topology and would pull unneeded RHCOS artifacts. diff --git a/scripts/airgap-post-install.sh b/scripts/airgap-post-install.sh index e0788432..6db90b48 100755 --- a/scripts/airgap-post-install.sh +++ b/scripts/airgap-post-install.sh @@ -195,14 +195,20 @@ print(yaml.dump_all(docs, default_flow_style=False)) done # Apply manually maintained ITMS (community-operator-pipeline-prod, intel, hashicorp) + # MIRROR_REGISTRY_HOST placeholders are substituted from $MIRROR_REGISTRY at apply + # time (FR-017: canonical identity is the DNS name, so the mirrors follow the env). local script_dir script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" local manual_itms="${script_dir}/../airgap/itms-manual-mirrors.yaml" if [[ -f "$manual_itms" ]]; then info " Applying manual ITMS: $(basename "$manual_itms") with NeverContactSource policy" - python3 -c " -import sys, yaml -docs = list(yaml.safe_load_all(open('$manual_itms'))) + MIRROR_REGISTRY="${MIRROR_REGISTRY}" python3 -c " +import sys, os, yaml +raw = open('$manual_itms').read() +mirror = os.environ.get('MIRROR_REGISTRY', '') +if mirror and 'MIRROR_REGISTRY_HOST:8443' in raw and mirror != 'MIRROR_REGISTRY_HOST:8443': + raw = raw.replace('MIRROR_REGISTRY_HOST:8443', mirror) +docs = list(yaml.safe_load_all(raw)) for doc in docs: if doc and doc.get('kind') == 'ImageTagMirrorSet': for entry in doc.get('spec', {}).get('imageTagMirrors', []): @@ -677,15 +683,16 @@ SVCEOF fi fi - # Report URLs - local host_ip - host_ip=$(hostname -I 2>/dev/null | awk '{print $1}' || echo "JUMP_HOST_IP") + # Report URLs — use the DNS name (FR-017): the node resolves it via lab DNS + # (the ignition's resolver), so a host address change does not break pulls. + local host_name + host_name=$(hostname -f 2>/dev/null || echo "JUMP_HOST_DNS") info "" info "Git HTTP URLs:" for repo_path in "${repos[@]}"; do local repo_name repo_name=$(basename "$repo_path") - info " http://${host_ip}:${GIT_HTTP_PORT}/${repo_name}.git" + info " http://${host_name}:${GIT_HTTP_PORT}/${repo_name}.git" done } diff --git a/scripts/enable-airgap-secrets.sh b/scripts/enable-airgap-secrets.sh index 83c521e6..2d465fe1 100755 --- a/scripts/enable-airgap-secrets.sh +++ b/scripts/enable-airgap-secrets.sh @@ -1,12 +1,18 @@ #!/usr/bin/env bash # Uncomments the disconnected/airgap-only secrets in a generated -# values-secret file: registryCaCert and bootstrap_secrets -# (mirror-registry-helm-oci). Both are commented out by default in -# values-secret.yaml.template because they only apply to airgap -# (disconnected mirror) deployments and reference local files -# (~/.coco-pattern/mirror-registry-ca-cert.pem, -# ~/.coco-pattern/mirror-registry-password) that don't exist on connected -# deployments. +# values-secret file: registryCaCert, bootstrap_secrets +# (mirror-registry-helm-oci) and tdxCollateral (offline DCAP collateral). +# All are commented out by default in values-secret.yaml.template because +# they only apply to airgap (disconnected mirror) deployments and reference +# local files (~/.coco-pattern/mirror-registry-ca-cert.pem, +# ~/.coco-pattern/mirror-registry-password, +# ~/.coco-pattern/dcap-offline/platform_collaterals.json) that don't exist +# on connected deployments. +# +# It also comments OUT intelPcsApiKey (connected-mode zero-touch PCS +# registration): in airgap mode PCK certificates are managed manually via +# scripts/dcap-pck.py, and load-secrets would otherwise fail on the +# connected-only key file. # # Usage: # ./scripts/enable-airgap-secrets.sh [path-to-values-secret-file] @@ -36,11 +42,29 @@ sed -i.bak ' /AIRGAP-MIRROR-SECRETS-END/b s/^\([[:space:]]*\)#/\1/ } +/# AIRGAP-TDX-COLLATERAL-START/,/# AIRGAP-TDX-COLLATERAL-END/ { + /AIRGAP-TDX-COLLATERAL-START/b + /AIRGAP-TDX-COLLATERAL-END/b + s/^\([[:space:]]*\)#/\1/ +} +' "${VALUES_FILE}" +rm -f "${VALUES_FILE}.bak" + +# Comment out intelPcsApiKey (connected-mode zero-touch registration only). +# In airgap mode PCK certificates are provisioned manually via +# scripts/dcap-pck.py; the connected key file must not be required. +sed -i.bak ' +/^ - name: intelPcsApiKey$/,/^[[:space:]]*$/ { + /^[[:space:]]*$/b + s/^/#/ +} ' "${VALUES_FILE}" rm -f "${VALUES_FILE}.bak" -echo "Uncommented airgap mirror secrets (registryCaCert, bootstrap_secrets) in ${VALUES_FILE}" +echo "Uncommented airgap mirror secrets (registryCaCert, bootstrap_secrets, tdxCollateral) in ${VALUES_FILE}" +echo "Commented out intelPcsApiKey (connected-mode zero-touch registration) in ${VALUES_FILE}" echo echo "Make sure these exist before 'make load-secrets' / 'make load-bootstrap':" echo " ~/.coco-pattern/mirror-registry-ca-cert.pem (make cache-registry-ca)" echo " ~/.coco-pattern/mirror-registry-password (make gen-mirror-helm-secret)" +echo " ~/.coco-pattern/dcap-offline/platform_collaterals.json (make collect-dcap-collateral)" diff --git a/values-secret.yaml.template b/values-secret.yaml.template index 3150c4f7..428f93c7 100644 --- a/values-secret.yaml.template +++ b/values-secret.yaml.template @@ -260,13 +260,16 @@ secrets: # Collected on a connected low-side host via: make collect-dcap-collateral # Loaded into Vault, delivered to KBS via ESO + kbsLocalCertCacheSpec # Refresh when collateral expires (~30-90 days) - # Uncomment after running 'make collect-dcap-collateral': + # Uncomment after running 'make collect-dcap-collateral' (or run + # 'make enable-airgap-secrets' for a disconnected deployment): + # AIRGAP-TDX-COLLATERAL-START #- name: tdxCollateral # vaultPrefixes: # - hub # fields: # - name: platform_collaterals.json # path: ~/.coco-pattern/dcap-offline/platform_collaterals.json + # AIRGAP-TDX-COLLATERAL-END # =================================================================== # Optional - Intel TDX DCAP connected mode (zero-touch platform