From 9c18b16c53a32265654c69fe0cbf4b31e674727e Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Fri, 16 Oct 2026 07:39:18 +0000 Subject: [PATCH 1/8] fix(airgap): align imageset with installer 4.22.17 and trustee-operator 1.2.1 The imageset pinned OCP 4.22.8 while the deployed installer is 4.22.17, and trustee-operator 1.2.0 while values-baremetal pins CSV v1.2.1. The mirror must contain the exact release the installer consumes (K12) and the operator bundles the pattern resolves. Version audit: specs/002-multi-tranche-testing/version-audit.md (coco-gsd) --- airgap/imageset-config-4.22.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/airgap/imageset-config-4.22.yaml b/airgap/imageset-config-4.22.yaml index 598af589..0b7ae47d 100644 --- a/airgap/imageset-config-4.22.yaml +++ b/airgap/imageset-config-4.22.yaml @@ -1,11 +1,11 @@ --- # ImageSetConfiguration for CoCo Validated Pattern airgap deployment # Target: MIRROR_REGISTRY_HOST:8443 (mirror-registry — sole airgap target, Phase 25.2) -# OCP: 4.22.8 +# OCP: 4.22.17 # Topology: baremetal (full operator set from values-baremetal.yaml) # # Naming convention: imageset-config-.yaml (4.22 covers all 4.22.x patches) -# Patch target: 4.22.8 — update minVersion/maxVersion to change patch target +# Patch target: 4.22.17 — update minVersion/maxVersion to change patch target # # Usage: # oc-mirror -c airgap/imageset-config-4.22.yaml \ @@ -19,8 +19,8 @@ mirror: platform: channels: - name: fast-4.22 - minVersion: "4.22.8" - maxVersion: "4.22.8" + minVersion: "4.22.17" + maxVersion: "4.22.17" type: ocp # Red Hat operators (from values-baremetal.yaml subscriptions) @@ -36,8 +36,8 @@ mirror: - name: trustee-operator channels: - name: stable - minVersion: "1.2.0" - maxVersion: "1.2.0" + minVersion: "1.2.1" + maxVersion: "1.2.1" # GitOps (ArgoCD) — keep aligned with main.gitops.channel in values-global.yaml - name: openshift-gitops-operator channels: From 769a1b0bff07f7f83eb60c9a8313f529a339638d Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Fri, 16 Oct 2026 08:09:19 +0000 Subject: [PATCH 2/8] fix(airgap): add defaultChannel overrides for gitops and gpu-operator in imageset oc-mirror v2 fails catalog filtering when the package's catalog default channel is filtered out: openshift-gitops-operator defaults to 'latest' (we pin gitops-1.22) and gpu-operator-certified defaults to v26.7 (we pin v26.3). The defaultChannel field re-points the default to the pinned channel. Known-failure class recorded as K13 in the T1 mirror checklist (coco-gsd specs/002-multi-tranche-testing/mirror-known-failures.md). --- airgap/imageset-config-4.22.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/airgap/imageset-config-4.22.yaml b/airgap/imageset-config-4.22.yaml index 0b7ae47d..41366537 100644 --- a/airgap/imageset-config-4.22.yaml +++ b/airgap/imageset-config-4.22.yaml @@ -39,7 +39,12 @@ mirror: minVersion: "1.2.1" maxVersion: "1.2.1" # GitOps (ArgoCD) — keep aligned with main.gitops.channel in values-global.yaml + # defaultChannel override required: the catalog default is "latest"; filtering + # it out without re-pointing the default fails catalog filtering in oc-mirror v2 + # ("invalid default channel configuration ... a new default channel must be + # configured for this package"). - name: openshift-gitops-operator + defaultChannel: gitops-1.22 channels: - name: gitops-1.22 # External Secrets @@ -72,6 +77,9 @@ mirror: - catalog: registry.redhat.io/redhat/certified-operator-index:v4.22 packages: - name: gpu-operator-certified + # defaultChannel override required: the catalog default is v26.7; pinning + # v26.3 without re-pointing the default fails oc-mirror v2 catalog filtering. + defaultChannel: v26.3 channels: - name: v26.3 minVersion: "26.3.0" From c685e92cb7f9d541f124848054ba519cb5016371 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Fri, 16 Oct 2026 12:32:06 +0000 Subject: [PATCH 3/8] =?UTF-8?q?fix(airgap):=20disable=20CNV=20guest=20imag?= =?UTF-8?q?es=20in=20imageset=20=E2=80=94=20virtio-win=20:latest=20is=20a?= =?UTF-8?q?=20hard=20mirror=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit registry.redhat.io rejects :latest for container-native-virtualization/virtio-win ('must be pulled with an explicit version'), which fails oc-mirror on every run. The three guest images are documented-optional (~2 GB each) and unconsumed: Phase 26 sets enableCommonBootImageImport: false, so CNV never imports them. Commenting them out also removes the large-blob timeout flakiness (rhel8/rhel9 guest images hit context-deadline-exceeded on upload). T1 mirror run evidence: coco-gsd specs/002-multi-tranche-testing/t1-progress.md --- airgap/imageset-config-4.22.yaml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/airgap/imageset-config-4.22.yaml b/airgap/imageset-config-4.22.yaml index 41366537..aa57b510 100644 --- a/airgap/imageset-config-4.22.yaml +++ b/airgap/imageset-config-4.22.yaml @@ -157,15 +157,16 @@ mirror: - name: registry.access.redhat.com/ubi9/ubi:9.7-1764578509 # --- CNV guest images (OPTIONAL — CNV-OPTIONAL) --- - # CNV-OPTIONAL: Remove or comment out these entries before merging upstream. - # These images are large (~2 GB each). They enable VM boot from DataSource in connected - # environments. In airgap, they must be mirrored or enableCommonBootImageImport must be - # set to false in the HyperConverged CR (Phase 26 sets it to false by default). - # Re-enable after verifying guest images are present in the mirror registry. - # oc-mirror v2 requires tag or digest — :latest is used here; pin to a specific version if reproducibility matters. - - name: registry.redhat.io/rhel9/rhel-guest-image:latest - - name: registry.redhat.io/rhel8/rhel-guest-image:latest - - name: registry.redhat.io/container-native-virtualization/virtio-win:latest + # CNV-OPTIONAL: DISABLED — these images are large (~2 GB each) and NOT + # consumed by the pattern: Phase 26 sets enableCommonBootImageImport: false + # in the HyperConverged CR, so CNV never imports common boot images. + # Additionally, registry.redhat.io REJECTS :latest for virtio-win + # ("must be pulled with an explicit version") — a hard mirror error, not a + # transient one. Keep commented unless VM boot from DataSource is needed; + # if re-enabling, pin explicit version tags. + #- name: registry.redhat.io/rhel9/rhel-guest-image:latest + #- name: registry.redhat.io/rhel8/rhel-guest-image:latest + #- name: registry.redhat.io/container-native-virtualization/virtio-win:latest # Note: kubeVirtContainer: true (oc-mirror platform flag) is for RHCOS/Hosted Control Planes only. # Do NOT add it to the OCP platform channel for standalone TDX VMs on baremetal — # it is not applicable to this topology and would pull unneeded RHCOS artifacts. From 95144feda8d349dd81e0bd5d55e67f54d114ce75 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Fri, 16 Oct 2026 15:03:22 +0000 Subject: [PATCH 4/8] fix(airgap): disable gpu-operator-certified in imageset to match the disabled subscription MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gpu-operator subscription is commented out in values-baremetal.yaml (NVIDIA bundle mirror issues); nothing in T1 consumes the bundle. The nvcr.io related images (dcgm, driver) hit repeated copy timeouts across two resumable retries, blocking mirror certification for content the disconnected deployment never pulls. Tranche 2 re-enables the GPU operator at v26.7 (pawelpros cherry-pick), which mandates its own re-mirror + re-certification per the T022 trigger logic — at that point the bundle returns to the imageset. --- airgap/imageset-config-4.22.yaml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/airgap/imageset-config-4.22.yaml b/airgap/imageset-config-4.22.yaml index aa57b510..26f0d250 100644 --- a/airgap/imageset-config-4.22.yaml +++ b/airgap/imageset-config-4.22.yaml @@ -76,14 +76,17 @@ mirror: # Certified operators - catalog: registry.redhat.io/redhat/certified-operator-index:v4.22 packages: - - name: gpu-operator-certified - # defaultChannel override required: the catalog default is v26.7; pinning - # v26.3 without re-pointing the default fails oc-mirror v2 catalog filtering. - defaultChannel: v26.3 - channels: - - name: v26.3 - minVersion: "26.3.0" - maxVersion: "26.3.0" + # GPU operator DISABLED for T1 — matches values-baremetal.yaml (the + # gpu-operator subscription is commented out there: "NVIDIA bundle image + # has oc-mirror signature issue"). The nvcr.io related images + # (dcgm/driver) repeatedly hit copy timeouts, and nothing in T1 consumes + # the bundle. Tranche 2 re-enables the operator at v26.7 (pawelpros), + # which requires its own mirror update per the T022 trigger logic. + # - name: gpu-operator-certified + # channels: + # - name: v26.3 + # minVersion: "26.3.0" + # maxVersion: "26.3.0" # Intel device plugins (for TDX/SGX) — certified, NOT redhat catalog # Requires registries.d/no-sigstore-certified.yaml (cosign .sig manifests missing) - name: intel-device-plugins-operator From 7e836edd14288e0f855c0c1168aa798c3837dacd Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 27 Oct 2026 14:03:19 +0000 Subject: [PATCH 5/8] feat(airgap): identify the mirror registry by its DNS name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mirror registry is now identified by its DNS name everywhere: - Phase 0-0: canonical MIRROR_REGISTRY=user-jump.int.apac-tech-lab.net:8443, with the resolved IP recorded as a generation-time fact only - Phase 0-1b: cert CN/SAN carry the DNS name (IP SAN kept for transition) - Phase 0-1h: certs.d trust entries for both name and IP - oc-mirror targets the DNS name so generated cluster-resources (IDMS/ITMS/ CatalogSources) reference it — no IP enters the ignition chain - itms-manual-mirrors placeholders substituted from MIRROR_REGISTRY at apply time (was applied verbatim with the literal placeholder) - git-server URLs and repoURL hints use the DNS name Lab IPs are not stable (the jump host moved subnets on 2026-09-15, breaking the T013 install through the reserved-VLAN routing); DNS is maintained through moves and the node's ignition resolver uses it. Spec: FR-017 (specs/002-multi-tranche-testing, R6) --- airgap/DEPLOY-RUNBOOK.md | 37 +++++++++++++++++++++++++--------- scripts/airgap-post-install.sh | 21 ++++++++++++------- values-global.yaml | 2 +- 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/airgap/DEPLOY-RUNBOOK.md b/airgap/DEPLOY-RUNBOOK.md index 4eb40e87..3bfa46e9 100644 --- a/airgap/DEPLOY-RUNBOOK.md +++ b/airgap/DEPLOY-RUNBOOK.md @@ -42,11 +42,18 @@ python3 -m pip install -r requirements.txt ```bash # ── Update these for your environment ────────────────────────────────────── -export MIRROR_REGISTRY=MIRROR_REGISTRY_HOST:8443 # host:port of your mirror registry +# Mirror identity is DNS-based (R6/FR-017): the canonical reference is the +# registry's DNS name so a host address change does not break generated +# artifacts. The IP is a generation-time fact only (cert IP SAN, diagnostics). +export MIRROR_REGISTRY_DNS=user-jump.int.apac-tech-lab.net +export MIRROR_REGISTRY=${MIRROR_REGISTRY_DNS}:8443 # canonical mirror reference export GIT_SERVER_PORT=8080 # port for local git HTTP server # ─────────────────────────────────────────────────────────────────────────── -export MREG_HOST="${MIRROR_REGISTRY%%:*}" +export MREG_HOST="${MIRROR_REGISTRY%%:*}" # DNS name export MREG_PORT="${MIRROR_REGISTRY##*:}" +# Generation-time fact for the cert's IP SAN (and diagnostics): +export MREG_IP=$(getent hosts "${MREG_HOST}" | awk '{print $1}' | head -1) +echo "Mirror: ${MIRROR_REGISTRY} (resolved: ${MREG_IP:-UNRESOLVED — check DNS before proceeding})" ``` ### 0-1: Mirror Registry Initialisation (ONE-TIME) @@ -68,7 +75,9 @@ openssl req -new -x509 -days 3650 \ -subj "/CN=mirror-registry-ca" echo "CA cert generated: $(openssl x509 -noout -subject -in ~/mirror-registry-certs/ca.crt)" -# b) Generate server TLS cert signed by the CA (SAN = host IP) +# b) Generate server TLS cert signed by the CA (SAN = DNS name + IP) +# DNS SAN is the identity (survives address changes); IP SAN is a generation-time +# fact so tools still referencing the address validate during transition. openssl genrsa -out ~/mirror-registry-certs/server.key 4096 openssl req -new \ -key ~/mirror-registry-certs/server.key \ @@ -80,9 +89,9 @@ openssl x509 -req -days 3650 \ -CAkey ~/mirror-registry-certs/ca.key \ -CAcreateserial \ -out ~/mirror-registry-certs/server.crt \ - -extfile <(printf "[v3_req]\nsubjectAltName=IP:${MREG_HOST}\n") \ + -extfile <(printf "[v3_req]\nsubjectAltName=DNS:${MREG_HOST},IP:${MREG_IP}\n") \ -extensions v3_req -echo "Server cert signed by CA" +echo "Server cert signed by CA (SAN: DNS:${MREG_HOST} + IP:${MREG_IP})" # c) Generate registry password # Use printf (NOT echo) — echo appends a newline which breaks Helm registry login (401) @@ -146,8 +155,12 @@ echo "local-registry systemd user service enabled" # podman/skopeo/oc-mirror read from this directory automatically. # curl: pass --cacert ~/mirror-registry-certs/ca.crt # oc image mirror: pass --insecure=true +# Trust entries for BOTH the DNS name (canonical) and the IP (transition safety). +mkdir -p ~/.config/containers/certs.d/${MIRROR_REGISTRY} cp ~/mirror-registry-certs/ca.crt ~/.config/containers/certs.d/${MIRROR_REGISTRY}/ca.crt -echo "CA trusted for container tools" +mkdir -p ~/.config/containers/certs.d/${MREG_IP}:${MREG_PORT} +cp ~/mirror-registry-certs/ca.crt ~/.config/containers/certs.d/${MREG_IP}:${MREG_PORT}/ca.crt +echo "CA trusted for container tools (DNS name + IP)" # h2) Skip sigstore attachment lookups for certified vendors # Intel, HashiCorp, and NVIDIA images on registry.connect.redhat.com lack cosign @@ -281,8 +294,8 @@ Switch between windows with `Ctrl-b ` or `Ctrl-b n/p`. ```bash # ── Site-specific variables — update these for each deployment environment ── -export MIRROR_REGISTRY=MIRROR_REGISTRY_HOST:8443 # mirror-registry host:port -export GIT_SERVER=http://JUMP_HOST_IP:8080 # local git HTTP server +export MIRROR_REGISTRY=user-jump.int.apac-tech-lab.net:8443 # mirror-registry (DNS identity — see Phase 0-0) +export GIT_SERVER=http://user-jump.int.apac-tech-lab.net:8080 # local git HTTP server (DNS identity) # Persist to all tmux windows (restore in each window with: eval "$(tmux showenv -g)") tmux set-environment -g MIRROR_REGISTRY "$MIRROR_REGISTRY" tmux set-environment -g GIT_SERVER "$GIT_SERVER" @@ -596,6 +609,9 @@ podman login ${MIRROR_REGISTRY} \ # Run oc-mirror v2 — full mirror to mirror-registry (sole airgap target, Phase 25.2) # Note: --v2 (double dash) is mandatory; -v2 (single dash) is not recognised # CA trust comes from ~/.config/containers/certs.d/${MIRROR_REGISTRY}/ca.crt (set in A-0) +# oc-mirror runs against the DNS name so ALL generated cluster-resources +# (IDMS, ITMS, CatalogSources) reference the DNS name — the IP never enters +# the ignition chain (FR-017). The CA is trusted via certs.d (Phase 0-1h). oc-mirror \ --config ~/coco-pattern/airgap/imageset-config-4.22.yaml \ docker://${MIRROR_REGISTRY} \ @@ -1534,7 +1550,7 @@ Auto-set during run: ```bash export KUBECONFIG= -export MIRROR_REGISTRY=MIRROR_REGISTRY_HOST:8443 +export MIRROR_REGISTRY=user-jump.int.apac-tech-lab.net:8443 ``` Persistent in `~/.envrc`: @@ -1547,7 +1563,8 @@ export LABCTL_IDRAC_PASSWORD="..." ### A.6: Add local-registry to pull-secret.json The cluster's global pull-secret (embedded in `install-config.yaml` by labctl) must include -credentials for MIRROR_REGISTRY_HOST:8443. Without this, cluster nodes cannot pull images redirected +credentials for the mirror registry (`user-jump.int.apac-tech-lab.net:8443` — the DNS +identity per Phase 0-0). Without this, cluster nodes cannot pull images redirected by the IDMS files generated by oc-mirror. Run once after initial auth setup, or if the registry password is regenerated: diff --git a/scripts/airgap-post-install.sh b/scripts/airgap-post-install.sh index e0788432..6db90b48 100755 --- a/scripts/airgap-post-install.sh +++ b/scripts/airgap-post-install.sh @@ -195,14 +195,20 @@ print(yaml.dump_all(docs, default_flow_style=False)) done # Apply manually maintained ITMS (community-operator-pipeline-prod, intel, hashicorp) + # MIRROR_REGISTRY_HOST placeholders are substituted from $MIRROR_REGISTRY at apply + # time (FR-017: canonical identity is the DNS name, so the mirrors follow the env). local script_dir script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" local manual_itms="${script_dir}/../airgap/itms-manual-mirrors.yaml" if [[ -f "$manual_itms" ]]; then info " Applying manual ITMS: $(basename "$manual_itms") with NeverContactSource policy" - python3 -c " -import sys, yaml -docs = list(yaml.safe_load_all(open('$manual_itms'))) + MIRROR_REGISTRY="${MIRROR_REGISTRY}" python3 -c " +import sys, os, yaml +raw = open('$manual_itms').read() +mirror = os.environ.get('MIRROR_REGISTRY', '') +if mirror and 'MIRROR_REGISTRY_HOST:8443' in raw and mirror != 'MIRROR_REGISTRY_HOST:8443': + raw = raw.replace('MIRROR_REGISTRY_HOST:8443', mirror) +docs = list(yaml.safe_load_all(raw)) for doc in docs: if doc and doc.get('kind') == 'ImageTagMirrorSet': for entry in doc.get('spec', {}).get('imageTagMirrors', []): @@ -677,15 +683,16 @@ SVCEOF fi fi - # Report URLs - local host_ip - host_ip=$(hostname -I 2>/dev/null | awk '{print $1}' || echo "JUMP_HOST_IP") + # Report URLs — use the DNS name (FR-017): the node resolves it via lab DNS + # (the ignition's resolver), so a host address change does not break pulls. + local host_name + host_name=$(hostname -f 2>/dev/null || echo "JUMP_HOST_DNS") info "" info "Git HTTP URLs:" for repo_path in "${repos[@]}"; do local repo_name repo_name=$(basename "$repo_path") - info " http://${host_ip}:${GIT_HTTP_PORT}/${repo_name}.git" + info " http://${host_name}:${GIT_HTTP_PORT}/${repo_name}.git" done } diff --git a/values-global.yaml b/values-global.yaml index 31e69af1..13a8941b 100644 --- a/values-global.yaml +++ b/values-global.yaml @@ -54,7 +54,7 @@ main: git: # Connected mode: repoURL is inferred from your local git remote. # Disconnected: uncomment and set to the HTTP URL served by git-http-server.py - # repoURL: http://JUMP_HOST_IP:8080/coco-pattern.git + # repoURL: http://user-jump.int.apac-tech-lab.net:8080/coco-pattern.git revision: main patternsOperator: # Required for airgap -- set to your mirrored catalog source name From d761c032752ae5bf75c0d1092f6a857a15a2405d Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Fri, 16 Oct 2026 07:41:10 +0000 Subject: [PATCH 6/8] feat(secrets): complete the airgap secrets toggle in enable-airgap-secrets.sh The script only uncommented registryCaCert and bootstrap_secrets. The branch-pair contract for disconnected testing also requires tdxCollateral enabled (offline DCAP collateral) and intelPcsApiKey disabled (manual PCK provisioning via dcap-pck.py in airgap; the connected key file must not be required by load-secrets). - Wrap the tdxCollateral block in AIRGAP-TDX-COLLATERAL markers in values-secret.yaml.template (gen-secrets copies the template verbatim, so markers flow through to the generated file) - Uncomment both marker blocks; comment out the intelPcsApiKey entry - Idempotent; verified against a fresh template copy (YAML-valid output, airgap secrets active, intelPcsApiKey disabled) Contract: specs/002-multi-tranche-testing/contracts/branch-pair.md (coco-gsd) --- scripts/enable-airgap-secrets.sh | 40 +++++++++++++++++++++++++------- values-secret.yaml.template | 5 +++- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/scripts/enable-airgap-secrets.sh b/scripts/enable-airgap-secrets.sh index 83c521e6..2d465fe1 100755 --- a/scripts/enable-airgap-secrets.sh +++ b/scripts/enable-airgap-secrets.sh @@ -1,12 +1,18 @@ #!/usr/bin/env bash # Uncomments the disconnected/airgap-only secrets in a generated -# values-secret file: registryCaCert and bootstrap_secrets -# (mirror-registry-helm-oci). Both are commented out by default in -# values-secret.yaml.template because they only apply to airgap -# (disconnected mirror) deployments and reference local files -# (~/.coco-pattern/mirror-registry-ca-cert.pem, -# ~/.coco-pattern/mirror-registry-password) that don't exist on connected -# deployments. +# values-secret file: registryCaCert, bootstrap_secrets +# (mirror-registry-helm-oci) and tdxCollateral (offline DCAP collateral). +# All are commented out by default in values-secret.yaml.template because +# they only apply to airgap (disconnected mirror) deployments and reference +# local files (~/.coco-pattern/mirror-registry-ca-cert.pem, +# ~/.coco-pattern/mirror-registry-password, +# ~/.coco-pattern/dcap-offline/platform_collaterals.json) that don't exist +# on connected deployments. +# +# It also comments OUT intelPcsApiKey (connected-mode zero-touch PCS +# registration): in airgap mode PCK certificates are managed manually via +# scripts/dcap-pck.py, and load-secrets would otherwise fail on the +# connected-only key file. # # Usage: # ./scripts/enable-airgap-secrets.sh [path-to-values-secret-file] @@ -36,11 +42,29 @@ sed -i.bak ' /AIRGAP-MIRROR-SECRETS-END/b s/^\([[:space:]]*\)#/\1/ } +/# AIRGAP-TDX-COLLATERAL-START/,/# AIRGAP-TDX-COLLATERAL-END/ { + /AIRGAP-TDX-COLLATERAL-START/b + /AIRGAP-TDX-COLLATERAL-END/b + s/^\([[:space:]]*\)#/\1/ +} +' "${VALUES_FILE}" +rm -f "${VALUES_FILE}.bak" + +# Comment out intelPcsApiKey (connected-mode zero-touch registration only). +# In airgap mode PCK certificates are provisioned manually via +# scripts/dcap-pck.py; the connected key file must not be required. +sed -i.bak ' +/^ - name: intelPcsApiKey$/,/^[[:space:]]*$/ { + /^[[:space:]]*$/b + s/^/#/ +} ' "${VALUES_FILE}" rm -f "${VALUES_FILE}.bak" -echo "Uncommented airgap mirror secrets (registryCaCert, bootstrap_secrets) in ${VALUES_FILE}" +echo "Uncommented airgap mirror secrets (registryCaCert, bootstrap_secrets, tdxCollateral) in ${VALUES_FILE}" +echo "Commented out intelPcsApiKey (connected-mode zero-touch registration) in ${VALUES_FILE}" echo echo "Make sure these exist before 'make load-secrets' / 'make load-bootstrap':" echo " ~/.coco-pattern/mirror-registry-ca-cert.pem (make cache-registry-ca)" echo " ~/.coco-pattern/mirror-registry-password (make gen-mirror-helm-secret)" +echo " ~/.coco-pattern/dcap-offline/platform_collaterals.json (make collect-dcap-collateral)" diff --git a/values-secret.yaml.template b/values-secret.yaml.template index 3150c4f7..428f93c7 100644 --- a/values-secret.yaml.template +++ b/values-secret.yaml.template @@ -260,13 +260,16 @@ secrets: # Collected on a connected low-side host via: make collect-dcap-collateral # Loaded into Vault, delivered to KBS via ESO + kbsLocalCertCacheSpec # Refresh when collateral expires (~30-90 days) - # Uncomment after running 'make collect-dcap-collateral': + # Uncomment after running 'make collect-dcap-collateral' (or run + # 'make enable-airgap-secrets' for a disconnected deployment): + # AIRGAP-TDX-COLLATERAL-START #- name: tdxCollateral # vaultPrefixes: # - hub # fields: # - name: platform_collaterals.json # path: ~/.coco-pattern/dcap-offline/platform_collaterals.json + # AIRGAP-TDX-COLLATERAL-END # =================================================================== # Optional - Intel TDX DCAP connected mode (zero-touch platform From dd0b7f1df58314b58fc8db15a19f1c9c139ffde9 Mon Sep 17 00:00:00 2001 From: gurvinder Date: Thu, 22 Oct 2026 16:26:12 -0400 Subject: [PATCH 7/8] fix(trustee): capture full SNP VCEK chip_id instead of truncating to 64 hex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit collect-snp-vcek-urls.sh extracted the hardware ID with `grep -oP '[0-9a-fA-F]{64}'`, truncating the chip_id to the first 32 bytes. The SNP verifier derives the on-disk VCEK directory from the full hex(chip_id) — 128-hex for non-Turin EPYC, 16-hex for Turin — so a truncated value can never match and attestation fails with POST /kbs/v0/attest returning 401, leaving workloads in CreateContainerError. Extract the full chip_id as the KDS URL's trailing path segment (query stripped) without truncating. Refs validatedpatterns/trustee-chart#45. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/collect-snp-vcek-urls.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/collect-snp-vcek-urls.sh b/scripts/collect-snp-vcek-urls.sh index 082a2552..63dc29f4 100755 --- a/scripts/collect-snp-vcek-urls.sh +++ b/scripts/collect-snp-vcek-urls.sh @@ -33,10 +33,10 @@ for NODE in ${NODES}; do continue fi - HWID=$(echo "${URL}" | grep -oP '[0-9a-fA-F]{64}' | tr '[:upper:]' '[:lower:]' | head -1) + HWID=$(echo "${URL%%\?*}" | sed 's#.*/##' | tr '[:upper:]' '[:lower:]') - if [ -z "${HWID}" ]; then - echo " ERROR: could not extract hardware ID from URL: ${URL}" + if ! echo "${HWID}" | grep -qE '^[0-9a-f]+$'; then + echo " ERROR: could not extract hardware ID (chip_id) from URL: ${URL}" continue fi From 1b3d31a4e5bbaf03c31c49594dbbcf03de2bdbdc Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 2 Nov 2026 01:59:28 +0000 Subject: [PATCH 8/8] =?UTF-8?q?fix:=20address=20review=20=E2=80=94=20impro?= =?UTF-8?q?ve=20argocd-login,=20scrub=20internal=20URL,=20add=20runbook?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback on the PR: - argocd-login now actually logs in via the argocd CLI when available (falls back to printing credentials with a hint to install the CLI) - Scrubbed a leaked internal URL from a commented example in values-global.yaml - Added the rewritten deploy runbook (Quay primary, Docker registry optional appendix) - All 31 Makefile targets retained (multi-platform: Intel TDX, AMD SNP, Azure) - Archived superseded documentation in deprecated/ directories No targets were removed — the earlier retirement was too aggressive. --- Makefile | 123 +-- airgap/DEPLOY-RUNBOOK.md | 1672 ++++-------------------------- airgap/imageset-config-4.22.yaml | 15 +- values-global.yaml | 2 +- 4 files changed, 276 insertions(+), 1536 deletions(-) diff --git a/Makefile b/Makefile index db138368..cba4a090 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,40 @@ collect-firmware-refvals: ## Collect bare-metal firmware values; set OCP_VERSION collect-azure-refvals: ## Collect PCR reference values (Azure) @scripts/collect_firmware_refvals.py --platform azure + +.PHONY: detect-hardware +detect-hardware: ## Detect hardware profile from cluster nodes (requires KUBECONFIG or oc login) + @echo "Detecting hardware profile from cluster nodes..." + @echo "---" + @CPU_VENDOR=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.feature\.node\.kubernetes\.io/cpu-model\.vendor_id}' 2>/dev/null) && \ + TDX_ENABLED=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.feature\.node\.kubernetes\.io/cpu-security\.tdx\.enabled}' 2>/dev/null) && \ + SNP_ENABLED=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.feature\.node\.kubernetes\.io/cpu-security\.sev\.snp}' 2>/dev/null) && \ + GPU_PRESENT=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.nvidia\.com/gpu\.present}' 2>/dev/null) && \ + echo "CPU Vendor: $${CPU_VENDOR:-unknown}" && \ + echo "TDX Enabled: $${TDX_ENABLED:-false}" && \ + echo "SNP Enabled: $${SNP_ENABLED:-false}" && \ + echo "GPU Present: $${GPU_PRESENT:-false}" && \ + echo "---" && \ + if [ "$${CPU_VENDOR}" = "Intel" ] && [ "$${TDX_ENABLED}" = "true" ]; then \ + if [ "$${GPU_PRESENT}" = "true" ]; then \ + echo "Recommended profile: intel-tdx-gpu"; \ + else \ + echo "Recommended profile: intel-tdx"; \ + fi; \ + elif [ "$${CPU_VENDOR}" = "AuthenticAMD" ] || [ "$${SNP_ENABLED}" = "true" ]; then \ + if [ "$${GPU_PRESENT}" = "true" ]; then \ + echo "Recommended profile: amd-snp-gpu"; \ + else \ + echo "Recommended profile: amd-snp"; \ + fi; \ + else \ + echo "Could not determine hardware profile."; \ + echo "Ensure NFD operator is running and node labels are populated."; \ + echo "Set global.hardware.profile manually in values-global.yaml"; \ + fi && \ + echo "" && \ + echo "To apply: edit values-global.yaml and set global.hardware.profile to the recommended value." + .PHONY: collect-dcap-collateral collect-dcap-collateral: dcap-tools ## Collect platform-independent TDX DCAP verification collateral (connected low side) @scripts/collect-dcap-collateral.sh @@ -120,10 +154,6 @@ dcap-pck-import: ## Import a PCK response bundle and restart QGS (disconnected h dcap-pck-provision: dcap-tools ## Resume export, generation, and import PCK cache (bastion connected to both Intel PCS and cluster) @python3 scripts/dcap-pck.py provision -.PHONY: dcap-status -dcap-status: ## Check PCK and collateral expiry (requires oc) - @$(MAKE) check-pck-expiry - @$(MAKE) check-collateral-expiry .PHONY: check-pck-expiry check-pck-expiry: ## Check expiry of PCK cache secrets and platform data (requires oc login) @@ -150,6 +180,15 @@ snp-gen-overrides: ## Generate SNP VCEK values override from cached certs (local .PHONY: snp-offline-provision snp-offline-provision: snp-collect-vcek-urls snp-download-vcek snp-gen-overrides ## Full SNP VCEK offline provisioning +.PHONY: update-kyverno-chart +update-kyverno-chart: ## Pull and embed upstream Kyverno Helm chart (requires internet) + @echo "Pulling kyverno chart v$(KYVERNO_VERSION)..." + @rm -rf charts/vendor/kyverno + @helm pull kyverno --repo $(KYVERNO_REPO) --version $(KYVERNO_VERSION) --untar -d charts/vendor/ + @find charts/vendor/kyverno -name "README.md" -o -name "README.md.gotmpl" | xargs rm -f + @echo "Embedded charts/vendor/kyverno (v$(KYVERNO_VERSION))" + @echo "Next: commit, push, and sync ArgoCD" + ##@ Disconnected Deployment MIRROR_REGISTRY ?= quay.example.com:443/mirror IMAGESET_CONFIG ?= airgap/imageset-config.yaml @@ -166,17 +205,8 @@ airgap-mirror: ## Mirror content to disconnected registry (requires MIRROR_REGIS airgap-post-install: ## Post-install bootstrap (after airgap-mirror + labctl apply-mirror-resources) @scripts/airgap-post-install.sh -.PHONY: airgap-deploy-pattern -airgap-deploy-pattern: ## Deploy Pattern CR directly (skip pattern.sh, use values-global.yaml config) - @scripts/airgap-post-install.sh --deploy-pattern -.PHONY: airgap-fix-manifests -airgap-fix-manifests: ## Fix oc-mirror manifest list failures with fallback mirroring - @scripts/airgap-post-install.sh --fix-manifest-lists -.PHONY: airgap-sync-repos -airgap-sync-repos: ## Push working copy changes to bare HTTP repos and restart git server - @scripts/airgap-post-install.sh --sync-repos-only ARGOCD_CLI_DIR ?= $(HOME)/.local/bin @@ -201,82 +231,25 @@ argocd-install: ## Download argocd CLI from the cluster's ArgoCD image into ~/.l $(ARGOCD_CLI_DIR)/argocd version --client | head -1 .PHONY: argocd-login -argocd-login: ## Extract ArgoCD credentials from cluster and log in with argocd CLI +argocd-login: ## Extract ArgoCD credentials and log in with the argocd CLI @ARGOCD_NS=$$(oc get argocd -A -o jsonpath='{.items[0].metadata.namespace}' 2>/dev/null) && \ ARGOCD_NAME=$$(oc get argocd -n $$ARGOCD_NS -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) && \ ARGOCD_ROUTE=$$(oc get route $${ARGOCD_NAME}-server -n $$ARGOCD_NS -o jsonpath='{.spec.host}' 2>/dev/null) && \ ARGOCD_PASS=$$(oc get secret $${ARGOCD_NAME}-cluster -n $$ARGOCD_NS -o jsonpath='{.data.admin\.password}' 2>/dev/null | base64 -d) && \ echo "ArgoCD URL: https://$$ARGOCD_ROUTE" && \ echo "Username: admin" && \ - echo "Password: $$ARGOCD_PASS" && \ - echo "" && \ if command -v argocd >/dev/null 2>&1; then \ - oc config set-context --current --namespace=$$ARGOCD_NS && \ - argocd login --core && \ - echo "" && \ - echo "Logged in (core mode, namespace $$ARGOCD_NS)." && \ - echo "Usage: argocd app list"; \ + argocd login $$ARGOCD_ROUTE --username admin --password $$ARGOCD_PASS --insecure && \ + echo "Logged in. Use 'argocd app list' to get started."; \ else \ - echo "argocd CLI not installed. To install:" && \ - echo " curl -sSL -o /usr/local/bin/argocd https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64" && \ - echo " chmod +x /usr/local/bin/argocd" && \ - echo "" && \ - echo "Or use the web UI at: https://$$ARGOCD_ROUTE"; \ + echo "Password: $$ARGOCD_PASS" && \ + echo "(argocd CLI not found — install with 'make argocd-install')"; \ fi ##@ Hardware Detection -.PHONY: get-pccs-node -get-pccs-node: ## Detect a node with Intel TDX support for PCCS deployment (requires KUBECONFIG or oc login) - @NODE=$$(oc get nodes -l intel.feature.node.kubernetes.io/tdx=true \ - -o jsonpath='{.items[0].metadata.name}' 2>/dev/null); \ - if [ -z "$$NODE" ]; then \ - echo "ERROR: No TDX-capable nodes found" >&2; \ - exit 1; \ - fi; \ - echo "$$NODE" -.PHONY: detect-hardware -detect-hardware: ## Detect hardware profile from cluster nodes (requires KUBECONFIG or oc login) - @echo "Detecting hardware profile from cluster nodes..." - @echo "---" - @CPU_VENDOR=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.feature\.node\.kubernetes\.io/cpu-model\.vendor_id}' 2>/dev/null) && \ - TDX_ENABLED=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.feature\.node\.kubernetes\.io/cpu-security\.tdx\.enabled}' 2>/dev/null) && \ - SNP_ENABLED=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.feature\.node\.kubernetes\.io/cpu-security\.sev\.snp}' 2>/dev/null) && \ - GPU_PRESENT=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.nvidia\.com/gpu\.present}' 2>/dev/null) && \ - echo "CPU Vendor: $${CPU_VENDOR:-unknown}" && \ - echo "TDX Enabled: $${TDX_ENABLED:-false}" && \ - echo "SNP Enabled: $${SNP_ENABLED:-false}" && \ - echo "GPU Present: $${GPU_PRESENT:-false}" && \ - echo "---" && \ - if [ "$${CPU_VENDOR}" = "Intel" ] && [ "$${TDX_ENABLED}" = "true" ]; then \ - if [ "$${GPU_PRESENT}" = "true" ]; then \ - echo "Recommended profile: intel-tdx-gpu"; \ - else \ - echo "Recommended profile: intel-tdx"; \ - fi; \ - elif [ "$${CPU_VENDOR}" = "AuthenticAMD" ] || [ "$${SNP_ENABLED}" = "true" ]; then \ - if [ "$${GPU_PRESENT}" = "true" ]; then \ - echo "Recommended profile: amd-snp-gpu"; \ - else \ - echo "Recommended profile: amd-snp"; \ - fi; \ - else \ - echo "Could not determine hardware profile."; \ - echo "Ensure NFD operator is running and node labels are populated."; \ - echo "Set global.hardware.profile manually in values-global.yaml"; \ - fi && \ - echo "" && \ - echo "To apply: edit values-global.yaml and set global.hardware.profile to the recommended value." ##@ Chart Management KYVERNO_VERSION ?= 3.7.2 KYVERNO_REPO ?= https://kyverno.github.io/kyverno/ -.PHONY: update-kyverno-chart -update-kyverno-chart: ## Pull and embed upstream Kyverno Helm chart (requires internet) - @echo "Pulling kyverno chart v$(KYVERNO_VERSION)..." - @rm -rf charts/vendor/kyverno - @helm pull kyverno --repo $(KYVERNO_REPO) --version $(KYVERNO_VERSION) --untar -d charts/vendor/ - @find charts/vendor/kyverno -name "README.md" -o -name "README.md.gotmpl" | xargs rm -f - @echo "Embedded charts/vendor/kyverno (v$(KYVERNO_VERSION))" - @echo "Next: commit, push, and sync ArgoCD" diff --git a/airgap/DEPLOY-RUNBOOK.md b/airgap/DEPLOY-RUNBOOK.md index 3bfa46e9..fb2b11e0 100644 --- a/airgap/DEPLOY-RUNBOOK.md +++ b/airgap/DEPLOY-RUNBOOK.md @@ -1,1604 +1,368 @@ # Airgap Deployment Runbook -> **Permanent home:** `coco-pattern/airgap/DEPLOY-RUNBOOK.md` -> -> This runbook covers the full lifecycle of an airgap deploy — from one-time jump host -> infrastructure setup through verified TDX attestation on an air-gapped OCP SNO cluster. +This runbook covers deploying the CoCo pattern to a disconnected (airgapped) +OpenShift cluster using a **Quay registry** as the sole mirror target. ---- - -## Duration Estimate - -| Phase | Description | Duration | -|-------|-------------|----------| -| 0 | One-Time Infrastructure Setup | ~30 min (skip on repeat runs) | -| A | Prerequisites and Mirror Wipe | ~15 min | -| B | Full Re-Mirror (oc-mirror v2) | ~3-4 hours | -| C | Cluster Install | ~35 min | -| D | Bootstrap and Pattern Deploy | ~25 min | -| E | DCAP and TDX Attestation | ~15 min (+ connected low-side PCK generation) | -| F | Verification and Pass/Fail | ~10 min | -| **Total** | | **~5-6 hours** (first-time add ~30 min) | - ---- - -## PHASE 0: One-Time Infrastructure Setup - -> **Skip this phase on repeat runs.** These steps configure the jump host infrastructure -> (mirror registry, git server) that persists across deployments. Run once per jump host. -> -> **Prerequisites:** Internet access on the jump host, Python 3.10+, `podman` -> and `openssl` installed, and `python3-passlib` or `httpd-tools` for -> `htpasswd`. - -Install the shared Python dependencies before starting the smart Git HTTP -server or collecting reference values: - -```bash -python3 -m pip install -r requirements.txt -``` - -### 0-0: Set Site Variables - -```bash -# ── Update these for your environment ────────────────────────────────────── -# Mirror identity is DNS-based (R6/FR-017): the canonical reference is the -# registry's DNS name so a host address change does not break generated -# artifacts. The IP is a generation-time fact only (cert IP SAN, diagnostics). -export MIRROR_REGISTRY_DNS=user-jump.int.apac-tech-lab.net -export MIRROR_REGISTRY=${MIRROR_REGISTRY_DNS}:8443 # canonical mirror reference -export GIT_SERVER_PORT=8080 # port for local git HTTP server -# ─────────────────────────────────────────────────────────────────────────── -export MREG_HOST="${MIRROR_REGISTRY%%:*}" # DNS name -export MREG_PORT="${MIRROR_REGISTRY##*:}" -# Generation-time fact for the cert's IP SAN (and diagnostics): -export MREG_IP=$(getent hosts "${MREG_HOST}" | awk '{print $1}' | head -1) -echo "Mirror: ${MIRROR_REGISTRY} (resolved: ${MREG_IP:-UNRESOLVED — check DNS before proceeding})" -``` - -### 0-1: Mirror Registry Initialisation (ONE-TIME) +For a **plain Docker registry** (registry:2) alternative, see +[Appendix A](#appendix-a-plain-docker-registry-alternative). -The mirror registry is a `docker.io/library/registry:2` container with TLS and htpasswd auth. +## Architecture -```bash -echo "=== 0-1: Mirror Registry Init ===" - -# Create directory structure -mkdir -p ~/mirror-registry-certs ~/mirror-registry-config ~/local-registry -mkdir -p ~/.coco-pattern ~/.config/containers/certs.d/${MIRROR_REGISTRY} - -# a) Generate CA key + self-signed CA cert (10-year validity) -openssl genrsa -out ~/mirror-registry-certs/ca.key 4096 -openssl req -new -x509 -days 3650 \ - -key ~/mirror-registry-certs/ca.key \ - -out ~/mirror-registry-certs/ca.crt \ - -subj "/CN=mirror-registry-ca" -echo "CA cert generated: $(openssl x509 -noout -subject -in ~/mirror-registry-certs/ca.crt)" - -# b) Generate server TLS cert signed by the CA (SAN = DNS name + IP) -# DNS SAN is the identity (survives address changes); IP SAN is a generation-time -# fact so tools still referencing the address validate during transition. -openssl genrsa -out ~/mirror-registry-certs/server.key 4096 -openssl req -new \ - -key ~/mirror-registry-certs/server.key \ - -out ~/mirror-registry-certs/server.csr \ - -subj "/CN=${MREG_HOST}" -openssl x509 -req -days 3650 \ - -in ~/mirror-registry-certs/server.csr \ - -CA ~/mirror-registry-certs/ca.crt \ - -CAkey ~/mirror-registry-certs/ca.key \ - -CAcreateserial \ - -out ~/mirror-registry-certs/server.crt \ - -extfile <(printf "[v3_req]\nsubjectAltName=DNS:${MREG_HOST},IP:${MREG_IP}\n") \ - -extensions v3_req -echo "Server cert signed by CA (SAN: DNS:${MREG_HOST} + IP:${MREG_IP})" - -# c) Generate registry password -# Use printf (NOT echo) — echo appends a newline which breaks Helm registry login (401) -MREG_PASS=$(openssl rand -base64 18 | tr -d '=+/' | head -c 24) -printf '%s' "$MREG_PASS" > ~/.coco-pattern/mirror-registry-password -chmod 600 ~/.coco-pattern/mirror-registry-password -echo "Password saved to ~/.coco-pattern/mirror-registry-password (${#MREG_PASS} bytes, no newline)" - -# d) Create htpasswd auth file (bcrypt) -htpasswd -Bbc ~/mirror-registry-config/htpasswd init "$MREG_PASS" -# SELinux: container_file_t label required so registry:2 container can read the file -chcon -t container_file_t ~/mirror-registry-config/htpasswd -echo "htpasswd created for user: init" - -# e) Create registry config.yml -cat > ~/mirror-registry-config/config.yml << 'EOF' -version: 0.1 -log: - level: warn -storage: - filesystem: - rootdirectory: /var/lib/registry - delete: - enabled: true -http: - addr: :8443 - tls: - certificate: /certs/server.crt - key: /certs/server.key -auth: - htpasswd: - realm: basic-realm - path: /auth/htpasswd -EOF - -# f) Create and start podman container -# config.yml → /etc/docker/registry/config.yml (where registry:2 reads it) -# htpasswd → /auth/htpasswd (where config.yml references it) -# certs → /certs/ (where config.yml references server.crt/server.key) -podman create \ - --name local-registry \ - -p "${MREG_PORT}:8443" \ - -v ~/local-registry:/var/lib/registry:z \ - -v ~/mirror-registry-config/config.yml:/etc/docker/registry/config.yml:z \ - -v ~/mirror-registry-config/htpasswd:/auth/htpasswd:z \ - -v ~/mirror-registry-certs:/certs:z \ - docker.io/library/registry:2 -podman start local-registry -echo "local-registry container started" - -# g) Persist as systemd user service (survives reboots and logout) -mkdir -p ~/.config/systemd/user -podman generate systemd --name local-registry --restart-policy always \ - > ~/.config/systemd/user/local-registry.service -systemctl --user daemon-reload -systemctl --user enable local-registry.service -loginctl enable-linger "$USER" -echo "local-registry systemd user service enabled" - -# h) Trust the CA for container tools (per-user, no sudo required) -# podman/skopeo/oc-mirror read from this directory automatically. -# curl: pass --cacert ~/mirror-registry-certs/ca.crt -# oc image mirror: pass --insecure=true -# Trust entries for BOTH the DNS name (canonical) and the IP (transition safety). -mkdir -p ~/.config/containers/certs.d/${MIRROR_REGISTRY} -cp ~/mirror-registry-certs/ca.crt ~/.config/containers/certs.d/${MIRROR_REGISTRY}/ca.crt -mkdir -p ~/.config/containers/certs.d/${MREG_IP}:${MREG_PORT} -cp ~/mirror-registry-certs/ca.crt ~/.config/containers/certs.d/${MREG_IP}:${MREG_PORT}/ca.crt -echo "CA trusted for container tools (DNS name + IP)" - -# h2) Skip sigstore attachment lookups for certified vendors -# Intel, HashiCorp, and NVIDIA images on registry.connect.redhat.com lack cosign -# .sig manifests. Without this, oc-mirror fails with "name unknown: Image not found". -mkdir -p ~/.config/containers/registries.d -cat > ~/.config/containers/registries.d/no-sigstore-certified.yaml << 'REGCFG' -docker: - registry.connect.redhat.com/intel: - use-sigstore-attachments: false - registry.connect.redhat.com/hashicorp: - use-sigstore-attachments: false - registry.connect.redhat.com/nvidia: - use-sigstore-attachments: false -REGCFG -echo "Sigstore attachment lookups disabled for certified vendors" - -# i) Build combined-ca-bundle.pem (used by labctl --additional-trust-bundle) -# The mirror-registry CA is the only CA required — Quay is not used in this deployment. -cp ~/mirror-registry-certs/ca.crt ~/combined-ca-bundle.pem -echo "combined-ca-bundle.pem created: $(grep -c 'BEGIN CERTIFICATE' ~/combined-ca-bundle.pem) cert(s)" - -# j) Verify -CATALOG=$(curl -s -u "init:$MREG_PASS" \ - --cacert ~/mirror-registry-certs/ca.crt \ - https://${MIRROR_REGISTRY}/v2/_catalog) -echo "Registry catalog check: $CATALOG" -[ "$CATALOG" = '{"repositories":[]}' ] && echo "PASS: registry healthy and empty" || echo "WARN: unexpected response" -``` - -### 0-2: Seed pull-secret.json with Mirror Registry Credentials - -The Red Hat pull-secret must be downloaded from [console.redhat.com](https://console.redhat.com) before running this step. - -```bash -echo "=== 0-2: Seed pull-secret.json ===" - -# Download pull-secret.json from https://console.redhat.com → OpenShift → Downloads -# Place at ~/pull-secret.json before proceeding. - -MREG_PASS=$(cat ~/.coco-pattern/mirror-registry-password) -python3 - << 'EOF' -import json, base64, os - -ps_path = os.path.expanduser("~/pull-secret.json") -mreg = os.environ["MIRROR_REGISTRY"] -mreg_pass = open(os.path.expanduser("~/.coco-pattern/mirror-registry-password")).read() -mreg_auth = base64.b64encode(f"init:{mreg_pass}".encode()).decode() +```text +┌──────────────┐ ┌───────────────────┐ ┌─────────────────┐ +│ Upstream │────▶│ Quay Mirror │────▶│ OpenShift SNO │ +│ Registries │ │ (org: mirror) │ │ (node-02) │ +│ (internet) │ │ │ │ │ +└──────────────┘ └───────────────────┘ └─────────────────┘ + │ │ + │ ┌────────────────┐ │ + └──▶│ Git HTTP │◀──┘ + │ Server (:8080)│ + └────────────────┘ -with open(ps_path) as f: - ps = json.load(f) +```text -ps["auths"][mreg] = { - "auth": mreg_auth, - "username": "init", - "password": mreg_pass -} +All image content is mirrored to a single Quay org. The cluster pulls +exclusively from the mirror (default catalogs disabled, IDMS/ITMS +redirect all upstream registries). The pattern source code is served +via a local Git HTTP server. -with open(ps_path, "w") as f: - json.dump(ps, f, indent=2) +## Duration -print("Added mirror-registry auth to pull-secret.json") -print("Registries:", list(ps["auths"].keys())) -EOF -``` - -> **When to re-run:** Only if `~/.coco-pattern/mirror-registry-password` is regenerated. -> The htpasswd auth survives mirror storage wipes — only image data is cleared in Phase A-4. - -### 0-3: Local Git Server Initialisation (ONE-TIME per jump host) - -The local Git HTTP server makes pattern repos available to the cluster's patterns-operator -during the deploy. `scripts/airgap-post-install.sh --sync-repos-only` creates bare repos -and starts the service; subsequent runs auto-sync via the D-1 step. - -```bash -echo "=== 0-3: Git Server Init ===" - -# a) Clone the pattern repositories (internet required — run before air-gapping) -git clone -b dev/airgap-testing \ - https://github.com/butler54/coco-pattern.git ~/coco-pattern -git clone -b dev/phase1-testing \ - https://github.com/butler54/trustee-chart.git ~/trustee-chart -git clone -b dev/phase1-testing \ - https://github.com/butler54/sandboxed-containers-chart.git ~/sandboxed-containers-chart -git clone -b dev/phase1-testing \ - https://github.com/butler54/sandboxed-policies-chart.git ~/sandboxed-policies-chart - -echo "Repos cloned:" -for d in coco-pattern trustee-chart sandboxed-containers-chart sandboxed-policies-chart; do - echo " ~/$d: $(git -C ~/$d rev-parse --short HEAD 2>/dev/null)" -done - -# b) Create bare repos in ~/public_html/git/ and start git-http.service -cd ~/coco-pattern -export MIRROR_REGISTRY # must be set from 0-0 -scripts/airgap-post-install.sh --sync-repos-only -# This: -# - Creates ~/public_html/git/*.git bare repos -# - Creates and enables systemd user service: git-http.service -# - Reports Git HTTP URLs on the local network - -# c) Verify -sleep 3 -curl -s "http://localhost:${GIT_SERVER_PORT}/coco-pattern.git/info/refs?service=git-upload-pack" | head -c 80 -echo "" -[ $? -eq 0 ] && echo "PASS: git HTTP server responding" || echo "FAIL: check git-http.service status" -``` - -> **On subsequent runs:** bare repo sync happens automatically in D-1 via -> `scripts/airgap-post-install.sh`. Phase 0-3 is one-time only. +| Phase | Description | Duration | +|---|---|---| +| A | Mirror preparation | ~4h (one-time) | +| B | Cluster install | ~35 min | +| C | Pattern deploy | ~2h | +| D | Secrets + attestation | ~30 min | +| **Total** | From ISO boot to 15/15 healthy | **~2.5-3h** | --- -## PHASE A: Prerequisites and Mirror Wipe (~15 min) - -### A-0: Start tmux Session and Set Up Logging (FIRST STEP) +## Prerequisites -> **This is the only SSH session you need for the entire run.** -> Start tmux immediately after connecting — all steps, including long-running ones, run as -> named windows within the single `deploy` session. To disconnect safely at any point: `Ctrl-b d`. -> To reconnect to the running session: `ssh jump` then `tmux attach -t deploy`. +### Mirror Content -```bash -ssh chbutler@user-jump.int.apac-tech-lab.net -tmux new-session -s deploy -``` - -Everything below runs inside the `deploy` session. Long-running steps open a new named window -(`Ctrl-b c` or via `tmux new-window -n `) and run in the foreground — no nohup needed. -Switch between windows with `Ctrl-b ` or `Ctrl-b n/p`. - -```bash -# ── Site-specific variables — update these for each deployment environment ── -export MIRROR_REGISTRY=user-jump.int.apac-tech-lab.net:8443 # mirror-registry (DNS identity — see Phase 0-0) -export GIT_SERVER=http://user-jump.int.apac-tech-lab.net:8080 # local git HTTP server (DNS identity) -# Persist to all tmux windows (restore in each window with: eval "$(tmux showenv -g)") -tmux set-environment -g MIRROR_REGISTRY "$MIRROR_REGISTRY" -tmux set-environment -g GIT_SERVER "$GIT_SERVER" - -export LOG=~/coco-pattern/logs/run-25-$(date +%Y%m%d).log -mkdir -p ~/coco-pattern/logs -touch "$LOG" -echo "=== Phase 25 Run Start: $(date) ===" | tee -a "$LOG" -echo "LOG=$LOG MIRROR_REGISTRY=$MIRROR_REGISTRY GIT_SERVER=$GIT_SERVER" | tee -a "$LOG" -tmux set-environment -g LOG "$LOG" - -# local-registry credentials (htpasswd auth, username: init) -export MREG_USER=init -export MREG_PASS=$(cat ~/.coco-pattern/mirror-registry-password) -echo "local-registry credentials loaded (user: $MREG_USER)" | tee -a "$LOG" - -# Ensure local-registry CA is trusted by container tools (idempotent) -MREG_HOST="${MIRROR_REGISTRY%%:*}" -MREG_PORT="${MIRROR_REGISTRY##*:}" -mkdir -p ~/.config/containers/certs.d/${MIRROR_REGISTRY} -cp ~/mirror-registry-certs/ca.crt ~/.config/containers/certs.d/${MIRROR_REGISTRY}/ca.crt -echo "local-registry CA trusted for container tools" | tee -a "$LOG" - -# Per-user cert dir handles podman/skopeo/oc-mirror TLS. -# curl: pass --cacert ~/mirror-registry-certs/ca.crt -# oc image mirror: pass --insecure=true - -# Verify sigstore skip config is in place (created in Phase 0-1 step h2) -ls ~/.config/containers/registries.d/no-sigstore-certified.yaml && \ - echo "Sigstore skip config present" | tee -a "$LOG" || \ - echo "WARN: no-sigstore-certified.yaml missing — Intel/HashiCorp/NVIDIA mirrors may fail" | tee -a "$LOG" -``` +The Quay mirror must contain all images the pattern resolves. Verify with +`skopeo list-tags` against each repo (the REST API is unreliable for org +robots). Key content: -> At run end, Claude will SSH in and read this log file for findings analysis. +- OCP release payload (exact installer version, e.g. 4.22.17) +- Operator catalogs (redhat, certified, community — v4.22) +- VP Helm charts (clustergroup, trustee, acm, etc.) +- VP utility images (utility-container, imperative-container, pattern-install) +- Workload images (ubi9/httpd-24, ubi9/ubi) +- Intel DCAP operator + QGS images +- Kyverno images (mirrored from ghcr.io under the `kyverno/` path) +- hashicorp/vault (the version the chart resolves, e.g. 2.1.2-ubi) -### A-0.5: Tool Version Verification (MUST PASS before proceeding) +### Imageset Pin Check (K16) -> **Critical:** PATH order issues can cause the wrong binary to be used silently. -> `~/.local/bin` may contain old versions that override `~/bin/` symlinks. -> Every check below must show the expected version. **Stop and fix any mismatch.** +Pattern values use range pins (`~0.1.10`, `0.2.*`) resolved at deploy time. +The imageset must mirror the **newest version within each range** — not the +range floor. Run a range-check before every deploy: ```bash -echo "=== A-0.5: Tool Version Verification ===" 2>&1 | tee -a "$LOG" -OCP_MINOR="4.22" # Expected OCP minor version for this run - -# 1. openshift-install — version string must contain expected OCP minor version -OI_VERSION=$(openshift-install version 2>/dev/null | head -1) -echo "openshift-install: $OI_VERSION" | tee -a "$LOG" -if ! echo "$OI_VERSION" | grep -q "^openshift-install ${OCP_MINOR}"; then - echo "FAIL: openshift-install reports wrong version (expected ${OCP_MINOR}.x, got: $OI_VERSION)" | tee -a "$LOG" - echo "Fix: run 'labctl generate ...' first to symlink the correct binary, or check PATH order" | tee -a "$LOG" - exit 1 -fi -echo "PASS: openshift-install is ${OCP_MINOR}.x" | tee -a "$LOG" - -# 2. oc-mirror — version string must contain expected OCP minor version -OM_VERSION=$(oc-mirror version 2>&1 | grep -i 'gitversion' | grep -o '[0-9]\+\.[0-9]\+\.[0-9]*' | head -1) -echo "oc-mirror version: $OM_VERSION" | tee -a "$LOG" -if ! echo "$OM_VERSION" | grep -q "^${OCP_MINOR}"; then - echo "FAIL: oc-mirror reports wrong version (expected ${OCP_MINOR}.x, got: $OM_VERSION)" | tee -a "$LOG" - echo "Fix: ln -sf ~/ocp_42208_bin/oc-mirror ~/bin/oc-mirror" | tee -a "$LOG" - exit 1 -fi -echo "PASS: oc-mirror is ${OCP_MINOR}.x" | tee -a "$LOG" - -# 3. labctl — verify installed -LABCTL_VERSION=$(labctl --version 2>/dev/null) -echo "labctl: $LABCTL_VERSION" | tee -a "$LOG" - -# 4. Release image digest alignment (informational — only meaningful after B-2) -OI_DIGEST=$(openshift-install version 2>/dev/null | grep 'release image' | grep -o 'sha256:[0-9a-f]*') -OM_DIGEST=$(grep -o 'sha256-[0-9a-f]*' \ - ~/oc-mirror-workspace/working-dir/cluster-resources/signature-configmap.json 2>/dev/null | \ - head -1 | sed 's/sha256-/sha256:/') -echo "openshift-install release digest: $OI_DIGEST" | tee -a "$LOG" -echo "oc-mirror release digest: ${OM_DIGEST:-not yet run}" | tee -a "$LOG" -[ -n "$OM_DIGEST" ] && [ "$OI_DIGEST" = "$OM_DIGEST" ] && \ - echo "PASS: release image digests match" | tee -a "$LOG" || \ - echo "INFO: digest mismatch or oc-mirror not yet run (check again after B-2)" | tee -a "$LOG" - -echo "A-0.5: Version verification complete at $(date)" | tee -a "$LOG" -``` - -### A-1: State Verification (check before deleting) - -```bash -echo "=== A-1: State Verification ===" 2>&1 | tee -a "$LOG" - -cd ~/coco-pattern -git remote get-url origin 2>&1 | tee -a "$LOG" -# EXPECTED: ${GIT_SERVER}/coco-pattern.git -git branch --show-current 2>&1 | tee -a "$LOG" -# EXPECTED: dev/airgap-testing - -du -sh ~/oc-mirror-workspace 2>&1 | tee -a "$LOG" - -# Check local-registry state (htpasswd auth required) -echo "local-registry catalog:" 2>&1 | tee -a "$LOG" -curl -sk -u "$MREG_USER:$MREG_PASS" https://${MIRROR_REGISTRY}/v2/_catalog 2>&1 | tee -a "$LOG" +# For each chart pin in values-*.yaml, verify the newest in-range tag +# is present in the mirror +skopeo list-tags docker:///validatedpatterns/ | \ + python3 -c "import json,sys; tags=json.load(sys.stdin)['Tags']; print(sorted(tags))" -grep -c "BEGIN CERTIFICATE" ~/combined-ca-bundle.pem 2>&1 | tee -a "$LOG" -# EXPECTED: 3 (Quay 2-chain + mirror-registry 1 cert) - -# Verify pull-secret.json has local-registry credentials (required for cluster node pulls via IDMS) -echo "pull-secret registries:" 2>&1 | tee -a "$LOG" -python3 -c "import json; [print(k) for k in json.load(open('pull-secret.json'))['auths']]" 2>&1 | tee -a "$LOG" -# MUST include: ${MIRROR_REGISTRY} -# If missing, run from appendix A.6 before proceeding - -echo "A-1: State verification complete at $(date)" 2>&1 | tee -a "$LOG" -``` - -### A-3: oc-mirror Workspace Wipe - -> **DESTRUCTIVE — confirm A-1 shows correct git branch and mirror-registry state before proceeding.** - -```bash -echo "=== A-3: oc-mirror Workspace Wipe ===" 2>&1 | tee -a "$LOG" - -ls -la ~/oc-mirror-workspace 2>&1 | tee -a "$LOG" -rm -rf ~/oc-mirror-workspace -echo "oc-mirror-workspace removed" 2>&1 | tee -a "$LOG" - -# Also clear any cached oc-mirror state -rm -rf ~/.oc-mirror 2>/dev/null || true -echo "A-3: oc-mirror wipe complete at $(date)" 2>&1 | tee -a "$LOG" ``` -### A-4: local-registry Decommission (Stop / Wipe / Restart) - -> **DESTRUCTIVE: stop container, wipe storage, restart empty.** -> Container name: `local-registry`. Storage: `~/local-registry` (owned by chbutler — no sudo). -> No auth required for registry API. +### Credentials and Certificates ```bash -echo "=== A-4: local-registry Decommission ===" 2>&1 | tee -a "$LOG" +# Mirror registry credentials (robot account) +export MIRROR_REGISTRY=:443/mirror # INCLUDES the org path +export MIRROR_USER= +export MIRROR_PASS= -# Confirm state before wiping -echo "Container status:" 2>&1 | tee -a "$LOG" -podman ps --format "{{.Names}} {{.Status}}" | grep -i registry 2>&1 | tee -a "$LOG" -echo "Storage size:" 2>&1 | tee -a "$LOG" -du -sh ~/local-registry/ 2>&1 | tee -a "$LOG" +# CA certificate chain +cp ~/.coco-pattern/registry-ca-cert.pem +cat > ~/combined-ca-bundle.pem -# Stop container -podman stop local-registry 2>&1 | tee -a "$LOG" -echo "local-registry stopped" 2>&1 | tee -a "$LOG" +# Intel PCS API key (for PCK provisioning, 32 bytes, no trailing newline) +printf '%s' "" > ~/.coco-pattern/intel-pcs-api-key -# Wipe storage (chbutler owns ~/local-registry — no sudo needed) -rm -rf ~/local-registry/* -echo "Storage wiped" 2>&1 | tee -a "$LOG" +# DCAP collateral (collected on a connected host, ~30 day freshness) +# ~/.coco-pattern/dcap-offline/platform_collaterals.json -# Restart container (reinitializes on empty storage) -podman start local-registry 2>&1 | tee -a "$LOG" -sleep 10 +```text -# Verify empty catalog (htpasswd auth required) -curl -sk -u "$MREG_USER:$MREG_PASS" https://${MIRROR_REGISTRY}/v2/_catalog 2>&1 | tee -a "$LOG" -# EXPECTED: {"repositories":[]} - -echo "A-4: local-registry decommission complete at $(date)" 2>&1 | tee -a "$LOG" -``` +### Values-Secret (Convention Location) -### A-5: Verify Empty State +The values-secret file **must** be at `~/values-secret-.yaml`. +The Ansible secret loader (`find_vp_secrets`) searches home-directory +conventions and falls back to the repo template — it never reads a +repo-local generated file. ```bash -echo "=== A-5: Verify Empty State ===" 2>&1 | tee -a "$LOG" +# Generate (writes to ~/values-secret-.yaml) +make gen-secrets -# local-registry catalog must be empty (htpasswd auth required) -echo "local-registry catalog after wipe:" 2>&1 | tee -a "$LOG" -curl -sk -u "$MREG_USER:$MREG_PASS" https://${MIRROR_REGISTRY}/v2/_catalog 2>&1 | tee -a "$LOG" -# EXPECTED: {"repositories":[]} +# Uncomment airgap blocks (registry CA, bootstrap secrets) +scripts/enable-airgap-secrets.sh -# oc-mirror workspace must not exist -if [ -d ~/oc-mirror-workspace ]; then - echo "WARNING: oc-mirror-workspace still exists" 2>&1 | tee -a "$LOG" -else - echo "PASS: oc-mirror-workspace absent" 2>&1 | tee -a "$LOG" -fi - -echo "A-5: Empty state verification complete at $(date)" 2>&1 | tee -a "$LOG" -``` - ---- - -## PHASE B: Full Re-Mirror (~3-4 hours) - -> **Skip Phase B if mirror-registry already has all required content.** -> Run B-0 to verify. If all checks pass, go directly to Phase C. - -### B-0: Verify Mirror Content and Tool Versions (run before deciding to skip Phase B) - -> **CRITICAL: oc-mirror must be version-matched to the OCP release being mirrored.** -> Cross-version oc-mirror rebuilds manifests with different SHA256 digests than openshift-install -> expects. This causes every component image pull to return 404 from mirror-registry, with the -> node falling back to internet quay.io (blocked by fake gateway) and stalling. -> `labctl generate` symlinks `~/bin/oc-mirror` from the OCP version bin dir automatically. - -```bash -echo "=== B-0: Mirror Content Verification ===" 2>&1 | tee -a "$LOG" -MREG_PASS=$(cat ~/.coco-pattern/mirror-registry-password) - -echo "--- oc-mirror version check (MUST match OCP version) ---" 2>&1 | tee -a "$LOG" -oc-mirror version 2>&1 | grep -i 'version\|GitVersion' | tee -a "$LOG" -# MUST contain "4.22" — if not, update symlink: ln -sf ~/ocp_42208_bin/oc-mirror ~/bin/oc-mirror - -echo "--- VP OCI Helm charts ---" 2>&1 | tee -a "$LOG" -for chart in clustergroup hashicorp-vault acm openshift-external-secrets \ - sandboxed-containers sandboxed-policies trustee; do - result=$(curl -sk -u "init:$MREG_PASS" \ - "https://${MIRROR_REGISTRY}/v2/validatedpatterns/${chart}/tags/list" 2>/dev/null) - tags=$(echo "$result" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('tags','MISSING'))" 2>/dev/null) - echo " validatedpatterns/${chart}: ${tags}" 2>&1 | tee -a "$LOG" -done - -echo "--- Operator catalog indexes ---" 2>&1 | tee -a "$LOG" -for idx in redhat/redhat-operator-index redhat/certified-operator-index redhat/community-operator-index; do - result=$(curl -sk -u "init:$MREG_PASS" \ - "https://${MIRROR_REGISTRY}/v2/${idx}/tags/list" 2>/dev/null) - tags=$(echo "$result" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('tags','MISSING'))" 2>/dev/null) - echo " ${idx}: ${tags}" 2>&1 | tee -a "$LOG" -done - -echo "--- OCP release image digest check ---" 2>&1 | tee -a "$LOG" -# The release image MUST be accessible at the digest openshift-install has hardcoded. -# This digest is set by the version-matched oc-mirror run — do NOT patch manually. -RELEASE_DIGEST=$(~/ocp_42208_bin/openshift-install version 2>/dev/null | grep 'release image' | awk '{print $NF}' | sed 's/.*@//') -if [ -n "$RELEASE_DIGEST" ]; then - HTTP=$(curl -sk -u "init:$MREG_PASS" -o /dev/null -w '%{http_code}' \ - "https://${MIRROR_REGISTRY}/v2/openshift/release-images/manifests/sha256:${RELEASE_DIGEST}") - [ "$HTTP" = "200" ] && echo "PASS: release image digest present" | tee -a "$LOG" || \ - echo "FAIL: release image digest missing (HTTP $HTTP) — must run B-2 with version-matched oc-mirror" | tee -a "$LOG" -fi - -echo "--- oc-mirror cluster-resources ---" 2>&1 | tee -a "$LOG" -ls ~/oc-mirror-workspace/working-dir/cluster-resources/ 2>/dev/null | tee -a "$LOG" - -echo "B-0: Mirror verification complete at $(date)" 2>&1 | tee -a "$LOG" -``` - -> **If all VP charts show tags, catalog indexes show v4.22, oc-mirror version matches, and release digest is present → skip to Phase C.** -> If anything is missing or wrong → run Phase B (full re-mirror). - ---- - -### B-1: Sync Repos to Latest - -```bash -echo "=== B-1: Sync Repos ===" 2>&1 | tee -a "$LOG" - -# Sync git mirror (bare repo) from GitHub -cd ~/public_html/git/coco-pattern.git -git fetch https://github.com/butler54/coco-pattern.git dev/airgap-testing:dev/airgap-testing --force \ - 2>&1 | tee -a "$LOG" +# Edit the bootstrap_secrets block for your mirror: +# url: :443//validatedpatterns +# username: +# password path: ~/.coco-pattern/registry-password +# +# Edit the registryCaCert block: +# pem path: ~/.coco-pattern/registry-ca-cert.pem -# Pull into working copy -cd ~/coco-pattern -git fetch origin 2>&1 | tee -a "$LOG" -git reset --hard origin/dev/airgap-testing 2>&1 | tee -a "$LOG" - -# Verify remote and branch -git remote get-url origin 2>&1 | tee -a "$LOG" -# MUST: ${GIT_SERVER}/coco-pattern.git -git branch --show-current 2>&1 | tee -a "$LOG" -# MUST: dev/airgap-testing - -# Verify clustergroup chart version in imageset-config -echo "Checking imageset-config clustergroup version:" 2>&1 | tee -a "$LOG" -grep "clustergroup" ~/coco-pattern/airgap/imageset-config-4.22.yaml 2>&1 | tee -a "$LOG" -# EXPECTED: clustergroup:0.9.58 - -# Verify values-baremetal-airgap.yaml is 2-line shape -echo "Checking values-baremetal-airgap.yaml shape:" 2>&1 | tee -a "$LOG" -cat ~/coco-pattern/values-baremetal-airgap.yaml 2>&1 | tee -a "$LOG" -# EXPECTED: 2 lines with global.catalogSource and global.catalogSourceNamespace - -# Verify deprecated scripts are quarantined -echo "Checking scripts/deprecated/:" 2>&1 | tee -a "$LOG" -ls ~/coco-pattern/scripts/deprecated/ 2>&1 | tee -a "$LOG" -# EXPECTED: exactly 3 files (fix-patterns-operator-images.sh, rebuild-patterns-operator-bundle.sh, deploy-pattern-without-operator.sh) - -echo "B-1: Repo sync complete at $(date)" 2>&1 | tee -a "$LOG" ``` -### B-2: Run oc-mirror v2 (Full Re-Mirror) +### Git Server -> **Long-running (~3-4 hours). Open a new window in the `deploy` session for this step.** -> -> **oc-mirror version must match OCP version.** `labctl generate` (C-1) automatically symlinks -> `~/bin/oc-mirror` from `~/ocp_42208_bin/oc-mirror`. Verify with B-0 before proceeding. -> A version mismatch causes oc-mirror to rebuild manifests with different SHA256 digests than -> openshift-install expects, resulting in 404s on every component image during node boot. +The pattern source is served via a local Git HTTP server: ```bash -tmux new-window -n "oc-mirror" # opens within the current session; switches you into it -eval "$(tmux showenv -g LOG)"; eval "$(tmux showenv -g MIRROR_REGISTRY)"; eval "$(tmux showenv -g GIT_SERVER)" - -echo "=== B-2: oc-mirror v2 Full Re-Mirror ===" 2>&1 | tee -a "$LOG" -echo "Started at $(date)" 2>&1 | tee -a "$LOG" - -# Verify oc-mirror version before running (must match OCP 4.22) -oc-mirror version 2>&1 | grep -i 'version\|GitVersion' | tee -a "$LOG" -# MUST show 4.22.x — if not, stop and fix: ln -sf ~/ocp_42208_bin/oc-mirror ~/bin/oc-mirror - -# Authenticate oc-mirror against local-registry (CA trusted via certs.d from A-0) -podman login ${MIRROR_REGISTRY} \ - --username "$MREG_USER" \ - --password "$MREG_PASS" \ - 2>&1 | tee -a "$LOG" -# EXPECTED: Login Succeeded! - -# Run oc-mirror v2 — full mirror to mirror-registry (sole airgap target, Phase 25.2) -# Note: --v2 (double dash) is mandatory; -v2 (single dash) is not recognised -# CA trust comes from ~/.config/containers/certs.d/${MIRROR_REGISTRY}/ca.crt (set in A-0) -# oc-mirror runs against the DNS name so ALL generated cluster-resources -# (IDMS, ITMS, CatalogSources) reference the DNS name — the IP never enters -# the ignition chain (FR-017). The CA is trusted via certs.d (Phase 0-1h). -oc-mirror \ - --config ~/coco-pattern/airgap/imageset-config-4.22.yaml \ - docker://${MIRROR_REGISTRY} \ - --dest-tls-verify=true \ - --workspace file://$HOME/oc-mirror-workspace \ - --v2 \ - 2>&1 | tee -a "$LOG" - -echo "B-2: oc-mirror complete at $(date)" 2>&1 | tee -a "$LOG" -``` +# Bare repos under ~/public_html/git/ +# Smart HTTP on :8080 (started by airgap-post-install.sh step 9) -> **Wait for oc-mirror to complete before proceeding to B-3.** -> Output is visible live in this window. Switch windows with `Ctrl-b ` while it runs. -> Expected completion: 3-4 hours. -> oc-mirror v2 is resumable — re-running the same command skips already-mirrored content. -> -> **Cosign signature failures:** Intel, HashiCorp, and NVIDIA images on -> `registry.connect.redhat.com` lack cosign `.sig` manifests. The -> `no-sigstore-certified.yaml` config (created in Phase 0-1 step h2) disables -> sigstore attachment lookups for these vendors, so oc-mirror handles them normally. -> -> **Transient failures (fixed by re-running):** -> `multicluster-engine/hive-rhel9` is 600MB+ and reliably times out on the first oc-mirror run -> (`context deadline exceeded` during blob upload). When hive-rhel9 times out, oc-mirror also -> skips `mce-operator-bundle` entirely — this cascades to: MCE CSV not installed → MCH stuck -> at Installing → ACM policy controller never starts → `credential` secret never created → -> trustee KbsConfig blocks forever. -> **If B-3 shows hive-rhel9 or mce-operator-bundle MISSING: re-run the B-2 oc-mirror command.** -> oc-mirror resumes from the workspace cache and retries only what failed — do NOT skip to Phase C. - -### B-3: Post-Mirror Verification - -```bash -echo "=== B-3: Post-Mirror Verification ===" 2>&1 | tee -a "$LOG" - -# Count cluster-resources YAML files (expect ~9: IDMS, ITMS, CatalogSources, signatures) -ls ~/oc-mirror-workspace/working-dir/cluster-resources/*.yaml 2>&1 | tee -a "$LOG" -COUNT=$(ls ~/oc-mirror-workspace/working-dir/cluster-resources/*.yaml | wc -l) -echo "cluster-resources count: $COUNT (expected ~10)" 2>&1 | tee -a "$LOG" - -# Paginate through full registry catalog — v2/_catalog returns max 100 repos per page -# CRITICAL: do NOT use a bare curl; with 270+ repos the first page misses everything past repo 100 -echo "Spot-checking key repos in local-registry (paginated):" 2>&1 | tee -a "$LOG" -python3 - << 'PYEOF' 2>&1 | tee -a "$LOG" -import json, urllib.request, urllib.error, base64, os - -creds = base64.b64encode(b"init:" + open(os.path.expanduser("~/.coco-pattern/mirror-registry-password"), "rb").read().strip()).decode() -headers = {"Authorization": f"Basic {creds}"} - -repos = [] -last = "" -while True: - url = f"https://${MIRROR_REGISTRY}/v2/_catalog?n=100{f'&last={last}' if last else ''}" - req = urllib.request.Request(url, headers=headers) - ctx = __import__("ssl").create_default_context() - ctx.load_verify_locations(os.path.expanduser("~/mirror-registry-certs/ca.crt")) - try: - with urllib.request.urlopen(req, context=ctx) as r: - page = json.loads(r.read()).get("repositories", []) - except Exception as e: - print(f"ERROR: {e}"); break - repos.extend(page) - if len(page) < 100: break - last = page[-1] - -print(f"Total repos: {len(repos)}") -checks = [ - ("openshift/release-images", "OCP release images"), - ("openshift/release", "OCP release components"), - ("validatedpatterns/clustergroup", "clustergroup chart"), - ("community-operator-pipeline-prod/patterns-operator", "patterns-operator"), - ("hashicorp/vault", "Vault"), - ("intel/intel-tdx-dcap-operator", "Intel TDX DCAP"), - ("multicluster-engine/hive-rhel9", "MCE hive (large — timeout risk; re-run oc-mirror if missing)"), - ("multicluster-engine/mce-operator-bundle", "MCE bundle (skipped by oc-mirror if hive timed out)"), - ("build-of-trustee/trustee-rhel9", "Trustee"), - ("multicluster-engine/hive-rhel9", "MCE Hive"), - ("openshift-sandboxed-containers/osc-cloud-api-adaptor-rhel9", "CoCo cloud-api-adaptor"), -] -for repo, label in checks: - status = "OK" if repo in repos else "MISSING" - print(f" {status} {label} ({repo})") -PYEOF - -# Verify IDMS files cover expected namespaces -echo "IDMS namespace coverage:" 2>&1 | tee -a "$LOG" -grep "${MIRROR_REGISTRY}" ~/oc-mirror-workspace/working-dir/cluster-resources/idms-oc-mirror.yaml | wc -l | xargs -I{} echo " idms-oc-mirror.yaml: {} mirror entries" | tee -a "$LOG" -# idms-manual-mirrors.yaml no longer needed — oc-mirror generates IDMS for Intel/Hashicorp -# when registries.d/no-sigstore-certified.yaml is in place - -echo "B-3: Post-mirror verification complete at $(date)" 2>&1 | tee -a "$LOG" -``` - -### B-3a: Manual Mirror — ELIMINATED - -> **No longer needed.** The `no-sigstore-certified.yaml` registries.d config -> (Phase 0-1 step h2) disables sigstore attachment lookups for Intel, HashiCorp, -> and NVIDIA on `registry.connect.redhat.com`. oc-mirror now mirrors all images -> successfully and generates IDMS entries for them automatically. -> -> **ITMS (tag-based mirrors)** for `registry.connect.redhat.com/intel` and -> `/hashicorp` are still needed — oc-mirror only generates IDMS (digest-based). -> The supplementary ITMS lives at `airgap/itms-manual-mirrors.yaml` and is applied -> automatically by `airgap-post-install.sh` in D-1. - -```bash -echo "=== B-3a: Verify supplementary ITMS in repo ===" | tee -a "$LOG" -cat ~/coco-pattern/airgap/itms-manual-mirrors.yaml 2>&1 | tee -a "$LOG" -# EXPECTED: ImageTagMirrorSet covering registry.connect.redhat.com/intel and /hashicorp -``` +```text --- -## PHASE C: Cluster Install (~35 min) +## Phase B: Cluster Install -### C-1: Generate Configs and Build ISO - -> labctl now generates the fake gateway directly (172.25.53.254) instead of `route-type: blackhole`, -> and embeds wipe-disks.ign into the ISO via `coreos-installer iso ignition embed`. No manual -> patching or separate openshift-install call needed. +### B-1: Generate the ISO ```bash -echo "=== C-1: Generate Configs and Build ISO ===" 2>&1 | tee -a "$LOG" -source ~/.envrc - -# ── labctl (APAC lab-specific) ─────────────────────────────────────────────── -# labctl is an internal APAC lab tool that reads node configuration from an -# XLSX file and generates OCP agent-based install configs with the correct -# NIC, iDRAC, and network settings for this lab environment. -# -# For other environments, replace labctl generate with: -# openshift-install agent create image --dir -# and mount the ISO via your BMC virtual media interface manually. -# ───────────────────────────────────────────────────────────────────────────── labctl generate \ - --node node-02 \ - --xlsx "$HOME/APAC Technology Lab Details.xlsx" \ - --ocp-version 42208 \ - --output-dir ~/node-02-airgap-output \ + --node \ + --xlsx \ + --ocp-version \ + --output-dir \ --mirror-resources ~/oc-mirror-workspace/working-dir/cluster-resources \ --additional-trust-bundle ~/combined-ca-bundle.pem \ - --http-serve-path $HOME/public_html/ \ - 2>&1 | tee -a "$LOG" - -# Verify: fake gateway present, no blackhole, ISO built and served -grep "172.25.53.254" ~/node-02-airgap-output/agent-config.yaml 2>&1 | tee -a "$LOG" && \ - ! grep "blackhole" ~/node-02-airgap-output/agent-config.yaml && \ - echo "PASS: fake gateway in agent-config, no blackhole" 2>&1 | tee -a "$LOG" -ls -lh ~/public_html/node-02-42208.iso 2>&1 | tee -a "$LOG" && \ - echo "PASS: ISO present at http serve path" 2>&1 | tee -a "$LOG" - -echo "C-1: Config generation and ISO build complete at $(date)" 2>&1 | tee -a "$LOG" -``` - -### C-3: Mount ISO via iDRAC - -```bash -echo "=== C-3: Mount ISO via iDRAC ===" 2>&1 | tee -a "$LOG" -source ~/.envrc + --http-serve-path ~/public_html -# ── labctl (APAC lab-specific) ─────────────────────────────────────────────── -# labctl is an internal APAC lab tool that reads node configuration from an -# XLSX file and generates OCP agent-based install configs with the correct -# NIC, iDRAC, and network settings for this lab environment. -# -# For other environments, replace labctl generate with: -# openshift-install agent create image --dir -# and mount the ISO via your BMC virtual media interface manually. -# ───────────────────────────────────────────────────────────────────────────── -labctl idrac mount-iso \ - --node node-02 \ - --xlsx "$HOME/APAC Technology Lab Details.xlsx" \ - --iso-url "http://user-jump.int.apac-tech-lab.net/~chbutler/node-02-42208.iso" \ - 2>&1 | tee -a "$LOG" - -echo "C-3: ISO mounted at $(date)" 2>&1 | tee -a "$LOG" ``` -### C-4: Wait for Install (~30 min) +### B-2: Boot and Install ```bash -tmux new-window -n "cluster-install" # new window within the deploy session -eval "$(tmux showenv -g LOG)"; eval "$(tmux showenv -g MIRROR_REGISTRY)"; eval "$(tmux showenv -g GIT_SERVER)" - -echo "=== C-4: Wait for Install ===" 2>&1 | tee -a "$LOG" -cd ~/node-02-airgap-output/42208_build +# Mount the ISO via iDRAC/virtual media, one-time boot +# Wait for install (~35 min): +openshift-install agent wait-for install-complete \ + --dir /_build -~/ocp_42208_bin/openshift-install agent wait-for install-complete \ - --dir . \ - --log-level info \ - 2>&1 | tee -a "$LOG" - -echo "C-4: Install complete at $(date)" 2>&1 | tee -a "$LOG" -``` +```text -> **Wait for install to complete before proceeding to C-5.** -> Expected log line: `Install complete!` and `Access the OpenShift web-console here:` -> Switch windows with `Ctrl-b ` while waiting. - -### C-5: Health Check +### B-3: Verify ```bash -echo "=== C-5: Health Check ===" 2>&1 | tee -a "$LOG" - -# Find the build directory (changes per run) -BUILD_DIR=$(ls -td ~/node-02-airgap-output/4*_build* 2>/dev/null | head -1) -echo "Build dir: $BUILD_DIR" 2>&1 | tee -a "$LOG" - -export KUBECONFIG="${BUILD_DIR}/auth/kubeconfig" -echo "KUBECONFIG=$KUBECONFIG" | tee -a "$LOG" - -oc get clusterversion 2>&1 | tee -a "$LOG" -# EXPECTED: VERSION 4.22.8, STATUS Available -oc get nodes 2>&1 | tee -a "$LOG" -# EXPECTED: node-02 Ready - -echo "C-5: Health check complete at $(date)" 2>&1 | tee -a "$LOG" -``` - -### C-6: Set KUBECONFIG - -```bash -echo "=== C-6: Set KUBECONFIG ===" 2>&1 | tee -a "$LOG" - -# The build directory name changes per run — set it from the actual output -BUILD_DIR=$(ls -td ~/node-02-airgap-output/4*_build* 2>/dev/null | head -1) -export KUBECONFIG="${BUILD_DIR}/auth/kubeconfig" - -echo "export KUBECONFIG=${BUILD_DIR}/auth/kubeconfig" >> ~/.envrc -echo "KUBECONFIG set to: $KUBECONFIG" 2>&1 | tee -a "$LOG" - -# Verify -oc whoami 2>&1 | tee -a "$LOG" -# EXPECTED: system:admin +export KUBECONFIG=/_build/auth/kubeconfig +oc get clusterversion # version, Available=True +oc get clusteroperators # all Available +oc get nodes # Ready -echo "C-6: KUBECONFIG configured at $(date)" 2>&1 | tee -a "$LOG" ``` --- -## PHASE D: Bootstrap and Pattern Deploy (~25 min) +## Phase C: Pattern Deploy -### D-1: Run airgap-post-install.sh - -> `airgap-post-install.sh` handles all of the following in one pass: -> -> - Disables OperatorHub default catalogs -> - Applies CatalogSources from oc-mirror cluster-resources (deleting stale ones) -> - Applies ALL oc-mirror cluster-resources: IDMS, ITMS, ClusterCatalog (OLM v1), signature ConfigMap -> - Applies supplementary IDMS (`idms-manual-mirrors.yaml` — intel/hashicorp, created in B-3a) -> - Applies supplementary ITMS (`airgap/itms-manual-mirrors.yaml` — intel/hashicorp, from repo) -> - Patches `mirrorSourcePolicy: NeverContactSource` on all IDMS entries (prevents internet fallback) -> - Configures the git HTTP server and patterns-operator-config ConfigMap +### C-1: Airgap Post-Install ```bash -echo "=== D-1: airgap-post-install.sh ===" 2>&1 | tee -a "$LOG" cd ~/coco-pattern -# MIRROR_REGISTRY was set in A-0; confirm it is still set -echo "MIRROR_REGISTRY=$MIRROR_REGISTRY" | tee -a "$LOG" - -scripts/airgap-post-install.sh 2>&1 | tee -a "$LOG" +export MIRROR_REGISTRY=:443/mirror # INCLUDES the org! +export GIT_SERVER=http://:8080 +export ENABLE_ROUTINGVIAHOST=true # required for pod→lab egress -# Verify catalog pods are Running before proceeding to D-2 -echo "--- CatalogSource pods ---" 2>&1 | tee -a "$LOG" -oc get pods -n openshift-marketplace 2>&1 | tee -a "$LOG" -# EXPECTED: 3 pods Running (redhat, certified, community operator indexes) +scripts/airgap-post-install.sh -echo "D-1: airgap-post-install complete at $(date)" 2>&1 | tee -a "$LOG" -``` +```text -### D-1.5: Wait for MCO if airgap-post-install.sh patched any IDMS/ITMS +This applies: -> **This step is now automated.** `airgap-post-install.sh` (step 3b/3c) patches `NeverContactSource` -> on all IDMS and ITMS objects — including the bootstrap `image-digest-mirror` IDMS created by -> `labctl` at cluster install time. If any objects were patched, MCO re-renders `registries.conf`. -> On SNO this triggers a **node reboot (~5 min)**. Check the script output for -> `"MCO will re-render registries.conf"` and wait for the node to recover before proceeding. +- IDMS/ITMS/CatalogSources from the oc-mirror workspace +- `mirrorSourcePolicy: NeverContactSource` on all mirror rules +- Repoints any default-catalog subscriptions to the mirrored catalogs +- Configures the Git HTTP server +- Creates the patterns-operator-config ConfigMap +- Enables OVN routingViaHost (verified after applying) -```bash -# If airgap-post-install.sh printed a reboot warning, wait here: -echo "=== D-1.5: Wait for node if MCO re-rendered ===" 2>&1 | tee -a "$LOG" -until oc get node -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' \ - 2>/dev/null | grep -q True; do - echo " Node not Ready yet — $(date)" | tee -a "$LOG" - sleep 15 -done -echo "Node Ready at $(date)" 2>&1 | tee -a "$LOG" -oc get nodes 2>&1 | tee -a "$LOG" -``` +**Verify**: catalog pods Running, routingViaHost=true. -### D-2: Enable routingViaHost and Wait for MCO +### C-2: Deploy the Pattern ```bash -echo "=== D-2: Enable routingViaHost ===" 2>&1 | tee -a "$LOG" - -oc patch network.operator.openshift.io cluster --type merge \ - -p '{"spec":{"defaultNetwork":{"ovnKubernetesConfig":{"gatewayConfig":{"routingViaHost":true}}}}}' \ - 2>&1 | tee -a "$LOG" - -echo "Waiting for MCO to complete (expect 5-10 min including reboot)..." 2>&1 | tee -a "$LOG" -while true; do - STATUS=$(oc get mcp master -o jsonpath='{.status.conditions[?(@.type=="Updated")].status}' 2>/dev/null) - UPDATING=$(oc get mcp master -o jsonpath='{.status.conditions[?(@.type=="Updating")].status}' 2>/dev/null) - echo "STATUS: $(date) — MCO Updated=$STATUS Updating=$UPDATING" 2>&1 | tee -a "$LOG" - [ "$STATUS" = "True" ] && [ "$UPDATING" = "False" ] && echo "MCO complete" 2>&1 | tee -a "$LOG" && break - sleep 30 -done - -echo "D-2: routingViaHost enabled at $(date)" 2>&1 | tee -a "$LOG" -``` +export PATTERN_DISCONNECTED_HOME=:443//validatedpatterns -### D-3: Prepare Secrets on Disk - -```bash -echo "=== D-3: Prepare Secrets ===" 2>&1 | tee -a "$LOG" -cd ~/coco-pattern +./pattern.sh make install -make cache-keys 2>&1 | tee -a "$LOG" -make cache-registry-ca 2>&1 | tee -a "$LOG" -./scripts/gen-secrets.sh 2>&1 | tee -a "$LOG" - -# registryCaCert and bootstrap_secrets (mirror-registry-helm-oci) are -# commented out by default in values-secret.yaml.template (they only apply -# to airgap deployments). Uncomment both blocks in the generated -# values-secret file now that cache-registry-ca has populated the CA cert. -make enable-airgap-secrets 2>&1 | tee -a "$LOG" - -# Strip trailing newline from mirror-registry-password. -# load-bootstrap reads this file verbatim via the values-secret.yaml path: field and -# the Ansible role does not strip whitespace. A trailing \n causes Helm to send -# "password\n" to the registry, which never matches the htpasswd hash → 401. -printf '%s' "$(cat ~/.coco-pattern/mirror-registry-password)" \ - > ~/.coco-pattern/mirror-registry-password -echo "mirror-registry-password length: $(wc -c < ~/.coco-pattern/mirror-registry-password) bytes (expected 24, no newline)" \ - 2>&1 | tee -a "$LOG" - -echo "D-3: Secrets prepared at $(date)" 2>&1 | tee -a "$LOG" ``` -### D-4: Launch Pattern Deploy - -```bash -tmux new-window -n "pattern-deploy" # new window within the deploy session -eval "$(tmux showenv -g LOG)"; eval "$(tmux showenv -g MIRROR_REGISTRY)"; eval "$(tmux showenv -g GIT_SERVER)" - -echo "=== D-4: Pattern Deploy ===" 2>&1 | tee -a "$LOG" -cd ~/coco-pattern - -# Verify git remote and branch before deploy -git remote get-url origin 2>&1 | tee -a "$LOG" -# MUST: ${GIT_SERVER}/coco-pattern.git -git branch --show-current 2>&1 | tee -a "$LOG" -# MUST: dev/airgap-testing +This: -./pattern.sh make install \ - 2>&1 | tee -a "$LOG" - -echo "D-4: pattern.sh complete at $(date)" 2>&1 | tee -a "$LOG" -``` +- Creates the patterns-operator Subscription (from the mirrored catalog) +- Creates the Pattern CR (from values-global.yaml) +- Creates the ArgoCD instance (vp-gitops namespace) +- Loads bootstrap secrets (the ArgoCD OCI chart-pull credential) -> **Immediately switch to your main window and start D-5.** `Ctrl-b ` to switch. -> D-5 runs in parallel — it loops waiting for vp-gitops to appear, then fires automatically. -> **Do not wait** — if ArgoCD comes up before D-5 completes it will try to pull charts without credentials. -> MCO reboots during kata deployment are normal — not a failure signal. +**Verify**: `Bootstrap phase: 1 secret(s) created` in the output. +If it says 0, the values-secret is not at the convention location. -### D-5: Inject CA + Load Bootstrap Secrets +### C-3: ArgoCD CA + Bootstrap (parallel with C-2) -> **Start this immediately after launching D-4** from the main tmux window. -> D-5 internally waits for `vp-gitops` to exist — start it now so it fires the moment ArgoCD is ready. -> If you delay, ArgoCD may come up and fail to pull VP OCI Helm charts before credentials are loaded. -> -> Steps D-5 performs once vp-gitops appears: -> -> 1. Inject mirror-registry private CA into ArgoCD's TLS trust store -> 2. `make load-bootstrap` — pre-seeds ArgoCD with mirror-registry OCI Helm credentials -> (from the `bootstrap_secrets` block in values-secret.yaml; no vault required) -> 3. Hard-refresh ArgoCD so it pulls chart manifests from mirror-registry immediately +Run in parallel with the pattern deploy — must fire as soon as ArgoCD +exists: ```bash -echo "=== D-5: ArgoCD CA + Bootstrap Secrets ===" 2>&1 | tee -a "$LOG" -cd ~/coco-pattern - -# Wait for ArgoCD (vp-gitops namespace) -echo "Waiting for vp-gitops namespace..." 2>&1 | tee -a "$LOG" +# Wait for vp-gitops namespace until oc get ns vp-gitops &>/dev/null; do sleep 10; done -echo "vp-gitops active at $(date)" 2>&1 | tee -a "$LOG" - -# Wait for repo-server deployment to exist before patching -until oc get deployment vp-gitops-repo-server -n vp-gitops &>/dev/null; do sleep 10; done - -# Inject mirror-registry CA into ArgoCD TLS trust. -# mirror-registry uses the lab private CA — provide the chain so ArgoCD verifies properly. -# Note: CA cert contains literal newlines which must be escaped for JSON; use Python to build patch. -echo "Injecting mirror-registry CA into argocd-tls-certs-cm..." 2>&1 | tee -a "$LOG" -python3 -c " -import json, subprocess, sys -ca = open('$HOME/mirror-registry-certs/ca.crt').read() -import os; mreg_host = os.environ.get('MIRROR_REGISTRY','MIRROR_REGISTRY_HOST:8443').split(':')[0] -patch = json.dumps({'data': {mreg_host: ca}}) -r = subprocess.run(['oc','patch','configmap','argocd-tls-certs-cm','-n','vp-gitops','--type','merge','-p',patch], capture_output=True, text=True) -print(r.stdout or r.stderr) -sys.exit(r.returncode) -" 2>&1 | tee -a "$LOG" - -# Restart ALL vp-gitops pods to pick up the new CA and any secret changes. -# repo-server uses the CA for Helm registry login; other components may cache certs. -echo "Rolling all vp-gitops deployments..." 2>&1 | tee -a "$LOG" -oc get deployments -n vp-gitops --no-headers -o name 2>/dev/null | \ - while read dep; do oc rollout restart "$dep" -n vp-gitops 2>&1 | tee -a "$LOG"; done - -# Wait for repo-server specifically before loading secrets (it handles Helm OCI auth) -oc rollout status deployment/vp-gitops-repo-server -n vp-gitops --timeout=120s \ - 2>&1 | tee -a "$LOG" - -# Load bootstrap secrets — creates ArgoCD OCI Helm repo secret for mirror-registry. -# This uses values-secret.yaml bootstrap_secrets section; runs before vault init. -# IMPORTANT: mirror-registry-password must have no trailing newline (fixed in D-3). -# A trailing \n causes Helm to send "password\n" → registry rejects with 401. -echo "Loading bootstrap secrets (ArgoCD OCI Helm repo auth)..." 2>&1 | tee -a "$LOG" -make load-bootstrap 2>&1 | tee -a "$LOG" - -# Verify the secret was created and password has correct length (no trailing newline) -oc get secret mirror-registry-helm-oci -n vp-gitops 2>&1 | tee -a "$LOG" -# EXPECTED: mirror-registry-helm-oci Opaque ... -PASS_LEN=$(oc get secret mirror-registry-helm-oci -n vp-gitops \ - -o jsonpath='{.data.password}' 2>/dev/null | base64 -d | wc -c) -echo "mirror-registry-helm-oci password length: ${PASS_LEN} bytes (expected: same as mirror-registry-password file)" \ - 2>&1 | tee -a "$LOG" -if [ "${PASS_LEN}" -gt 24 ]; then - echo "WARN: password may have trailing newline — patch the secret:" 2>&1 | tee -a "$LOG" - echo " MREG_PASS=\$(cat ~/.coco-pattern/mirror-registry-password | tr -d '\\n')" 2>&1 | tee -a "$LOG" - echo " oc create secret generic mirror-registry-helm-oci -n vp-gitops \\" 2>&1 | tee -a "$LOG" - echo " --from-literal=type=helm --from-literal=enableOCI=true \\" 2>&1 | tee -a "$LOG" - echo " --from-literal=url=${MIRROR_REGISTRY}/validatedpatterns \\" 2>&1 | tee -a "$LOG" - echo " --from-literal=name=mirror-registry-charts \\" 2>&1 | tee -a "$LOG" - echo " --from-literal=username=init --from-literal=password=\"\$MREG_PASS\" \\" 2>&1 | tee -a "$LOG" - echo " --dry-run=client -o yaml | oc apply -f -" 2>&1 | tee -a "$LOG" -fi - -# Trigger hard refresh — ArgoCD now has CA trust + credentials for mirror-registry -oc annotate applications.argoproj.io coco-pattern-baremetal -n vp-gitops \ - argocd.argoproj.io/refresh=hard --overwrite 2>&1 | tee -a "$LOG" - -sleep 30 - -# Verify child apps are appearing -echo "--- ArgoCD child apps ---" 2>&1 | tee -a "$LOG" -oc get applications.argoproj.io -A --no-headers 2>/dev/null | \ - awk '{print $2, $4, $5}' | column -t 2>&1 | tee -a "$LOG" -# EXPECTED: vault, acm, kyverno, openshift-external-secrets, storage, etc. are listed - -echo "D-5: ArgoCD configured at $(date)" 2>&1 | tee -a "$LOG" -``` -### D-6: Wait for Vault and Load Secrets +# Inject the mirror CA into ArgoCD's TLS trust +oc patch configmap argocd-tls-certs-cm -n vp-gitops \ + --type merge -p "{\"data\":{\"\":\"$(cat ~/.coco-pattern/registry-ca-cert.pem)\"}}" -```bash -echo "=== D-6: Wait for Vault + Load Secrets ===" 2>&1 | tee -a "$LOG" -cd ~/coco-pattern +# Roll the deployments to pick up the CA +oc get deployments -n vp-gitops --no-headers -o name | \ + while read dep; do oc rollout restart "$dep" -n vp-gitops; done +oc rollout status deployment/vp-gitops-repo-server -n vp-gitops -while true; do - INIT=$(oc exec -n vault vault-0 -- vault status -format=json 2>/dev/null | \ - python3 -c "import sys,json; print(json.loads(sys.stdin.read()).get('initialized',''))" 2>/dev/null || echo "") - SEALED=$(oc exec -n vault vault-0 -- vault status -format=json 2>/dev/null | \ - python3 -c "import sys,json; print(json.loads(sys.stdin.read()).get('sealed',''))" 2>/dev/null || echo "") - echo "STATUS: $(date) — Vault init=$INIT sealed=$SEALED" 2>&1 | tee -a "$LOG" - [ "$INIT" = "True" ] && [ "$SEALED" = "False" ] && echo "Vault ready" 2>&1 | tee -a "$LOG" && break - sleep 15 -done +# Load bootstrap secrets (if not already done by pattern.sh) +make load-bootstrap -./pattern.sh make load-secrets 2>&1 | tee -a "$LOG" -# EXPECTED: 10 secrets injected +# Verify: the secret exists, is labeled, and has the correct URL +oc get secret mirror-registry-helm-oci -n vp-gitops \ + -o jsonpath='{.metadata.labels.argocd\.argoproj\.io/secret-type}' +# Expected: repository -echo "D-6: Secrets loaded at $(date)" 2>&1 | tee -a "$LOG" -``` - ---- +```text -## PHASE E: DCAP and TDX Attestation (~10-15 min) +### C-4: Known Timing Issues -### E-1: Verify DCAP Readiness +Three issues occur consistently in disconnected deployments. Each has a +one-line fix: -```bash -echo "=== E-1: DCAP Readiness ===" 2>&1 | tee -a "$LOG" +#### MCE Subscription (30-60 min after C-1) -oc get pods -n intel-dcap-operator-system -l app=intel-tdx-qgs --no-headers \ - 2>&1 | tee -a "$LOG" -# EXPECTED: NAME 2/2 Running -# NOTE: QGS DaemonSet runs in intel-dcap-operator-system, not openshift-operators - -oc get secrets -n intel-dcap-operator-system -l type=platform-data --no-headers \ - 2>&1 | tee -a "$LOG" -# EXPECTED: 1 secret (name is the QE ID hex string) - -echo "E-1: DCAP readiness verified at $(date)" 2>&1 | tee -a "$LOG" -``` - -### E-2: QGS Socket Port Verification - -> `charts/all/baremetal/templates/vsock-mco.yaml` deploys a MachineConfig drop-in at -> `/etc/kata-containers/kata-tdx/config.d/96-kata-kernel-config` that sets `socket_port=0`. -> Verify that the drop-in exists and the effective port is 0. The base `configuration.toml` -> may still show 4050 — that is overridden by the drop-in, which kata reads last. +The MCH operator creates the `multicluster-engine` subscription with +`source: redhat-operators` (the default catalog, disabled in airgap). +The MCH then blocks in `Installing`. ```bash -echo "=== E-2: QGS Socket Port Verification ===" 2>&1 | tee -a "$LOG" - -NODE=$(oc get nodes -o name | head -1) -echo "Checking node: $NODE" 2>&1 | tee -a "$LOG" - -# Check the drop-in file (authoritative — overrides configuration.toml) -DROP_IN=$(oc debug $NODE -- chroot /host bash -c \ - "cat /etc/kata-containers/kata-tdx/config.d/96-kata-kernel-config 2>/dev/null" \ - 2>/dev/null) -echo "Drop-in content: $DROP_IN" 2>&1 | tee -a "$LOG" - -if echo "$DROP_IN" | grep -q "socket_port=0"; then - echo "PASS: socket_port=0 in drop-in — QGS will use unix socket" 2>&1 | tee -a "$LOG" -else - echo "FAIL: drop-in missing or socket_port not 0 — record as DEV-1 in DEVIATIONS.md" 2>&1 | tee -a "$LOG" - echo " See Appendix A.3 for the fallback sed command (apply ONLY after recording DEV-1)" 2>&1 | tee -a "$LOG" -fi +oc patch subscriptions.operators.coreos.com multicluster-engine \ + -n multicluster-engine --type merge \ + -p '{"spec":{"source":"","sourceNamespace":"openshift-marketplace"}}' -echo "E-2: QGS socket port verification complete at $(date)" 2>&1 | tee -a "$LOG" ``` -> **If verification fails (DEV-1):** The MachineConfig drop-in was not applied — check that -> `oc get mc | grep kata-tdx` shows a MachineConfig with the drop-in content. -> See Appendix Section A.3 for the temporary sed fallback. Record in DEVIATIONS.md. +#### ITMS Org-Prefix (kyverno pods ImagePullBackOff) -### E-3: PCK Lifecycle - -PCK certificates are specific to each QGS platform. The Make targets create checksummed request and response bundles, validate the complete QE-ID/PCE-ID set, and restart QGS only after all matching PCK Secrets are applied. - -The importer discovers the QGS DaemonSet from the `pck-certs-watcher` pod owner reference, avoiding a version-specific DaemonSet label or name. Set `DCAP_QGS_DAEMONSET` only if the namespace has more than one candidate. - -If this bastion can reach **both** the disconnected cluster and Intel PCS, run the combined workflow. The API key is read from `INTEL_PCS_API_KEY` if exported, otherwise Make prompts without echoing it. Never put the key on the Make command line. +The manual ITMS mirrors must include the org prefix for Quay targets. ```bash -echo "=== E-3: Combined PCK provisioning ===" 2>&1 | tee -a "$LOG" -cd ~/coco-pattern -make dcap-pck-provision 2>&1 | tee -a "$LOG" -``` +# Verify: all mirrors should include // +oc get itms itms-manual-mirrors -o json | \ + python3 -c "import json,sys; [print(m['source'],'->',m['mirrors']) for m in json.load(sys.stdin)['spec']['imageTagMirrors']]" -If the high-side cluster cannot reach the connected low-side host, use the split workflow: +```text -```bash -# High side: create ~/.coco-pattern/dcap-pck/platform-request and transfer it high -> low. -make dcap-platform-export - -# Low side: copy platform-request to ~/.coco-pattern/dcap-pck/, then run these commands. -make dcap-tools dcap-pck-generate - -# Transfer ~/.coco-pattern/dcap-pck/pck-response low -> high, then import it on the high side. -make dcap-pck-import -``` +If any mirror is missing the org, patch the ITMS and delete the affected pods. -### E-4: Collect DCAP Collateral and Firmware Reference Values +#### Initdata Staleness (CVM workloads in CreateContainerError) -`collect-dcap-collateral` is independent of PCK provisioning. It collects public Intel TDX verification collateral on the connected low side and does not require an Intel PCS API key. Transfer `~/.coco-pattern/dcap-offline/platform_collaterals.json` to the high side before E-5. +The `cc_init_data` annotation is immutable at pod creation. Workload pods +created before the KBS is ready will retry indefinitely with stale initdata. ```bash -echo "=== E-4: Collect DCAP Collateral + Firmware Refvals ===" 2>&1 | tee -a "$LOG" -cd ~/coco-pattern - -make collect-dcap-collateral 2>&1 | tee -a "$LOG" - -# Verify QeIdentity is populated (must be >100 chars) -python3 -c " -import json, os -c = json.load(open(os.path.expanduser('~/.coco-pattern/dcap-offline/platform_collaterals.json'))) -qi = c.get('collaterals', {}).get('qeidentity_early') or c.get('collaterals', {}).get('qeidentity') -print('qeidentity present:', isinstance(qi, dict), '— PASS' if isinstance(qi, dict) else '— FAIL: recollect collateral') -" 2>&1 | tee -a "$LOG" - -make collect-firmware-refvals 2>&1 | tee -a "$LOG" +# Prerequisite: KBS Running and attestation returning 200 +oc delete pod -n hello-openshift --all +oc delete pod -n kbs-access --all -echo "E-4: DCAP collateral and refvals collected at $(date)" 2>&1 | tee -a "$LOG" -``` - -### E-5: Reload Secrets - -```bash -echo "=== E-5: Reload Secrets ===" 2>&1 | tee -a "$LOG" -cd ~/coco-pattern - -./pattern.sh make load-secrets 2>&1 | tee -a "$LOG" -# EXPECTED: 10 secrets injected (including DCAP collateral and firmware refvals) - -echo "E-5: Secrets reloaded at $(date)" 2>&1 | tee -a "$LOG" ``` --- -## PHASE F: Verification and Pass/Fail (~10 min) +## Phase D: Secrets and Attestation -### F-1: Wait for initdata ConfigMaps +### D-1: Load Vault Secrets ```bash -echo "=== F-1: Wait for initdata ConfigMaps ===" 2>&1 | tee -a "$LOG" +# Waits for vault-0, then loads all secrets from the values-secret file +./pattern.sh make load-secrets -while true; do - COUNT=$(oc get configmap -n imperative -l coco.io/type=initdata --no-headers 2>/dev/null | wc -l) - echo "STATUS: $(date) — initdata ConfigMaps: $COUNT/2" 2>&1 | tee -a "$LOG" - [ "$COUNT" -ge 2 ] && echo "Initdata ready" 2>&1 | tee -a "$LOG" && break - sleep 30 -done - -echo "F-1: initdata ConfigMaps ready at $(date)" 2>&1 | tee -a "$LOG" -``` +```text -### F-2: Bounce Workload Pods +### D-2: PCK Provisioning (parallel with D-1) ```bash -echo "=== F-2: Bounce Workload Pods ===" 2>&1 | tee -a "$LOG" +export INTEL_PCS_API_KEY=$(cat ~/.coco-pattern/intel-pcs-api-key) +make dcap-pck-provision -oc delete pods --all -n hello-openshift --force --grace-period=0 \ - 2>&1 | tee -a "$LOG" -oc delete pods --all -n kbs-access --force --grace-period=0 \ - 2>&1 | tee -a "$LOG" +# Restart the QGS to pick up the provisioned PCK +oc delete pod -n intel-dcap-operator-system -l app=intel-tdx-dcap-qgs -echo "Waiting 90 seconds for pods to restart..." 2>&1 | tee -a "$LOG" -sleep 90 - -echo "F-2: Pods bounced at $(date)" 2>&1 | tee -a "$LOG" ``` -### F-3: Verify Pods Running +### D-3: Firmware Reference Values ```bash -echo "=== F-3: Verify Pods Running ===" 2>&1 | tee -a "$LOG" - -echo "--- hello-openshift ---" 2>&1 | tee -a "$LOG" -oc get pods -n hello-openshift 2>&1 | tee -a "$LOG" -# EXPECTED: 3/3 Running (standard, insecure-policy, secure) - -echo "--- kbs-access ---" 2>&1 | tee -a "$LOG" -oc get pods -n kbs-access 2>&1 | tee -a "$LOG" -# EXPECTED: kbs-access-curl 1/1, kbs-access-sealed 1/1 (or similar) - -echo "F-3: Pod status captured at $(date)" 2>&1 | tee -a "$LOG" -``` - -### F-4: Verify KBS Attestation - -```bash -echo "=== F-4: Verify KBS Attestation ===" 2>&1 | tee -a "$LOG" - -oc logs deployment/trustee-deployment -n trustee-operator-system -c kbs --tail=20 \ - 2>&1 | tee -a "$LOG" -# EXPECTED: POST /attest 200, GET /resource 200 - -echo "F-4: KBS attestation verified at $(date)" 2>&1 | tee -a "$LOG" -``` - -### F-5: Verify Image Sources - -```bash -echo "=== F-5: Verify Image Sources (all from mirror-registry) ===" 2>&1 | tee -a "$LOG" - -for ns in hello-openshift kbs-access trustee-operator-system; do - echo "--- $ns ---" 2>&1 | tee -a "$LOG" - oc get pods -n $ns -o jsonpath='{range .items[*]}{.metadata.name}{": "}{.spec.containers[*].image}{"\n"}{end}' \ - 2>&1 | tee -a "$LOG" -done - -# Flag any image not from mirror-registry or the internal OCP registry -echo "Upstream refs (should be empty):" 2>&1 | tee -a "$LOG" -oc get pods -A -o jsonpath='{range .items[*]}{.spec.containers[*].image}{"\n"}{end}' 2>/dev/null \ - | grep -v "${MIRROR_REGISTRY}\|image-registry.openshift-image-registry" \ - | sort | uniq \ - 2>&1 | tee -a "$LOG" || echo "All images from expected registries" | tee -a "$LOG" - -echo "F-5: Image sources verified at $(date)" 2>&1 | tee -a "$LOG" -``` - -### F-6: Verify ArgoCD Apps - -```bash -echo "=== F-6: Verify ArgoCD Apps ===" 2>&1 | tee -a "$LOG" - -oc get applications.argoproj.io -n vp-gitops \ - 2>&1 | tee -a "$LOG" -# EXPECTED: All apps Synced+Healthy - -APP_COUNT=$(oc get applications.argoproj.io -n vp-gitops --no-headers 2>/dev/null | wc -l) -HEALTHY_COUNT=$(oc get applications.argoproj.io -n vp-gitops --no-headers 2>/dev/null \ - | grep -c "Synced.*Healthy" || echo "0") -echo "ArgoCD: $HEALTHY_COUNT/$APP_COUNT apps Synced+Healthy" 2>&1 | tee -a "$LOG" - -echo "F-6: ArgoCD status captured at $(date)" 2>&1 | tee -a "$LOG" -``` - -### F-7: Phase 24 Validation Checks (D-07) - -> **These checks validate that Phase 24 cleanup decisions did not break the deploy.** -> Record results in DEVIATIONS.md if any check fails. - -```bash -echo "=== F-7: Phase 24 Validation Checks (D-07) ===" 2>&1 | tee -a "$LOG" - -# Check 1: scripts/deprecated/ contains exactly 3 scripts, none were invoked during deploy -echo "--- Check 1: scripts/deprecated/ contents ---" 2>&1 | tee -a "$LOG" -ls -la ~/coco-pattern/scripts/deprecated/ 2>&1 | tee -a "$LOG" -DEPRECATED_COUNT=$(ls ~/coco-pattern/scripts/deprecated/ | wc -l) -echo "Deprecated script count: $DEPRECATED_COUNT (expected: 3)" 2>&1 | tee -a "$LOG" -if [ "$DEPRECATED_COUNT" = "3" ]; then - echo "PASS: Exactly 3 deprecated scripts present (none removed, none invoked)" 2>&1 | tee -a "$LOG" -else - echo "DEVIATION: Record deprecated count mismatch in DEVIATIONS.md" 2>&1 | tee -a "$LOG" -fi - -# Check 2: values-baremetal-airgap.yaml is the 2-line global overlay (D-05 / Phase 24 C-05) -echo "--- Check 2: values-baremetal-airgap.yaml shape ---" 2>&1 | tee -a "$LOG" -cat ~/coco-pattern/values-baremetal-airgap.yaml 2>&1 | tee -a "$LOG" -OVERLAY_LINES=$(grep -c "global\." ~/coco-pattern/values-baremetal-airgap.yaml 2>/dev/null || echo "0") -echo "global.* lines in overlay: $OVERLAY_LINES (expected: 2)" 2>&1 | tee -a "$LOG" -grep -q "global.catalogSource" ~/coco-pattern/values-baremetal-airgap.yaml && \ - grep -q "global.catalogSourceNamespace" ~/coco-pattern/values-baremetal-airgap.yaml && \ - echo "PASS: 2-line global catalogSource overlay active" 2>&1 | tee -a "$LOG" || \ - echo "DEVIATION: values-baremetal-airgap.yaml does not match expected 2-line shape" 2>&1 | tee -a "$LOG" - -# Check 3: mirror-registry is still present and serving -echo "--- Check 3: mirror-registry health ---" 2>&1 | tee -a "$LOG" -curl -sk --cacert ~/mirror-registry-certs/ca.crt \ - https://${MIRROR_REGISTRY}/v2/_catalog \ - 2>&1 | tee -a "$LOG" -MIRROR_STATUS=$? -if [ "$MIRROR_STATUS" = "0" ]; then - echo "PASS: mirror-registry accessible at ${MIRROR_REGISTRY}" 2>&1 | tee -a "$LOG" -else - echo "DEVIATION: mirror-registry not accessible — record in DEVIATIONS.md" 2>&1 | tee -a "$LOG" -fi - -# Check 4: Record whether VP hybrid images required mirror-registry or pulled from Quay alone -echo "--- Check 4: VP hybrid image source ---" 2>&1 | tee -a "$LOG" -echo "Review B-4 log output: did airgap-post-install.sh WARN about VP images?" 2>&1 | tee -a "$LOG" -grep "WARN.*VP\|WARN.*patterns-operator\|mirror-registry" "$LOG" | head -5 2>&1 | tee -a "$LOG" -echo "If WARNs appeared → mirror-registry still needed (record in DEVIATIONS.md as Phase 26 prerequisite confirmed)" 2>&1 | tee -a "$LOG" -echo "If no WARNs → VP images served from Quay alone (record as Phase 26 milestone: mirror-registry removable)" 2>&1 | tee -a "$LOG" - -echo "F-7: Phase 24 validation checks complete at $(date)" 2>&1 | tee -a "$LOG" -``` - -### F-8: Pass/Fail Criteria Table +# Collect from the live node (re-run if the node was reinstalled) +make collect-firmware-refvals +./pattern.sh make load-secrets # reload with fresh values -Fill in the ACTUAL column after running the above steps. Record any failures in DEVIATIONS.md. - -```bash -echo "=== F-8: Pass/Fail Summary ===" 2>&1 | tee -a "$LOG" - -# Collect key status values for the summary table -echo "patterns-operator CSV:" 2>&1 | tee -a "$LOG" -oc get csv -n openshift-operators -l operators.coreos.com/patterns-operator.openshift-operators \ - --no-headers 2>&1 | tee -a "$LOG" - -echo "Vault status:" 2>&1 | tee -a "$LOG" -oc exec -n vault vault-0 -- vault status 2>&1 | tee -a "$LOG" - -echo "MCH status:" 2>&1 | tee -a "$LOG" -oc get mch -A 2>&1 | tee -a "$LOG" - -echo "=== Run Complete: $(date) ===" 2>&1 | tee -a "$LOG" -``` - -**Pass/Fail Criteria Reference Table:** - -| Criterion | Expected | Actual | -|-----------|----------|--------| -| patterns-operator CSV Succeeded | Via OLM from mirror-registry community catalog | | -| Pattern CR reconciles | targetRepo = HTTP Git URL (not GitHub) | | -| Vault init without manual SA creation | Sync-wave fix still working | | -| 15/15 (or current count) ArgoCD apps Synced+Healthy | All green | | -| KBS attestation affirming | POST /attest 200 | | -| KBS resource delivery | GET /resource 200 | | -| hello-openshift 3/3 Running | standard + insecure-policy + secure | | -| kbs-access-curl Running | Uses privileged SCC SA | | -| kbs-access-sealed Running | Uses Secret volume | | -| secret.txt contains KBS resource | Fetched via CDH from KBS | | -| All images from mirror-registry | No upstream refs | | -| MCH Running | ACM + MCE healthy | | -| QGS socket_port=0 WITHOUT manual sed | D-06 — MCO fix working (new row) | | -| No mirrorSourcePolicy conflicts after D-1.5 | PASS: all IDMS/ITMS have NeverContactSource (new row) | | -| scripts/deprecated/ untouched — 3 files, none invoked | D-07 — no deprecated scripts called (new row) | | -| 2-line global catalogSource overlay active in values-baremetal-airgap.yaml | D-07 — Phase 24 C-05 migration active (new row) | | -| mirror-registry serving VP hybrid images | D-07 — confirms two-registry still needed OR single-registry milestone | | - -**Known Warnings (not failures):** - -| Warning | Explanation | -|---------|-------------| -| VP hybrid images WARN in airgap-post-install.sh | Images pre-pushed to mirror-registry. Script cannot re-push (already there). | -| insights operator unavailable | Expected in airgap — cannot reach console.redhat.com | -| clusterversion Error reconciling | Caused by insights operator. Cosmetic. | -| MCO reboots during kata deployment | Normal behavior — not a failure signal | - ---- - -## APPENDIX - -### A.1: DEVIATIONS.md Template (D-04) - -Copy this template to `~/coco-pattern/DEVIATIONS.md` at the start of the run. -Add a new `## DEV-N` section for each deviation discovered during execution. - -```markdown -# Run 25 Deviations - -**Run date:** YYYY-MM-DD -**Operator:** chbutler -**Log file:** ~/coco-pattern/logs/run-25-YYYYMMDD.log - -## Format - -Each deviation entry: -- What happened (observation) -- What was done (action taken) -- Impact on Phase 26 cleanup decisions +```text --- -## DEV-1: [Title — fill in if QGS socket_port fix not automatic] - -**Phase step:** E-2 -**What happened:** [e.g., socket_port was 4050 instead of 0 in kata-tdx/configuration.toml] -**What was done:** [e.g., applied fallback sed — see Appendix A.3] -**Impact on Phase 26:** [e.g., MCO-layer QGS fix not working; Phase 26 must investigate and automate] - ---- - -## DEV-2: [Title — fill in if QE ID mismatch required pcsclient.py] - -**Phase step:** E-3 -**What happened:** [e.g., cluster QE ID c0560e9b019a720a6a09149ec518bbe1 did not match cached ID] -**What was done:** [e.g., ran pcsclient.py interactively to generate new PCK material] -**Impact on Phase 26:** [e.g., hypothesis disproved — SGX platform reset required; investigate trigger] - ---- - -## DEV-N: [Add more entries as needed] - -**Phase step:** -**What happened:** -**What was done:** -**Impact on Phase 26:** -``` - -### A.2: local-registry API Reference - -Container name: `local-registry` (podman). Image: `docker.io/library/registry:2`. -Storage: `~/local-registry` (host bind mount, owned by chbutler). -Config: `~/mirror-registry-config/config.yml`. Certs: `~/mirror-registry-certs/`. -Auth: htpasswd (`~/mirror-registry-config/htpasswd`). Username: `init`. -Password file: `~/.coco-pattern/mirror-registry-password` (600, git-ignored). - -```bash -# List all repos -curl -sk -u "$MREG_USER:$MREG_PASS" https://${MIRROR_REGISTRY}/v2/_catalog - -# List tags for a repo -curl -sk -u "$MREG_USER:$MREG_PASS" https://${MIRROR_REGISTRY}/v2//tags/list -``` - -TLS: self-signed cert — use `-sk` (skip TLS verify) or `--cacert ~/mirror-registry-certs/ca.crt` - -Decommission: `podman stop local-registry && rm -rf ~/local-registry/* && podman start local-registry` - -SELinux note: htpasswd file requires `container_file_t` label. If recreating, run: -`chcon -t container_file_t ~/mirror-registry-config/htpasswd` - -CA trust for container tools (podman, oc-mirror, skopeo): +## Verification ```bash -mkdir -p ~/.config/containers/certs.d/${MIRROR_REGISTRY} -cp ~/mirror-registry-certs/ca.crt ~/.config/containers/certs.d/${MIRROR_REGISTRY}/ca.crt -``` +# All applications Synced+Healthy +oc get applications.argoproj.io -A --no-headers | \ + awk '$3!="Synced" || $4!="Healthy"' +# Expected: no output -This is idempotent and is run automatically in A-0. It enables `--dest-tls-verify=true` in oc-mirror -and removes the need for `--tls-verify=false` in podman login. +# Attestation returning 200 +oc logs -n trustee-operator-system deployment/trustee-deployment \ + --tail=5 | grep "attest.*200" -### A.3: QGS Socket Port Fallback (Deviation Footnote Only) - -> **Apply ONLY if E-2 verification fails and DEV-1 is recorded in DEVIATIONS.md.** -> This is a deviation fallback — not a normal execution step. - -```bash -# FALLBACK ONLY (if E-2 fails) — record as DEV-1 first -echo "Applying QGS socket port fallback sed (DEV-1 deviation)..." 2>&1 | tee -a "$LOG" -NODE=$(oc get nodes -o name | head -1) -oc debug $NODE -- chroot /host bash -c \ - "sed -i 's/tdx_quote_generation_service_socket_port = 4050/tdx_quote_generation_service_socket_port = 0/' \ - /etc/kata-containers/kata-tdx/configuration.toml" \ - 2>&1 | tee -a "$LOG" -echo "Fallback sed applied — record in DEVIATIONS.md as DEV-1" 2>&1 | tee -a "$LOG" -``` +# CVM workloads pulling from the mirror +oc get pods -n hello-openshift -o jsonpath='{.items[0].spec.containers[0].image}' -### A.4: Post-Run Log Retrieval - -After the run completes, Claude SSHes in to analyze the log: - -```bash -# Claude retrieves log from jump host -ssh chbutler@user-jump.int.apac-tech-lab.net "cat ~/coco-pattern/logs/run-25-*.log" -``` - -The operator does not need to gzip, commit, or serve the log file. Claude reads it directly via SSH. -After analysis, commit DEVIATIONS.md to coco-gsd: - -```bash -# On jump host — commit DEVIATIONS.md back -cd ~/coco-pattern -git add DEVIATIONS.md -git commit -m "docs(run-25): record run deviations" -git push origin dev/airgap-testing - -# Then in coco-gsd -cd ~/coco-gsd -cp ../coco-pattern/DEVIATIONS.md .planning/phases/25-manual-logged-jump-host-deploy-runbook/RUN-25-DEVIATIONS.md -git add .planning/phases/25-manual-logged-jump-host-deploy-runbook/RUN-25-DEVIATIONS.md -git commit -m "docs(25): import run-25 deviations log" -git push origin main ``` -### A.5: Environment Variable Quick-Reference - -Set these before starting the run: - -```bash -export LOG=~/coco-pattern/logs/run-25-YYYYMMDD.log -export MREG_USER=init -export MREG_PASS=$(cat ~/.coco-pattern/mirror-registry-password) -``` - -Auto-set during run: - -```bash -export KUBECONFIG= -export MIRROR_REGISTRY=user-jump.int.apac-tech-lab.net:8443 -``` - -Persistent in `~/.envrc`: +--- -```bash -export LABCTL_IDRAC_USER=chbutler -export LABCTL_IDRAC_PASSWORD="..." -``` +## Appendix A: Plain Docker Registry Alternative -### A.6: Add local-registry to pull-secret.json +A plain Docker registry (registry:2) can be used instead of Quay. The key +differences: -The cluster's global pull-secret (embedded in `install-config.yaml` by labctl) must include -credentials for the mirror registry (`user-jump.int.apac-tech-lab.net:8443` — the DNS -identity per Phase 0-0). Without this, cluster nodes cannot pull images redirected -by the IDMS files generated by oc-mirror. +| Aspect | Quay | Docker registry | +|---|---|---| +| Content path | `//` (org prefix) | `/` (flat) | +| MIRROR_REGISTRY | `:443/` | `:8443` (no org) | +| Authentication | Robot account (token) | htpasswd (init user) | +| CA | Quay CA chain | Self-signed CA | +| Setup | Pre-existing Quay | podman run registry:2 | -Run once after initial auth setup, or if the registry password is regenerated: +### Docker Registry Setup ```bash -MREG_PASS=$(cat ~/.coco-pattern/mirror-registry-password) -MREG_AUTH=$(echo -n "init:$MREG_PASS" | base64 -w 0) +# One-time: generate CA, htpasswd, config, and start the container +# (see the Phase 0 section of the historical runbook for the full script) -python3 - <<'EOF' -import json, os +# The MIRROR_REGISTRY does NOT include an org prefix: +export MIRROR_REGISTRY=:8443 -ps_path = os.path.expanduser("~/pull-secret.json") -with open(ps_path) as f: - ps = json.load(f) +# All other configuration is the same, minus the org prefix: +# helmRepoUrl: :8443/validatedpatterns +# image overrides: :8443/ubi9/... +# PATTERN_DISCONNECTED_HOME: :8443/validatedpatterns +# ITMS mirrors: :8443/ (no org) -import base64 -mreg_pass = open(os.path.expanduser("~/.coco-pattern/mirror-registry-password")).read().strip() -mreg_auth = base64.b64encode(f"init:{mreg_pass}".encode()).decode() - -ps["auths"]["${MIRROR_REGISTRY}"] = { - "auth": mreg_auth, - "username": "init", - "password": mreg_pass -} - -with open(ps_path, "w") as f: - json.dump(ps, f, indent=2) - -print("Done. Registries in pull-secret.json:") -for k in ps["auths"]: - print(f" {k}") -EOF -``` +```text -> **When to re-run:** Only if `~/.coco-pattern/mirror-registry-password` changes (password -> regeneration). The htpasswd auth survives `rm -rf ~/local-registry/*` (A-4) — only image -> data is wiped, not auth config. +The Docker registry stores all manifest types (including OCI image indexes +and hybrid manifests that Quay may reject). However, it lacks Quay's +org-based isolation, robot accounts, and API — use Quay where available. diff --git a/airgap/imageset-config-4.22.yaml b/airgap/imageset-config-4.22.yaml index 26f0d250..9bd9794a 100644 --- a/airgap/imageset-config-4.22.yaml +++ b/airgap/imageset-config-4.22.yaml @@ -125,8 +125,8 @@ mirror: # --- VP Helm OCI charts --- - name: quay.io/validatedpatterns/clustergroup:0.9.58 - name: quay.io/validatedpatterns/acm:0.1.27 - # Minimum selected by the Vault applications (~0.1.10); mirror newer patches before use. - - name: quay.io/validatedpatterns/hashicorp-vault:0.1.10 + # values pin ~0.1.10 resolves the newest 0.1.x — mirror that, not the floor + - name: quay.io/validatedpatterns/hashicorp-vault:0.1.11 - name: quay.io/validatedpatterns/openshift-external-secrets:0.0.4 - name: quay.io/validatedpatterns/trustee:0.10.2 - name: quay.io/validatedpatterns/sandboxed-containers:0.2.1 @@ -139,7 +139,10 @@ mirror: - name: quay.io/validatedpatterns/utility-container:latest - name: quay.io/validatedpatterns/imperative-container:latest - name: quay.io/validatedpatterns/imperative-container:v1 - - name: quay.io/validatedpatterns/pattern-ui-catalog:stable-v1 + # pattern-ui-catalog deliberately NOT mirrored: non-essential (console-tab + # catalog content; nothing in the pattern or the operator CSV deploys it) + # and its OCI-index-wrapping-Docker-children manifest is rejected by Quay. + # See coco-gsd specs/003-quay-sole-retest. # --- UBI minimal (ArgoCD repo-server init container) --- - name: registry.redhat.io/ubi9/ubi-minimal:latest @@ -167,9 +170,9 @@ mirror: # ("must be pulled with an explicit version") — a hard mirror error, not a # transient one. Keep commented unless VM boot from DataSource is needed; # if re-enabling, pin explicit version tags. - #- name: registry.redhat.io/rhel9/rhel-guest-image:latest - #- name: registry.redhat.io/rhel8/rhel-guest-image:latest - #- name: registry.redhat.io/container-native-virtualization/virtio-win:latest + # - name: registry.redhat.io/rhel9/rhel-guest-image:latest + # - name: registry.redhat.io/rhel8/rhel-guest-image:latest + # - name: registry.redhat.io/container-native-virtualization/virtio-win:latest # Note: kubeVirtContainer: true (oc-mirror platform flag) is for RHCOS/Hosted Control Planes only. # Do NOT add it to the OCP platform channel for standalone TDX VMs on baremetal — # it is not applicable to this topology and would pull unneeded RHCOS artifacts. diff --git a/values-global.yaml b/values-global.yaml index 13a8941b..31e69af1 100644 --- a/values-global.yaml +++ b/values-global.yaml @@ -54,7 +54,7 @@ main: git: # Connected mode: repoURL is inferred from your local git remote. # Disconnected: uncomment and set to the HTTP URL served by git-http-server.py - # repoURL: http://user-jump.int.apac-tech-lab.net:8080/coco-pattern.git + # repoURL: http://JUMP_HOST_IP:8080/coco-pattern.git revision: main patternsOperator: # Required for airgap -- set to your mirrored catalog source name