Skip to content
37 changes: 27 additions & 10 deletions airgap/DEPLOY-RUNBOOK.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,18 @@ python3 -m pip install -r requirements.txt

```bash
# ── Update these for your environment ──────────────────────────────────────
export MIRROR_REGISTRY=MIRROR_REGISTRY_HOST:8443 # host:port of your mirror registry
# Mirror identity is DNS-based (R6/FR-017): the canonical reference is the
# registry's DNS name so a host address change does not break generated
# artifacts. The IP is a generation-time fact only (cert IP SAN, diagnostics).
export MIRROR_REGISTRY_DNS=user-jump.int.apac-tech-lab.net
export MIRROR_REGISTRY=${MIRROR_REGISTRY_DNS}:8443 # canonical mirror reference
export GIT_SERVER_PORT=8080 # port for local git HTTP server
# ───────────────────────────────────────────────────────────────────────────
export MREG_HOST="${MIRROR_REGISTRY%%:*}"
export MREG_HOST="${MIRROR_REGISTRY%%:*}" # DNS name
export MREG_PORT="${MIRROR_REGISTRY##*:}"
# Generation-time fact for the cert's IP SAN (and diagnostics):
export MREG_IP=$(getent hosts "${MREG_HOST}" | awk '{print $1}' | head -1)
echo "Mirror: ${MIRROR_REGISTRY} (resolved: ${MREG_IP:-UNRESOLVED — check DNS before proceeding})"
```

### 0-1: Mirror Registry Initialisation (ONE-TIME)
Expand All @@ -68,7 +75,9 @@ openssl req -new -x509 -days 3650 \
-subj "/CN=mirror-registry-ca"
echo "CA cert generated: $(openssl x509 -noout -subject -in ~/mirror-registry-certs/ca.crt)"

# b) Generate server TLS cert signed by the CA (SAN = host IP)
# b) Generate server TLS cert signed by the CA (SAN = DNS name + IP)
# DNS SAN is the identity (survives address changes); IP SAN is a generation-time
# fact so tools still referencing the address validate during transition.
openssl genrsa -out ~/mirror-registry-certs/server.key 4096
openssl req -new \
-key ~/mirror-registry-certs/server.key \
Expand All @@ -80,9 +89,9 @@ openssl x509 -req -days 3650 \
-CAkey ~/mirror-registry-certs/ca.key \
-CAcreateserial \
-out ~/mirror-registry-certs/server.crt \
-extfile <(printf "[v3_req]\nsubjectAltName=IP:${MREG_HOST}\n") \
-extfile <(printf "[v3_req]\nsubjectAltName=DNS:${MREG_HOST},IP:${MREG_IP}\n") \
-extensions v3_req
echo "Server cert signed by CA"
echo "Server cert signed by CA (SAN: DNS:${MREG_HOST} + IP:${MREG_IP})"

# c) Generate registry password
# Use printf (NOT echo) — echo appends a newline which breaks Helm registry login (401)
Expand Down Expand Up @@ -146,8 +155,12 @@ echo "local-registry systemd user service enabled"
# podman/skopeo/oc-mirror read from this directory automatically.
# curl: pass --cacert ~/mirror-registry-certs/ca.crt
# oc image mirror: pass --insecure=true
# Trust entries for BOTH the DNS name (canonical) and the IP (transition safety).
mkdir -p ~/.config/containers/certs.d/${MIRROR_REGISTRY}
cp ~/mirror-registry-certs/ca.crt ~/.config/containers/certs.d/${MIRROR_REGISTRY}/ca.crt
echo "CA trusted for container tools"
mkdir -p ~/.config/containers/certs.d/${MREG_IP}:${MREG_PORT}
cp ~/mirror-registry-certs/ca.crt ~/.config/containers/certs.d/${MREG_IP}:${MREG_PORT}/ca.crt
echo "CA trusted for container tools (DNS name + IP)"

# h2) Skip sigstore attachment lookups for certified vendors
# Intel, HashiCorp, and NVIDIA images on registry.connect.redhat.com lack cosign
Expand Down Expand Up @@ -281,8 +294,8 @@ Switch between windows with `Ctrl-b <number>` or `Ctrl-b n/p`.

```bash
# ── Site-specific variables — update these for each deployment environment ──
export MIRROR_REGISTRY=MIRROR_REGISTRY_HOST:8443 # mirror-registry host:port
export GIT_SERVER=http://JUMP_HOST_IP:8080 # local git HTTP server
export MIRROR_REGISTRY=user-jump.int.apac-tech-lab.net:8443 # mirror-registry (DNS identity — see Phase 0-0)
export GIT_SERVER=http://user-jump.int.apac-tech-lab.net:8080 # local git HTTP server (DNS identity)
# Persist to all tmux windows (restore in each window with: eval "$(tmux showenv -g)")
tmux set-environment -g MIRROR_REGISTRY "$MIRROR_REGISTRY"
tmux set-environment -g GIT_SERVER "$GIT_SERVER"
Expand Down Expand Up @@ -596,6 +609,9 @@ podman login ${MIRROR_REGISTRY} \
# Run oc-mirror v2 — full mirror to mirror-registry (sole airgap target, Phase 25.2)
# Note: --v2 (double dash) is mandatory; -v2 (single dash) is not recognised
# CA trust comes from ~/.config/containers/certs.d/${MIRROR_REGISTRY}/ca.crt (set in A-0)
# oc-mirror runs against the DNS name so ALL generated cluster-resources
# (IDMS, ITMS, CatalogSources) reference the DNS name — the IP never enters
# the ignition chain (FR-017). The CA is trusted via certs.d (Phase 0-1h).
oc-mirror \
--config ~/coco-pattern/airgap/imageset-config-4.22.yaml \
docker://${MIRROR_REGISTRY} \
Expand Down Expand Up @@ -1534,7 +1550,7 @@ Auto-set during run:

```bash
export KUBECONFIG=<set in C-6 from build dir>
export MIRROR_REGISTRY=MIRROR_REGISTRY_HOST:8443
export MIRROR_REGISTRY=user-jump.int.apac-tech-lab.net:8443
```

Persistent in `~/.envrc`:
Expand All @@ -1547,7 +1563,8 @@ export LABCTL_IDRAC_PASSWORD="..."
### A.6: Add local-registry to pull-secret.json

The cluster's global pull-secret (embedded in `install-config.yaml` by labctl) must include
credentials for MIRROR_REGISTRY_HOST:8443. Without this, cluster nodes cannot pull images redirected
credentials for the mirror registry (`user-jump.int.apac-tech-lab.net:8443` — the DNS
identity per Phase 0-0). Without this, cluster nodes cannot pull images redirected
by the IDMS files generated by oc-mirror.

Run once after initial auth setup, or if the registry password is regenerated:
Expand Down
52 changes: 32 additions & 20 deletions airgap/imageset-config-4.22.yaml
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
---
# ImageSetConfiguration for CoCo Validated Pattern airgap deployment
# Target: MIRROR_REGISTRY_HOST:8443 (mirror-registry — sole airgap target, Phase 25.2)
# OCP: 4.22.8
# OCP: 4.22.17
# Topology: baremetal (full operator set from values-baremetal.yaml)
#
# Naming convention: imageset-config-<minor>.yaml (4.22 covers all 4.22.x patches)
# Patch target: 4.22.8 — update minVersion/maxVersion to change patch target
# Patch target: 4.22.17 — update minVersion/maxVersion to change patch target
#
# Usage:
# oc-mirror -c airgap/imageset-config-4.22.yaml \
Expand All @@ -19,8 +19,8 @@ mirror:
platform:
channels:
- name: fast-4.22
minVersion: "4.22.8"
maxVersion: "4.22.8"
minVersion: "4.22.17"
maxVersion: "4.22.17"
type: ocp

# Red Hat operators (from values-baremetal.yaml subscriptions)
Expand All @@ -36,10 +36,15 @@ mirror:
- name: trustee-operator
channels:
- name: stable
minVersion: "1.2.0"
maxVersion: "1.2.0"
minVersion: "1.2.1"
maxVersion: "1.2.1"
# GitOps (ArgoCD) — keep aligned with main.gitops.channel in values-global.yaml
# defaultChannel override required: the catalog default is "latest"; filtering
# it out without re-pointing the default fails catalog filtering in oc-mirror v2
# ("invalid default channel configuration ... a new default channel must be
# configured for this package").
- name: openshift-gitops-operator
defaultChannel: gitops-1.22
channels:
- name: gitops-1.22
# External Secrets
Expand Down Expand Up @@ -71,11 +76,17 @@ mirror:
# Certified operators
- catalog: registry.redhat.io/redhat/certified-operator-index:v4.22
packages:
- name: gpu-operator-certified
channels:
- name: v26.3
minVersion: "26.3.0"
maxVersion: "26.3.0"
# GPU operator DISABLED for T1 — matches values-baremetal.yaml (the
# gpu-operator subscription is commented out there: "NVIDIA bundle image
# has oc-mirror signature issue"). The nvcr.io related images
# (dcgm/driver) repeatedly hit copy timeouts, and nothing in T1 consumes
# the bundle. Tranche 2 re-enables the operator at v26.7 (pawelpros),
# which requires its own mirror update per the T022 trigger logic.
# - name: gpu-operator-certified
# channels:
# - name: v26.3
# minVersion: "26.3.0"
# maxVersion: "26.3.0"
# Intel device plugins (for TDX/SGX) — certified, NOT redhat catalog
# Requires registries.d/no-sigstore-certified.yaml (cosign .sig manifests missing)
- name: intel-device-plugins-operator
Expand Down Expand Up @@ -149,15 +160,16 @@ mirror:
- name: registry.access.redhat.com/ubi9/ubi:9.7-1764578509

# --- CNV guest images (OPTIONAL — CNV-OPTIONAL) ---
# CNV-OPTIONAL: Remove or comment out these entries before merging upstream.
# These images are large (~2 GB each). They enable VM boot from DataSource in connected
# environments. In airgap, they must be mirrored or enableCommonBootImageImport must be
# set to false in the HyperConverged CR (Phase 26 sets it to false by default).
# Re-enable after verifying guest images are present in the mirror registry.
# oc-mirror v2 requires tag or digest — :latest is used here; pin to a specific version if reproducibility matters.
- name: registry.redhat.io/rhel9/rhel-guest-image:latest
- name: registry.redhat.io/rhel8/rhel-guest-image:latest
- name: registry.redhat.io/container-native-virtualization/virtio-win:latest
# CNV-OPTIONAL: DISABLED — these images are large (~2 GB each) and NOT
# consumed by the pattern: Phase 26 sets enableCommonBootImageImport: false
# in the HyperConverged CR, so CNV never imports common boot images.
# Additionally, registry.redhat.io REJECTS :latest for virtio-win
# ("must be pulled with an explicit version") — a hard mirror error, not a
# transient one. Keep commented unless VM boot from DataSource is needed;
# if re-enabling, pin explicit version tags.
# - name: registry.redhat.io/rhel9/rhel-guest-image:latest
# - name: registry.redhat.io/rhel8/rhel-guest-image:latest
# - name: registry.redhat.io/container-native-virtualization/virtio-win:latest
# Note: kubeVirtContainer: true (oc-mirror platform flag) is for RHCOS/Hosted Control Planes only.
# Do NOT add it to the OCP platform channel for standalone TDX VMs on baremetal —
# it is not applicable to this topology and would pull unneeded RHCOS artifacts.
21 changes: 14 additions & 7 deletions scripts/airgap-post-install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -195,14 +195,20 @@ print(yaml.dump_all(docs, default_flow_style=False))
done

# Apply manually maintained ITMS (community-operator-pipeline-prod, intel, hashicorp)
# MIRROR_REGISTRY_HOST placeholders are substituted from $MIRROR_REGISTRY at apply
# time (FR-017: canonical identity is the DNS name, so the mirrors follow the env).
local script_dir
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
local manual_itms="${script_dir}/../airgap/itms-manual-mirrors.yaml"
if [[ -f "$manual_itms" ]]; then
info " Applying manual ITMS: $(basename "$manual_itms") with NeverContactSource policy"
python3 -c "
import sys, yaml
docs = list(yaml.safe_load_all(open('$manual_itms')))
MIRROR_REGISTRY="${MIRROR_REGISTRY}" python3 -c "
import sys, os, yaml
raw = open('$manual_itms').read()
mirror = os.environ.get('MIRROR_REGISTRY', '')
if mirror and 'MIRROR_REGISTRY_HOST:8443' in raw and mirror != 'MIRROR_REGISTRY_HOST:8443':
raw = raw.replace('MIRROR_REGISTRY_HOST:8443', mirror)
docs = list(yaml.safe_load_all(raw))
for doc in docs:
if doc and doc.get('kind') == 'ImageTagMirrorSet':
for entry in doc.get('spec', {}).get('imageTagMirrors', []):
Expand Down Expand Up @@ -677,15 +683,16 @@ SVCEOF
fi
fi

# Report URLs
local host_ip
host_ip=$(hostname -I 2>/dev/null | awk '{print $1}' || echo "JUMP_HOST_IP")
# Report URLs — use the DNS name (FR-017): the node resolves it via lab DNS
# (the ignition's resolver), so a host address change does not break pulls.
local host_name
host_name=$(hostname -f 2>/dev/null || echo "JUMP_HOST_DNS")
info ""
info "Git HTTP URLs:"
for repo_path in "${repos[@]}"; do
local repo_name
repo_name=$(basename "$repo_path")
info " http://${host_ip}:${GIT_HTTP_PORT}/${repo_name}.git"
info " http://${host_name}:${GIT_HTTP_PORT}/${repo_name}.git"
done
}

Expand Down
40 changes: 32 additions & 8 deletions scripts/enable-airgap-secrets.sh
Original file line number Diff line number Diff line change
@@ -1,12 +1,18 @@
#!/usr/bin/env bash
# Uncomments the disconnected/airgap-only secrets in a generated
# values-secret file: registryCaCert and bootstrap_secrets
# (mirror-registry-helm-oci). Both are commented out by default in
# values-secret.yaml.template because they only apply to airgap
# (disconnected mirror) deployments and reference local files
# (~/.coco-pattern/mirror-registry-ca-cert.pem,
# ~/.coco-pattern/mirror-registry-password) that don't exist on connected
# deployments.
# values-secret file: registryCaCert, bootstrap_secrets
# (mirror-registry-helm-oci) and tdxCollateral (offline DCAP collateral).
# All are commented out by default in values-secret.yaml.template because
# they only apply to airgap (disconnected mirror) deployments and reference
# local files (~/.coco-pattern/mirror-registry-ca-cert.pem,
# ~/.coco-pattern/mirror-registry-password,
# ~/.coco-pattern/dcap-offline/platform_collaterals.json) that don't exist
# on connected deployments.
#
# It also comments OUT intelPcsApiKey (connected-mode zero-touch PCS
# registration): in airgap mode PCK certificates are managed manually via
# scripts/dcap-pck.py, and load-secrets would otherwise fail on the
# connected-only key file.
#
# Usage:
# ./scripts/enable-airgap-secrets.sh [path-to-values-secret-file]
Expand Down Expand Up @@ -36,11 +42,29 @@ sed -i.bak '
/AIRGAP-MIRROR-SECRETS-END/b
s/^\([[:space:]]*\)#/\1/
}
/# AIRGAP-TDX-COLLATERAL-START/,/# AIRGAP-TDX-COLLATERAL-END/ {
/AIRGAP-TDX-COLLATERAL-START/b
/AIRGAP-TDX-COLLATERAL-END/b
s/^\([[:space:]]*\)#/\1/
}
' "${VALUES_FILE}"
rm -f "${VALUES_FILE}.bak"

# Comment out intelPcsApiKey (connected-mode zero-touch registration only).
# In airgap mode PCK certificates are provisioned manually via
# scripts/dcap-pck.py; the connected key file must not be required.
sed -i.bak '
/^ - name: intelPcsApiKey$/,/^[[:space:]]*$/ {
/^[[:space:]]*$/b
s/^/#/
}
' "${VALUES_FILE}"
rm -f "${VALUES_FILE}.bak"

echo "Uncommented airgap mirror secrets (registryCaCert, bootstrap_secrets) in ${VALUES_FILE}"
echo "Uncommented airgap mirror secrets (registryCaCert, bootstrap_secrets, tdxCollateral) in ${VALUES_FILE}"
echo "Commented out intelPcsApiKey (connected-mode zero-touch registration) in ${VALUES_FILE}"
echo
echo "Make sure these exist before 'make load-secrets' / 'make load-bootstrap':"
echo " ~/.coco-pattern/mirror-registry-ca-cert.pem (make cache-registry-ca)"
echo " ~/.coco-pattern/mirror-registry-password (make gen-mirror-helm-secret)"
echo " ~/.coco-pattern/dcap-offline/platform_collaterals.json (make collect-dcap-collateral)"
2 changes: 1 addition & 1 deletion values-global.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ main:
git:
# Connected mode: repoURL is inferred from your local git remote.
# Disconnected: uncomment and set to the HTTP URL served by git-http-server.py
# repoURL: http://JUMP_HOST_IP:8080/coco-pattern.git
# repoURL: http://user-jump.int.apac-tech-lab.net:8080/coco-pattern.git
revision: main
patternsOperator:
# Required for airgap -- set to your mirrored catalog source name
Expand Down
5 changes: 4 additions & 1 deletion values-secret.yaml.template
Original file line number Diff line number Diff line change
Expand Up @@ -260,13 +260,16 @@ secrets:
# Collected on a connected low-side host via: make collect-dcap-collateral
# Loaded into Vault, delivered to KBS via ESO + kbsLocalCertCacheSpec
# Refresh when collateral expires (~30-90 days)
# Uncomment after running 'make collect-dcap-collateral':
# Uncomment after running 'make collect-dcap-collateral' (or run
# 'make enable-airgap-secrets' for a disconnected deployment):
# AIRGAP-TDX-COLLATERAL-START
#- name: tdxCollateral
# vaultPrefixes:
# - hub
# fields:
# - name: platform_collaterals.json
# path: ~/.coco-pattern/dcap-offline/platform_collaterals.json
# AIRGAP-TDX-COLLATERAL-END

# ===================================================================
# Optional - Intel TDX DCAP connected mode (zero-touch platform
Expand Down
Loading