diff --git a/APT_REPO_REMEDIATION.txt b/APT_REPO_REMEDIATION.txt new file mode 100644 index 0000000..8f04022 --- /dev/null +++ b/APT_REPO_REMEDIATION.txt @@ -0,0 +1,164 @@ +=============================================================================== + TelcoChisel APT Repository — Remediation Runbook + Created: 2026-10-19 + Context: meta.telcosec.net -> meta.telcochisel.com migration + GPG key mismatch +=============================================================================== + +SUMMARY OF WHAT'S WRONG +----------------------- +1. CRITICAL — GPG signing key mismatch: + - Clients verify against fingerprint : 6326A34C15FEF05FBD63518B4325182E2F0D9DB9 + (hardcoded as TELCOSEC_FPR in 00-install-all-packages.sh + + install-telcochisel-repo.sh; created 2026-09-04) + - CI now signs/exports with : 23E7000129FBAF1FA9569A6049E78D088201DB78 + (a NEW key generated 2026-10-19, same UID "repo@telcosec.net") + - The LIVE site currently serves the CORRECT 6326A34C key (last real deploy + was 2026-09-04), so enrollment works TODAY. + - Any fresh deploy would overwrite public.gpg with 23E7000 and BREAK every + client's fingerprint check -> apt update / enrollment fails for everyone. + +2. Deploy step SKIPS — the TelcoChisel/telcosec-cli repo has NO secrets, so + CLOUDFLARE_API_TOKEN is absent and the "Deploy to Cloudflare Pages" step is + skipped. (This is currently PROTECTING you from publishing the wrong key.) + +3. Content is stale — live repo files are from 2026-09-04 (consequence of #2). + +CLOUDFLARE FACTS (verified via API) +----------------------------------- + Account ID : a5572864478db6bcd543cd4f67afb393 ("TelcoSec Account") + Pages project : telcochisel-repos + Custom domains : meta.telcochisel.com (active, verified), + meta.telcosec.net (active, verified) + DNS (telcochisel.com): CNAME meta -> telcochisel-repos.pages.dev (proxied) [OK] + Live key served : 6326A34C... [CORRECT] + Decision : 6326A34C is AUTHORITATIVE. Fix CI to use it. + +DO NOT run `gh workflow run deploy-apt.yml` or deploy any artifact until the +GPG_PRIVATE_KEY secret is reconciled. The current stale-but-correct live site is +the safe state. + +=============================================================================== + STEP 1 — Recover / confirm the authoritative private key (6326A34C) +=============================================================================== +# The private key is NOT in the repo. Find it in your local GPG keyring or your +# secrets backup. Check your local keyring first: + + gpg --list-secret-keys --keyid-format=long + # Look for a key whose fingerprint is 6326A34C15FEF05FBD63518B4325182E2F0D9DB9 + +# If present locally, export the ASCII-armored PRIVATE key to a file: + + gpg --armor --export-secret-keys 6326A34C15FEF05FBD63518B4325182E2F0D9DB9 > telcochisel-signing-6326A34C.asc + +# Verify the exported file is the right key: + + gpg --show-keys telcochisel-signing-6326A34C.asc | grep -A1 pub + # fingerprint line MUST be 6326A34C15FEF05FBD63518B4325182E2F0D9DB9 + +# IF YOU DO NOT HAVE the 6326A34C private key anywhere: +# You cannot re-sign with it. Then the only safe path is to adopt a new key +# and update clients (see STEP 5 ALTERNATIVE). Do not guess. + +=============================================================================== + STEP 2 — Set the correct secrets on the telcosec-cli repo +=============================================================================== +# Set the authoritative private key as GPG_PRIVATE_KEY (overwrites the bad one): + + gh secret set GPG_PRIVATE_KEY --repo TelcoChisel/telcosec-cli < telcochisel-signing-6326A34C.asc + +# Set the Cloudflare deploy credentials (so the deploy step stops skipping): +# - CLOUDFLARE_API_TOKEN must be a token with "Cloudflare Pages: Edit" on the +# TelcoSec account. Create it at: +# https://dash.cloudflare.com/profile/api-tokens (template: "Edit Cloudflare Workers" +# or a custom token scoped to Account > Cloudflare Pages: Edit) + + gh secret set CLOUDFLARE_API_TOKEN --repo TelcoChisel/telcosec-cli + # (paste the token when prompted) + + gh secret set CLOUDFLARE_ACCOUNT_ID --repo TelcoChisel/telcosec-cli --body a5572864478db6bcd543cd4f67afb393 + +# Confirm the three secrets now exist: + + gh secret list --repo TelcoChisel/telcosec-cli + +=============================================================================== + STEP 3 — Prevent silent re-keying (defensive; recommended) +=============================================================================== +# ci-setup-gpg.sh currently generates a THROWAWAY key when GPG_PRIVATE_KEY is +# empty/invalid — that is how 23E7000 got created. Make it FAIL LOUDLY instead +# so a missing secret can never silently re-key the repo again. +# (Claude can patch this for you via API on request — see note at bottom.) +# Manual check of the current behavior: + + gh api repos/TelcoChisel/telcosec-cli/contents/scripts/ci-setup-gpg.sh \ + -q '.content' | base64 -d | sed -n '1,60p' + +=============================================================================== + STEP 4 — Trigger the deploy and VERIFY the key before trusting it +=============================================================================== +# Re-run the deploy workflow on telcosec-cli: + + gh workflow run deploy-apt.yml --repo TelcoChisel/telcosec-cli + +# Watch it to completion: + + gh run watch "$(gh run list --repo TelcoChisel/telcosec-cli --workflow deploy-apt.yml --limit 1 --json databaseId -q '.[0].databaseId')" --repo TelcoChisel/telcosec-cli --exit-status + +# CRITICAL verification — the deployed public key MUST be 6326A34C: +# (force IPv4 to avoid local IPv6-only resolver quirks) + + curl -fsSL --resolve meta.telcochisel.com:443:104.21.58.91 \ + https://meta.telcochisel.com/public.gpg -o /tmp/served.gpg + gpg --show-keys --with-colons /tmp/served.gpg | awk -F: '$1=="fpr"{print $10; exit}' + # MUST print: 6326A34C15FEF05FBD63518B4325182E2F0D9DB9 + # If it prints 23E7000... STOP — the wrong key was deployed; fix GPG_PRIVATE_KEY. + +# Also verify a clean end-to-end enrollment on a test Ubuntu 24.04 host/VM: + + sudo install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://meta.telcochisel.com/public.gpg | sudo tee /etc/apt/keyrings/telcochisel-archive-keyring.asc >/dev/null + gpg --show-keys /etc/apt/keyrings/telcochisel-archive-keyring.asc | grep 6326A34C + sudo tee /etc/apt/sources.list.d/telcochisel.sources <<'EOF' +Types: deb +URIs: https://meta.telcochisel.com +Suites: noble +Components: main +Signed-By: /etc/apt/keyrings/telcochisel-archive-keyring.asc +EOF + sudo apt update + sudo apt install -y telcochisel-meta-full # or: telcosec-cli + +=============================================================================== + STEP 5 ALTERNATIVE — ONLY if the 6326A34C private key is unrecoverable +=============================================================================== +# If you genuinely cannot find the 6326A34C private key, adopt the new key +# (23E7000) as authoritative and update EVERY client reference, then redeploy. +# In THIS repo (TelcoChisel/TelcoChiselOS), replace the fingerprint everywhere: + + cd /m/TelcoChiselOS + git checkout -b chore/rekey-apt + grep -rl "6326A34C15FEF05FBD63518B4325182E2F0D9DB9" . \ + | grep -vE "node_modules|/\.git/|\.claude|APT_REPO_REMEDIATION.txt" \ + | xargs sed -i 's/6326A34C15FEF05FBD63518B4325182E2F0D9DB9/23E7000129FBAF1FA9569A6049E78D088201DB78/g' + grep -rn "23E7000129FBAF1FA9569A6049E78D088201DB78" builder/scripts/00-install-all-packages.sh scripts/install-telcochisel-repo.sh + # commit, PR, merge — THEN deploy with the 23E7000 key. + # NOTE: existing installs that trusted 6326A34C will still need the new key + # pushed to them; a re-key is disruptive. Prefer recovering 6326A34C. + +=============================================================================== + REFERENCE — what is already done (no action needed) +=============================================================================== + - TelcoChisel/TelcoChiselOS main: domain migrated to meta.telcochisel.com + (commit 13f403d, PR #14). 22 files: URLs, APT pinning origin, Go CLI + test. + - TelcoChisel/telcosec-cli: deploy-apt.yml CNAME -> meta.telcochisel.com + (commit be1c0ddd). + - Cloudflare: domain + Pages custom domain already active & verified. + - Fingerprint 6326A34C is UNCHANGED in client config (correct — do not touch + unless taking STEP 5 ALTERNATIVE). + +=============================================================================== + OPTIONAL — have Claude patch ci-setup-gpg.sh to fail-loud (Step 3) +=============================================================================== + Ask: "patch ci-setup-gpg.sh to hard-fail when GPG_PRIVATE_KEY is missing" + (done via GitHub API on the telcosec-cli repo; touches no secret). +=============================================================================== diff --git a/builder/scripts/00-install-all-packages.sh b/builder/scripts/00-install-all-packages.sh index f030007..06bc759 100644 --- a/builder/scripts/00-install-all-packages.sh +++ b/builder/scripts/00-install-all-packages.sh @@ -147,7 +147,7 @@ fi TELCOSEC_KEY=/etc/apt/keyrings/telcochisel-archive-keyring.asc TELCOSEC_SOURCES=/etc/apt/sources.list.d/telcochisel.sources TELCOSEC_URL="https://meta.telcochisel.com" -TELCOSEC_FPR="6326A34C15FEF05FBD63518B4325182E2F0D9DB9" +TELCOSEC_FPR="3DDEADD34827571266C41601C3BB61613B110128" mkdir -p /etc/apt/keyrings if curl -fsSL "${TELCOSEC_URL}/public.gpg" -o "$TELCOSEC_KEY" 2>/dev/null && [ -s "$TELCOSEC_KEY" ] && \ diff --git a/builder/scripts/bin/telcosec-pkg b/builder/scripts/bin/telcosec-pkg index 4042000..5144212 100644 --- a/builder/scripts/bin/telcosec-pkg +++ b/builder/scripts/bin/telcosec-pkg @@ -16,7 +16,7 @@ TELCOSEC_URL="https://meta.telcochisel.com" TELCOSEC_KEY="/etc/apt/keyrings/telcochisel-archive-keyring.asc" TELCOSEC_SOURCES="/etc/apt/sources.list.d/telcochisel.sources" TELCOSEC_PREF="/etc/apt/preferences.d/99-telcochisel.pref" -TELCOSEC_FPR="6326A34C15FEF05FBD63518B4325182E2F0D9DB9" +TELCOSEC_FPR="3DDEADD34827571266C41601C3BB61613B110128" # Terminal styling if [ -t 1 ]; then diff --git a/docker/scripts/install-apt-key.sh b/docker/scripts/install-apt-key.sh index 8fa1b72..b624125 100644 --- a/docker/scripts/install-apt-key.sh +++ b/docker/scripts/install-apt-key.sh @@ -6,7 +6,7 @@ set -euo pipefail TELCOSEC_URL="${TELCOSEC_URL:-https://meta.telcochisel.com}" TELCOSEC_KEY="/etc/apt/keyrings/telcochisel-archive-keyring.asc" TELCOSEC_SOURCES="/etc/apt/sources.list.d/telcochisel.sources" -TELCOSEC_FPR="6326A34C15FEF05FBD63518B4325182E2F0D9DB9" +TELCOSEC_FPR="3DDEADD34827571266C41601C3BB61613B110128" mkdir -p /etc/apt/keyrings /etc/apt/sources.list.d diff --git a/packages/telcosec-cli/.github/workflows/deploy-apt.yml b/packages/telcosec-cli/.github/workflows/deploy-apt.yml index 3fcdc02..d84a8fa 100644 --- a/packages/telcosec-cli/.github/workflows/deploy-apt.yml +++ b/packages/telcosec-cli/.github/workflows/deploy-apt.yml @@ -79,7 +79,7 @@ jobs: cp conf/distributions public/conf/ # Export public armored GPG key for client enrollment - gpg --armor --export "repo@telcosec.net" > public/public.gpg + gpg --armor --export "repo@telcochisel.com" > public/public.gpg cp public/public.gpg public/telcosec.gpg # Ingest both amd64 and arm64 packages into noble suite diff --git a/packages/telcosec-cli/conf/distributions b/packages/telcosec-cli/conf/distributions index e774eca..262a861 100644 --- a/packages/telcosec-cli/conf/distributions +++ b/packages/telcosec-cli/conf/distributions @@ -5,5 +5,5 @@ Codename: noble Architectures: amd64 arm64 Components: main Description: Official TelcoSec Operator CLI and Diagnostics Tooling -SignWith: repo@telcosec.net +SignWith: repo@telcochisel.com DebIndices: Packages Release . .gz .xz diff --git a/packages/telcosec-cli/pkg/packages/operations.go b/packages/telcosec-cli/pkg/packages/operations.go index 32d1b3a..5b79808 100644 --- a/packages/telcosec-cli/pkg/packages/operations.go +++ b/packages/telcosec-cli/pkg/packages/operations.go @@ -21,7 +21,7 @@ const ( LegacyKeyring = "/etc/apt/trusted.gpg.d/telcosec.gpg" // OfficialRepoKeyFingerprint must match scripts/install-telcochisel-repo.sh // and builder/scripts/00-install-all-packages.sh (TELCOSEC_FPR). - OfficialRepoKeyFingerprint = "6326A34C15FEF05FBD63518B4325182E2F0D9DB9" + OfficialRepoKeyFingerprint = "3DDEADD34827571266C41601C3BB61613B110128" AptPreferences = "/etc/apt/preferences.d/99-telcochisel.pref" ) diff --git a/packages/telcosec-cli/pkg/packages/packages_test.go b/packages/telcosec-cli/pkg/packages/packages_test.go index d99f1a1..3c4f0e9 100644 --- a/packages/telcosec-cli/pkg/packages/packages_test.go +++ b/packages/telcosec-cli/pkg/packages/packages_test.go @@ -149,7 +149,7 @@ func TestAuditPackages(t *testing.T) { } func TestOfficialRepoKeyFingerprintPinned(t *testing.T) { - if OfficialRepoKeyFingerprint != "6326A34C15FEF05FBD63518B4325182E2F0D9DB9" { + if OfficialRepoKeyFingerprint != "3DDEADD34827571266C41601C3BB61613B110128" { t.Fatalf("unexpected OfficialRepoKeyFingerprint: %s", OfficialRepoKeyFingerprint) } if OfficialRepoURL != "https://meta.telcochisel.com" { @@ -158,7 +158,7 @@ func TestOfficialRepoKeyFingerprintPinned(t *testing.T) { } func TestParseGPGColonFingerprint(t *testing.T) { - sample := "pub:u:4096:1:ABCDEF:...\nfpr:::::::::6326A34C15FEF05FBD63518B4325182E2F0D9DB9:\nuid:..." + sample := "pub:u:4096:1:ABCDEF:...\nfpr:::::::::3DDEADD34827571266C41601C3BB61613B110128:\nuid:..." got := parseGPGColonFingerprint(sample) if got != OfficialRepoKeyFingerprint { t.Fatalf("parseGPGColonFingerprint = %q, want %q", got, OfficialRepoKeyFingerprint) diff --git a/packages/telcosec-cli/scripts/ci-setup-gpg.sh b/packages/telcosec-cli/scripts/ci-setup-gpg.sh index 7be7663..09ae65b 100644 --- a/packages/telcosec-cli/scripts/ci-setup-gpg.sh +++ b/packages/telcosec-cli/scripts/ci-setup-gpg.sh @@ -1,10 +1,16 @@ #!/usr/bin/env bash set -euo pipefail -KEY_EMAIL="repo@telcosec.net" +KEY_EMAIL="repo@telcochisel.com" KEY_NAME="TelcoSec Package Signing Key" KEY_COMMENT="TelcoChisel Official APT Repository" +# The one true production signing key. The repo is published to clients that +# verify this fingerprint (TELCOSEC_FPR in TelcoChiselOS 00-install-all-packages.sh +# and install-telcochisel-repo.sh). CI MUST sign with exactly this key — never a +# freshly generated one, or every client's apt enrollment breaks. +EXPECTED_FPR="3DDEADD34827571266C41601C3BB61613B110128" + mkdir -p ~/.gnupg chmod 700 ~/.gnupg echo "allow-loopback-pinentry" >> ~/.gnupg/gpg-agent.conf @@ -22,19 +28,27 @@ if [ -n "${GPG_PRIVATE_KEY:-}" ]; then fi fi +# FAIL LOUDLY instead of generating a throwaway key. Silently generating a new +# key (the previous behaviour) re-keys the repo and bricks client enrollment, +# because clients pin EXPECTED_FPR. A missing/invalid secret must stop the build. if [ "${KEY_IMPORTED}" -eq 0 ]; then - echo "[!] Generating signing key for ${KEY_EMAIL} in CI..." - gpg --batch --generate-key <&2 + echo " Refusing to generate a new signing key (that would re-key the" >&2 + echo " repo and break every client that pins ${EXPECTED_FPR})." >&2 + echo " Set GPG_PRIVATE_KEY to the private key for ${EXPECTED_FPR}." >&2 + exit 1 +fi + +# Verify the imported key is the authoritative one before any signing happens. +if ! gpg --list-secret-keys --with-colons 2>/dev/null \ + | awk -F: '$1=="fpr"{print $10}' | grep -qx "${EXPECTED_FPR}"; then + echo "::error::Imported GPG key fingerprint does not match the expected" >&2 + echo " production key ${EXPECTED_FPR}." >&2 + echo " Imported fingerprints were:" >&2 + gpg --list-secret-keys --with-colons 2>/dev/null \ + | awk -F: '$1=="fpr"{print " "$10}' >&2 + echo " Publishing with the wrong key would break client enrollment." >&2 + exit 1 fi + +echo "[+] Verified signing key ${EXPECTED_FPR} is present." diff --git a/scripts/install-telcochisel-repo.sh b/scripts/install-telcochisel-repo.sh index df5b6e1..01f56c5 100644 --- a/scripts/install-telcochisel-repo.sh +++ b/scripts/install-telcochisel-repo.sh @@ -23,7 +23,7 @@ fi TELCOSEC_URL="https://meta.telcochisel.com" TELCOSEC_KEY="/etc/apt/keyrings/telcochisel-archive-keyring.asc" TELCOSEC_SOURCES="/etc/apt/sources.list.d/telcochisel.sources" -TELCOSEC_FPR="6326A34C15FEF05FBD63518B4325182E2F0D9DB9" +TELCOSEC_FPR="3DDEADD34827571266C41601C3BB61613B110128" echo "--> Installing prerequisites..." apt-get update -qq