perf(ci): shard the helper suites so Package invariants fits its cap

.github/workflows/hydra-gates-package.yml

@@ -53,23 +53,32 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  invariants:
-    name: "Package invariants"
+  # The helper suites run on SHARDS legs, balanced by measured run time
+  # (hydra-gates/tests/helper-suite-timings.tsv). Run one after another they
+  # took over 30 minutes by 2026-11-03 (141 suites; the slowest three,
+  # test_gate_acceptance_matrix, test_gate_45_to_55_acceptance and
+  # test_gate_crashed_checker_is_not_a_finding, about 10 minutes together),
+  # so this job hit its timeout on every pull request and read CANCELLED.
+  #
+  # `Package invariants` stays the one name the ruleset requires. It is the
+  # aggregate below, and it fails unless every discovered suite has exactly one
+  # PASS (or QUARANTINED) verdict from exactly one shard: a shard that is
+  # cancelled, crashes or loses a suite cannot read as green.
+  invariants-shard:
+    name: "Package invariants (shard ${{ matrix.shard }} of 6)"
     runs-on: ubuntu-latest
-    # Measured 2026-09-03 across five runs of this job: the ones that finished
-    # took 10.0 and 10.4 minutes; three others ran to exactly 15.3 and were
-    # killed. A 15-minute cap on a 10-minute suite with that spread times out
-    # routinely, and GitHub reports a timeout as CANCELLED — which reads as
-    # "superseded by a newer push" rather than "your suite was killed", so the
-    # failures were being dismissed. It was flapping on `main` and on two other
-    # branches, not on any one change.
-    #
-    # 30 leaves real headroom without hiding a suite that has genuinely started
-    # to hang: double the observed worst finish.
+    # Each shard carries about a sixth of the suite (5 to 6 minutes measured).
+    # The cap is the job's old 30, kept per leg, not raised.
     timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [0, 1, 2, 3, 4, 5]
     steps:
       - uses: actions/checkout@v4
+
       - name: Run the invariant suite
+        if: matrix.shard == 0
         run: bash hydra-gates/tests/test-hydra-gates-bin.sh
 
       # FIRST, with no node_modules at all — this is the only state in which the
@@ -78,41 +87,59 @@ jobs:
       # once ajv is installed below, `NODE_PATH=/nonexistent` no longer hides it
       # (node still resolves from ./node_modules) and that assertion self-skips.
       - name: "Manifest validator: the no-Ajv degraded contract"
+        if: matrix.shard == 0
         run: bash hydra-gates/scripts/lib/test_check_manifest.sh
 
       # Ajv is what gates 22 and 53 validate manifests WITH. Without it the
       # vendored validator can only run its structural lint, and the suite below
-      # would skip exactly the paths worth testing. It is a devDependency of
-      # every fleet app already; installing it here means CI exercises the same
-      # code path a real `npm ci` repo does.
+      # would skip exactly the paths worth testing.
       - name: Install ajv for the manifest-validation helpers
         run: npm install --no-save --no-audit --no-fund ajv ajv-formats
 
-      # EVERY scripts/lib/test_* suite, discovered rather than listed.
-      #
-      # This step used to name three files by hand. The package ships 19, so 15
-      # of them were executed by nothing at all — they existed, they looked like
-      # coverage, and no job ever ran them. Two were pointing at fixture
-      # directories that had never existed:
-      #
-      #   test_check_manifest.sh          green its whole life (a missing
-      #                                   manifest path makes the validator
-      #                                   print "Tier 0, skipping" and exit 0,
-      #                                   which is what 2 of its 3 assertions
-      #                                   expected) — fixed 2026-08-03
-      #   test_check_manifest_crossref.js red its whole life, unseen, because
-      #                                   nothing ran it — fixed 2026-08-04
-      #
-      # A hand-written list of test names decays exactly like a hand-written
-      # paths filter: adding a file does not add it to the list and the omission
-      # is invisible. The runner discovers instead, so a new scripts/lib/test_*
-      # is enforced the moment it lands, with no workflow edit to forget. Suites
-      # may be quarantined only with a stated reason, and the quarantine cannot
-      # rot — an entry that no longer exists, or that starts passing, fails the
-      # step.
-      - name: All helper suites (discovered, not listed)
+      # EVERY scripts/lib/test_* suite, discovered rather than listed, and
+      # split over the shards by run-helper-suites.sh itself. See the header of
+      # that script for why the list is discovered and how the quarantine
+      # cannot rot.
+      - name: Helper suites on this shard (discovered, not listed)
+        env:
+          HELPER_SHARDS: "6"
+          HELPER_SHARD: ${{ matrix.shard }}
+          HELPER_SUITE_REPORT: ${{ runner.temp }}/helper-suites-shard-${{ matrix.shard }}.tsv
         run: bash hydra-gates/tests/run-helper-suites.sh
 
+      - name: Keep this shard's verdicts for the aggregate
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: helper-suites-shard-${{ matrix.shard }}
+          path: ${{ runner.temp }}/helper-suites-shard-${{ matrix.shard }}.tsv
+          if-no-files-found: error
+          retention-days: 3
+
+  invariants:
+    name: "Package invariants"
+    runs-on: ubuntu-latest
+    needs: [invariants-shard]
+    if: ${{ !cancelled() }}
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: The partition puts every suite on exactly one shard
+        run: python3 hydra-gates/tests/test-partition-helper-suites.py
+
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: helper-suites-shard-*
+          path: ${{ runner.temp }}/shards
+          merge-multiple: true
+
+      - name: Every discovered suite passed on exactly one shard
+        env:
+          SHARD_RESULT: ${{ needs.invariants-shard.result }}
+          SHARD_DIR: ${{ runner.temp }}/shards
+        run: python3 hydra-gates/tests/check-helper-shard-reports.py hydra-gates/scripts/lib "${SHARD_DIR}" 6 "${SHARD_RESULT}"
+
   install-from-published-location:
     name: "Install from published location (php:8.3-cli)"
     runs-on: ubuntu-latest

hydra-gates/scripts/lib/test_gate_acceptance_matrix.sh

@@ -107,12 +107,46 @@ echo "== runner declares ${#DECLARED[@]} gates =="
 # ---------------------------------------------------------------------------
 _OUT=""
 _LOGDIR=""
+# Every bundle arm is a read-only fixture and an independent runner process, so
+# _prefetch runs them all up front, HYDRA_ACCEPT_JOBS at a time (default: the
+# CPU count; 1 runs nothing up front). _run then reads the stored output
+# instead of starting the runner again; grading is unchanged and in order.
+_PREFETCH=""
+_prefetch_one() {  # <fixture-dir> <key>
+    mkdir -p "${_PREFETCH}/$2.logs"
+    HYDRA_GATE_LOG_DIR="${_PREFETCH}/$2.logs" \
+        HYDRA_OR_GATE_BLOCK_AFTER_EPOCH=0 \
+        bash "${RUNNER}" "$1" > "${_PREFETCH}/$2.out" 2>&1 || true
+}
+_prefetch() {  # <fixture-dir>...
+    local _jobs="${HYDRA_ACCEPT_JOBS:-$(nproc 2>/dev/null || echo 1)}" _d _n=0
+    [ "${_jobs}" -gt 1 ] 2>/dev/null || return 0
+    _PREFETCH="$(mktemp -d "${TMPDIR:-/tmp}/hydra-accept-pre.XXXXXXXX")"
+    for _d in "$@"; do
+        _prefetch_one "${_d}" "$(_key "${_d}")" &
+        _n=$((_n + 1))
+        if [ "${_n}" -ge "${_jobs}" ]; then
+            wait -n
+            _n=$((_n - 1))
+        fi
+    done
+    wait
+}
+_key() { printf '%s' "${1#"${BUNDLES}/"}" | tr '/' '_'; }
+
 _run() {  # <fixture-dir> [runner args...]
     local _dir="$1"; shift
-    _LOGDIR="$(mktemp -d "${TMPDIR:-/tmp}/hydra-accept.XXXXXXXX")"
-    _OUT="$(HYDRA_GATE_LOG_DIR="${_LOGDIR}" \
-        HYDRA_OR_GATE_BLOCK_AFTER_EPOCH=0 \
-        bash "${RUNNER}" "$@" "${_dir}" 2>&1 || true)"
+    local _k
+    _k="$(_key "${_dir}")"
+    if [ -n "${_PREFETCH}" ] && [ "$#" -eq 0 ] && [ -f "${_PREFETCH}/${_k}.out" ]; then
+        _LOGDIR="${_PREFETCH}/${_k}.logs"
+        _OUT="$(cat "${_PREFETCH}/${_k}.out")"
+    else
+        _LOGDIR="$(mktemp -d "${TMPDIR:-/tmp}/hydra-accept.XXXXXXXX")"
+        _OUT="$(HYDRA_GATE_LOG_DIR="${_LOGDIR}" \
+            HYDRA_OR_GATE_BLOCK_AFTER_EPOCH=0 \
+            bash "${RUNNER}" "$@" "${_dir}" 2>&1 || true)"
+    fi
     # An abort before the summary leaves per-gate PASS lines on stdout and
     # reads exactly like a clean run. Refuse to grade a run that did not finish.
     if ! printf '%s' "${_OUT}" | grep -q '^\[hydra-gates\] COVERAGE:'; then
@@ -259,6 +293,14 @@ fi
 echo "== discovered ${#BUNDLE_DIRS[@]} fixture bundle(s) =="
 echo
 
+_ARMS=()
+for _bundle in "${BUNDLE_DIRS[@]}"; do
+    for _arm in planted clean; do
+        [ -d "${BUNDLES}/${_bundle}/${_arm}" ] && _ARMS+=("${BUNDLES}/${_bundle}/${_arm}")
+    done
+done
+_prefetch "${_ARMS[@]}"
+
 COVERED=()
 
 for _bundle in "${BUNDLE_DIRS[@]}"; do

hydra-gates/tests/check-helper-shard-reports.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: EUPL-1.2
+"""Decide `Package invariants` from the verdicts of every helper-suite shard.
+
+A sharded run must not read green when a suite ran on no shard, ran twice,
+failed, or when a shard never reported. So this checks, against a fresh
+discovery of scripts/lib/test_*:
+
+  * every shard 0..N-1 left its report (helper-suites-shard-<i>.tsv);
+  * every discovered suite has exactly one verdict, across all shards;
+  * every verdict is PASS or QUARANTINED, and no report names an unknown suite;
+  * the shard matrix itself succeeded.
+
+Usage: check-helper-shard-reports.py <lib-dir> <reports-dir> <shards> <matrix-result>
+Exit:  0 all hold, 1 otherwise (each problem printed).
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+
+
+def discover(lib: str) -> set[str]:
+    return {n for n in os.listdir(lib) if n.startswith("test_") and os.path.isfile(os.path.join(lib, n))}
+
+
+def problems(lib: str, reports: str, shards: int, matrix_result: str) -> list[str]:
+    found = []
+    if matrix_result != "success":
+        found.append(f"the shard matrix ended '{matrix_result}', not 'success'")
+    seen: dict[str, list[str]] = {}
+    for i in range(shards):
+        path = os.path.join(reports, f"helper-suites-shard-{i}.tsv")
+        if not os.path.isfile(path):
+            found.append(f"shard {i} left no report ({path})")
+            continue
+        with open(path, encoding="utf-8") as fh:
+            for line in fh:
+                parts = line.rstrip("\n").split("\t")
+                if len(parts) != 2:
+                    continue
+                verdict, name = parts
+                seen.setdefault(name, []).append(f"{verdict} on shard {i}")
+                if verdict not in ("PASS", "QUARANTINED"):
+                    found.append(f"{name}: {verdict} on shard {i}")
+    suites = discover(lib)
+    if not suites:
+        found.append(f"discovered zero suites under {lib}")
+    for name in sorted(suites - seen.keys()):
+        found.append(f"{name} ran on no shard")
+    for name in sorted(seen.keys() - suites):
+        found.append(f"{name} was reported but is not a discovered suite")
+    for name, verdicts in sorted(seen.items()):
+        if len(verdicts) > 1:
+            found.append(f"{name} ran {len(verdicts)} times: {', '.join(verdicts)}")
+    return found
+
+
+def main(argv: list[str]) -> int:
+    if len(argv) != 5 or not argv[3].isdigit():
+        print(__doc__, file=sys.stderr)
+        return 1
+    found = problems(argv[1], argv[2], int(argv[3]), argv[4])
+    for line in found:
+        print(f"::error::{line}")
+    if found:
+        return 1
+    print(f"OK: {len(discover(argv[1]))} suites, each with one passing verdict, over {argv[3]} shards.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))