From 6bb8df8bb856c831c4e28bc98ac3a840b05fc231 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Tue, 3 Nov 2026 14:42:45 +0100 Subject: [PATCH 1/2] perf(ci): shard the helper suites so Package invariants fits its cap The 141 helper suites ran one after another for over 30 minutes, so Package invariants hit its timeout on every pull request. Split them over six legs balanced by measured run time, keep Package invariants as an aggregate that fails unless every discovered suite passed on exactly one shard, print each suite's time, and run the acceptance matrix's read-only fixture arms in parallel. Assisted-by: ClaudeCode:claude-opus-5-5 --- .github/workflows/hydra-gates-package.yml | 103 ++++++++----- .../lib/test_gate_acceptance_matrix.sh | 50 ++++++- .../tests/check-helper-shard-reports.py | 74 +++++++++ hydra-gates/tests/helper-suite-timings.tsv | 141 ++++++++++++++++++ hydra-gates/tests/partition_helper_suites.py | 60 ++++++++ hydra-gates/tests/run-helper-suites.sh | 45 +++++- .../tests/test-partition-helper-suites.py | 130 ++++++++++++++++ 7 files changed, 559 insertions(+), 44 deletions(-) create mode 100644 hydra-gates/tests/check-helper-shard-reports.py create mode 100644 hydra-gates/tests/helper-suite-timings.tsv create mode 100644 hydra-gates/tests/partition_helper_suites.py create mode 100644 hydra-gates/tests/test-partition-helper-suites.py diff --git a/.github/workflows/hydra-gates-package.yml b/.github/workflows/hydra-gates-package.yml index 5281f2ce6..e42afb639 100644 --- a/.github/workflows/hydra-gates-package.yml +++ b/.github/workflows/hydra-gates-package.yml @@ -53,23 +53,32 @@ concurrency: cancel-in-progress: true jobs: - invariants: - name: "Package invariants" + # The helper suites run on SHARDS legs, balanced by measured run time + # (hydra-gates/tests/helper-suite-timings.tsv). Run one after another they + # took over 30 minutes by 2026-11-03 (141 suites; the slowest three, + # test_gate_acceptance_matrix, test_gate_45_to_55_acceptance and + # test_gate_crashed_checker_is_not_a_finding, about 10 minutes together), + # so this job hit its timeout on every pull request and read CANCELLED. + # + # `Package invariants` stays the one name the ruleset requires. It is the + # aggregate below, and it fails unless every discovered suite has exactly one + # PASS (or QUARANTINED) verdict from exactly one shard: a shard that is + # cancelled, crashes or loses a suite cannot read as green. + invariants-shard: + name: "Package invariants (shard ${{ matrix.shard }} of 6)" runs-on: ubuntu-latest - # Measured 2026-09-03 across five runs of this job: the ones that finished - # took 10.0 and 10.4 minutes; three others ran to exactly 15.3 and were - # killed. A 15-minute cap on a 10-minute suite with that spread times out - # routinely, and GitHub reports a timeout as CANCELLED — which reads as - # "superseded by a newer push" rather than "your suite was killed", so the - # failures were being dismissed. It was flapping on `main` and on two other - # branches, not on any one change. - # - # 30 leaves real headroom without hiding a suite that has genuinely started - # to hang: double the observed worst finish. + # Each shard carries about a sixth of the suite (5 to 6 minutes measured). + # The cap is the job's old 30, kept per leg, not raised. timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2, 3, 4, 5] steps: - uses: actions/checkout@v4 + - name: Run the invariant suite + if: matrix.shard == 0 run: bash hydra-gates/tests/test-hydra-gates-bin.sh # FIRST, with no node_modules at all — this is the only state in which the @@ -78,41 +87,59 @@ jobs: # once ajv is installed below, `NODE_PATH=/nonexistent` no longer hides it # (node still resolves from ./node_modules) and that assertion self-skips. - name: "Manifest validator: the no-Ajv degraded contract" + if: matrix.shard == 0 run: bash hydra-gates/scripts/lib/test_check_manifest.sh # Ajv is what gates 22 and 53 validate manifests WITH. Without it the # vendored validator can only run its structural lint, and the suite below - # would skip exactly the paths worth testing. It is a devDependency of - # every fleet app already; installing it here means CI exercises the same - # code path a real `npm ci` repo does. + # would skip exactly the paths worth testing. - name: Install ajv for the manifest-validation helpers run: npm install --no-save --no-audit --no-fund ajv ajv-formats - # EVERY scripts/lib/test_* suite, discovered rather than listed. - # - # This step used to name three files by hand. The package ships 19, so 15 - # of them were executed by nothing at all — they existed, they looked like - # coverage, and no job ever ran them. Two were pointing at fixture - # directories that had never existed: - # - # test_check_manifest.sh green its whole life (a missing - # manifest path makes the validator - # print "Tier 0, skipping" and exit 0, - # which is what 2 of its 3 assertions - # expected) — fixed 2026-08-03 - # test_check_manifest_crossref.js red its whole life, unseen, because - # nothing ran it — fixed 2026-08-04 - # - # A hand-written list of test names decays exactly like a hand-written - # paths filter: adding a file does not add it to the list and the omission - # is invisible. The runner discovers instead, so a new scripts/lib/test_* - # is enforced the moment it lands, with no workflow edit to forget. Suites - # may be quarantined only with a stated reason, and the quarantine cannot - # rot — an entry that no longer exists, or that starts passing, fails the - # step. - - name: All helper suites (discovered, not listed) + # EVERY scripts/lib/test_* suite, discovered rather than listed, and + # split over the shards by run-helper-suites.sh itself. See the header of + # that script for why the list is discovered and how the quarantine + # cannot rot. + - name: Helper suites on this shard (discovered, not listed) + env: + HELPER_SHARDS: "6" + HELPER_SHARD: ${{ matrix.shard }} + HELPER_SUITE_REPORT: ${{ runner.temp }}/helper-suites-shard-${{ matrix.shard }}.tsv run: bash hydra-gates/tests/run-helper-suites.sh + - name: Keep this shard's verdicts for the aggregate + if: always() + uses: actions/upload-artifact@v4 + with: + name: helper-suites-shard-${{ matrix.shard }} + path: ${{ runner.temp }}/helper-suites-shard-${{ matrix.shard }}.tsv + if-no-files-found: error + retention-days: 3 + + invariants: + name: "Package invariants" + runs-on: ubuntu-latest + needs: [invariants-shard] + if: ${{ !cancelled() }} + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - name: The partition puts every suite on exactly one shard + run: python3 hydra-gates/tests/test-partition-helper-suites.py + + - uses: actions/download-artifact@v4 + with: + pattern: helper-suites-shard-* + path: ${{ runner.temp }}/shards + merge-multiple: true + + - name: Every discovered suite passed on exactly one shard + env: + SHARD_RESULT: ${{ needs.invariants-shard.result }} + SHARD_DIR: ${{ runner.temp }}/shards + run: python3 hydra-gates/tests/check-helper-shard-reports.py hydra-gates/scripts/lib "${SHARD_DIR}" 6 "${SHARD_RESULT}" + install-from-published-location: name: "Install from published location (php:8.3-cli)" runs-on: ubuntu-latest diff --git a/hydra-gates/scripts/lib/test_gate_acceptance_matrix.sh b/hydra-gates/scripts/lib/test_gate_acceptance_matrix.sh index b2000da1e..236213263 100644 --- a/hydra-gates/scripts/lib/test_gate_acceptance_matrix.sh +++ b/hydra-gates/scripts/lib/test_gate_acceptance_matrix.sh @@ -107,12 +107,46 @@ echo "== runner declares ${#DECLARED[@]} gates ==" # --------------------------------------------------------------------------- _OUT="" _LOGDIR="" +# Every bundle arm is a read-only fixture and an independent runner process, so +# _prefetch runs them all up front, HYDRA_ACCEPT_JOBS at a time (default: the +# CPU count; 1 runs nothing up front). _run then reads the stored output +# instead of starting the runner again; grading is unchanged and in order. +_PREFETCH="" +_prefetch_one() { # + mkdir -p "${_PREFETCH}/$2.logs" + HYDRA_GATE_LOG_DIR="${_PREFETCH}/$2.logs" \ + HYDRA_OR_GATE_BLOCK_AFTER_EPOCH=0 \ + bash "${RUNNER}" "$1" > "${_PREFETCH}/$2.out" 2>&1 || true +} +_prefetch() { # ... + local _jobs="${HYDRA_ACCEPT_JOBS:-$(nproc 2>/dev/null || echo 1)}" _d _n=0 + [ "${_jobs}" -gt 1 ] 2>/dev/null || return 0 + _PREFETCH="$(mktemp -d "${TMPDIR:-/tmp}/hydra-accept-pre.XXXXXXXX")" + for _d in "$@"; do + _prefetch_one "${_d}" "$(_key "${_d}")" & + _n=$((_n + 1)) + if [ "${_n}" -ge "${_jobs}" ]; then + wait -n + _n=$((_n - 1)) + fi + done + wait +} +_key() { printf '%s' "${1#"${BUNDLES}/"}" | tr '/' '_'; } + _run() { # [runner args...] local _dir="$1"; shift - _LOGDIR="$(mktemp -d "${TMPDIR:-/tmp}/hydra-accept.XXXXXXXX")" - _OUT="$(HYDRA_GATE_LOG_DIR="${_LOGDIR}" \ - HYDRA_OR_GATE_BLOCK_AFTER_EPOCH=0 \ - bash "${RUNNER}" "$@" "${_dir}" 2>&1 || true)" + local _k + _k="$(_key "${_dir}")" + if [ -n "${_PREFETCH}" ] && [ "$#" -eq 0 ] && [ -f "${_PREFETCH}/${_k}.out" ]; then + _LOGDIR="${_PREFETCH}/${_k}.logs" + _OUT="$(cat "${_PREFETCH}/${_k}.out")" + else + _LOGDIR="$(mktemp -d "${TMPDIR:-/tmp}/hydra-accept.XXXXXXXX")" + _OUT="$(HYDRA_GATE_LOG_DIR="${_LOGDIR}" \ + HYDRA_OR_GATE_BLOCK_AFTER_EPOCH=0 \ + bash "${RUNNER}" "$@" "${_dir}" 2>&1 || true)" + fi # An abort before the summary leaves per-gate PASS lines on stdout and # reads exactly like a clean run. Refuse to grade a run that did not finish. if ! printf '%s' "${_OUT}" | grep -q '^\[hydra-gates\] COVERAGE:'; then @@ -259,6 +293,14 @@ fi echo "== discovered ${#BUNDLE_DIRS[@]} fixture bundle(s) ==" echo +_ARMS=() +for _bundle in "${BUNDLE_DIRS[@]}"; do + for _arm in planted clean; do + [ -d "${BUNDLES}/${_bundle}/${_arm}" ] && _ARMS+=("${BUNDLES}/${_bundle}/${_arm}") + done +done +_prefetch "${_ARMS[@]}" + COVERED=() for _bundle in "${BUNDLE_DIRS[@]}"; do diff --git a/hydra-gates/tests/check-helper-shard-reports.py b/hydra-gates/tests/check-helper-shard-reports.py new file mode 100644 index 000000000..59d58b27b --- /dev/null +++ b/hydra-gates/tests/check-helper-shard-reports.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: EUPL-1.2 +"""Decide `Package invariants` from the verdicts of every helper-suite shard. + +A sharded run must not read green when a suite ran on no shard, ran twice, +failed, or when a shard never reported. So this checks, against a fresh +discovery of scripts/lib/test_*: + + * every shard 0..N-1 left its report (helper-suites-shard-.tsv); + * every discovered suite has exactly one verdict, across all shards; + * every verdict is PASS or QUARANTINED, and no report names an unknown suite; + * the shard matrix itself succeeded. + +Usage: check-helper-shard-reports.py +Exit: 0 all hold, 1 otherwise (each problem printed). +""" + +from __future__ import annotations + +import os +import sys + + +def discover(lib: str) -> set[str]: + return {n for n in os.listdir(lib) if n.startswith("test_") and os.path.isfile(os.path.join(lib, n))} + + +def problems(lib: str, reports: str, shards: int, matrix_result: str) -> list[str]: + found = [] + if matrix_result != "success": + found.append(f"the shard matrix ended '{matrix_result}', not 'success'") + seen: dict[str, list[str]] = {} + for i in range(shards): + path = os.path.join(reports, f"helper-suites-shard-{i}.tsv") + if not os.path.isfile(path): + found.append(f"shard {i} left no report ({path})") + continue + with open(path, encoding="utf-8") as fh: + for line in fh: + parts = line.rstrip("\n").split("\t") + if len(parts) != 2: + continue + verdict, name = parts + seen.setdefault(name, []).append(f"{verdict} on shard {i}") + if verdict not in ("PASS", "QUARANTINED"): + found.append(f"{name}: {verdict} on shard {i}") + suites = discover(lib) + if not suites: + found.append(f"discovered zero suites under {lib}") + for name in sorted(suites - seen.keys()): + found.append(f"{name} ran on no shard") + for name in sorted(seen.keys() - suites): + found.append(f"{name} was reported but is not a discovered suite") + for name, verdicts in sorted(seen.items()): + if len(verdicts) > 1: + found.append(f"{name} ran {len(verdicts)} times: {', '.join(verdicts)}") + return found + + +def main(argv: list[str]) -> int: + if len(argv) != 5 or not argv[3].isdigit(): + print(__doc__, file=sys.stderr) + return 1 + found = problems(argv[1], argv[2], int(argv[3]), argv[4]) + for line in found: + print(f"::error::{line}") + if found: + return 1 + print(f"OK: {len(discover(argv[1]))} suites, each with one passing verdict, over {argv[3]} shards.") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/hydra-gates/tests/helper-suite-timings.tsv b/hydra-gates/tests/helper-suite-timings.tsv new file mode 100644 index 000000000..468450ecf --- /dev/null +++ b/hydra-gates/tests/helper-suite-timings.tsv @@ -0,0 +1,141 @@ +test_build_effective_manifest.js 1 +test_check_admin_router.py 1 +test_check_apphost_autoload_prelude.py 1 +test_check_aria_hidden_focusable.py 1 +test_check_autocomplete.py 1 +test_check_button_name.py 1 +test_check_coding_standard_adoption.sh 2 +test_check_connections_declaration.js 2 +test_check_contract_coverage.py 1 +test_check_contract_surface_shift.sh 1 +test_check_cross_app_schema_slug.py 1 +test_check_csrf_callers.py 1 +test_check_csrf_removal.py 1 +test_check_custom_widget_ratchet.py 8 +test_check_dashboard_antipattern.py 1 +test_check_detail_page_cross_app.py 1 +test_check_detail_page_icon_registry.py 1 +test_check_detail_page_placement.py 1 +test_check_duplicate_index_pages.js 1 +test_check_duplicate_page_refresh.sh 1 +test_check_e2e_coverage.py 1 +test_check_e2e_skips.sh 1 +test_check_exclusion_evidence.py 1 +test_check_forbidden_patterns.py 1 +test_check_form_labels.py 1 +test_check_header_action_budget.py 1 +test_check_icon_vocabulary.py 1 +test_check_initial_state.py 1 +test_check_js_call_sites.py 1 +test_check_license_triangle.py 1 +test_check_link_text.py 1 +test_check_listener_placement.py 1 +test_check_manifest.sh 5 +test_check_manifest_ajv_collapse.js 1 +test_check_manifest_copy_style.py 1 +test_check_manifest_crossref.js 3 +test_check_manifest_l10n_coverage.py 1 +test_check_markup_a11y.py 1 +test_check_migration_version_bump.py 1 +test_check_nc_select_labels.py 1 +test_check_newman_reach.py 1 +test_check_no_admin_idor.py 1 +test_check_notification_dialect.py 1 +test_check_npm_supply_chain_config.sh 1 +test_check_openregister_dependency_shape.py 1 +test_check_orphan_auth.py 1 +test_check_orphaned_write_capability.py 1 +test_check_phantom_cross_app_rpc.py 1 +test_check_procedure_code.py 1 +test_check_public_endpoint_throttling.py 1 +test_check_register_auth_dialect.py 1 +test_check_register_handler_resolution.py 1 +test_check_register_version_bump.py 1 +test_check_relation_dialect.py 1 +test_check_schema_property_meta.py 1 +test_check_security_cochange.py 1 +test_check_seed_required_slugs.py 1 +test_check_semantic_auth.py 1 +test_check_shipped_object_properties.py 1 +test_check_sig_floor.py 2 +test_check_sig_maintainability.py 9 +test_check_spec_anchors.py 1 +test_check_spec_coverage.py 1 +test_check_stale_fleet_app_id.py 1 +test_check_store_and_settings_surface.py 1 +test_check_stub_run_body.py 1 +test_check_table_headers.py 1 +test_check_unclosable_gate.py 1 +test_check_unsafe_auth_resolver.py 1 +test_check_visual_coverage.py 1 +test_checker_patterns_match_documented_shapes.py 1 +test_detect_redundant_controllers.py 1 +test_diff_schema_strictness.js 1 +test_exclusion_reason.py 1 +test_exclusion_reason_adjacent.py 1 +test_filter_preexisting_methods.py 1 +test_gate100_setup_demo_first_scope.sh 22 +test_gate101_demo_data_scope.sh 11 +test_gate108_shipped_object_properties_scope.sh 17 +test_gate110_migration_version_bump_scope.sh 22 +test_gate111_flow_node_taxonomy.sh 1 +test_gate114_header_action_budget_scope.sh 22 +test_gate116_connections_declaration_scope.sh 21 +test_gate119_procedure_code_scope.sh 29 +test_gate120_register_version_scope.sh 16 +test_gate121_register_auth_dialect_scope.sh 16 +test_gate122_sig_maintainability_scope.sh 17 +test_gate123_sig_floor_scope.sh 11 +test_gate16_spec_coverage_scope.sh 30 +test_gate19_coverage_credibility.sh 3 +test_gate67_contract_parity_scope.sh 3 +test_gate7_verb_object_guards.sh 1 +test_gate98_repair_registration_scope.sh 20 +test_gate_13_multiline_dialog.sh 29 +test_gate_1_11_honest_verdicts.sh 59 +test_gate_20_comment_masking.sh 14 +test_gate_29_gitignore_negation.sh 10 +test_gate_3_14_comment_evidence.sh 26 +test_gate_3_bodiless_declaration.sh 11 +test_gate_3_contract_param.sh 15 +test_gate_3_external_contract_and_tabs.sh 20 +test_gate_45_selector_parity.sh 103 +test_gate_45_stylesheet_scope.sh 50 +test_gate_45_to_55_acceptance.sh 166 +test_gate_46_tests_scope.sh 46 +test_gate_4_composer_audit_transport_retry.sh 11 +test_gate_5661_empty_scope_is_not_a_pass.sh 21 +test_gate_95_adr_number_collision.sh 1 +test_gate_a11y_helper_wiring.sh 100 +test_gate_a11y_markup_scope.sh 15 +test_gate_acceptance_matrix.sh 268 +test_gate_base_ref_delivery_channel.sh 14 +test_gate_crashed_checker_is_not_a_finding.sh 165 +test_gate_diff_provenance.sh 17 +test_gate_discarded_counts_and_empty_deltas.sh 38 +test_gate_empty_scope_never_passes.sh 41 +test_gate_errexit_discipline.sh 3 +test_gate_exit_code_semantics.sh 8 +test_gate_fq_route_names.sh 7 +test_gate_glob_recursion_fixtures.sh 4 +test_gate_license_triangle_scope.sh 20 +test_gate_local_base_is_honoured.sh 10 +test_gate_mainline_push_audits_everything.sh 7 +test_gate_monitoring_and_skiplink.sh 18 +test_gate_numbers_unique.sh 1 +test_gate_or_objectservice_surface.sh 3 +test_gate_orphaned_capability_fixtures.sh 1 +test_gate_route_auth.sh 46 +test_gate_route_registration.sh 46 +test_gate_scope_matrix.sh 12 +test_gates_23_33_never_green_over_nothing.sh 40 +test_generate_mock_register.py 2 +test_js_scope.js 5 +test_lib_commonjs_scope.sh 7 +test_manifest_diff_scope.py 3 +test_manifest_scope_filter.js 1 +test_or_abstraction_exceptions.sh 22 +test_or_abstraction_pdok.sh 7 +test_php_template_scope.py 1 +test_source_scope.py 1 +test_vendored_schema_freshness.js 8 diff --git a/hydra-gates/tests/partition_helper_suites.py b/hydra-gates/tests/partition_helper_suites.py new file mode 100644 index 000000000..19e1b27e0 --- /dev/null +++ b/hydra-gates/tests/partition_helper_suites.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: EUPL-1.2 +"""Split the helper suites over N shards, balanced by their measured run time. + +Reads suite names on stdin, one per line, and prints `\t` for +every one of them, so each suite lands on exactly one shard. The split is +longest-first: suites are taken in descending weight (name breaks ties) and +each goes to the shard with the least weight so far (lowest index breaks +ties). The output depends only on the names, the weights and N. + +Weights come from helper-suite-timings.tsv (`\t`, measured in +CI). A suite missing from the table still runs: it gets DEFAULT_WEIGHT, so a +stale table can only make the shards less even, never drop a suite. + +Usage: partition_helper_suites.py < names +""" + +from __future__ import annotations + +import sys + +DEFAULT_WEIGHT = 20 + + +def read_weights(path: str) -> dict[str, int]: + weights: dict[str, int] = {} + try: + with open(path, encoding="utf-8") as fh: + for line in fh: + parts = line.rstrip("\n").split("\t") + if len(parts) == 2 and parts[1].isdigit(): + weights[parts[0]] = int(parts[1]) + except OSError: + pass + return weights + + +def partition(names: list[str], weights: dict[str, int], shards: int) -> list[tuple[int, str]]: + load = [0] * shards + out = [] + order = sorted(set(names), key=lambda n: (-weights.get(n, DEFAULT_WEIGHT), n)) + for name in order: + shard = min(range(shards), key=lambda i: (load[i], i)) + load[shard] += weights.get(name, DEFAULT_WEIGHT) + out.append((shard, name)) + return sorted(out, key=lambda row: row[1]) + + +def main(argv: list[str]) -> int: + if len(argv) != 3 or not argv[2].isdigit() or int(argv[2]) < 1: + print("usage: partition_helper_suites.py ", file=sys.stderr) + return 2 + names = [line.strip() for line in sys.stdin if line.strip()] + for shard, name in partition(names, read_weights(argv[1]), int(argv[2])): + print(f"{shard}\t{name}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/hydra-gates/tests/run-helper-suites.sh b/hydra-gates/tests/run-helper-suites.sh index 0ea78d886..4d28cb3ab 100755 --- a/hydra-gates/tests/run-helper-suites.sh +++ b/hydra-gates/tests/run-helper-suites.sh @@ -30,6 +30,14 @@ # * a quarantined suite that now PASSES -> hard failure (un-quarantine it) # So the list can only shrink, and it tells you why each entry is on it. # +# SHARDS +# ------ +# HELPER_SHARDS= HELPER_SHARD= runs only the suites partition_helper_suites.py +# puts on shard i of n (balanced by helper-suite-timings.tsv). Every discovered +# suite is on exactly one shard, and the quarantine check of an entry runs on +# the shard that owns it. HELPER_SUITE_REPORT= writes `\t` +# per suite, which the CI aggregate job compares against discovery. +# # Run: bash hydra-gates/tests/run-helper-suites.sh (exit 0 = all green) set -uo pipefail @@ -101,6 +109,25 @@ if [ "${#SUITES[@]}" -eq 0 ]; then fi echo "== discovered ${#SUITES[@]} helper suite(s) under scripts/lib/ ==" + +SHARDS="${HELPER_SHARDS:-1}" +SHARD="${HELPER_SHARD:-0}" +if ! [[ "${SHARDS}" =~ ^[1-9][0-9]*$ && "${SHARD}" =~ ^[0-9]+$ ]] || [ "${SHARD}" -ge "${SHARDS}" ]; then + echo "FAIL — HELPER_SHARD=${SHARD} of HELPER_SHARDS=${SHARDS} is not a shard." + exit 1 +fi +if [ "${SHARDS}" -gt 1 ]; then + mapfile -t SUITES < <(printf '%s\n' "${SUITES[@]}" \ + | python3 "${PKG_ROOT}/tests/partition_helper_suites.py" "${PKG_ROOT}/tests/helper-suite-timings.tsv" "${SHARDS}" \ + | awk -F'\t' -v s="${SHARD}" '$1 == s { print $2 }') + if [ "${#SUITES[@]}" -eq 0 ]; then + echo "FAIL — shard ${SHARD} of ${SHARDS} was given no suites; the partition is broken." + exit 1 + fi + echo "== shard ${SHARD} of ${SHARDS}: ${#SUITES[@]} suite(s) ==" +fi +REPORT="${HELPER_SUITE_REPORT:-/dev/null}" +: > "${REPORT}" echo _failed=() @@ -110,6 +137,7 @@ _skipped=() for name in "${SUITES[@]}"; do if reason="$(is_quarantined "${name}")"; then _skipped+=("${name}") + printf 'QUARANTINED\t%s\n' "${name}" >> "${REPORT}" echo "QUARANTINED — ${name}" echo " ${reason}" continue @@ -120,11 +148,15 @@ for name in "${SUITES[@]}"; do _failed+=("${name}") continue fi + _t0=${SECONDS} if ( cd "${PKG_ROOT}" && "${cmd}" "scripts/lib/${name}" ) > "${LOGDIR}/helper_${name}.log" 2>&1; then - echo "PASS — ${name}" + echo "PASS — ${name} ($((SECONDS - _t0))s)" + printf 'PASS\t%s\n' "${name}" >> "${REPORT}" _passed=$((_passed + 1)) else - echo "FAIL — ${name} (exit $?)" + _rc=$? + echo "FAIL — ${name} (exit ${_rc}, $((SECONDS - _t0))s)" + printf 'FAIL\t%s\n' "${name}" >> "${REPORT}" sed 's/^/ /' "${LOGDIR}/helper_${name}.log" | tail -40 _failed+=("${name}") fi @@ -136,6 +168,15 @@ echo "== quarantine integrity ==" _rot=0 for entry in "${QUARANTINE[@]}"; do name="${entry%%|*}" + # On a sharded run, the shard that owns the entry checks it. A stale entry + # (no such file) is on no shard, so shard 0 reports it. + if [ "${SHARDS}" -gt 1 ] && [ -f "${LIB}/${name}" ] \ + && ! printf '%s\n' "${SUITES[@]}" | grep -qxF "${name}"; then + continue + fi + if [ "${SHARDS}" -gt 1 ] && [ ! -f "${LIB}/${name}" ] && [ "${SHARD}" -ne 0 ]; then + continue + fi if [ ! -f "${LIB}/${name}" ]; then echo "FAIL — quarantined suite '${name}' no longer exists. Remove it from QUARANTINE." _rot=$((_rot + 1)) diff --git a/hydra-gates/tests/test-partition-helper-suites.py b/hydra-gates/tests/test-partition-helper-suites.py new file mode 100644 index 000000000..b70d260d4 --- /dev/null +++ b/hydra-gates/tests/test-partition-helper-suites.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: EUPL-1.2 +"""Tests for the helper-suite sharding: the partition and the aggregate check. + +The partition must put every suite on exactly one shard, whatever the timings +table says, and the aggregate must refuse each way a sharded run can lose a +verdict. Each refusal has its own case, so a check that stopped refusing is +seen here first. + +Run: python3 hydra-gates/tests/test-partition-helper-suites.py (exit 0 = green) +""" + +from __future__ import annotations + +import importlib.util +import os +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def _load(file_name: str, mod_name: str): + spec = importlib.util.spec_from_file_location(mod_name, os.path.join(HERE, file_name)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +part = _load("partition_helper_suites.py", "_part") +agg = _load("check-helper-shard-reports.py", "_agg") +LIB = os.path.join(HERE, "..", "scripts", "lib") +TIMINGS = os.path.join(HERE, "helper-suite-timings.tsv") + + +class PartitionTest(unittest.TestCase): + + def test_every_real_suite_is_on_exactly_one_shard(self): + names = sorted(agg.discover(LIB)) + rows = part.partition(names, part.read_weights(TIMINGS), 6) + self.assertEqual(sorted(n for _s, n in rows), names) + self.assertTrue(all(0 <= s < 6 for s, _n in rows)) + + def test_a_suite_missing_from_the_table_still_runs(self): + rows = part.partition(["test_a.sh", "test_new.sh"], {"test_a.sh": 100}, 2) + self.assertEqual(sorted(n for _s, n in rows), ["test_a.sh", "test_new.sh"]) + + def test_the_split_is_deterministic(self): + names = [f"test_{i}.sh" for i in range(40)] + weights = {n: (i * 7) % 13 + 1 for i, n in enumerate(names)} + first = part.partition(names, weights, 5) + self.assertEqual(first, part.partition(list(reversed(names)), weights, 5)) + + def test_heavy_suites_go_to_different_shards(self): + rows = dict((n, s) for s, n in part.partition( + ["test_a.sh", "test_b.sh", "test_c.sh", "test_d.sh"], + {"test_a.sh": 300, "test_b.sh": 290, "test_c.sh": 5, "test_d.sh": 5}, 2)) + self.assertNotEqual(rows["test_a.sh"], rows["test_b.sh"]) + + def test_the_measured_table_balances_the_real_suites(self): + weights = part.read_weights(TIMINGS) + names = sorted(agg.discover(LIB)) + load = [0] * 6 + for shard, name in part.partition(names, weights, 6): + load[shard] += weights.get(name, part.DEFAULT_WEIGHT) + self.assertLess(max(load), 1.25 * sum(load) / 6, load) + + +class AggregateTest(unittest.TestCase): + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.lib = os.path.join(self._tmp.name, "lib") + self.rep = os.path.join(self._tmp.name, "rep") + os.makedirs(self.lib) + os.makedirs(self.rep) + for name in ("test_a.sh", "test_b.py", "test_c.js"): + open(os.path.join(self.lib, name), "w").close() + + def tearDown(self): + self._tmp.cleanup() + + def _report(self, shard, rows): + with open(os.path.join(self.rep, f"helper-suites-shard-{shard}.tsv"), "w") as fh: + for verdict, name in rows: + fh.write(f"{verdict}\t{name}\n") + + def _check(self, result="success"): + return agg.problems(self.lib, self.rep, 2, result) + + def test_a_complete_green_run_passes(self): + self._report(0, [("PASS", "test_a.sh"), ("QUARANTINED", "test_c.js")]) + self._report(1, [("PASS", "test_b.py")]) + self.assertEqual(self._check(), []) + + def test_a_suite_on_no_shard_fails(self): + self._report(0, [("PASS", "test_a.sh")]) + self._report(1, [("PASS", "test_b.py")]) + self.assertEqual(self._check(), ["test_c.js ran on no shard"]) + + def test_a_suite_on_two_shards_fails(self): + self._report(0, [("PASS", "test_a.sh"), ("PASS", "test_c.js")]) + self._report(1, [("PASS", "test_b.py"), ("PASS", "test_c.js")]) + self.assertEqual(len(self._check()), 1) + self.assertIn("test_c.js ran 2 times", self._check()[0]) + + def test_a_failed_suite_fails(self): + self._report(0, [("PASS", "test_a.sh"), ("FAIL", "test_c.js")]) + self._report(1, [("PASS", "test_b.py")]) + self.assertEqual(self._check(), ["test_c.js: FAIL on shard 0"]) + + def test_a_shard_without_a_report_fails(self): + self._report(0, [("PASS", "test_a.sh"), ("PASS", "test_b.py"), ("PASS", "test_c.js")]) + found = self._check() + self.assertEqual(len(found), 1) + self.assertIn("shard 1 left no report", found[0]) + + def test_an_unknown_suite_in_a_report_fails(self): + self._report(0, [("PASS", "test_a.sh"), ("PASS", "test_c.js"), ("PASS", "test_gone.sh")]) + self._report(1, [("PASS", "test_b.py")]) + self.assertEqual(self._check(), ["test_gone.sh was reported but is not a discovered suite"]) + + def test_a_matrix_that_did_not_succeed_fails(self): + self._report(0, [("PASS", "test_a.sh"), ("PASS", "test_c.js")]) + self._report(1, [("PASS", "test_b.py")]) + self.assertEqual(self._check("cancelled"), ["the shard matrix ended 'cancelled', not 'success'"]) + + +if __name__ == "__main__": + unittest.main() From ebaafd503d8ce04a023d2f929b582fd2ee2cc6ec Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Tue, 3 Nov 2026 15:57:28 +0100 Subject: [PATCH 2/2] perf(ci): rebalance helper shards from the measured run Run 41646864752 put 146 s of suites on shard 0 and 393 s on shard 5, because the table still held the pre-prefetch time of the acceptance matrix. With the measured per-suite seconds the split predicts 302 to 303 s per shard. The quarantined suite keeps its old weight. Assisted-by: ClaudeCode:claude-opus-5-5 --- hydra-gates/tests/helper-suite-timings.tsv | 220 ++++++++++----------- 1 file changed, 110 insertions(+), 110 deletions(-) diff --git a/hydra-gates/tests/helper-suite-timings.tsv b/hydra-gates/tests/helper-suite-timings.tsv index 468450ecf..ec9065a5e 100644 --- a/hydra-gates/tests/helper-suite-timings.tsv +++ b/hydra-gates/tests/helper-suite-timings.tsv @@ -1,141 +1,141 @@ -test_build_effective_manifest.js 1 -test_check_admin_router.py 1 -test_check_apphost_autoload_prelude.py 1 -test_check_aria_hidden_focusable.py 1 +test_build_effective_manifest.js 0 +test_check_admin_router.py 0 +test_check_apphost_autoload_prelude.py 0 +test_check_aria_hidden_focusable.py 0 test_check_autocomplete.py 1 -test_check_button_name.py 1 +test_check_button_name.py 0 test_check_coding_standard_adoption.sh 2 -test_check_connections_declaration.js 2 +test_check_connections_declaration.js 3 test_check_contract_coverage.py 1 -test_check_contract_surface_shift.sh 1 +test_check_contract_surface_shift.sh 0 test_check_cross_app_schema_slug.py 1 -test_check_csrf_callers.py 1 -test_check_csrf_removal.py 1 -test_check_custom_widget_ratchet.py 8 -test_check_dashboard_antipattern.py 1 +test_check_csrf_callers.py 0 +test_check_csrf_removal.py 0 +test_check_custom_widget_ratchet.py 16 +test_check_dashboard_antipattern.py 0 test_check_detail_page_cross_app.py 1 test_check_detail_page_icon_registry.py 1 -test_check_detail_page_placement.py 1 -test_check_duplicate_index_pages.js 1 -test_check_duplicate_page_refresh.sh 1 +test_check_detail_page_placement.py 0 +test_check_duplicate_index_pages.js 0 +test_check_duplicate_page_refresh.sh 0 test_check_e2e_coverage.py 1 test_check_e2e_skips.sh 1 -test_check_exclusion_evidence.py 1 -test_check_forbidden_patterns.py 1 -test_check_form_labels.py 1 -test_check_header_action_budget.py 1 -test_check_icon_vocabulary.py 1 -test_check_initial_state.py 1 -test_check_js_call_sites.py 1 -test_check_license_triangle.py 1 +test_check_exclusion_evidence.py 0 +test_check_forbidden_patterns.py 0 +test_check_form_labels.py 0 +test_check_header_action_budget.py 0 +test_check_icon_vocabulary.py 0 +test_check_initial_state.py 0 +test_check_js_call_sites.py 0 +test_check_license_triangle.py 0 test_check_link_text.py 1 -test_check_listener_placement.py 1 +test_check_listener_placement.py 0 test_check_manifest.sh 5 test_check_manifest_ajv_collapse.js 1 test_check_manifest_copy_style.py 1 test_check_manifest_crossref.js 3 -test_check_manifest_l10n_coverage.py 1 -test_check_markup_a11y.py 1 -test_check_migration_version_bump.py 1 -test_check_nc_select_labels.py 1 -test_check_newman_reach.py 1 -test_check_no_admin_idor.py 1 -test_check_notification_dialect.py 1 +test_check_manifest_l10n_coverage.py 0 +test_check_markup_a11y.py 0 +test_check_migration_version_bump.py 0 +test_check_nc_select_labels.py 0 +test_check_newman_reach.py 0 +test_check_no_admin_idor.py 0 +test_check_notification_dialect.py 0 test_check_npm_supply_chain_config.sh 1 -test_check_openregister_dependency_shape.py 1 +test_check_openregister_dependency_shape.py 0 test_check_orphan_auth.py 1 -test_check_orphaned_write_capability.py 1 -test_check_phantom_cross_app_rpc.py 1 -test_check_procedure_code.py 1 -test_check_public_endpoint_throttling.py 1 -test_check_register_auth_dialect.py 1 -test_check_register_handler_resolution.py 1 -test_check_register_version_bump.py 1 +test_check_orphaned_write_capability.py 0 +test_check_phantom_cross_app_rpc.py 0 +test_check_procedure_code.py 0 +test_check_public_endpoint_throttling.py 0 +test_check_register_auth_dialect.py 0 +test_check_register_handler_resolution.py 0 +test_check_register_version_bump.py 0 test_check_relation_dialect.py 1 test_check_schema_property_meta.py 1 test_check_security_cochange.py 1 test_check_seed_required_slugs.py 1 -test_check_semantic_auth.py 1 -test_check_shipped_object_properties.py 1 +test_check_semantic_auth.py 0 +test_check_shipped_object_properties.py 0 test_check_sig_floor.py 2 -test_check_sig_maintainability.py 9 -test_check_spec_anchors.py 1 -test_check_spec_coverage.py 1 +test_check_sig_maintainability.py 12 +test_check_spec_anchors.py 0 +test_check_spec_coverage.py 0 test_check_stale_fleet_app_id.py 1 -test_check_store_and_settings_surface.py 1 -test_check_stub_run_body.py 1 -test_check_table_headers.py 1 -test_check_unclosable_gate.py 1 -test_check_unsafe_auth_resolver.py 1 -test_check_visual_coverage.py 1 +test_check_store_and_settings_surface.py 0 +test_check_stub_run_body.py 0 +test_check_table_headers.py 0 +test_check_unclosable_gate.py 0 +test_check_unsafe_auth_resolver.py 0 +test_check_visual_coverage.py 2 test_checker_patterns_match_documented_shapes.py 1 -test_detect_redundant_controllers.py 1 +test_detect_redundant_controllers.py 0 test_diff_schema_strictness.js 1 test_exclusion_reason.py 1 -test_exclusion_reason_adjacent.py 1 -test_filter_preexisting_methods.py 1 -test_gate100_setup_demo_first_scope.sh 22 -test_gate101_demo_data_scope.sh 11 -test_gate108_shipped_object_properties_scope.sh 17 -test_gate110_migration_version_bump_scope.sh 22 -test_gate111_flow_node_taxonomy.sh 1 -test_gate114_header_action_budget_scope.sh 22 -test_gate116_connections_declaration_scope.sh 21 -test_gate119_procedure_code_scope.sh 29 -test_gate120_register_version_scope.sh 16 -test_gate121_register_auth_dialect_scope.sh 16 -test_gate122_sig_maintainability_scope.sh 17 -test_gate123_sig_floor_scope.sh 11 -test_gate16_spec_coverage_scope.sh 30 -test_gate19_coverage_credibility.sh 3 +test_exclusion_reason_adjacent.py 0 +test_filter_preexisting_methods.py 0 +test_gate100_setup_demo_first_scope.sh 32 +test_gate101_demo_data_scope.sh 15 +test_gate108_shipped_object_properties_scope.sh 26 +test_gate110_migration_version_bump_scope.sh 20 +test_gate111_flow_node_taxonomy.sh 0 +test_gate114_header_action_budget_scope.sh 28 +test_gate116_connections_declaration_scope.sh 27 +test_gate119_procedure_code_scope.sh 26 +test_gate120_register_version_scope.sh 20 +test_gate121_register_auth_dialect_scope.sh 15 +test_gate122_sig_maintainability_scope.sh 16 +test_gate123_sig_floor_scope.sh 15 +test_gate16_spec_coverage_scope.sh 38 +test_gate19_coverage_credibility.sh 4 test_gate67_contract_parity_scope.sh 3 -test_gate7_verb_object_guards.sh 1 -test_gate98_repair_registration_scope.sh 20 -test_gate_13_multiline_dialog.sh 29 -test_gate_1_11_honest_verdicts.sh 59 -test_gate_20_comment_masking.sh 14 -test_gate_29_gitignore_negation.sh 10 -test_gate_3_14_comment_evidence.sh 26 -test_gate_3_bodiless_declaration.sh 11 -test_gate_3_contract_param.sh 15 -test_gate_3_external_contract_and_tabs.sh 20 -test_gate_45_selector_parity.sh 103 -test_gate_45_stylesheet_scope.sh 50 -test_gate_45_to_55_acceptance.sh 166 -test_gate_46_tests_scope.sh 46 -test_gate_4_composer_audit_transport_retry.sh 11 -test_gate_5661_empty_scope_is_not_a_pass.sh 21 +test_gate7_verb_object_guards.sh 0 +test_gate98_repair_registration_scope.sh 17 +test_gate_13_multiline_dialog.sh 35 +test_gate_1_11_honest_verdicts.sh 76 +test_gate_20_comment_masking.sh 18 +test_gate_29_gitignore_negation.sh 8 +test_gate_3_14_comment_evidence.sh 24 +test_gate_3_bodiless_declaration.sh 10 +test_gate_3_contract_param.sh 17 +test_gate_3_external_contract_and_tabs.sh 26 +test_gate_45_selector_parity.sh 87 +test_gate_45_stylesheet_scope.sh 65 +test_gate_45_to_55_acceptance.sh 145 +test_gate_46_tests_scope.sh 57 +test_gate_4_composer_audit_transport_retry.sh 14 +test_gate_5661_empty_scope_is_not_a_pass.sh 26 test_gate_95_adr_number_collision.sh 1 -test_gate_a11y_helper_wiring.sh 100 -test_gate_a11y_markup_scope.sh 15 -test_gate_acceptance_matrix.sh 268 +test_gate_a11y_helper_wiring.sh 122 +test_gate_a11y_markup_scope.sh 12 +test_gate_acceptance_matrix.sh 124 test_gate_base_ref_delivery_channel.sh 14 -test_gate_crashed_checker_is_not_a_finding.sh 165 -test_gate_diff_provenance.sh 17 -test_gate_discarded_counts_and_empty_deltas.sh 38 -test_gate_empty_scope_never_passes.sh 41 +test_gate_crashed_checker_is_not_a_finding.sh 210 +test_gate_diff_provenance.sh 20 +test_gate_discarded_counts_and_empty_deltas.sh 48 +test_gate_empty_scope_never_passes.sh 50 test_gate_errexit_discipline.sh 3 -test_gate_exit_code_semantics.sh 8 -test_gate_fq_route_names.sh 7 +test_gate_exit_code_semantics.sh 10 +test_gate_fq_route_names.sh 8 test_gate_glob_recursion_fixtures.sh 4 -test_gate_license_triangle_scope.sh 20 -test_gate_local_base_is_honoured.sh 10 -test_gate_mainline_push_audits_everything.sh 7 -test_gate_monitoring_and_skiplink.sh 18 -test_gate_numbers_unique.sh 1 +test_gate_license_triangle_scope.sh 26 +test_gate_local_base_is_honoured.sh 13 +test_gate_mainline_push_audits_everything.sh 9 +test_gate_monitoring_and_skiplink.sh 16 +test_gate_numbers_unique.sh 0 test_gate_or_objectservice_surface.sh 3 test_gate_orphaned_capability_fixtures.sh 1 -test_gate_route_auth.sh 46 -test_gate_route_registration.sh 46 -test_gate_scope_matrix.sh 12 -test_gates_23_33_never_green_over_nothing.sh 40 -test_generate_mock_register.py 2 -test_js_scope.js 5 -test_lib_commonjs_scope.sh 7 -test_manifest_diff_scope.py 3 -test_manifest_scope_filter.js 1 -test_or_abstraction_exceptions.sh 22 -test_or_abstraction_pdok.sh 7 -test_php_template_scope.py 1 -test_source_scope.py 1 -test_vendored_schema_freshness.js 8 +test_gate_route_auth.sh 40 +test_gate_route_registration.sh 59 +test_gate_scope_matrix.sh 15 +test_gates_23_33_never_green_over_nothing.sh 25 +test_generate_mock_register.py 0 +test_js_scope.js 1 +test_lib_commonjs_scope.sh 0 +test_manifest_diff_scope.py 0 +test_manifest_scope_filter.js 0 +test_or_abstraction_exceptions.sh 2 +test_or_abstraction_pdok.sh 1 +test_php_template_scope.py 0 +test_source_scope.py 0 +test_vendored_schema_freshness.js 1