diff --git a/.github/workflows/hydra-gates-package.yml b/.github/workflows/hydra-gates-package.yml index 5281f2ce6..e42afb639 100644 --- a/.github/workflows/hydra-gates-package.yml +++ b/.github/workflows/hydra-gates-package.yml @@ -53,23 +53,32 @@ concurrency: cancel-in-progress: true jobs: - invariants: - name: "Package invariants" + # The helper suites run on SHARDS legs, balanced by measured run time + # (hydra-gates/tests/helper-suite-timings.tsv). Run one after another they + # took over 30 minutes by 2026-11-03 (141 suites; the slowest three, + # test_gate_acceptance_matrix, test_gate_45_to_55_acceptance and + # test_gate_crashed_checker_is_not_a_finding, about 10 minutes together), + # so this job hit its timeout on every pull request and read CANCELLED. + # + # `Package invariants` stays the one name the ruleset requires. It is the + # aggregate below, and it fails unless every discovered suite has exactly one + # PASS (or QUARANTINED) verdict from exactly one shard: a shard that is + # cancelled, crashes or loses a suite cannot read as green. + invariants-shard: + name: "Package invariants (shard ${{ matrix.shard }} of 6)" runs-on: ubuntu-latest - # Measured 2026-09-03 across five runs of this job: the ones that finished - # took 10.0 and 10.4 minutes; three others ran to exactly 15.3 and were - # killed. A 15-minute cap on a 10-minute suite with that spread times out - # routinely, and GitHub reports a timeout as CANCELLED — which reads as - # "superseded by a newer push" rather than "your suite was killed", so the - # failures were being dismissed. It was flapping on `main` and on two other - # branches, not on any one change. - # - # 30 leaves real headroom without hiding a suite that has genuinely started - # to hang: double the observed worst finish. + # Each shard carries about a sixth of the suite (5 to 6 minutes measured). + # The cap is the job's old 30, kept per leg, not raised. timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2, 3, 4, 5] steps: - uses: actions/checkout@v4 + - name: Run the invariant suite + if: matrix.shard == 0 run: bash hydra-gates/tests/test-hydra-gates-bin.sh # FIRST, with no node_modules at all — this is the only state in which the @@ -78,41 +87,59 @@ jobs: # once ajv is installed below, `NODE_PATH=/nonexistent` no longer hides it # (node still resolves from ./node_modules) and that assertion self-skips. - name: "Manifest validator: the no-Ajv degraded contract" + if: matrix.shard == 0 run: bash hydra-gates/scripts/lib/test_check_manifest.sh # Ajv is what gates 22 and 53 validate manifests WITH. Without it the # vendored validator can only run its structural lint, and the suite below - # would skip exactly the paths worth testing. It is a devDependency of - # every fleet app already; installing it here means CI exercises the same - # code path a real `npm ci` repo does. + # would skip exactly the paths worth testing. - name: Install ajv for the manifest-validation helpers run: npm install --no-save --no-audit --no-fund ajv ajv-formats - # EVERY scripts/lib/test_* suite, discovered rather than listed. - # - # This step used to name three files by hand. The package ships 19, so 15 - # of them were executed by nothing at all — they existed, they looked like - # coverage, and no job ever ran them. Two were pointing at fixture - # directories that had never existed: - # - # test_check_manifest.sh green its whole life (a missing - # manifest path makes the validator - # print "Tier 0, skipping" and exit 0, - # which is what 2 of its 3 assertions - # expected) — fixed 2026-08-03 - # test_check_manifest_crossref.js red its whole life, unseen, because - # nothing ran it — fixed 2026-08-04 - # - # A hand-written list of test names decays exactly like a hand-written - # paths filter: adding a file does not add it to the list and the omission - # is invisible. The runner discovers instead, so a new scripts/lib/test_* - # is enforced the moment it lands, with no workflow edit to forget. Suites - # may be quarantined only with a stated reason, and the quarantine cannot - # rot — an entry that no longer exists, or that starts passing, fails the - # step. - - name: All helper suites (discovered, not listed) + # EVERY scripts/lib/test_* suite, discovered rather than listed, and + # split over the shards by run-helper-suites.sh itself. See the header of + # that script for why the list is discovered and how the quarantine + # cannot rot. + - name: Helper suites on this shard (discovered, not listed) + env: + HELPER_SHARDS: "6" + HELPER_SHARD: ${{ matrix.shard }} + HELPER_SUITE_REPORT: ${{ runner.temp }}/helper-suites-shard-${{ matrix.shard }}.tsv run: bash hydra-gates/tests/run-helper-suites.sh + - name: Keep this shard's verdicts for the aggregate + if: always() + uses: actions/upload-artifact@v4 + with: + name: helper-suites-shard-${{ matrix.shard }} + path: ${{ runner.temp }}/helper-suites-shard-${{ matrix.shard }}.tsv + if-no-files-found: error + retention-days: 3 + + invariants: + name: "Package invariants" + runs-on: ubuntu-latest + needs: [invariants-shard] + if: ${{ !cancelled() }} + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - name: The partition puts every suite on exactly one shard + run: python3 hydra-gates/tests/test-partition-helper-suites.py + + - uses: actions/download-artifact@v4 + with: + pattern: helper-suites-shard-* + path: ${{ runner.temp }}/shards + merge-multiple: true + + - name: Every discovered suite passed on exactly one shard + env: + SHARD_RESULT: ${{ needs.invariants-shard.result }} + SHARD_DIR: ${{ runner.temp }}/shards + run: python3 hydra-gates/tests/check-helper-shard-reports.py hydra-gates/scripts/lib "${SHARD_DIR}" 6 "${SHARD_RESULT}" + install-from-published-location: name: "Install from published location (php:8.3-cli)" runs-on: ubuntu-latest diff --git a/hydra-gates/scripts/lib/test_gate_acceptance_matrix.sh b/hydra-gates/scripts/lib/test_gate_acceptance_matrix.sh index b2000da1e..236213263 100644 --- a/hydra-gates/scripts/lib/test_gate_acceptance_matrix.sh +++ b/hydra-gates/scripts/lib/test_gate_acceptance_matrix.sh @@ -107,12 +107,46 @@ echo "== runner declares ${#DECLARED[@]} gates ==" # --------------------------------------------------------------------------- _OUT="" _LOGDIR="" +# Every bundle arm is a read-only fixture and an independent runner process, so +# _prefetch runs them all up front, HYDRA_ACCEPT_JOBS at a time (default: the +# CPU count; 1 runs nothing up front). _run then reads the stored output +# instead of starting the runner again; grading is unchanged and in order. +_PREFETCH="" +_prefetch_one() { # + mkdir -p "${_PREFETCH}/$2.logs" + HYDRA_GATE_LOG_DIR="${_PREFETCH}/$2.logs" \ + HYDRA_OR_GATE_BLOCK_AFTER_EPOCH=0 \ + bash "${RUNNER}" "$1" > "${_PREFETCH}/$2.out" 2>&1 || true +} +_prefetch() { # ... + local _jobs="${HYDRA_ACCEPT_JOBS:-$(nproc 2>/dev/null || echo 1)}" _d _n=0 + [ "${_jobs}" -gt 1 ] 2>/dev/null || return 0 + _PREFETCH="$(mktemp -d "${TMPDIR:-/tmp}/hydra-accept-pre.XXXXXXXX")" + for _d in "$@"; do + _prefetch_one "${_d}" "$(_key "${_d}")" & + _n=$((_n + 1)) + if [ "${_n}" -ge "${_jobs}" ]; then + wait -n + _n=$((_n - 1)) + fi + done + wait +} +_key() { printf '%s' "${1#"${BUNDLES}/"}" | tr '/' '_'; } + _run() { # [runner args...] local _dir="$1"; shift - _LOGDIR="$(mktemp -d "${TMPDIR:-/tmp}/hydra-accept.XXXXXXXX")" - _OUT="$(HYDRA_GATE_LOG_DIR="${_LOGDIR}" \ - HYDRA_OR_GATE_BLOCK_AFTER_EPOCH=0 \ - bash "${RUNNER}" "$@" "${_dir}" 2>&1 || true)" + local _k + _k="$(_key "${_dir}")" + if [ -n "${_PREFETCH}" ] && [ "$#" -eq 0 ] && [ -f "${_PREFETCH}/${_k}.out" ]; then + _LOGDIR="${_PREFETCH}/${_k}.logs" + _OUT="$(cat "${_PREFETCH}/${_k}.out")" + else + _LOGDIR="$(mktemp -d "${TMPDIR:-/tmp}/hydra-accept.XXXXXXXX")" + _OUT="$(HYDRA_GATE_LOG_DIR="${_LOGDIR}" \ + HYDRA_OR_GATE_BLOCK_AFTER_EPOCH=0 \ + bash "${RUNNER}" "$@" "${_dir}" 2>&1 || true)" + fi # An abort before the summary leaves per-gate PASS lines on stdout and # reads exactly like a clean run. Refuse to grade a run that did not finish. if ! printf '%s' "${_OUT}" | grep -q '^\[hydra-gates\] COVERAGE:'; then @@ -259,6 +293,14 @@ fi echo "== discovered ${#BUNDLE_DIRS[@]} fixture bundle(s) ==" echo +_ARMS=() +for _bundle in "${BUNDLE_DIRS[@]}"; do + for _arm in planted clean; do + [ -d "${BUNDLES}/${_bundle}/${_arm}" ] && _ARMS+=("${BUNDLES}/${_bundle}/${_arm}") + done +done +_prefetch "${_ARMS[@]}" + COVERED=() for _bundle in "${BUNDLE_DIRS[@]}"; do diff --git a/hydra-gates/tests/check-helper-shard-reports.py b/hydra-gates/tests/check-helper-shard-reports.py new file mode 100644 index 000000000..59d58b27b --- /dev/null +++ b/hydra-gates/tests/check-helper-shard-reports.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: EUPL-1.2 +"""Decide `Package invariants` from the verdicts of every helper-suite shard. + +A sharded run must not read green when a suite ran on no shard, ran twice, +failed, or when a shard never reported. So this checks, against a fresh +discovery of scripts/lib/test_*: + + * every shard 0..N-1 left its report (helper-suites-shard-.tsv); + * every discovered suite has exactly one verdict, across all shards; + * every verdict is PASS or QUARANTINED, and no report names an unknown suite; + * the shard matrix itself succeeded. + +Usage: check-helper-shard-reports.py +Exit: 0 all hold, 1 otherwise (each problem printed). +""" + +from __future__ import annotations + +import os +import sys + + +def discover(lib: str) -> set[str]: + return {n for n in os.listdir(lib) if n.startswith("test_") and os.path.isfile(os.path.join(lib, n))} + + +def problems(lib: str, reports: str, shards: int, matrix_result: str) -> list[str]: + found = [] + if matrix_result != "success": + found.append(f"the shard matrix ended '{matrix_result}', not 'success'") + seen: dict[str, list[str]] = {} + for i in range(shards): + path = os.path.join(reports, f"helper-suites-shard-{i}.tsv") + if not os.path.isfile(path): + found.append(f"shard {i} left no report ({path})") + continue + with open(path, encoding="utf-8") as fh: + for line in fh: + parts = line.rstrip("\n").split("\t") + if len(parts) != 2: + continue + verdict, name = parts + seen.setdefault(name, []).append(f"{verdict} on shard {i}") + if verdict not in ("PASS", "QUARANTINED"): + found.append(f"{name}: {verdict} on shard {i}") + suites = discover(lib) + if not suites: + found.append(f"discovered zero suites under {lib}") + for name in sorted(suites - seen.keys()): + found.append(f"{name} ran on no shard") + for name in sorted(seen.keys() - suites): + found.append(f"{name} was reported but is not a discovered suite") + for name, verdicts in sorted(seen.items()): + if len(verdicts) > 1: + found.append(f"{name} ran {len(verdicts)} times: {', '.join(verdicts)}") + return found + + +def main(argv: list[str]) -> int: + if len(argv) != 5 or not argv[3].isdigit(): + print(__doc__, file=sys.stderr) + return 1 + found = problems(argv[1], argv[2], int(argv[3]), argv[4]) + for line in found: + print(f"::error::{line}") + if found: + return 1 + print(f"OK: {len(discover(argv[1]))} suites, each with one passing verdict, over {argv[3]} shards.") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/hydra-gates/tests/helper-suite-timings.tsv b/hydra-gates/tests/helper-suite-timings.tsv new file mode 100644 index 000000000..ec9065a5e --- /dev/null +++ b/hydra-gates/tests/helper-suite-timings.tsv @@ -0,0 +1,141 @@ +test_build_effective_manifest.js 0 +test_check_admin_router.py 0 +test_check_apphost_autoload_prelude.py 0 +test_check_aria_hidden_focusable.py 0 +test_check_autocomplete.py 1 +test_check_button_name.py 0 +test_check_coding_standard_adoption.sh 2 +test_check_connections_declaration.js 3 +test_check_contract_coverage.py 1 +test_check_contract_surface_shift.sh 0 +test_check_cross_app_schema_slug.py 1 +test_check_csrf_callers.py 0 +test_check_csrf_removal.py 0 +test_check_custom_widget_ratchet.py 16 +test_check_dashboard_antipattern.py 0 +test_check_detail_page_cross_app.py 1 +test_check_detail_page_icon_registry.py 1 +test_check_detail_page_placement.py 0 +test_check_duplicate_index_pages.js 0 +test_check_duplicate_page_refresh.sh 0 +test_check_e2e_coverage.py 1 +test_check_e2e_skips.sh 1 +test_check_exclusion_evidence.py 0 +test_check_forbidden_patterns.py 0 +test_check_form_labels.py 0 +test_check_header_action_budget.py 0 +test_check_icon_vocabulary.py 0 +test_check_initial_state.py 0 +test_check_js_call_sites.py 0 +test_check_license_triangle.py 0 +test_check_link_text.py 1 +test_check_listener_placement.py 0 +test_check_manifest.sh 5 +test_check_manifest_ajv_collapse.js 1 +test_check_manifest_copy_style.py 1 +test_check_manifest_crossref.js 3 +test_check_manifest_l10n_coverage.py 0 +test_check_markup_a11y.py 0 +test_check_migration_version_bump.py 0 +test_check_nc_select_labels.py 0 +test_check_newman_reach.py 0 +test_check_no_admin_idor.py 0 +test_check_notification_dialect.py 0 +test_check_npm_supply_chain_config.sh 1 +test_check_openregister_dependency_shape.py 0 +test_check_orphan_auth.py 1 +test_check_orphaned_write_capability.py 0 +test_check_phantom_cross_app_rpc.py 0 +test_check_procedure_code.py 0 +test_check_public_endpoint_throttling.py 0 +test_check_register_auth_dialect.py 0 +test_check_register_handler_resolution.py 0 +test_check_register_version_bump.py 0 +test_check_relation_dialect.py 1 +test_check_schema_property_meta.py 1 +test_check_security_cochange.py 1 +test_check_seed_required_slugs.py 1 +test_check_semantic_auth.py 0 +test_check_shipped_object_properties.py 0 +test_check_sig_floor.py 2 +test_check_sig_maintainability.py 12 +test_check_spec_anchors.py 0 +test_check_spec_coverage.py 0 +test_check_stale_fleet_app_id.py 1 +test_check_store_and_settings_surface.py 0 +test_check_stub_run_body.py 0 +test_check_table_headers.py 0 +test_check_unclosable_gate.py 0 +test_check_unsafe_auth_resolver.py 0 +test_check_visual_coverage.py 2 +test_checker_patterns_match_documented_shapes.py 1 +test_detect_redundant_controllers.py 0 +test_diff_schema_strictness.js 1 +test_exclusion_reason.py 1 +test_exclusion_reason_adjacent.py 0 +test_filter_preexisting_methods.py 0 +test_gate100_setup_demo_first_scope.sh 32 +test_gate101_demo_data_scope.sh 15 +test_gate108_shipped_object_properties_scope.sh 26 +test_gate110_migration_version_bump_scope.sh 20 +test_gate111_flow_node_taxonomy.sh 0 +test_gate114_header_action_budget_scope.sh 28 +test_gate116_connections_declaration_scope.sh 27 +test_gate119_procedure_code_scope.sh 26 +test_gate120_register_version_scope.sh 20 +test_gate121_register_auth_dialect_scope.sh 15 +test_gate122_sig_maintainability_scope.sh 16 +test_gate123_sig_floor_scope.sh 15 +test_gate16_spec_coverage_scope.sh 38 +test_gate19_coverage_credibility.sh 4 +test_gate67_contract_parity_scope.sh 3 +test_gate7_verb_object_guards.sh 0 +test_gate98_repair_registration_scope.sh 17 +test_gate_13_multiline_dialog.sh 35 +test_gate_1_11_honest_verdicts.sh 76 +test_gate_20_comment_masking.sh 18 +test_gate_29_gitignore_negation.sh 8 +test_gate_3_14_comment_evidence.sh 24 +test_gate_3_bodiless_declaration.sh 10 +test_gate_3_contract_param.sh 17 +test_gate_3_external_contract_and_tabs.sh 26 +test_gate_45_selector_parity.sh 87 +test_gate_45_stylesheet_scope.sh 65 +test_gate_45_to_55_acceptance.sh 145 +test_gate_46_tests_scope.sh 57 +test_gate_4_composer_audit_transport_retry.sh 14 +test_gate_5661_empty_scope_is_not_a_pass.sh 26 +test_gate_95_adr_number_collision.sh 1 +test_gate_a11y_helper_wiring.sh 122 +test_gate_a11y_markup_scope.sh 12 +test_gate_acceptance_matrix.sh 124 +test_gate_base_ref_delivery_channel.sh 14 +test_gate_crashed_checker_is_not_a_finding.sh 210 +test_gate_diff_provenance.sh 20 +test_gate_discarded_counts_and_empty_deltas.sh 48 +test_gate_empty_scope_never_passes.sh 50 +test_gate_errexit_discipline.sh 3 +test_gate_exit_code_semantics.sh 10 +test_gate_fq_route_names.sh 8 +test_gate_glob_recursion_fixtures.sh 4 +test_gate_license_triangle_scope.sh 26 +test_gate_local_base_is_honoured.sh 13 +test_gate_mainline_push_audits_everything.sh 9 +test_gate_monitoring_and_skiplink.sh 16 +test_gate_numbers_unique.sh 0 +test_gate_or_objectservice_surface.sh 3 +test_gate_orphaned_capability_fixtures.sh 1 +test_gate_route_auth.sh 40 +test_gate_route_registration.sh 59 +test_gate_scope_matrix.sh 15 +test_gates_23_33_never_green_over_nothing.sh 25 +test_generate_mock_register.py 0 +test_js_scope.js 1 +test_lib_commonjs_scope.sh 0 +test_manifest_diff_scope.py 0 +test_manifest_scope_filter.js 0 +test_or_abstraction_exceptions.sh 2 +test_or_abstraction_pdok.sh 1 +test_php_template_scope.py 0 +test_source_scope.py 0 +test_vendored_schema_freshness.js 1 diff --git a/hydra-gates/tests/partition_helper_suites.py b/hydra-gates/tests/partition_helper_suites.py new file mode 100644 index 000000000..19e1b27e0 --- /dev/null +++ b/hydra-gates/tests/partition_helper_suites.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: EUPL-1.2 +"""Split the helper suites over N shards, balanced by their measured run time. + +Reads suite names on stdin, one per line, and prints `\t` for +every one of them, so each suite lands on exactly one shard. The split is +longest-first: suites are taken in descending weight (name breaks ties) and +each goes to the shard with the least weight so far (lowest index breaks +ties). The output depends only on the names, the weights and N. + +Weights come from helper-suite-timings.tsv (`\t`, measured in +CI). A suite missing from the table still runs: it gets DEFAULT_WEIGHT, so a +stale table can only make the shards less even, never drop a suite. + +Usage: partition_helper_suites.py < names +""" + +from __future__ import annotations + +import sys + +DEFAULT_WEIGHT = 20 + + +def read_weights(path: str) -> dict[str, int]: + weights: dict[str, int] = {} + try: + with open(path, encoding="utf-8") as fh: + for line in fh: + parts = line.rstrip("\n").split("\t") + if len(parts) == 2 and parts[1].isdigit(): + weights[parts[0]] = int(parts[1]) + except OSError: + pass + return weights + + +def partition(names: list[str], weights: dict[str, int], shards: int) -> list[tuple[int, str]]: + load = [0] * shards + out = [] + order = sorted(set(names), key=lambda n: (-weights.get(n, DEFAULT_WEIGHT), n)) + for name in order: + shard = min(range(shards), key=lambda i: (load[i], i)) + load[shard] += weights.get(name, DEFAULT_WEIGHT) + out.append((shard, name)) + return sorted(out, key=lambda row: row[1]) + + +def main(argv: list[str]) -> int: + if len(argv) != 3 or not argv[2].isdigit() or int(argv[2]) < 1: + print("usage: partition_helper_suites.py ", file=sys.stderr) + return 2 + names = [line.strip() for line in sys.stdin if line.strip()] + for shard, name in partition(names, read_weights(argv[1]), int(argv[2])): + print(f"{shard}\t{name}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/hydra-gates/tests/run-helper-suites.sh b/hydra-gates/tests/run-helper-suites.sh index 0ea78d886..4d28cb3ab 100755 --- a/hydra-gates/tests/run-helper-suites.sh +++ b/hydra-gates/tests/run-helper-suites.sh @@ -30,6 +30,14 @@ # * a quarantined suite that now PASSES -> hard failure (un-quarantine it) # So the list can only shrink, and it tells you why each entry is on it. # +# SHARDS +# ------ +# HELPER_SHARDS= HELPER_SHARD= runs only the suites partition_helper_suites.py +# puts on shard i of n (balanced by helper-suite-timings.tsv). Every discovered +# suite is on exactly one shard, and the quarantine check of an entry runs on +# the shard that owns it. HELPER_SUITE_REPORT= writes `\t` +# per suite, which the CI aggregate job compares against discovery. +# # Run: bash hydra-gates/tests/run-helper-suites.sh (exit 0 = all green) set -uo pipefail @@ -101,6 +109,25 @@ if [ "${#SUITES[@]}" -eq 0 ]; then fi echo "== discovered ${#SUITES[@]} helper suite(s) under scripts/lib/ ==" + +SHARDS="${HELPER_SHARDS:-1}" +SHARD="${HELPER_SHARD:-0}" +if ! [[ "${SHARDS}" =~ ^[1-9][0-9]*$ && "${SHARD}" =~ ^[0-9]+$ ]] || [ "${SHARD}" -ge "${SHARDS}" ]; then + echo "FAIL — HELPER_SHARD=${SHARD} of HELPER_SHARDS=${SHARDS} is not a shard." + exit 1 +fi +if [ "${SHARDS}" -gt 1 ]; then + mapfile -t SUITES < <(printf '%s\n' "${SUITES[@]}" \ + | python3 "${PKG_ROOT}/tests/partition_helper_suites.py" "${PKG_ROOT}/tests/helper-suite-timings.tsv" "${SHARDS}" \ + | awk -F'\t' -v s="${SHARD}" '$1 == s { print $2 }') + if [ "${#SUITES[@]}" -eq 0 ]; then + echo "FAIL — shard ${SHARD} of ${SHARDS} was given no suites; the partition is broken." + exit 1 + fi + echo "== shard ${SHARD} of ${SHARDS}: ${#SUITES[@]} suite(s) ==" +fi +REPORT="${HELPER_SUITE_REPORT:-/dev/null}" +: > "${REPORT}" echo _failed=() @@ -110,6 +137,7 @@ _skipped=() for name in "${SUITES[@]}"; do if reason="$(is_quarantined "${name}")"; then _skipped+=("${name}") + printf 'QUARANTINED\t%s\n' "${name}" >> "${REPORT}" echo "QUARANTINED — ${name}" echo " ${reason}" continue @@ -120,11 +148,15 @@ for name in "${SUITES[@]}"; do _failed+=("${name}") continue fi + _t0=${SECONDS} if ( cd "${PKG_ROOT}" && "${cmd}" "scripts/lib/${name}" ) > "${LOGDIR}/helper_${name}.log" 2>&1; then - echo "PASS — ${name}" + echo "PASS — ${name} ($((SECONDS - _t0))s)" + printf 'PASS\t%s\n' "${name}" >> "${REPORT}" _passed=$((_passed + 1)) else - echo "FAIL — ${name} (exit $?)" + _rc=$? + echo "FAIL — ${name} (exit ${_rc}, $((SECONDS - _t0))s)" + printf 'FAIL\t%s\n' "${name}" >> "${REPORT}" sed 's/^/ /' "${LOGDIR}/helper_${name}.log" | tail -40 _failed+=("${name}") fi @@ -136,6 +168,15 @@ echo "== quarantine integrity ==" _rot=0 for entry in "${QUARANTINE[@]}"; do name="${entry%%|*}" + # On a sharded run, the shard that owns the entry checks it. A stale entry + # (no such file) is on no shard, so shard 0 reports it. + if [ "${SHARDS}" -gt 1 ] && [ -f "${LIB}/${name}" ] \ + && ! printf '%s\n' "${SUITES[@]}" | grep -qxF "${name}"; then + continue + fi + if [ "${SHARDS}" -gt 1 ] && [ ! -f "${LIB}/${name}" ] && [ "${SHARD}" -ne 0 ]; then + continue + fi if [ ! -f "${LIB}/${name}" ]; then echo "FAIL — quarantined suite '${name}' no longer exists. Remove it from QUARANTINE." _rot=$((_rot + 1)) diff --git a/hydra-gates/tests/test-partition-helper-suites.py b/hydra-gates/tests/test-partition-helper-suites.py new file mode 100644 index 000000000..b70d260d4 --- /dev/null +++ b/hydra-gates/tests/test-partition-helper-suites.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: EUPL-1.2 +"""Tests for the helper-suite sharding: the partition and the aggregate check. + +The partition must put every suite on exactly one shard, whatever the timings +table says, and the aggregate must refuse each way a sharded run can lose a +verdict. Each refusal has its own case, so a check that stopped refusing is +seen here first. + +Run: python3 hydra-gates/tests/test-partition-helper-suites.py (exit 0 = green) +""" + +from __future__ import annotations + +import importlib.util +import os +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def _load(file_name: str, mod_name: str): + spec = importlib.util.spec_from_file_location(mod_name, os.path.join(HERE, file_name)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +part = _load("partition_helper_suites.py", "_part") +agg = _load("check-helper-shard-reports.py", "_agg") +LIB = os.path.join(HERE, "..", "scripts", "lib") +TIMINGS = os.path.join(HERE, "helper-suite-timings.tsv") + + +class PartitionTest(unittest.TestCase): + + def test_every_real_suite_is_on_exactly_one_shard(self): + names = sorted(agg.discover(LIB)) + rows = part.partition(names, part.read_weights(TIMINGS), 6) + self.assertEqual(sorted(n for _s, n in rows), names) + self.assertTrue(all(0 <= s < 6 for s, _n in rows)) + + def test_a_suite_missing_from_the_table_still_runs(self): + rows = part.partition(["test_a.sh", "test_new.sh"], {"test_a.sh": 100}, 2) + self.assertEqual(sorted(n for _s, n in rows), ["test_a.sh", "test_new.sh"]) + + def test_the_split_is_deterministic(self): + names = [f"test_{i}.sh" for i in range(40)] + weights = {n: (i * 7) % 13 + 1 for i, n in enumerate(names)} + first = part.partition(names, weights, 5) + self.assertEqual(first, part.partition(list(reversed(names)), weights, 5)) + + def test_heavy_suites_go_to_different_shards(self): + rows = dict((n, s) for s, n in part.partition( + ["test_a.sh", "test_b.sh", "test_c.sh", "test_d.sh"], + {"test_a.sh": 300, "test_b.sh": 290, "test_c.sh": 5, "test_d.sh": 5}, 2)) + self.assertNotEqual(rows["test_a.sh"], rows["test_b.sh"]) + + def test_the_measured_table_balances_the_real_suites(self): + weights = part.read_weights(TIMINGS) + names = sorted(agg.discover(LIB)) + load = [0] * 6 + for shard, name in part.partition(names, weights, 6): + load[shard] += weights.get(name, part.DEFAULT_WEIGHT) + self.assertLess(max(load), 1.25 * sum(load) / 6, load) + + +class AggregateTest(unittest.TestCase): + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.lib = os.path.join(self._tmp.name, "lib") + self.rep = os.path.join(self._tmp.name, "rep") + os.makedirs(self.lib) + os.makedirs(self.rep) + for name in ("test_a.sh", "test_b.py", "test_c.js"): + open(os.path.join(self.lib, name), "w").close() + + def tearDown(self): + self._tmp.cleanup() + + def _report(self, shard, rows): + with open(os.path.join(self.rep, f"helper-suites-shard-{shard}.tsv"), "w") as fh: + for verdict, name in rows: + fh.write(f"{verdict}\t{name}\n") + + def _check(self, result="success"): + return agg.problems(self.lib, self.rep, 2, result) + + def test_a_complete_green_run_passes(self): + self._report(0, [("PASS", "test_a.sh"), ("QUARANTINED", "test_c.js")]) + self._report(1, [("PASS", "test_b.py")]) + self.assertEqual(self._check(), []) + + def test_a_suite_on_no_shard_fails(self): + self._report(0, [("PASS", "test_a.sh")]) + self._report(1, [("PASS", "test_b.py")]) + self.assertEqual(self._check(), ["test_c.js ran on no shard"]) + + def test_a_suite_on_two_shards_fails(self): + self._report(0, [("PASS", "test_a.sh"), ("PASS", "test_c.js")]) + self._report(1, [("PASS", "test_b.py"), ("PASS", "test_c.js")]) + self.assertEqual(len(self._check()), 1) + self.assertIn("test_c.js ran 2 times", self._check()[0]) + + def test_a_failed_suite_fails(self): + self._report(0, [("PASS", "test_a.sh"), ("FAIL", "test_c.js")]) + self._report(1, [("PASS", "test_b.py")]) + self.assertEqual(self._check(), ["test_c.js: FAIL on shard 0"]) + + def test_a_shard_without_a_report_fails(self): + self._report(0, [("PASS", "test_a.sh"), ("PASS", "test_b.py"), ("PASS", "test_c.js")]) + found = self._check() + self.assertEqual(len(found), 1) + self.assertIn("shard 1 left no report", found[0]) + + def test_an_unknown_suite_in_a_report_fails(self): + self._report(0, [("PASS", "test_a.sh"), ("PASS", "test_c.js"), ("PASS", "test_gone.sh")]) + self._report(1, [("PASS", "test_b.py")]) + self.assertEqual(self._check(), ["test_gone.sh was reported but is not a discovered suite"]) + + def test_a_matrix_that_did_not_succeed_fails(self): + self._report(0, [("PASS", "test_a.sh"), ("PASS", "test_c.js")]) + self._report(1, [("PASS", "test_b.py")]) + self.assertEqual(self._check("cancelled"), ["the shard matrix ended 'cancelled', not 'success'"]) + + +if __name__ == "__main__": + unittest.main()