From ba569ec7699b5f7132013e789b673dc58db8136a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 17:55:40 +0000 Subject: [PATCH] fix(dispatch-runner): give the predicate verdict to Hypatia, and record no_op and unverified (E0-02) The runner collapsed each verdict to an outcome and made no call for no_op and unverified, so Hypatia never saw an attempt that fixed nothing. Hypatia's `mix hypatia.record_outcome` now takes --verdict (hypatia E0-01), and stores no_op and unverified as audit rows that carry no credit. The runner learns once per run whether Hypatia knows the option, from the help text of the task and never from an exit code: a Hypatia from before E0-01 ignores an option that it does not know, so a no_op call would be recorded as a success. * Hypatia names --verdict: each of the 5 verdicts is passed with the outcome of the pair table (success for no_op and unverified), and no_op and unverified are recorded. A refusal passes no verdict. * An older Hypatia: no --verdict and no call for no_op or unverified, exactly the calls that the runner made before (checked against the previous script). * The help command fails: the run stops with a visible error. A Hypatia that does not answer is not an older Hypatia. * One line per run says which mode it uses. The automation quarantine (exit 78, before the manifest) and the licence refusal gate are not changed. tests/outcome-verdict-test.sh gets a stub mix on PATH for the 4 cases: 60 checks. Co-Authored-By: Claude --- scripts/dispatch-runner.sh | 63 +++++++++++++++++++++++--- tests/outcome-verdict-test.sh | 85 +++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 5 deletions(-) diff --git a/scripts/dispatch-runner.sh b/scripts/dispatch-runner.sh index 28777917..3be4e087 100755 --- a/scripts/dispatch-runner.sh +++ b/scripts/dispatch-runner.sh @@ -219,11 +219,43 @@ HYPATIA_HOME="${HYPATIA_HOME:-${REPOS_BASE}/hypatia}" HYPATIA_OUTCOME_REPORT="${HYPATIA_OUTCOME_REPORT:-on}" HYPATIA_BATCH_ID="${HYPATIA_BATCH_ID:-}" +# Does this Hypatia know `--verdict`? Learned ONCE per run, from the help text of +# the task and never from an exit code: a Hypatia from before E0-01 ignores an +# option that it does not know, so a no_op call would be recorded as a success. +# verdict Hypatia names --verdict: every verdict is passed to it, and no_op and +# unverified are recorded (they carry no credit). +# legacy an older Hypatia: no --verdict, and no call for no_op or unverified +# (the behaviour before this check, exactly). +# off no calls are made at all (reporting off, no checkout, no mix). +# A help command that fails is not an older Hypatia: the run stops, visibly. +HYPATIA_VERDICT_MODE="off" +hypatia_detect_verdict_mode() { + HYPATIA_VERDICT_MODE="off" + [[ "$HYPATIA_OUTCOME_REPORT" == "on" ]] || return 0 + [[ -d "$HYPATIA_HOME" ]] || return 0 + command -v mix &>/dev/null || return 0 + + local help_text rc=0 + help_text=$(cd "$HYPATIA_HOME" && mix help hypatia.record_outcome 2>&1) || rc=$? + if [[ "$rc" -ne 0 ]]; then + HYPATIA_VERDICT_MODE="error" + echo "ERROR: 'mix help hypatia.record_outcome' failed (exit $rc) in $HYPATIA_HOME: Hypatia does not answer, and a Hypatia that does not answer is not an older Hypatia. Stopping." >&2 + return 1 + fi + if grep -q -- '--verdict' <<< "$help_text"; then + HYPATIA_VERDICT_MODE="verdict" + else + HYPATIA_VERDICT_MODE="legacy" + fi + echo "hypatia outcome mode: $HYPATIA_VERDICT_MODE" +} + hypatia_record_outcome() { local recipe_id="$1" local repo="$2" local file="$3" local outcome="$4" + local verdict="${5:-}" # Opt-out short-circuit. if [[ "$HYPATIA_OUTCOME_REPORT" != "on" ]]; then @@ -249,6 +281,13 @@ hypatia_record_outcome() { local file_arg="${file:-.}" [[ -z "$file_arg" ]] && file_arg="." + # The verdict goes along only when Hypatia knows the option, and only the + # 5 verdicts of the pair table (a refusal has none). + local verdict_args=() + if [[ "$HYPATIA_VERDICT_MODE" == "verdict" && -n "$verdict" ]]; then + verdict_args=(--verdict "$verdict") + fi + local rc=0 ( cd "$HYPATIA_HOME" @@ -256,7 +295,8 @@ hypatia_record_outcome() { --recipe "$recipe_id" \ --repo "$repo" \ --file "$file_arg" \ - --outcome "$outcome" + --outcome "$outcome" \ + "${verdict_args[@]}" ) || rc=$? case "$rc" in @@ -336,27 +376,40 @@ record_outcome() { # failed / incomplete → failure (the fix ran but the work remains) # refused_* → false_positive (the recipe is misfiring, not # the code; surfaces this to auto-quarantine) - # no_op / unverified → skip: no evidence either way, so it must not - # move the recipe's confidence. - local hypatia_outcome + # no_op / unverified → with a Hypatia that knows --verdict: recorded, as + # success WITH the verdict, which Hypatia stores as + # an audit row with no credit (E0-01). With an older + # Hypatia: skipped, as before, because without the + # verdict it would be recorded as a success. + local hypatia_outcome verdict_arg="" case "$verdict" in verified) hypatia_outcome="success" + verdict_arg="$verdict" echo "$repo" >> "$RESCAN_REPOS_FILE" ;; failed|incomplete) hypatia_outcome="failure" + verdict_arg="$verdict" ;; refused_license_policy|refused_*) hypatia_outcome="false_positive" ;; + no_op|unverified) + [[ "$HYPATIA_VERDICT_MODE" == "verdict" ]] || return 0 + hypatia_outcome="success" + verdict_arg="$verdict" + ;; *) return 0 ;; esac - hypatia_record_outcome "$recipe_id" "$repo" "$file" "$hypatia_outcome" || true + hypatia_record_outcome "$recipe_id" "$repo" "$file" "$hypatia_outcome" "$verdict_arg" || true } +# Learn once per run which of the 2 calls Hypatia takes. Stop if it does not answer. +hypatia_detect_verdict_mode || exit 1 + # --- Execute a single manifest entry --- execute_entry() { local entry="$1" diff --git a/tests/outcome-verdict-test.sh b/tests/outcome-verdict-test.sh index 00997731..134ea650 100755 --- a/tests/outcome-verdict-test.sh +++ b/tests/outcome-verdict-test.sh @@ -256,6 +256,91 @@ for PARALLEL in 1 2; do "EXECUTED=$EXECUTED SUCCEEDED=$SUCCEEDED FAILED=$FAILED SKIPPED=$SKIPPED" done +# --- 10. the verdict goes to Hypatia when Hypatia knows the option (E0-02) - +# A stub `mix` on PATH records its arguments and answers `mix help +# hypatia.record_outcome` as the case needs. The mode is learned once, from the +# help text and never from an exit code: an older Hypatia ignores an option that +# it does not know, so a no_op call would be recorded as a success. The main +# loop is not run; it sits behind the quarantine (exit 78). +stub_dir="$TMP/stub-bin" +mkdir -p "$stub_dir" "$TMP/hy-home" +cat > "$stub_dir/mix" <<'STUB' +#!/usr/bin/env bash +if [[ "$1" == "help" ]]; then + case "$STUB_HELP" in + verdict) echo "mix hypatia.record_outcome --recipe R --verdict verified|no_op|incomplete|unverified|failed"; exit 0 ;; + legacy) echo "mix hypatia.record_outcome --recipe R --repo X --file F --outcome O"; exit 0 ;; + *) echo "mix: cannot start" >&2; exit 3 ;; + esac +fi +echo "$*" >> "$STUB_LOG" +exit 0 +STUB +chmod +x "$stub_dir/mix" + +# run_case [on|off]: runs the runner's own contract +# block in a subshell, as the runner does: detect first, stop if that fails, +# then record each verdict and one refusal. +run_case() { + local mode="$1" report="${2:-on}" + : > "$TMP/stub.log" + ( + export PATH="$stub_dir:$PATH" STUB_HELP="$mode" STUB_LOG="$TMP/stub.log" + export HYPATIA_HOME="$TMP/hy-home" HYPATIA_OUTCOME_REPORT="$report" REPOS_BASE="$TMP/repos" + OUTCOME_FILE="$TMP/case-outcomes.jsonl" + OUTCOME_FILE_CENTRAL="$OUTCOME_FILE" + RESCAN_REPOS_FILE="$TMP/case-rescan" + : > "$OUTCOME_FILE" + : > "$RESCAN_REPOS_FILE" + # shellcheck disable=SC1091 + source "$TMP/extracted.sh" + rc=0 + hypatia_detect_verdict_mode || rc=$? + echo "detect_rc=$rc mode=${HYPATIA_VERDICT_MODE:-unset}" + if [[ "$rc" -eq 0 ]]; then + for v in verified no_op unverified incomplete failed; do + record_outcome PA009 recipe-x repo f "$v" 0 3 0 + done + record_outcome PA009 recipe-lic repo "" refused_license_policy + fi + ) > "$TMP/case.out" 2>&1 || true +} +calls() { cat "$TMP/stub.log"; } + +run_case verdict +check "knows --verdict: mode is verdict" "detect_rc=0 mode=verdict" "$(grep '^detect_rc' "$TMP/case.out")" +check "knows --verdict: the mode is logged once" 1 "$(grep -c '^hypatia outcome mode: verdict' "$TMP/case.out")" +check "knows --verdict: 6 calls, one for each verdict and the refusal" 6 "$(calls | wc -l | tr -d ' ')" +for pair in verified:success no_op:success unverified:success incomplete:failure failed:failure; do + v="${pair%%:*}" o="${pair##*:}" + check "knows --verdict: $v is passed with outcome $o" \ + "hypatia.record_outcome --recipe recipe-x --repo repo --file f --outcome $o --verdict $v" \ + "$(calls | grep -- "--verdict $v\$" || true)" +done +check "knows --verdict: a refusal has no --verdict" \ + "hypatia.record_outcome --recipe recipe-lic --repo repo --file . --outcome false_positive" \ + "$(calls | grep -- 'recipe-lic' || true)" + +run_case legacy +check "older Hypatia: mode is legacy" "detect_rc=0 mode=legacy" "$(grep '^detect_rc' "$TMP/case.out")" +check "older Hypatia: the mode is logged once" 1 "$(grep -c '^hypatia outcome mode: legacy' "$TMP/case.out")" +check "older Hypatia: no --verdict in any call" 0 "$(calls | grep -c -- '--verdict' || true)" +check "older Hypatia: no call for no_op or unverified, 4 calls in all" 4 "$(calls | wc -l | tr -d ' ')" +check "older Hypatia: the calls are the calls of today" \ + "success failure failure false_positive" \ + "$(calls | sed -E 's/.*--outcome ([a-z_]+).*/\1/' | tr '\n' ' ' | sed 's/ $//')" + +run_case fail +check "help fails: detection fails" "detect_rc=1 mode=error" "$(grep '^detect_rc' "$TMP/case.out")" +check "help fails: a visible error says why" 1 "$(grep -c 'does not answer' "$TMP/case.out")" +check "help fails: no record call is made" 0 "$(calls | wc -l | tr -d ' ')" +check "the runner stops when detection fails (before any call)" 1 \ + "$(grep -c '^hypatia_detect_verdict_mode || exit 1$' "$ROOT/scripts/dispatch-runner.sh")" + +run_case fail off +check "reporting off: no help call and no record call" "detect_rc=0 mode=off 0" \ + "$(grep '^detect_rc' "$TMP/case.out") $(calls | wc -l | tr -d ' ')" + echo "" echo "Results: $pass passed, $fail failed" [[ "$fail" -eq 0 ]]