diff --git a/scripts/dispatch-runner.sh b/scripts/dispatch-runner.sh index 28777917..3be4e087 100755 --- a/scripts/dispatch-runner.sh +++ b/scripts/dispatch-runner.sh @@ -219,11 +219,43 @@ HYPATIA_HOME="${HYPATIA_HOME:-${REPOS_BASE}/hypatia}" HYPATIA_OUTCOME_REPORT="${HYPATIA_OUTCOME_REPORT:-on}" HYPATIA_BATCH_ID="${HYPATIA_BATCH_ID:-}" +# Does this Hypatia know `--verdict`? Learned ONCE per run, from the help text of +# the task and never from an exit code: a Hypatia from before E0-01 ignores an +# option that it does not know, so a no_op call would be recorded as a success. +# verdict Hypatia names --verdict: every verdict is passed to it, and no_op and +# unverified are recorded (they carry no credit). +# legacy an older Hypatia: no --verdict, and no call for no_op or unverified +# (the behaviour before this check, exactly). +# off no calls are made at all (reporting off, no checkout, no mix). +# A help command that fails is not an older Hypatia: the run stops, visibly. +HYPATIA_VERDICT_MODE="off" +hypatia_detect_verdict_mode() { + HYPATIA_VERDICT_MODE="off" + [[ "$HYPATIA_OUTCOME_REPORT" == "on" ]] || return 0 + [[ -d "$HYPATIA_HOME" ]] || return 0 + command -v mix &>/dev/null || return 0 + + local help_text rc=0 + help_text=$(cd "$HYPATIA_HOME" && mix help hypatia.record_outcome 2>&1) || rc=$? + if [[ "$rc" -ne 0 ]]; then + HYPATIA_VERDICT_MODE="error" + echo "ERROR: 'mix help hypatia.record_outcome' failed (exit $rc) in $HYPATIA_HOME: Hypatia does not answer, and a Hypatia that does not answer is not an older Hypatia. Stopping." >&2 + return 1 + fi + if grep -q -- '--verdict' <<< "$help_text"; then + HYPATIA_VERDICT_MODE="verdict" + else + HYPATIA_VERDICT_MODE="legacy" + fi + echo "hypatia outcome mode: $HYPATIA_VERDICT_MODE" +} + hypatia_record_outcome() { local recipe_id="$1" local repo="$2" local file="$3" local outcome="$4" + local verdict="${5:-}" # Opt-out short-circuit. if [[ "$HYPATIA_OUTCOME_REPORT" != "on" ]]; then @@ -249,6 +281,13 @@ hypatia_record_outcome() { local file_arg="${file:-.}" [[ -z "$file_arg" ]] && file_arg="." + # The verdict goes along only when Hypatia knows the option, and only the + # 5 verdicts of the pair table (a refusal has none). + local verdict_args=() + if [[ "$HYPATIA_VERDICT_MODE" == "verdict" && -n "$verdict" ]]; then + verdict_args=(--verdict "$verdict") + fi + local rc=0 ( cd "$HYPATIA_HOME" @@ -256,7 +295,8 @@ hypatia_record_outcome() { --recipe "$recipe_id" \ --repo "$repo" \ --file "$file_arg" \ - --outcome "$outcome" + --outcome "$outcome" \ + "${verdict_args[@]}" ) || rc=$? case "$rc" in @@ -336,27 +376,40 @@ record_outcome() { # failed / incomplete → failure (the fix ran but the work remains) # refused_* → false_positive (the recipe is misfiring, not # the code; surfaces this to auto-quarantine) - # no_op / unverified → skip: no evidence either way, so it must not - # move the recipe's confidence. - local hypatia_outcome + # no_op / unverified → with a Hypatia that knows --verdict: recorded, as + # success WITH the verdict, which Hypatia stores as + # an audit row with no credit (E0-01). With an older + # Hypatia: skipped, as before, because without the + # verdict it would be recorded as a success. + local hypatia_outcome verdict_arg="" case "$verdict" in verified) hypatia_outcome="success" + verdict_arg="$verdict" echo "$repo" >> "$RESCAN_REPOS_FILE" ;; failed|incomplete) hypatia_outcome="failure" + verdict_arg="$verdict" ;; refused_license_policy|refused_*) hypatia_outcome="false_positive" ;; + no_op|unverified) + [[ "$HYPATIA_VERDICT_MODE" == "verdict" ]] || return 0 + hypatia_outcome="success" + verdict_arg="$verdict" + ;; *) return 0 ;; esac - hypatia_record_outcome "$recipe_id" "$repo" "$file" "$hypatia_outcome" || true + hypatia_record_outcome "$recipe_id" "$repo" "$file" "$hypatia_outcome" "$verdict_arg" || true } +# Learn once per run which of the 2 calls Hypatia takes. Stop if it does not answer. +hypatia_detect_verdict_mode || exit 1 + # --- Execute a single manifest entry --- execute_entry() { local entry="$1" diff --git a/tests/outcome-verdict-test.sh b/tests/outcome-verdict-test.sh index 00997731..134ea650 100755 --- a/tests/outcome-verdict-test.sh +++ b/tests/outcome-verdict-test.sh @@ -256,6 +256,91 @@ for PARALLEL in 1 2; do "EXECUTED=$EXECUTED SUCCEEDED=$SUCCEEDED FAILED=$FAILED SKIPPED=$SKIPPED" done +# --- 10. the verdict goes to Hypatia when Hypatia knows the option (E0-02) - +# A stub `mix` on PATH records its arguments and answers `mix help +# hypatia.record_outcome` as the case needs. The mode is learned once, from the +# help text and never from an exit code: an older Hypatia ignores an option that +# it does not know, so a no_op call would be recorded as a success. The main +# loop is not run; it sits behind the quarantine (exit 78). +stub_dir="$TMP/stub-bin" +mkdir -p "$stub_dir" "$TMP/hy-home" +cat > "$stub_dir/mix" <<'STUB' +#!/usr/bin/env bash +if [[ "$1" == "help" ]]; then + case "$STUB_HELP" in + verdict) echo "mix hypatia.record_outcome --recipe R --verdict verified|no_op|incomplete|unverified|failed"; exit 0 ;; + legacy) echo "mix hypatia.record_outcome --recipe R --repo X --file F --outcome O"; exit 0 ;; + *) echo "mix: cannot start" >&2; exit 3 ;; + esac +fi +echo "$*" >> "$STUB_LOG" +exit 0 +STUB +chmod +x "$stub_dir/mix" + +# run_case [on|off]: runs the runner's own contract +# block in a subshell, as the runner does: detect first, stop if that fails, +# then record each verdict and one refusal. +run_case() { + local mode="$1" report="${2:-on}" + : > "$TMP/stub.log" + ( + export PATH="$stub_dir:$PATH" STUB_HELP="$mode" STUB_LOG="$TMP/stub.log" + export HYPATIA_HOME="$TMP/hy-home" HYPATIA_OUTCOME_REPORT="$report" REPOS_BASE="$TMP/repos" + OUTCOME_FILE="$TMP/case-outcomes.jsonl" + OUTCOME_FILE_CENTRAL="$OUTCOME_FILE" + RESCAN_REPOS_FILE="$TMP/case-rescan" + : > "$OUTCOME_FILE" + : > "$RESCAN_REPOS_FILE" + # shellcheck disable=SC1091 + source "$TMP/extracted.sh" + rc=0 + hypatia_detect_verdict_mode || rc=$? + echo "detect_rc=$rc mode=${HYPATIA_VERDICT_MODE:-unset}" + if [[ "$rc" -eq 0 ]]; then + for v in verified no_op unverified incomplete failed; do + record_outcome PA009 recipe-x repo f "$v" 0 3 0 + done + record_outcome PA009 recipe-lic repo "" refused_license_policy + fi + ) > "$TMP/case.out" 2>&1 || true +} +calls() { cat "$TMP/stub.log"; } + +run_case verdict +check "knows --verdict: mode is verdict" "detect_rc=0 mode=verdict" "$(grep '^detect_rc' "$TMP/case.out")" +check "knows --verdict: the mode is logged once" 1 "$(grep -c '^hypatia outcome mode: verdict' "$TMP/case.out")" +check "knows --verdict: 6 calls, one for each verdict and the refusal" 6 "$(calls | wc -l | tr -d ' ')" +for pair in verified:success no_op:success unverified:success incomplete:failure failed:failure; do + v="${pair%%:*}" o="${pair##*:}" + check "knows --verdict: $v is passed with outcome $o" \ + "hypatia.record_outcome --recipe recipe-x --repo repo --file f --outcome $o --verdict $v" \ + "$(calls | grep -- "--verdict $v\$" || true)" +done +check "knows --verdict: a refusal has no --verdict" \ + "hypatia.record_outcome --recipe recipe-lic --repo repo --file . --outcome false_positive" \ + "$(calls | grep -- 'recipe-lic' || true)" + +run_case legacy +check "older Hypatia: mode is legacy" "detect_rc=0 mode=legacy" "$(grep '^detect_rc' "$TMP/case.out")" +check "older Hypatia: the mode is logged once" 1 "$(grep -c '^hypatia outcome mode: legacy' "$TMP/case.out")" +check "older Hypatia: no --verdict in any call" 0 "$(calls | grep -c -- '--verdict' || true)" +check "older Hypatia: no call for no_op or unverified, 4 calls in all" 4 "$(calls | wc -l | tr -d ' ')" +check "older Hypatia: the calls are the calls of today" \ + "success failure failure false_positive" \ + "$(calls | sed -E 's/.*--outcome ([a-z_]+).*/\1/' | tr '\n' ' ' | sed 's/ $//')" + +run_case fail +check "help fails: detection fails" "detect_rc=1 mode=error" "$(grep '^detect_rc' "$TMP/case.out")" +check "help fails: a visible error says why" 1 "$(grep -c 'does not answer' "$TMP/case.out")" +check "help fails: no record call is made" 0 "$(calls | wc -l | tr -d ' ')" +check "the runner stops when detection fails (before any call)" 1 \ + "$(grep -c '^hypatia_detect_verdict_mode || exit 1$' "$ROOT/scripts/dispatch-runner.sh")" + +run_case fail off +check "reporting off: no help call and no record call" "detect_rc=0 mode=off 0" \ + "$(grep '^detect_rc' "$TMP/case.out") $(calls | wc -l | tr -d ' ')" + echo "" echo "Results: $pass passed, $fail failed" [[ "$fail" -eq 0 ]]