Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 58 additions & 5 deletions scripts/dispatch-runner.sh
Original file line number Diff line number Diff line change
Expand Up @@ -219,11 +219,43 @@ HYPATIA_HOME="${HYPATIA_HOME:-${REPOS_BASE}/hypatia}"
HYPATIA_OUTCOME_REPORT="${HYPATIA_OUTCOME_REPORT:-on}"
HYPATIA_BATCH_ID="${HYPATIA_BATCH_ID:-}"

# Does this Hypatia know `--verdict`? Learned ONCE per run, from the help text of
# the task and never from an exit code: a Hypatia from before E0-01 ignores an
# option that it does not know, so a no_op call would be recorded as a success.
# verdict Hypatia names --verdict: every verdict is passed to it, and no_op and
# unverified are recorded (they carry no credit).
# legacy an older Hypatia: no --verdict, and no call for no_op or unverified
# (the behaviour before this check, exactly).
# off no calls are made at all (reporting off, no checkout, no mix).
# A help command that fails is not an older Hypatia: the run stops, visibly.
HYPATIA_VERDICT_MODE="off"
hypatia_detect_verdict_mode() {
HYPATIA_VERDICT_MODE="off"
[[ "$HYPATIA_OUTCOME_REPORT" == "on" ]] || return 0
[[ -d "$HYPATIA_HOME" ]] || return 0
command -v mix &>/dev/null || return 0

local help_text rc=0
help_text=$(cd "$HYPATIA_HOME" && mix help hypatia.record_outcome 2>&1) || rc=$?
if [[ "$rc" -ne 0 ]]; then
HYPATIA_VERDICT_MODE="error"
echo "ERROR: 'mix help hypatia.record_outcome' failed (exit $rc) in $HYPATIA_HOME: Hypatia does not answer, and a Hypatia that does not answer is not an older Hypatia. Stopping." >&2
return 1
fi
if grep -q -- '--verdict' <<< "$help_text"; then
HYPATIA_VERDICT_MODE="verdict"
else
HYPATIA_VERDICT_MODE="legacy"
fi
echo "hypatia outcome mode: $HYPATIA_VERDICT_MODE"
}

hypatia_record_outcome() {
local recipe_id="$1"
local repo="$2"
local file="$3"
local outcome="$4"
local verdict="${5:-}"

# Opt-out short-circuit.
if [[ "$HYPATIA_OUTCOME_REPORT" != "on" ]]; then
Expand All @@ -249,14 +281,22 @@ hypatia_record_outcome() {
local file_arg="${file:-.}"
[[ -z "$file_arg" ]] && file_arg="."

# The verdict goes along only when Hypatia knows the option, and only the
# 5 verdicts of the pair table (a refusal has none).
local verdict_args=()
if [[ "$HYPATIA_VERDICT_MODE" == "verdict" && -n "$verdict" ]]; then
verdict_args=(--verdict "$verdict")
fi

local rc=0
(
cd "$HYPATIA_HOME"
mix hypatia.record_outcome \
--recipe "$recipe_id" \
--repo "$repo" \
--file "$file_arg" \
--outcome "$outcome"
--outcome "$outcome" \
"${verdict_args[@]}"
) || rc=$?

case "$rc" in
Expand Down Expand Up @@ -336,27 +376,40 @@ record_outcome() {
# failed / incomplete → failure (the fix ran but the work remains)
# refused_* → false_positive (the recipe is misfiring, not
# the code; surfaces this to auto-quarantine)
# no_op / unverified → skip: no evidence either way, so it must not
# move the recipe's confidence.
local hypatia_outcome
# no_op / unverified → with a Hypatia that knows --verdict: recorded, as
# success WITH the verdict, which Hypatia stores as
# an audit row with no credit (E0-01). With an older
# Hypatia: skipped, as before, because without the
# verdict it would be recorded as a success.
local hypatia_outcome verdict_arg=""
case "$verdict" in
verified)
hypatia_outcome="success"
verdict_arg="$verdict"
echo "$repo" >> "$RESCAN_REPOS_FILE"
;;
failed|incomplete)
hypatia_outcome="failure"
verdict_arg="$verdict"
;;
refused_license_policy|refused_*)
hypatia_outcome="false_positive"
;;
no_op|unverified)
[[ "$HYPATIA_VERDICT_MODE" == "verdict" ]] || return 0
hypatia_outcome="success"
verdict_arg="$verdict"
;;
*)
return 0
;;
esac
hypatia_record_outcome "$recipe_id" "$repo" "$file" "$hypatia_outcome" || true
hypatia_record_outcome "$recipe_id" "$repo" "$file" "$hypatia_outcome" "$verdict_arg" || true
}

# Learn once per run which of the 2 calls Hypatia takes. Stop if it does not answer.
hypatia_detect_verdict_mode || exit 1

# --- Execute a single manifest entry ---
execute_entry() {
local entry="$1"
Expand Down
85 changes: 85 additions & 0 deletions tests/outcome-verdict-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,91 @@ for PARALLEL in 1 2; do
"EXECUTED=$EXECUTED SUCCEEDED=$SUCCEEDED FAILED=$FAILED SKIPPED=$SKIPPED"
done

# --- 10. the verdict goes to Hypatia when Hypatia knows the option (E0-02) -
# A stub `mix` on PATH records its arguments and answers `mix help
# hypatia.record_outcome` as the case needs. The mode is learned once, from the
# help text and never from an exit code: an older Hypatia ignores an option that
# it does not know, so a no_op call would be recorded as a success. The main
# loop is not run; it sits behind the quarantine (exit 78).
stub_dir="$TMP/stub-bin"
mkdir -p "$stub_dir" "$TMP/hy-home"
cat > "$stub_dir/mix" <<'STUB'
#!/usr/bin/env bash
if [[ "$1" == "help" ]]; then
case "$STUB_HELP" in
verdict) echo "mix hypatia.record_outcome --recipe R --verdict verified|no_op|incomplete|unverified|failed"; exit 0 ;;
legacy) echo "mix hypatia.record_outcome --recipe R --repo X --file F --outcome O"; exit 0 ;;
*) echo "mix: cannot start" >&2; exit 3 ;;
esac
fi
echo "$*" >> "$STUB_LOG"
exit 0
STUB
chmod +x "$stub_dir/mix"

# run_case <what the stub's help says> [on|off]: runs the runner's own contract
# block in a subshell, as the runner does: detect first, stop if that fails,
# then record each verdict and one refusal.
run_case() {
local mode="$1" report="${2:-on}"
: > "$TMP/stub.log"
(
export PATH="$stub_dir:$PATH" STUB_HELP="$mode" STUB_LOG="$TMP/stub.log"
export HYPATIA_HOME="$TMP/hy-home" HYPATIA_OUTCOME_REPORT="$report" REPOS_BASE="$TMP/repos"
OUTCOME_FILE="$TMP/case-outcomes.jsonl"
OUTCOME_FILE_CENTRAL="$OUTCOME_FILE"
RESCAN_REPOS_FILE="$TMP/case-rescan"
: > "$OUTCOME_FILE"
: > "$RESCAN_REPOS_FILE"
# shellcheck disable=SC1091
source "$TMP/extracted.sh"
rc=0
hypatia_detect_verdict_mode || rc=$?
echo "detect_rc=$rc mode=${HYPATIA_VERDICT_MODE:-unset}"
if [[ "$rc" -eq 0 ]]; then
for v in verified no_op unverified incomplete failed; do
record_outcome PA009 recipe-x repo f "$v" 0 3 0
done
record_outcome PA009 recipe-lic repo "" refused_license_policy
fi
) > "$TMP/case.out" 2>&1 || true
}
calls() { cat "$TMP/stub.log"; }

run_case verdict
check "knows --verdict: mode is verdict" "detect_rc=0 mode=verdict" "$(grep '^detect_rc' "$TMP/case.out")"
check "knows --verdict: the mode is logged once" 1 "$(grep -c '^hypatia outcome mode: verdict' "$TMP/case.out")"
check "knows --verdict: 6 calls, one for each verdict and the refusal" 6 "$(calls | wc -l | tr -d ' ')"
for pair in verified:success no_op:success unverified:success incomplete:failure failed:failure; do
v="${pair%%:*}" o="${pair##*:}"
check "knows --verdict: $v is passed with outcome $o" \
"hypatia.record_outcome --recipe recipe-x --repo repo --file f --outcome $o --verdict $v" \
"$(calls | grep -- "--verdict $v\$" || true)"
done
check "knows --verdict: a refusal has no --verdict" \
"hypatia.record_outcome --recipe recipe-lic --repo repo --file . --outcome false_positive" \
"$(calls | grep -- 'recipe-lic' || true)"

run_case legacy
check "older Hypatia: mode is legacy" "detect_rc=0 mode=legacy" "$(grep '^detect_rc' "$TMP/case.out")"
check "older Hypatia: the mode is logged once" 1 "$(grep -c '^hypatia outcome mode: legacy' "$TMP/case.out")"
check "older Hypatia: no --verdict in any call" 0 "$(calls | grep -c -- '--verdict' || true)"
check "older Hypatia: no call for no_op or unverified, 4 calls in all" 4 "$(calls | wc -l | tr -d ' ')"
check "older Hypatia: the calls are the calls of today" \
"success failure failure false_positive" \
"$(calls | sed -E 's/.*--outcome ([a-z_]+).*/\1/' | tr '\n' ' ' | sed 's/ $//')"

run_case fail
check "help fails: detection fails" "detect_rc=1 mode=error" "$(grep '^detect_rc' "$TMP/case.out")"
check "help fails: a visible error says why" 1 "$(grep -c 'does not answer' "$TMP/case.out")"
check "help fails: no record call is made" 0 "$(calls | wc -l | tr -d ' ')"
check "the runner stops when detection fails (before any call)" 1 \
"$(grep -c '^hypatia_detect_verdict_mode || exit 1$' "$ROOT/scripts/dispatch-runner.sh")"

run_case fail off
check "reporting off: no help call and no record call" "detect_rc=0 mode=off 0" \
"$(grep '^detect_rc' "$TMP/case.out") $(calls | wc -l | tr -d ' ')"

echo ""
echo "Results: $pass passed, $fail failed"
[[ "$fail" -eq 0 ]]
Loading