diff --git a/.claude/skills/triage-ci-failures/SKILL.md b/.claude/skills/triage-ci-failures/SKILL.md new file mode 100644 index 0000000000..6bb6eb3149 --- /dev/null +++ b/.claude/skills/triage-ci-failures/SKILL.md @@ -0,0 +1,40 @@ +--- +name: triage-ci-failures +description: Use when a quarto-cli PR has red CI checks or a workflow run failed, to find which tests failed on which OSes and why, without loading raw job logs into context. Covers dependency-bump PRs (Pandoc, Typst, Deno) where many assertions break at once. +--- + +# Triage CI Failures + +A red PR on quarto-cli fans out into dozens of failed jobs (test buckets × OSes), each log several hundred KB. Check-run annotations only name the bucket, and `gh run view --log` silently truncates near 750 KB. The script below downloads the failed-job logs to a cache directory and prints a compact digest: one row per unique failing test, the OSes it failed on, and the assertion that broke. + +## Step 1: Digest the failures + +```bash +bash .claude/skills/triage-ci-failures/scripts/ci-failures.sh +bash .claude/skills/triage-ci-failures/scripts/ci-failures.sh --run # one run, e.g. a scheduled run on main +``` + +The PR form takes every failing run on the PR's head commit. Logs are cached per job id under `$CI_FAILURES_DIR` (default `${TMPDIR:-/tmp}/quarto-ci-failures`), so re-running after a new push only fetches the new jobs. + +The output has up to three sections: + +- **unique failing tests**: `[OSes] `, then one representative `[verify]` check and its message (up to 6 lines, including `noErrors` diagnostics). The excerpt identifies its OS and job; the row lists every contributing log in the cache directory. Missing output (`No output created message`, a missing path) may indicate a render failure, so the row includes the captured render output from the test's `OUTPUT:` tail. A row with no excerpt may have failed during prereq or teardown and names the log to grep. +- **step errors**: `##[error]` lines from failed jobs that had no failing test (runtime-boundary, lint, setup). +- **failed jobs with no recognised failure**: a log shape the script does not parse (Playwright and feature-format-matrix jobs are unverified). Grep the printed log path for the cause (`grep -n -E 'Error|FAIL' | head`) rather than reading the log whole. + +Step 1 is done when every failed job is accounted for by a test row, a step error, or a cause found by grep in its log. + +## Step 2: Group rows by cause + +Several rows may share one cause, such as a Pandoc change affecting several docx crossref tests. Compare diagnostics in the listed logs before grouping failures across OSes; the representative excerpt may not describe every failure. Investigate failures limited to one OS for platform differences or intermittent behavior; see `dev-docs/debugging-flaky-tests.md`. + +## Step 3: Classify each cause before changing any test + +For each cause, decide **drift** or **regression** before touching an assertion or snapshot: + +- **Drift**: the dependency changed its output in a way that is correct or neutral for users (reordered attributes, hashed bookmark names, new whitespace). Update the assertion or snapshot, and say in the commit which upstream change caused it. +- **Regression**: the rendered result is now wrong for users (missing content, broken cross-reference, a document that no longer renders). Fix Quarto or report upstream; the test stays as it is. + +Decide from the rendered output, not the assertion text: render the test document with the dev build and inspect it (`.claude/rules/testing/smoke-all-tests.md` covers how smoke-all tests declare their checks). For a dependency bump, the dependency's changelog usually names the change. + +Step 3 is done when every cause carries a drift-or-regression verdict with the evidence behind it. Track one work item per cause, so each verdict and fix can be reviewed on its own. diff --git a/.claude/skills/triage-ci-failures/scripts/ci-failures.sh b/.claude/skills/triage-ci-failures/scripts/ci-failures.sh new file mode 100644 index 0000000000..1c1079fcac --- /dev/null +++ b/.claude/skills/triage-ci-failures/scripts/ci-failures.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# Summarise the failing CI jobs of a PR (or of one workflow run) without +# loading raw logs: one row per unique failing test, deduplicated across OSes. +# +# Usage: ci-failures.sh [repo] +# ci-failures.sh --run [repo] +# +# Logs are cached per job id under $CI_FAILURES_DIR +# (default: ${TMPDIR:-/tmp}/quarto-ci-failures). +set -euo pipefail + +if [ "${1:-}" = "--run" ]; then + RUNS="${2:?run id required}"; REPO="${3:-quarto-dev/quarto-cli}" + echo "Run $RUNS" +else + PR="${1:?usage: ci-failures.sh | --run [repo]}" + REPO="${2:-quarto-dev/quarto-cli}" + SHA="$(gh pr view "$PR" --repo "$REPO" --json headRefOid --jq .headRefOid)" + RUNS="$(gh run list --repo "$REPO" --commit "$SHA" --limit 100 \ + --json databaseId,conclusion --jq '.[] | select(.conclusion=="failure") | .databaseId')" + echo "PR #$PR head ${SHA:0:9}: failing runs: $(echo $RUNS)" + [ -n "$RUNS" ] || { echo "No failing runs."; exit 0; } +fi + +D="${CI_FAILURES_DIR:-${TMPDIR:-/tmp}/quarto-ci-failures}"; mkdir -p "$D" +JOBS="$D/jobs-$$.tsv"; : > "$JOBS" +trap 'rm -f "$JOBS"' EXIT + +for RUN in $RUNS; do + gh api "repos/$REPO/actions/runs/$RUN/jobs?per_page=100&filter=latest" --paginate \ + --jq '.jobs[] | select(.conclusion=="failure") | [.id, .name] | @tsv' >> "$JOBS" +done +# Linux jobs first, so excerpts show forward-slash paths. +awk -F'\t' '{print ($2 ~ /ubuntu|linux/ ? 0 : 1) "\t" $0}' "$JOBS" \ + | sort -s -k1,1 | cut -f2- > "$JOBS.sorted" && mv "$JOBS.sorted" "$JOBS" +echo "Failed jobs: $(wc -l < "$JOBS" | tr -d ' ') Logs: $D" + +# Download job logs directly to avoid truncation. +while IFS=$'\t' read -r id name; do + f="$D/$id.log" + [ -s "$f" ] && continue + # Publish only a complete download, so a failed fetch is retried next run. + gh api --allow-escape-sequences "repos/$REPO/actions/jobs/$id/logs" \ + | sed -e 's/\x1b\[[0-9;]*m//g' -e 's/^[0-9T:.-]*Z //' > "$f.part" && mv "$f.part" "$f" +done < "$JOBS" + +# Pass 1: one record per (failure, job): kind \t key \t os \t job-id \t excerpt (lines joined by \037) +while IFS=$'\t' read -r id name; do + # "Run smoke ()" may carry a suffix such as " with timed file" (test-smokes.yml). + os="$(printf '%s' "$name" | sed -nE 's/.*\(([^)]*)\).*/\1/p')"; os="${os:-$name}" + group="$(printf '%s' "$name" | sed -E 's/ *\([^)]*\)//')" + awk -v os="$os" -v id="$id" -v group="$group" ' + function norm(s) { sub(/^\[smoke\] > /, "", s); gsub(/\\/, "/", s); return s } + function flush( i, n, out, seen, start) { + if (title == "") return + out = verify msg + if (notfound && nout > 0) { + # Missing output: include the captured render output. + out = out "\037render output (OUTPUT tail):" + n = 0; start = nout + for (i = nout; i >= 1 && n < 6; i--) if (!(outl[i] in seen)) { seen[outl[i]] = 1; n++; start = i } + delete seen + for (i = start; i <= nout; i++) if (!(outl[i] in seen)) { seen[outl[i]] = 1; out = out "\037 " outl[i] } + } + print "T\t" norm(title) "\t" os "\t" id "\t" out + title = ""; verify = ""; msg = ""; m1 = ""; nmsg = 0; inmsg = 0; notfound = 0; nout = 0; inout = 0; hdr = 0 + } + / \.\.\. FAILED \(/ { t = $0; sub(/ \.\.\. FAILED.*/, "", t); print "T\t" norm(t) "\t" os "\t" id "\t"; next } + /━━━ TEST FAILURE: / { flush(); title = substr($0, index($0, "TEST FAILURE: ") + 14); next } + title != "" { + if ($0 ~ /^\[verify\] > /) { verify = substr($0, 1, 200); inmsg = 1; next } + # The assertion message runs from [verify] to the stack trace, whose first line is + # ": ". noErrors-style messages carry the real + # diagnostics on later lines, possibly after blank lines. + if (inmsg) { + if ($0 ~ /^[ \t]*$/) next + if ($0 ~ /^OUTPUT:/) inmsg = 0 + else if ($0 ~ /^[ \t]+at / || (m1 != "" && $0 ~ /^[A-Za-z]+: / && substr($0, index($0, ": ") + 2) == m1)) { inmsg = 0; next } + } + if (inmsg) { + if (m1 == "") m1 = $0 + if (nmsg < 6) { msg = msg "\037" substr($0, 1, 240); nmsg++ } + # XPath and layout verifiers end with "... follows:" and then dump the whole document. + if ($0 ~ /follows:[ \t]*$/) inmsg = 0 + # Missing output: include the captured render output. + if ($0 ~ /NotFound|os error 2|cannot find the file|No output created message|^Path .* doesn.t exist/) notfound = 1 + next + } + if ($0 ~ /^OUTPUT:/) { inout = 1; next } + if ($0 ~ /Stack trace:|throw new AssertionError/) { inout = 0; next } + if (inout) { + l = $0; sub(/^[ \t]+/, "", l); sub(/[ \t]+$/, "", l) + # Skip the echoed "pandoc" options and "metadata" blocks; each ends at a blank line. + if (l == "pandoc" || l == "metadata") { hdr = 1; next } + if (l == "") { hdr = 0; next } + if (!hdr && l !~ /^[0-9]* *\|/ && l != "^") outl[++nout] = substr(l, 1, 240) + } + } + /^##\[error\]/ && !/Process completed with exit code/ { + print "E\t" substr($0, 10, 300) "\t" os "\t" id "\t" group + } + END { flush() } + ' "$D/$id.log" +done < "$JOBS" > "$D/records-$$.tsv" + +# Pass 2: aggregate into one row per test (and per step error), listing the OSes that failed it. +awk -F'\t' -v jobs="$JOBS" -v dir="$D" ' + function addos(list, os) { return index(", " list ", ", ", " os ", ") ? list : (list == "" ? os : list ", " os) } + BEGIN { while ((getline l < jobs) > 0) { split(l, a, "\t"); jname[a[1]] = a[2]; jorder[++nj] = a[1] } } + { seenjob[$4] = 1 } + $1 == "T" { + testjob[$4] = 1 + if (!($2 in tos)) order[++nt] = $2 + tos[$2] = addos(tos[$2], $3) + if (ex[$2] == "" && $5 != "") { ex[$2] = $5; exos[$2] = $3; exjob[$2] = $4 } + if (!($2 in tjob)) tjob[$2] = $4 + if (!logseen[$2, $4]) { + logseen[$2, $4] = 1 + tlogs[$2] = tlogs[$2] (tlogs[$2] == "" ? "" : "; ") "[" $3 "] " $4 ".log" + } + } + $1 == "E" { ejob[++nr] = $4; ekey[nr] = $5 "\t" $2; eosr[nr] = $3 } + END { + printf "\n== %d unique failing tests\n", nt + for (i = 1; i <= nt; i++) { + t = order[i]; printf "\n[%s] %s\n", tos[t], t + if (ex[t] != "") printf " representative excerpt (%s, job %s):\n", exos[t], exjob[t] + n = split(ex[t], lines, "\037"); for (j = 1; j <= n; j++) if (lines[j] != "") print " " lines[j] + printf " logs: %s\n", tlogs[t] + # prereq and teardown-only failures print no TEST FAILURE block. + if (ex[t] == "") printf " no TEST FAILURE block (prereq or teardown failure?); grep %s/%s.log\n", dir, tjob[t] + } + # Hide step annotations when this job already has test rows. + for (r = 1; r <= nr; r++) if (!(ejob[r] in testjob)) { + k = ekey[r]; if (!(k in eos)) eorder[++ne] = k + eos[k] = addos(eos[k], eosr[r]) + } + if (ne) printf "\n== %d step errors\n", ne + for (i = 1; i <= ne; i++) { split(eorder[i], p, "\t"); printf "\n[%s] %s\n %s\n", eos[eorder[i]], p[1], p[2] } + m = 0 + for (i = 1; i <= nj; i++) if (!(jorder[i] in seenjob)) { + if (!m++) print "\n== Failed jobs with no recognised failure (grep their log for the cause)" + printf "%s\n %s/%s.log\n", jname[jorder[i]], dir, jorder[i] + } + } +' "$D/records-$$.tsv" +rm -f "$D/records-$$.tsv"