From 7508a29cfbf5989f8dc21aedc12ea4d4d188ae7d Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Fri, 30 Oct 2026 19:01:38 -0300 Subject: [PATCH 01/31] ci: lint the workflows and templates with actionlint --- .github/workflows/test.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0450a2e..be73f60 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,7 +3,7 @@ name: Tests # This repo's own checks. A bug here reaches every repo that pins a tag from it, so the logic in the # actions is unit tested rather than only exercised by a real release. # -# Run it locally: `npm install` once, then `node --test`. +# Run it locally: `npm install` once, then `node --test`. For the lint job, run `actionlint`. on: push: @@ -40,3 +40,22 @@ jobs: # finds every *.test.js under the repository on all of them. - name: Run the unit tests run: node --test + + # The workflows and templates never run in this repository the way a plugin runs them, so a typo + # in one would only show up in someone else's CI. actionlint also runs shellcheck on every run: + # block it finds. + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + + # Pinned, so a new actionlint rule cannot fail a pull request that did not change anything. + # `queue` is ignored because GitHub accepts it in `concurrency` (release-tags.yml uses it) + # and actionlint 1.7.12 does not know it yet. + - name: Run actionlint + run: | + bash <(curl -sSfL https://raw.githubusercontent.com/rhysd/actionlint/v1.7.12/scripts/download-actionlint.bash) 1.7.12 + ./actionlint -ignore 'unexpected key "queue" for "concurrency" section' \ + .github/workflows/*.yml templates/workflows/*.yml From 85d651a7592c9abe7933331e626911bb65ad238a Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Fri, 30 Oct 2026 19:06:02 -0300 Subject: [PATCH 02/31] feat(plugin-check): add the shared Plugin Check workflow and report action --- .github/actions/plugin-check-report/README.md | 221 +++++++++ .../actions/plugin-check-report/action.yml | 195 ++++++++ .../plugin-check-report/fetch-baseline.sh | 148 ++++++ .../actions/plugin-check-report/ratchet.sh | 286 ++++++++++++ .github/workflows/plugin-check.yml | 441 ++++++++++++++++++ .github/workflows/test.yml | 8 +- .gitignore | 3 + templates/workflows/plugin-check.yml | 40 ++ .../fetch-baseline.test.js | 265 +++++++++++ .../fixtures/baseline.json | 9 + .../plugin-check-report/fixtures/clean.txt | 1 + .../plugin-check-report/fixtures/empty.txt | 0 .../fixtures/errors-down.txt | 4 + .../fixtures/errors-up.txt | 4 + .../plugin-check-report/fixtures/garbage.txt | 3 + .../fixtures/rule-up-total-same.txt | 4 + .../fixtures/same-as-baseline.txt | 4 + .../fixtures/warnings-up.txt | 4 + .../plugin-check-report/ratchet.test.js | 297 ++++++++++++ 19 files changed, 1936 insertions(+), 1 deletion(-) create mode 100644 .github/actions/plugin-check-report/README.md create mode 100644 .github/actions/plugin-check-report/action.yml create mode 100755 .github/actions/plugin-check-report/fetch-baseline.sh create mode 100755 .github/actions/plugin-check-report/ratchet.sh create mode 100644 .github/workflows/plugin-check.yml create mode 100644 templates/workflows/plugin-check.yml create mode 100644 tests/actions/plugin-check-report/fetch-baseline.test.js create mode 100644 tests/actions/plugin-check-report/fixtures/baseline.json create mode 100644 tests/actions/plugin-check-report/fixtures/clean.txt create mode 100644 tests/actions/plugin-check-report/fixtures/empty.txt create mode 100644 tests/actions/plugin-check-report/fixtures/errors-down.txt create mode 100644 tests/actions/plugin-check-report/fixtures/errors-up.txt create mode 100644 tests/actions/plugin-check-report/fixtures/garbage.txt create mode 100644 tests/actions/plugin-check-report/fixtures/rule-up-total-same.txt create mode 100644 tests/actions/plugin-check-report/fixtures/same-as-baseline.txt create mode 100644 tests/actions/plugin-check-report/fixtures/warnings-up.txt create mode 100644 tests/actions/plugin-check-report/ratchet.test.js diff --git a/.github/actions/plugin-check-report/README.md b/.github/actions/plugin-check-report/README.md new file mode 100644 index 0000000..b63148c --- /dev/null +++ b/.github/actions/plugin-check-report/README.md @@ -0,0 +1,221 @@ +# plugin-check-report + +Compares a [WordPress.org Plugin Check](https://github.com/WordPress/plugin-check) results file with +a baseline: the counts of an earlier run of the same workflow. It saves this run's counts as the +next baseline and writes a short body for the pull request comment. + +Most repositories do not use this action directly. They call the reusable workflow +[`.github/workflows/plugin-check.yml`](../../workflows/plugin-check.yml), which builds the plugin, +runs [wordpress/plugin-check-action](https://github.com/WordPress/plugin-check-action), runs this +action and posts the comment. A caller to copy is in +[`templates/workflows/plugin-check.yml`](../../../templates/workflows/plugin-check.yml): + +```yaml +jobs: + plugin-check: + permissions: + actions: read + contents: read + issues: write + pull-requests: write + uses: stellarwp/plugin-toolbox/.github/workflows/plugin-check.yml@v1 + with: + plugin-slug: my-plugin +``` + +## What the repository needs + +- A `.puprc` whose `build` makes the plugin as it ships, and whose `zip_name` equals `plugin-slug`. + The zip's top folder is named after `zip_name`, and Plugin Check takes the slug from the folder + name. +- A `pup` Composer script that puts `pup.phar` in `bin/` and runs it, as GiveWP and Kadence Blocks + have: + + ```json + "pup": [ + "test -f ./bin/pup.phar || curl -o bin/pup.phar -L -C - https://github.com/stellarwp/pup/releases/download/2.0.0/pup.phar", + "@php bin/pup.phar" + ] + ``` + +- A `.distignore` (or pup's default ignore list) that keeps dev-only files out of the zip. +- The caller file named `plugin-check.yml`. See [The baseline](#the-baseline). + +The JavaScript toolchain comes from files the repository already has: `.nvmrc` sets up Node, +`package-lock.json` turns on the npm cache, and `bun.lock` or `bun.lockb` sets up bun. + +## How the plugin is built + +By default, with pup, the same steps the release zip uses, from the root of the checkout: + +```bash +composer run -q pup -- --version > /dev/null # downloads bin/pup.phar +composer -- pup build +version=$(php -d display_errors=0 bin/pup.phar get-version --dev) +composer -- pup package "$version" +name=$(php -d display_errors=0 bin/pup.phar zip-name "$version") +unzip -q "$name.zip" -d "$PLUGIN_CHECK_BUILD_DIR" +``` + +So `.puprc` and `.distignore` decide what is checked. If the release needs a step the check should +see too, such as making the POT file, put it in `.puprc`'s `build`, not in the workflow. + +### `build-command`, for a repository without pup + +`build-command` replaces the pup steps. Prefer `.puprc` when the repository has one: a custom +command is a second place that defines the build, and the two drift apart. + +The contract: + +- It runs with bash, from the root of the checkout, with `-e` and `pipefail`. +- PHP, Node or bun are set up before it, as for pup. The `composer-token` and `package-token` + secrets are in its environment. +- It must leave the plugin, as it ships, in `$PLUGIN_CHECK_BUILD_DIR//`, with the main + plugin file inside: `.php`, or `main-file` when the name differs. + +After the build, pup or not, the workflow checks that `$PLUGIN_CHECK_BUILD_DIR//` +exists, and fails naming what it found instead. A folder, not a zip, because the Plugin Check action +takes a folder and its slug from the folder name; asking for a zip would only add an unzip and a +guess at the top folder. + +```yaml + with: + plugin-slug: my-plugin + build-command: | + composer install --no-dev + npm ci && npm run build + mkdir -p "$PLUGIN_CHECK_BUILD_DIR/my-plugin" + rsync -a --exclude-from=.distignore ./ "$PLUGIN_CHECK_BUILD_DIR/my-plugin/" +``` + +## The baseline + +The baseline is never a committed file. It is the `baseline.json` in the `plugin-check-report` +artifact of an earlier run of the same workflow file: + +- A pull request compares with the run for its base commit. For a stacked pull request that is + the run of the pull request below it. A run still going is waited for, up to 20 minutes. +- If that run has no baseline (it was cancelled, say), the newest run on the target branch is used. + More errors than that older run only warn, because the target may have changed since. +- A push compares with the run before it on the same branch. +- With no earlier run at all, nothing is compared and the check passes with a warning. A failed + lookup (an API or download error) fails instead, so an outage never passes as "no baseline". + +The earlier runs are found by the caller's workflow file name, read from `GITHUB_WORKFLOW_REF`. +Renaming the caller file starts again with no baseline. Artifacts expire after the repository's +retention period (90 days by default), so a branch with no run for that long starts with none too. + +For a pull request to have a baseline, the branch it targets needs runs: list it under `push` in +the caller. + +## Modes + +- `ratchet` (default): fail only when a rule has more errors than the baseline. More warnings, or + fewer of either, only add a notice. The totals can go down while one rule goes up, so the check + looks at each rule. +- `strict`: fail on any error. The comment still compares with the baseline. Switch to it once a + plugin reaches zero errors; a new plugin with zero errors should start there. + +## The comment + +One comment per pull request, replaced on every run, found by a hidden first line: +``. It says "in progress" while the check runs, then shows the +result, the totals, up to five rules that went up, and links to the run and the artifact. It never +grows past 15 lines. Pull requests from forks get no comment: their token cannot write. + +## The check name + +A job in a called workflow reports as ` / `. With the template's job id, +the check to require in branch protection is `plugin-check / Plugin Check`. + +## Inputs of the workflow + +| Input | Default | What it does | +|---|---|---| +| `plugin-slug` | required | The plugin's folder name. Equals `zip_name` in `.puprc` | +| `main-file` | `.php` | The main plugin file, checked after the build | +| `build-command` | empty: pup | A bash command that builds the plugin instead of pup. See [`build-command`](#build-command-for-a-repository-without-pup) | +| `runs-on` | `ubuntu-latest` | Runner of the job that builds and checks | +| `php-version` | `7.4` | PHP that builds `vendor/`. Use the oldest the plugin supports | +| `exclude-checks` | empty | Checks to skip, one per line, passed to the Plugin Check action | +| `cache-paths` | empty | Build caches kept between runs, one path per line, relative to the repository | +| `mode` | `ratchet` | `ratchet` or `strict`. See [Modes](#modes) | +| `submodules` | `false` | Passed to `actions/checkout` | + +| Secret | Becomes, in the build step only | +|---|---| +| `composer-token` | `COMPOSER_AUTH` with a `github-oauth` entry | +| `package-token` | `NODE_AUTH_TOKEN`, `BUN_AUTH_TOKEN` and `GITHUB_TOKEN` | + +The secrets never reach the job's environment, so the third-party Plugin Check step that runs later +in the same job does not see them there. + +### Permissions the caller grants + +```yaml +permissions: + actions: read # read earlier runs and download their baseline + contents: read # check out the repository + issues: write # the sticky comment + pull-requests: write # the sticky comment +``` + +A called workflow can only lower these. If the caller grants less than a job asks for, GitHub does +not start the run. The job that runs the Plugin Check action lowers itself to `contents: read`. + +## Inputs of the action + +| Input | Required | Default | What it does | +|---|---|---|---| +| `results` | yes | | The results file. A missing file fails the compare | +| `comment-file` | no | none | Where to write the comment body | +| `mode` | no | `ratchet` | `ratchet` or `strict` | +| `comment-marker` | no | `` | First line of the comment body | +| `workflow-file` | no | from `GITHUB_WORKFLOW_REF` | Whose runs hold the baseline | +| `baseline-wait` | no | `1200` | Seconds to wait for the base commit's run | +| `badge-url` | no | none | Image under the status line | +| `token` | no | `github.token` | Needs `actions: read` | + +| Output | What it is | +|---|---| +| `artifact-url` | The `plugin-check-report` artifact of this run | +| `baseline-url` | The run the baseline came from. Empty with no baseline | + +## Running it locally + +You need `bash`, `jq`, and for the baseline the GitHub CLI signed in (`gh auth login`). + +1. Build the plugin and run Plugin Check in a local WordPress, for example with wp-env: + + ```bash + npx wp-env start + npx wp-env run cli wp plugin check my-plugin --format=json > results.txt + ``` + + Keep only the command's output in the file. A notice from wp-env at the top fails the ratchet + with "not Plugin Check output", on purpose. + +2. Get a baseline from the branch you compare with: + + ```bash + GITHUB_REPOSITORY=my-org/my-plugin \ + bash .github/actions/plugin-check-report/fetch-baseline.sh develop baseline.json + ``` + +3. Compare: + + ```bash + PLUGIN_CHECK_BASELINE=baseline.json \ + bash .github/actions/plugin-check-report/ratchet.sh --comment-file comment.md results.txt + ``` + + Exit 0 means the check passes. `comment.md` holds what the pull request would show. Add + `PLUGIN_CHECK_MODE=strict` to see the strict result. Skip step 2 to see the counts with no + baseline. + +## How it is tested + +`tests/actions/plugin-check-report/` runs both scripts with bash from `node --test`. `ratchet.sh` +reads the sample results files in `fixtures/`; `fetch-baseline.sh` talks to a stand-in `gh` that +answers from JSON files and records each call. Run `npm install` once, then `node --test`. They need +`jq`, which GitHub's Ubuntu runners have. diff --git a/.github/actions/plugin-check-report/action.yml b/.github/actions/plugin-check-report/action.yml new file mode 100644 index 0000000..343802e --- /dev/null +++ b/.github/actions/plugin-check-report/action.yml @@ -0,0 +1,195 @@ +# Compares a WordPress.org Plugin Check results file with a baseline: the counts of an earlier run +# of the same workflow, never a committed file. Only a rule with more errors than the baseline +# fails (ratchet mode), or any error at all (strict mode). It also saves this run's counts as the +# next baseline and writes the short body for the pull request comment. +# +# It does not run Plugin Check and it does not post the comment. The reusable workflow +# .github/workflows/plugin-check.yml does both, so this action stays usable with a results file +# from anywhere. +# +# The work is in two bash scripts next to this file, ratchet.sh and fetch-baseline.sh. They are +# run by path, so there is nothing to build: the file in the repository is the file that runs. +name: Plugin Check report +description: Compares Plugin Check results with an earlier run's baseline, and saves a new baseline + +inputs: + results: + description: >- + Path of the results file wordpress/plugin-check-action writes, `plugin-check-results.txt`. A + missing file fails the compare step: that is how a broken build or a broken check shows up. + required: true + comment-file: + description: >- + Where to write the short Markdown body for the pull request comment. Empty writes none. The + file is only written when the compare runs, so a caller can tell "no result" from a result. + required: false + default: '' + mode: + description: >- + `ratchet`: fail only when a rule has more errors than the baseline. `strict`: fail on any + error, baseline or not. + required: false + default: 'ratchet' + comment-marker: + description: >- + Hidden first line of the comment body, which the caller searches for to find its own + comment. Empty uses ``. + required: false + default: '' + workflow-file: + description: >- + File name of the workflow whose earlier runs hold the baseline, e.g. `plugin-check.yml`. + Empty reads it from GITHUB_WORKFLOW_REF, which names the calling workflow's file when this + runs inside a reusable workflow. + required: false + default: '' + baseline-wait: + description: >- + Seconds a pull request waits for the run of its base commit to finish, when that run is + still going. + required: false + default: '1200' + badge-url: + description: 'Image URL shown under the status line of the comment. Empty shows none.' + required: false + default: '' + token: + description: >- + Token that lists earlier runs and downloads their artifacts. It needs `actions: read`. + required: false + default: ${{ github.token }} + +outputs: + artifact-url: + description: 'Link to the plugin-check-report artifact this run uploaded. Empty when there was nothing to upload.' + value: ${{ steps.artifact.outputs.artifact-url }} + baseline-url: + description: 'Link to the run the baseline came from. Empty when there was no baseline.' + value: ${{ steps.baseline.outputs.url }} + +runs: + using: 'composite' + steps: + # One folder per run under RUNNER_TEMP, cleared first, so a second use of the action in the same + # job never reads the first one's files. + - name: Set file paths + id: paths + shell: bash + env: + INPUT_WORKFLOW_FILE: ${{ inputs.workflow-file }} + INPUT_COMMENT_MARKER: ${{ inputs.comment-marker }} + run: | + dir="$RUNNER_TEMP/plugin-check-report" + rm -rf "$dir" + mkdir -p "$dir/report" "$dir/baseline" + + # In a called workflow GITHUB_WORKFLOW_REF is the caller's, like + # impress-org/givewp/.github/workflows/plugin-check.yml@refs/pull/12/merge. The runs that + # hold the baseline belong to that file, not to the toolbox's workflow. + workflow=$INPUT_WORKFLOW_FILE + if [ -z "$workflow" ]; then + ref=${GITHUB_WORKFLOW_REF%%@*} + workflow=${ref##*/} + fi + + { + echo "report=$dir/report" + echo "baseline=$dir/baseline/baseline.json" + echo "workflow=$workflow" + echo "marker=${INPUT_COMMENT_MARKER:-}" + } >> "$GITHUB_OUTPUT" + echo "Baselines are read from the runs of $workflow." + + # A baseline made from this run, uploaded even when the compare below fails. Later runs + # download it as their baseline: pull requests into this branch, and the next push. + - name: Write a new baseline + shell: bash + env: + RESULTS: ${{ inputs.results }} + REPORT_DIR: ${{ steps.paths.outputs.report }} + run: | + if [ -f "$RESULTS" ]; then + cp "$RESULTS" "$REPORT_DIR/" + PLUGIN_CHECK_BASELINE="$REPORT_DIR/baseline.json" bash "$GITHUB_ACTION_PATH/ratchet.sh" --update "$RESULTS" + fi + + # The name and the baseline.json inside it are what fetch-baseline.sh looks for. Keep both: + # they are also what GiveWP's runs from before the shared workflow uploaded, so its old + # baselines keep working. + - name: Upload the report + id: artifact + uses: actions/upload-artifact@v7 + with: + name: plugin-check-report + path: ${{ steps.paths.outputs.report }} + if-no-files-found: ignore + + # The baseline: what the branch the code goes into looks like now, from the artifact of an + # earlier run on it. A pull request takes the run for its base commit; for a stacked pull + # request that is the run of the pull request below it. When that run is still going, it is + # waited for. When it has no baseline (it was cancelled by a newer push, say), the latest run on + # the base branch is used instead, and more errors than it are only reported, not failed: that + # run is older than the target. A push takes the run before it on the same branch, so a merge + # that adds errors fails there, where the whole team sees it. + # With no run to take it from, ratchet.sh compares nothing and passes with a warning. A failed + # lookup (GitHub API or download error) fails the step instead, so an outage never passes as + # "no baseline". + - name: Get the baseline + id: baseline + shell: bash + env: + GH_TOKEN: ${{ inputs.token }} + EVENT: ${{ github.event_name }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + WAIT: ${{ inputs.baseline-wait }} + WORKFLOW: ${{ steps.paths.outputs.workflow }} + BASELINE: ${{ steps.paths.outputs.baseline }} + run: | + fetch="$GITHUB_ACTION_PATH/fetch-baseline.sh" + status=0 + if [ "$EVENT" = pull_request ]; then + label="Target branch" + found=$(bash "$fetch" --workflow "$WORKFLOW" --sha "$BASE_SHA" --wait "$WAIT" "$BASE_REF" "$BASELINE") || status=$? + else + label="Previous run" + found=$(bash "$fetch" --workflow "$WORKFLOW" --before-run "$GITHUB_RUN_ID" "$GITHUB_REF_NAME" "$BASELINE") || status=$? + fi + case "$status" in + 0) ;; + 1) echo "No earlier run has a baseline."; exit 0 ;; + *) exit "$status" ;; + esac + read -r _ url match <<< "$found" + advisory=false + if [ "$match" = older ]; then + label="Target branch, older run" + advisory=true + fi + { + echo "label=$label" + echo "url=$url" + echo "advisory=$advisory" + } >> "$GITHUB_OUTPUT" + echo "Baseline from $url ($match)" + + # Fails the step when the mode says so. Also writes the comment body and the same text to the + # job summary. + - name: Compare with the baseline + shell: bash + env: + RESULTS: ${{ inputs.results }} + COMMENT_FILE: ${{ inputs.comment-file }} + PLUGIN_CHECK_MODE: ${{ inputs.mode }} + PLUGIN_CHECK_BASELINE: ${{ steps.paths.outputs.baseline }} + PLUGIN_CHECK_ADVISORY: ${{ steps.baseline.outputs.advisory }} + COMMENT_MARKER: ${{ steps.paths.outputs.marker }} + BASELINE_LABEL: ${{ steps.baseline.outputs.label }} + BASELINE_URL: ${{ steps.baseline.outputs.url }} + COMMENT_BADGE: ${{ inputs.badge-url }} + ARTIFACT_URL: ${{ steps.artifact.outputs.artifact-url }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + args=() + [ -z "$COMMENT_FILE" ] || args=(--comment-file "$COMMENT_FILE") + bash "$GITHUB_ACTION_PATH/ratchet.sh" "${args[@]}" "$RESULTS" diff --git a/.github/actions/plugin-check-report/fetch-baseline.sh b/.github/actions/plugin-check-report/fetch-baseline.sh new file mode 100755 index 0000000..c4b4718 --- /dev/null +++ b/.github/actions/plugin-check-report/fetch-baseline.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# +# Downloads the baseline.json that the Plugin Check workflow wrote for a branch: the +# plugin-check-report artifact of an earlier run. That run checked the branch as it really is, so +# the baseline always matches it. Nothing is committed, so two branches worked on at the same time +# can never leave a stale baseline behind. See this action's README. +# +# Usage: +# fetch-baseline.sh [--workflow ] [--sha ] [--wait ] [--before-run ] [] +# +# --workflow is the file name of the workflow whose runs hold the baseline (default: +# plugin-check.yml). When the check runs as a reusable workflow, the runs belong to the CALLER's +# file, so the action passes that name, read from GITHUB_WORKFLOW_REF. +# +# Runs are tried in this order, and the first one with the artifact wins: +# 1. runs for --sha (a pull request's base commit), when given. With --wait, a run for that +# commit that is still going is waited for, up to that many seconds. A run that does not +# show up at all is waited for only a short time (GRACE seconds), since the commit may never +# have had one; +# 2. the latest runs on , newest first. +# --before-run keeps only runs started before that one (run ids only grow), so a run never +# compares with itself, and a re-run of an older push never compares with a newer one. +# +# defaults to baseline.json next to this script, which ratchet.sh reads by default +# and git ignores. +# +# Prints " ". is "exact" when the run is for --sha (or no --sha +# was given), and "older" when --sha was given but only an older run on the branch had a baseline. +# +# Exit status: +# 0 baseline written +# 1 no run has the artifact (none finished yet, or all expired) +# 2 the lookup itself failed (GitHub API or download error), so "no baseline" is not known +# +# Needs the GitHub CLI, signed in (GH_TOKEN in CI) with read access to Actions. + +set -euo pipefail + +workflow=plugin-check.yml +sha="" +wait_for=0 +before_run="" +# How long to wait for a run for --sha to show up at all. +GRACE=120 +while [ $# -gt 0 ]; do + case "$1" in + --workflow) workflow=${2:?"--workflow needs a file name"}; shift 2 ;; + --sha) sha=${2:?"--sha needs a commit"}; shift 2 ;; + --wait) wait_for=${2:?"--wait needs seconds"}; shift 2 ;; + --before-run) before_run=${2:?"--before-run needs a run id"}; shift 2 ;; + *) break ;; + esac +done + +branch=${1:?"Usage: $0 [--workflow ] [--sha ] [--wait ] [--before-run ] []"} +output=${2:-"$(cd "$(dirname "$0")" && pwd)/baseline.json"} + +lookup_failed() { + echo "Could not look up Plugin Check runs: $1" >&2 + exit 2 +} + +# Any other command that fails (mktemp, mkdir, cp, ...) is a failure too, never "no baseline": +# exit 1 is kept for the one place below that means it. +trap 'echo "fetch-baseline.sh failed at line $LINENO." >&2; exit 2' ERR + +repo=${GITHUB_REPOSITORY:-$(gh repo view --json nameWithOwner --jq .nameWithOwner)} || lookup_failed "no repository" +# Only a bare file name: it goes into an API path, and a path or a ref there would ask for some +# other resource. +case "$workflow" in + */* | *@* | "") lookup_failed "--workflow must be a file name such as plugin-check.yml, not '$workflow'" ;; +esac +runs="repos/$repo/actions/workflows/$workflow/runs" + +# The run for the base commit may still be going: a pull request opened right after a merge starts +# at the same time as the push run on the target branch. Waiting for it beats comparing with an +# older run. +if [ -n "$sha" ] && [ "$wait_for" -gt 0 ]; then + started=$(date +%s) + deadline=$((started + wait_for)) + while :; do + # " " for the commit. + counts=$(gh api "$runs?head_sha=$sha&per_page=20" \ + --jq '"\(.workflow_runs | length) \([.workflow_runs[] | select(.status != "completed")] | length)"') \ + || lookup_failed "runs for $sha" + read -r total going <<< "$counts" + # A run can take a moment to show up after its push. After GRACE seconds with none, the + # commit never had one. + if [ "$total" -eq 0 ] && [ "$(date +%s)" -ge $((started + GRACE)) ]; then + break + fi + [ "$total" -eq 0 ] || [ "$going" -gt 0 ] || break + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "The run for $sha is still going after $wait_for seconds; trying older runs." >&2 + break + fi + sleep 30 + done +fi + +# " " lines, best first. Captured, not streamed, so a failed API call +# stops here instead of looking like an empty list. +candidates="" +if [ -n "$sha" ]; then + candidates=$(gh api "$runs?status=completed&head_sha=$sha&per_page=10" \ + --jq '.workflow_runs[] | "\(.id) \(.html_url) \(.head_sha)"') || lookup_failed "runs for $sha" +fi +# Newest first by run id (ids only grow), not by the order the API happens to return. The 50 +# newest runs are plenty: if none of them has a baseline, an older one would be stale anyway. +on_branch=$(gh api "$runs?status=completed&branch=$branch&per_page=50" \ + --jq '.workflow_runs[] | "\(.id) \(.html_url) \(.head_sha)"' | sort -k1,1nr) || lookup_failed "runs on $branch" +candidates+=${candidates:+$'\n'}$on_branch + +tried=" " +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +while read -r id url head; do + [ -n "$id" ] || continue + if [ -n "$before_run" ] && [ "$id" -ge "$before_run" ]; then + continue + fi + case "$tried" in *" $id "*) continue ;; esac + tried+="$id " + + # A run that broke before the report job, or whose artifact expired, has nothing to give. + found=$(gh api "repos/$repo/actions/runs/$id/artifacts?name=plugin-check-report" \ + --jq '[.artifacts[] | select(.expired | not)] | length') || lookup_failed "artifacts of run $id" + [ "$found" -gt 0 ] || continue + + rm -rf "$tmp/report" + gh run download "$id" --repo "$repo" --name plugin-check-report --dir "$tmp/report" > /dev/null \ + || lookup_failed "download of the artifact of run $id" + [ -s "$tmp/report/baseline.json" ] || continue + + match=exact + if [ -n "$sha" ] && [ "$head" != "$sha" ]; then + match=older + fi + + mkdir -p "$(dirname "$output")" + cp "$tmp/report/baseline.json" "$output" + echo "$id $url $match" + exit 0 +done <<< "$candidates" + +echo "No Plugin Check run on $branch${sha:+ or $sha} has a baseline." >&2 +exit 1 diff --git a/.github/actions/plugin-check-report/ratchet.sh b/.github/actions/plugin-check-report/ratchet.sh new file mode 100755 index 0000000..be260c2 --- /dev/null +++ b/.github/actions/plugin-check-report/ratchet.sh @@ -0,0 +1,286 @@ +#!/usr/bin/env bash +# +# Plugin Check ratchet. Counts ERROR and WARNING results per rule code and compares them with a +# baseline: the counts of an earlier run, fetched with fetch-baseline.sh. Only a rule with more +# ERRORS than the baseline fails. More warnings only add a notice. Fewer of either passes. With no +# baseline at all, nothing is compared and the check passes with a warning. In strict mode any +# error fails, baseline or not. See this action's README. +# +# Usage: +# ratchet.sh [--comment-file ] compare the results with the baseline +# ratchet.sh --update write the baseline from the results +# +# is the output of `wp plugin check --format=json`: for each file, a +# "FILE: " line and then a line holding a JSON array of that file's results. +# +# --comment-file writes a short Markdown summary for the pull request comment. The same summary +# goes to the job summary when GITHUB_STEP_SUMMARY is set. +# +# Environment (all optional): +# PLUGIN_CHECK_MODE "ratchet" (default): fail only when a rule has more errors than the +# baseline. "strict": fail when there is any error at all. +# COMMENT_MARKER hidden first line of the comment (default: "") +# PLUGIN_CHECK_BASELINE baseline path (default: baseline.json next to this script, where +# fetch-baseline.sh puts it) +# BASELINE_LABEL what the baseline is, for the comment's column (default: "Baseline") +# BASELINE_URL link to the run the baseline came from +# PLUGIN_CHECK_ADVISORY "true" when the baseline is from an older commit than the one the pull +# request targets. Rules with more errors are then reported as warnings +# and do not fail the check: the target may have changed since. +# COMMENT_BADGE image URL shown under the status line +# RUN_URL link to the workflow run +# ARTIFACT_URL link to the uploaded results and fresh baseline + +set -euo pipefail + +# The pull request comment must stay short. Everything else is behind the links. +MAX_COMMENT_LINES=15 +MAX_RULES_LISTED=5 + +# Hidden line that marks the sticky comment. The workflow searches for it, so no other comment +# is ever replaced. The workflow passes the same value it searches for. +COMMENT_MARKER=${COMMENT_MARKER:-''} + +# Checked before anything else, so a typo in the caller never runs as the default. +mode=${PLUGIN_CHECK_MODE:-ratchet} +case "$mode" in + ratchet | strict) ;; + *) + echo "::error title=Plugin Check::Unknown mode '$mode'. Use 'ratchet' or 'strict'." + exit 1 + ;; +esac + +update=false +comment_file="" +while [ $# -gt 0 ]; do + case "$1" in + --update) update=true; shift ;; + --comment-file) comment_file=${2:?"--comment-file needs a path"}; shift 2 ;; + *) break ;; + esac +done + +results=${1:?"Usage: $0 [--update | --comment-file ] "} +baseline=${PLUGIN_CHECK_BASELINE:-"$(cd "$(dirname "$0")" && pwd)/baseline.json"} + +if [ ! -f "$results" ]; then + echo "::error title=Plugin Check::Results file not found: $results" + exit 1 +fi + +# A results file that holds anything else (an HTML database error, a PHP fatal) means the check +# itself broke. Without this guard that file would count as "no results" and pass. +unexpected=$({ grep -v -E '^(FILE: |\[|Success:)' "$results" || true; } | grep -v -E '^[[:space:]]*$' | head -n 1 | cut -c 1-200 || true) +if [ -n "$unexpected" ]; then + echo "::error title=Plugin Check::The results file is not Plugin Check output, so the check did not run. First unexpected line: $unexpected" + exit 1 +fi + +# A clean run prints "Success: ..." and a run with results prints "FILE: ..." lines. An empty +# file means the command failed with its error on stderr, which the results file does not hold. +if ! grep -q -E '^(FILE: |Success:)' "$results"; then + echo "::error title=Plugin Check::The results file has no Plugin Check output, so the check did not run." + exit 1 +fi + +# Only the JSON lines matter. grep exits 1 when there are none (a clean run), which is fine. +counts=$({ grep '^\[' "$results" || true; } | jq -s -S ' + def per_code($type): + map(select(.type == $type)) + | group_by(.code) + | map({ key: .[0].code, value: length }) + | from_entries; + (add // []) as $all + | { errors: ($all | per_code("ERROR")), warnings: ($all | per_code("WARNING")) } +') + +if [ "$update" = true ]; then + printf '%s\n' "$counts" > "$baseline" + echo "Wrote $baseline: $(jq '.errors | length' <<< "$counts") error codes, $(jq '.warnings | length' <<< "$counts") warning codes" + exit 0 +fi + +# No earlier run to compare with: report the counts and pass. Failing here would block every pull +# request whenever the target branch has no finished run (its first run, or expired artifacts). +if [ ! -f "$baseline" ]; then + echo "::warning title=Plugin Check::No baseline to compare with ($baseline), so nothing was compared. Run the workflow on the target branch to make one." + baseline=$(mktemp) + jq -n '{errors: {}, warnings: {}}' > "$baseline" + no_baseline=true +else + no_baseline=false +fi + +# One tab-separated row per kind and rule code found in either file: +# kind, code, baseline count, current count. Rules that went up come first, errors before +# warnings, biggest increase first, so the comment can show the top few. +rows=$(jq -r -n --argjson now "$counts" --slurpfile base "$baseline" ' + $base[0] as $base + | [ + ("errors", "warnings") as $kind + | ((($now[$kind] // {}) + ($base[$kind] // {})) | keys[]) as $code + | { kind: $kind, code: $code, base: ($base[$kind][$code] // 0), now: ($now[$kind][$code] // 0) } + ] + | sort_by((if .now > .base then 0 else 1 end), (if .kind == "errors" then 0 else 1 end), -(.now - .base)) + | .[] + | [.kind, .code, .base, .now] + | @tsv +') + +failed=false +errors_base=0 +errors_now=0 +warnings_base=0 +warnings_now=0 +went_down=0 +went_up=0 +errors_up=0 +up_list="" + +while IFS=$'\t' read -r kind code allowed found; do + [ -n "$code" ] || continue + + if [ "$kind" = errors ]; then + errors_base=$((errors_base + allowed)) + errors_now=$((errors_now + found)) + label="" + else + warnings_base=$((warnings_base + allowed)) + warnings_now=$((warnings_now + found)) + label=" warning" + fi + + # With no baseline, only the totals mean anything. + [ "$no_baseline" = false ] || continue + + if [ "$found" -gt "$allowed" ]; then + went_up=$((went_up + 1)) + if [ "$went_up" -le "$MAX_RULES_LISTED" ]; then + up_list+="${up_list:+, }\`$code\`$label (+$((found - allowed)))" + fi + if [ "$kind" = errors ]; then + errors_up=$((errors_up + 1)) + if [ "${PLUGIN_CHECK_ADVISORY:-}" = true ]; then + echo "::warning title=Plugin Check::$code has $found errors, an older run of the target branch has $allowed. Not enforced, because the target may have changed since." + else + failed=true + echo "::error title=Plugin Check::$code has $found errors, the baseline has $allowed. The full list is in the plugin-check-results artifact." + fi + else + echo "::notice title=Plugin Check::$code has $found warnings, the baseline has $allowed. Warnings do not fail the check." + fi + elif [ "$found" -lt "$allowed" ]; then + went_down=$((went_down + 1)) + echo "::notice title=Plugin Check::$code ($kind) went down from $allowed to $found." + fi +done <<< "$rows" + +# Strict mode ignores the baseline for pass or fail: any error fails. The counts and the table +# still compare with the baseline, so the comment shows what changed. +if [ "$mode" = strict ]; then + if [ "$errors_now" -gt 0 ]; then + failed=true + echo "::error title=Plugin Check::$errors_now errors found. Strict mode fails on any error." + else + failed=false + fi +fi + +# "+3", "-2" or "±0". +change() { + local diff=$(($1 - $2)) + if [ "$diff" -gt 0 ]; then echo "+$diff"; elif [ "$diff" -lt 0 ]; then echo "$diff"; else echo "±0"; fi +} + +# The totals can go down while a rule goes up, so the status names the rules, not the totals. +if [ "$mode" = strict ] && [ "$failed" = true ]; then + status="❌ Plugin Check failed: $errors_now errors, and strict mode allows none" +elif [ "$mode" = strict ]; then + status="✅ Plugin Check passed: no errors" +elif [ "$no_baseline" = true ]; then + status="⚠️ Plugin Check ran, but there was no baseline to compare with" +elif [ "$errors_up" -gt 0 ] && [ "${PLUGIN_CHECK_ADVISORY:-}" = true ]; then + status="⚠️ Plugin Check passed, not enforced: $errors_up rule(s) have more errors than an older run of the target branch" +elif [ "$failed" = true ]; then + if [ "$errors_up" -eq 1 ]; then + status="❌ Plugin Check failed: 1 rule has more errors than the baseline allows" + else + status="❌ Plugin Check failed: $errors_up rules have more errors than the baseline allows" + fi +else + status="✅ Plugin Check passed" +fi + +run_url=${RUN_URL:-} +if [ -z "$run_url" ] && [ -n "${GITHUB_RUN_ID:-}" ]; then + run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" +fi +if [ -n "${ARTIFACT_URL:-}" ]; then + links="📎 [Full results and new baseline](${ARTIFACT_URL})" + [ -z "$run_url" ] || links+=" · [Run]($run_url)" +elif [ -n "$run_url" ]; then + links="📎 [Run]($run_url)" +else + links="📎 Local run, see $results" +fi + +body="$COMMENT_MARKER"$'\n'"$status"$'\n\n' +if [ -n "${COMMENT_BADGE:-}" ]; then + body+="![Plugin Check finished](${COMMENT_BADGE})"$'\n\n' +fi +# The column is named after where the baseline came from ("Target branch", "Previous run"), and +# links to that run when known. +column=${BASELINE_LABEL:-Baseline} +if [ -n "${BASELINE_URL:-}" ]; then + column="[$column](${BASELINE_URL})" +fi +if [ "$no_baseline" = true ]; then + body+="| | Now |"$'\n' + body+="|:--|--:|"$'\n' + body+="| Errors | $errors_now |"$'\n' + body+="| Warnings | $warnings_now |"$'\n\n' +else + body+="| | Now | $column | Change |"$'\n' + body+="|:--|--:|--:|:--|"$'\n' + body+="| Errors | $errors_now | $errors_base | $(change "$errors_now" "$errors_base") |"$'\n' + body+="| Warnings | $warnings_now | $warnings_base | $(change "$warnings_now" "$warnings_base") |"$'\n\n' +fi +if [ "$went_up" -gt "$MAX_RULES_LISTED" ]; then + body+="Went up (top $MAX_RULES_LISTED of $went_up): $up_list"$'\n' +elif [ "$went_up" -gt 0 ]; then + body+="Went up: $up_list"$'\n' +fi +if [ "$went_down" -gt 0 ]; then + body+="Rules that went down: $went_down."$'\n' +fi +# A blank line separates the notes above from the links, when there are notes. +if [ "$went_up" -gt 0 ] || [ "$went_down" -gt 0 ]; then + body+=$'\n' +fi +body+="$links" + +# Hard cap: whatever happens above, the comment never grows past MAX_COMMENT_LINES. +body=$(head -n "$MAX_COMMENT_LINES" <<< "$body") + +if [ -n "$comment_file" ]; then + printf '%s\n' "$body" > "$comment_file" +fi +if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + printf '%s\n' "$body" >> "$GITHUB_STEP_SUMMARY" +fi + +if [ "$failed" = true ]; then + if [ "$mode" = strict ]; then + echo "Plugin Check found errors, and strict mode allows none." + else + echo "Plugin Check found more errors than the baseline allows." + fi + exit 1 +fi + +if [ "$no_baseline" = true ]; then + echo "Plugin Check ran with no baseline, so nothing was compared." +else + echo "Plugin Check error counts are within the baseline." +fi diff --git a/.github/workflows/plugin-check.yml b/.github/workflows/plugin-check.yml new file mode 100644 index 0000000..32f32fb --- /dev/null +++ b/.github/workflows/plugin-check.yml @@ -0,0 +1,441 @@ +name: Plugin Check + +# Runs WordPress.org's Plugin Check against the built plugin: the same files the release zip ships, +# so tests, dotfiles, node_modules and other dev-only files cannot cause false positives. +# +# A reusable workflow. A plugin repository calls it from a thin workflow of its own; copy +# templates/workflows/plugin-check.yml. Ported from GiveWP's plugin-check.yml. +# +# The plugin is built with pup by default (`composer -- pup build`, then `pup package`), so the +# repository's .puprc and .distignore decide what is checked, the same as the release. A repository +# without pup passes `build-command` instead; see the plugin-check-report README. +# +# Three jobs, so the third-party action never holds a token that can write: +# start posts "in progress" on the pull request. +# check builds the plugin and runs wordpress/plugin-check-action. Read-only token. +# report compares the results with the baseline and replaces the comment with the result. +# +# The action also tries to post its own comment. With a read-only token that call fails (it logs +# "Failed to post PR comment" and carries on), so the only comment is ours. +# +# The caller has to grant every permission a job below asks for: a called workflow can lower the +# token's permissions, never raise them, and GitHub refuses to start the run when a job asks for +# more than the caller gave. `concurrency` is the caller's too: here it would be evaluated with the +# caller's context and could clash with the caller's own group. +# +# Run it locally: see "Running it locally" in .github/actions/plugin-check-report/README.md. In +# short, `wp plugin check --format=json > results.txt` in a local WordPress, then +# `bash .github/actions/plugin-check-report/ratchet.sh results.txt`. + +on: + workflow_call: + inputs: + plugin-slug: + description: >- + The plugin's folder name, e.g. `give`. It has to equal the top folder of the zip pup + builds (`zip_name` in .puprc), because Plugin Check takes the slug from the folder name. + required: true + type: string + main-file: + description: 'The main plugin file inside the plugin folder. Empty means `.php`.' + required: false + type: string + default: '' + build-command: + description: >- + Empty (the default) builds with pup. Otherwise a bash command that runs in the root of + the checkout instead of pup, and leaves the plugin as it ships in + "$PLUGIN_CHECK_BUILD_DIR//". Prefer .puprc when the repository has one: this + is a second place that defines the build. + required: false + type: string + default: '' + runs-on: + description: 'Runner label for the job that builds and checks. The two small jobs run on ubuntu-latest.' + required: false + type: string + default: 'ubuntu-latest' + php-version: + description: >- + PHP that builds vendor/. Use the oldest version the plugin supports, as its release does, + so Composer resolves dependencies that still run there. + required: false + type: string + default: '7.4' + exclude-checks: + description: 'Checks to skip, one per line. Passed to wordpress/plugin-check-action as it is.' + required: false + type: string + default: '' + cache-paths: + description: >- + Build caches to keep between runs, one path per line, relative to the repository root, + e.g. `.cache/webpack`. Empty keeps none. A path inside node_modules does not survive: + pup's build reinstalls it. + required: false + type: string + default: '' + mode: + description: >- + `ratchet`: fail only when a rule has more errors than the baseline. `strict`: fail on any + error. Not called `strict` because wordpress/plugin-check-action has a `strict` input + that means something else. + required: false + type: string + default: 'ratchet' + submodules: + description: 'Passed to actions/checkout: `false`, `true` or `recursive`.' + required: false + type: string + default: 'false' + secrets: + composer-token: + description: >- + GitHub token for private Composer packages. Becomes COMPOSER_AUTH for the build step + only. + required: false + package-token: + description: >- + Token for private npm or bun packages. Becomes NODE_AUTH_TOKEN, BUN_AUTH_TOKEN and + GITHUB_TOKEN for the build step only. + required: false + +env: + # Hidden line that marks the sticky pull request comment. The report action writes it as the + # first line of the body, and both comment steps search for it. Named after the repository, so + # two plugin repositories never share a marker. + COMMENT_MARKER: '' + # Only pull requests from this repository get a comment: a fork's token is read-only. + COMMENT: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} + +jobs: + start: + name: Plugin Check start + if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + issues: write + pull-requests: write + steps: + # One comment per pull request, found by a hidden marker line and replaced on every run. The + # badge time is Eastern Time: %Z prints EST in winter and EDT in summer. + - name: Make the start badge + id: badge + run: | + stamp=$(LC_ALL=C TZ='America/New_York' date '+%m/%d %I:%M %p %Z') + stamp=${stamp// /%20} + stamp=${stamp//\//%2F} + echo "url=https://img.shields.io/badge/Check%20Started-${stamp}-blue" >> "$GITHUB_OUTPUT" + + - name: Find the Plugin Check comment + id: find_comment + continue-on-error: true + uses: peter-evans/find-comment@v4 + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: ${{ env.COMMENT_MARKER }} + + - name: Post "in progress" + continue-on-error: true + uses: peter-evans/create-or-update-comment@v5 + with: + comment-id: ${{ steps.find_comment.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + edit-mode: replace + body: | + ${{ env.COMMENT_MARKER }} + ⏳ Plugin Check in progress + + ![Plugin Check started](${{ steps.badge.outputs.url }}) + + check: + name: Plugin Check run + runs-on: ${{ inputs.runs-on }} + + # Build, WordPress start and the check together take about 15 minutes for GiveWP. The cap is + # for a container that stops answering. + timeout-minutes: 30 + + # Read-only on purpose: this job runs a third-party action. Its own attempt to post a comment + # fails with this token, which is what keeps the pull request to one comment. + permissions: + contents: read + + env: + PLUGIN_CHECK_BUILD_DIR: ${{ github.workspace }}/build + SLUG: ${{ inputs.plugin-slug }} + MAIN_FILE: ${{ inputs.main-file || format('{0}.php', inputs.plugin-slug) }} + + steps: + # The repo goes into source/, so the workspace root has no .wp-env.json. A repo's own file + # mounts the repo root as a plugin, and the action would merge into it. + - uses: actions/checkout@v7 + with: + path: source + submodules: ${{ inputs.submodules }} + # Nothing here pushes over git, and the job goes on to run third-party code. + persist-credentials: false + + # The action's wp-env pulls these images when it starts, and pulling them is part of the + # time that step takes. Started now, in the background, the download overlaps with the + # build. The images are the ones wp-env's generated files name (FROM wordpress..., FROM + # wordpress:cli..., image: mariadb:lts); if a later wp-env names others, these pulls are just + # wasted, never wrong. A failed pull only means wp-env pulls the image itself, so this step + # cannot fail the job. + - name: Start pulling the wp-env Docker images + continue-on-error: true + env: + IMAGES: wordpress wordpress:cli mariadb:lts + run: | + ( + set +e + start=$(date +%s) + for image in $IMAGES; do + docker pull --quiet "$image" || echo "Could not pull $image, wp-env will pull it." & + done + wait + echo "Pulled in $(( $(date +%s) - start )) seconds." + touch "$RUNNER_TEMP/docker-pull.done" + ) > "$RUNNER_TEMP/docker-pull.log" 2>&1 < /dev/null & + + - name: Set Up PHP + uses: shivammathur/setup-php@v2 + with: + php-version: ${{ inputs.php-version }} + coverage: none + + - name: Find the Composer cache + id: composer-cache + working-directory: source + run: echo "dir=$(composer config cache-files-dir)" >> "$GITHUB_OUTPUT" + + - name: Cache dependencies + uses: actions/cache@v6 + with: + path: ${{ steps.composer-cache.outputs.dir }} + key: composer-${{ hashFiles('source/composer.lock') }} + + # The JavaScript toolchain is picked from the files the repository already has, not from an + # input: .nvmrc for Node, package-lock.json for the npm cache, bun.lock or bun.lockb for bun. + - name: Set Up Node + if: ${{ hashFiles('source/.nvmrc') != '' }} + uses: actions/setup-node@v7 + with: + node-version-file: source/.nvmrc + cache: ${{ hashFiles('source/package-lock.json') != '' && 'npm' || '' }} + cache-dependency-path: source/package-lock.json + + - name: Set Up Bun + if: ${{ hashFiles('source/bun.lock', 'source/bun.lockb') != '' }} + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + # actions/cache takes paths relative to the workspace, and the repository sits in source/. + # A path that leaves the checkout is refused: it would cache files this job did not build. + - name: List the cache paths + id: cache-paths + if: ${{ inputs.cache-paths != '' }} + env: + CACHE_PATHS: ${{ inputs.cache-paths }} + run: | + { + echo 'paths<> "$GITHUB_OUTPUT" + + # Each commit saves a new entry; the restore key picks the newest one for the same lockfile. + # A stale cache must be safe for the tool that owns it (webpack checks every entry against + # the files). + - name: Restore the build cache + id: build-cache + if: ${{ inputs.cache-paths != '' }} + uses: actions/cache/restore@v6 + with: + path: ${{ steps.cache-paths.outputs.paths }} + key: plugin-check-${{ runner.os }}-${{ hashFiles('source/package-lock.json', 'source/bun.lock', 'source/bun.lockb') }}-${{ github.sha }} + restore-keys: | + plugin-check-${{ runner.os }}-${{ hashFiles('source/package-lock.json', 'source/bun.lock', 'source/bun.lockb') }}- + + # The same pup steps the release zip uses, so .puprc and .distignore decide what is checked. + # The secrets reach this step only, never the job, so the third-party action later in this + # job does not see them in its environment. + - name: Build the plugin + working-directory: source + env: + BUILD_COMMAND: ${{ inputs.build-command }} + COMPOSER_TOKEN: ${{ secrets.composer-token }} + PACKAGE_TOKEN: ${{ secrets.package-token }} + run: | + if [ -n "$COMPOSER_TOKEN" ]; then + COMPOSER_AUTH=$(jq -cn --arg token "$COMPOSER_TOKEN" '{"github-oauth": {"github.com": $token}}') + export COMPOSER_AUTH + fi + if [ -n "$PACKAGE_TOKEN" ]; then + export NODE_AUTH_TOKEN="$PACKAGE_TOKEN" BUN_AUTH_TOKEN="$PACKAGE_TOKEN" GITHUB_TOKEN="$PACKAGE_TOKEN" + fi + unset COMPOSER_TOKEN PACKAGE_TOKEN + mkdir -p "$PLUGIN_CHECK_BUILD_DIR" + + if [ -n "$BUILD_COMMAND" ]; then + echo "Building with build-command." + bash -eo pipefail -c "$BUILD_COMMAND" + exit 0 + fi + + composer run -q pup -- --version > /dev/null + composer -- pup build + version=$(php -d display_errors=0 bin/pup.phar get-version --dev) + composer -- pup package "$version" + name=$(php -d display_errors=0 bin/pup.phar zip-name "$version") + unzip -q "$name.zip" -d "$PLUGIN_CHECK_BUILD_DIR" + + # The one contract every build has to meet, pup or not: a folder named after the slug, with + # the main plugin file in it. Plugin Check takes the slug from the folder name, so a wrong + # folder would check nothing useful and still look green. + - name: Check the built plugin + env: + CACHE_PATHS: ${{ inputs.cache-paths }} + run: | + dir="$PLUGIN_CHECK_BUILD_DIR/$SLUG" + if [ ! -f "$dir/$MAIN_FILE" ]; then + echo "::error title=Plugin Check::The build did not leave $SLUG/$MAIN_FILE in $PLUGIN_CHECK_BUILD_DIR. Found: $(find "$PLUGIN_CHECK_BUILD_DIR" -mindepth 1 -maxdepth 1 -printf '%f ')" + exit 1 + fi + # A cache that ends up in the plugin would be checked, and shipped by the release. + while IFS= read -r line; do + line=$(echo "$line" | xargs) + [ -n "$line" ] || continue + if [ -e "$dir/$line" ]; then + echo "::error title=Plugin Check::The cache path $line is inside the built plugin. Add it to .distignore." + exit 1 + fi + done <<< "$CACHE_PATHS" + echo "Checking $dir ($(find "$dir" -type f | wc -l | xargs) files)." + + # Saved now, before the third-party action below runs, so nothing it does can end up in a + # cache that later runs restore. Skipped when this exact commit already saved one. + - name: Save the build cache + if: ${{ inputs.cache-paths != '' && steps.build-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v6 + with: + path: ${{ steps.cache-paths.outputs.paths }} + key: ${{ steps.build-cache.outputs.cache-primary-key }} + + # Waits for the pulls started earlier, so wp-env does not start a second download of an + # image that is still coming, and prints how long they took. Gives up after two minutes and + # lets wp-env carry on; never fails the job. + - name: Wait for the Docker image pulls + continue-on-error: true + timeout-minutes: 3 + run: | + for _ in $(seq 120); do + [ -f "$RUNNER_TEMP/docker-pull.done" ] && break + sleep 1 + done + cat "$RUNNER_TEMP/docker-pull.log" 2>/dev/null || true + + # Fails on any error, which is expected while a plugin still has known problems: the report + # job decides pass or fail from the counts. The action writes its results to + # $RUNNER_TEMP/plugin-check-results.txt and uploads them as the plugin-check-results + # artifact. Its `slug` input is not passed: it would rename that artifact. + - name: Run Plugin Check + continue-on-error: true + uses: wordpress/plugin-check-action@v1 + with: + build-dir: ${{ github.workspace }}/build/${{ inputs.plugin-slug }} + exclude-checks: ${{ inputs.exclude-checks }} + + report: + name: Plugin Check + needs: [start, check] + # "start" is skipped for forks and non-pull-request runs, which would skip this job too. + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + # Up to 20 minutes of it can be waiting for the target branch's run. + timeout-minutes: 30 + + # Writes one sticky comment on the pull request, and reads an earlier run's artifact as the + # baseline. No checkout: the scripts come with the action. + permissions: + actions: read + issues: write + pull-requests: write + + steps: + - name: Set file paths + run: | + { + echo "RESULTS=$RUNNER_TEMP/results/plugin-check-results.txt" + echo "COMMENT_BODY=$RUNNER_TEMP/plugin-check-comment.md" + } >> "$GITHUB_ENV" + + # The action uploaded its results under this name. Missing when the build or the check broke + # before then: the compare reports that. + - name: Download the results + continue-on-error: true + uses: actions/download-artifact@v8 + with: + name: plugin-check-results + path: ${{ runner.temp }}/results + + - name: Make the finish badge + id: finish_badge + run: | + stamp=$(LC_ALL=C TZ='America/New_York' date '+%m/%d %I:%M %p %Z') + stamp=${stamp// /%20} + stamp=${stamp//\//%2F} + echo "url=https://img.shields.io/badge/Check%20Finished-${stamp}-darkgreen" >> "$GITHUB_OUTPUT" + + # Full path, never ./: in a called workflow a relative path resolves against the caller's + # repository. + - name: Compare with the baseline + uses: stellarwp/plugin-toolbox/.github/actions/plugin-check-report@v1 + with: + results: ${{ env.RESULTS }} + comment-file: ${{ env.COMMENT_BODY }} + comment-marker: ${{ env.COMMENT_MARKER }} + mode: ${{ inputs.mode }} + badge-url: ${{ steps.finish_badge.outputs.url }} + + # When the build or the check broke, the action never wrote a body. Say so instead of leaving + # "in progress" behind. + - name: Prepare the comment + if: ${{ !cancelled() && env.COMMENT == 'true' }} + env: + BADGE: ${{ steps.finish_badge.outputs.url }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + if [ ! -s "$COMMENT_BODY" ]; then + printf '%s\n❌ Plugin Check could not run\n\n![Plugin Check finished](%s)\n\n📎 [Run](%s)\n' "$COMMENT_MARKER" "$BADGE" "$RUN_URL" > "$COMMENT_BODY" + fi + + - name: Find the Plugin Check comment + id: find_comment + if: ${{ !cancelled() && env.COMMENT == 'true' }} + continue-on-error: true + uses: peter-evans/find-comment@v4 + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: ${{ env.COMMENT_MARKER }} + + - name: Post the result + if: ${{ !cancelled() && env.COMMENT == 'true' }} + continue-on-error: true + uses: peter-evans/create-or-update-comment@v5 + with: + comment-id: ${{ steps.find_comment.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + edit-mode: replace + body-path: ${{ env.COMMENT_BODY }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index be73f60..ed1acb0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,7 +3,8 @@ name: Tests # This repo's own checks. A bug here reaches every repo that pins a tag from it, so the logic in the # actions is unit tested rather than only exercised by a real release. # -# Run it locally: `npm install` once, then `node --test`. For the lint job, run `actionlint`. +# Run it locally: `npm install` once, then `node --test`. For the lint job, `actionlint` and +# `shellcheck .github/actions/*/*.sh`. on: push: @@ -59,3 +60,8 @@ jobs: bash <(curl -sSfL https://raw.githubusercontent.com/rhysd/actionlint/v1.7.12/scripts/download-actionlint.bash) 1.7.12 ./actionlint -ignore 'unexpected key "queue" for "concurrency" section' \ .github/workflows/*.yml templates/workflows/*.yml + + # actionlint does not read composite actions or standalone scripts. shellcheck comes with the + # runner image. + - name: Run shellcheck on the action scripts + run: shellcheck .github/actions/*/*.sh diff --git a/.gitignore b/.gitignore index 04f3b70..b58b9f1 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ npm-shrinkwrap.json bun.lock bun.lockb yarn.lock + +# fetch-baseline.sh writes the baseline here when run by hand (see the plugin-check-report README). +.github/actions/plugin-check-report/baseline.json diff --git a/templates/workflows/plugin-check.yml b/templates/workflows/plugin-check.yml new file mode 100644 index 0000000..01e6133 --- /dev/null +++ b/templates/workflows/plugin-check.yml @@ -0,0 +1,40 @@ +name: Plugin Check + +# Runs WordPress.org's Plugin Check on the plugin as it ships, and compares the error counts with +# an earlier run: a pull request with the branch it targets, a push with the run before it. +# +# A file to copy into .github/workflows/ of a plugin repository. Keep the file name +# plugin-check.yml: earlier runs of this file hold the baselines, and a new name starts with none. +# The work is in stellarwp/plugin-toolbox/.github/workflows/plugin-check.yml. The repository needs +# a .puprc whose zip_name equals plugin-slug, and a `pup` Composer script; see the +# plugin-check-report action's README. +# +# Run it locally: `wp plugin check --format=json > results.txt` in a local WordPress, then +# the ratchet script on that file. See "Running it locally" in the action's README. + +on: + pull_request: + # The branches pull requests target. A push there makes the baseline those pull requests read. + push: + branches: [main] + workflow_dispatch: + +# A new push to a pull request cancels the run still going for the older commit. +concurrency: + group: plugin-check-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + plugin-check: + # The most any job of the shared workflow asks for. A called workflow cannot go above this. + permissions: + actions: read # read earlier runs and download their baseline + contents: read # check out the repository + issues: write # the sticky pull request comment + pull-requests: write # the sticky pull request comment + uses: stellarwp/plugin-toolbox/.github/workflows/plugin-check.yml@v1 + with: + plugin-slug: my-plugin diff --git a/tests/actions/plugin-check-report/fetch-baseline.test.js b/tests/actions/plugin-check-report/fetch-baseline.test.js new file mode 100644 index 0000000..8b5e228 --- /dev/null +++ b/tests/actions/plugin-check-report/fetch-baseline.test.js @@ -0,0 +1,265 @@ +'use strict' + +const { describe, it } = require('node:test') +const assert = require('node:assert') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') +const { spawnSync } = require('node:child_process') + +const FETCH = path.join(__dirname, '../../../.github/actions/plugin-check-report/fetch-baseline.sh') + +const REPO = 'impress-org/givewp' +const SHA = 'a'.repeat(40) +const OLD_SHA = 'b'.repeat(40) + +/** + * A stand-in for the GitHub CLI, put first on PATH. + * + * `gh api --jq ` answers from a JSON file named after the path, and applies the + * filter with the real jq, so the script's own filters run as they would against GitHub. A path + * with no file answers the way gh does for a 404: a message on stderr and exit 1. That is how a + * test makes the lookup fail. + * + * `gh run download --dir ` copies the folder artifact-, standing in for the + * plugin-check-report artifact of that run. + * + * Every call is appended to a log, so a test can assert which requests were sent. + */ +const FAKE_GH = `#!/usr/bin/env bash +printf '%s\\n' "$*" >> "$FAKE_GH_LOG" +if [ "$1" = api ]; then + key=$(printf '%s' "$2" | tr -c 'A-Za-z0-9' '_') + file="$FAKE_GH_DIR/$key.json" + if [ ! -f "$file" ]; then + echo "gh: Not Found (HTTP 404)" >&2 + exit 1 + fi + if [ "$3" = --jq ]; then + jq -r "$4" "$file" + else + cat "$file" + fi +elif [ "$1" = run ] && [ "$2" = download ]; then + id=$3 + while [ $# -gt 0 ]; do + [ "$1" = --dir ] && dir=$2 + shift + done + [ -d "$FAKE_GH_DIR/artifact-$id" ] || { echo "no artifact for run $id" >&2; exit 1; } + mkdir -p "$dir" + cp -R "$FAKE_GH_DIR/artifact-$id/." "$dir/" +else + echo "fake gh: no answer for: $*" >&2 + exit 1 +fi +` + +/** + * The file name the stand-in reads for an API path. + * + * @param {string} apiPath The path as the script passes it to `gh api`. + * + * @returns {string} The file name, every character other than a letter or digit made `_`. + */ +function keyFor(apiPath) { + return `${apiPath.replace(/[^A-Za-z0-9]/g, '_')}.json` +} + +/** + * The API paths fetch-baseline.sh asks for. + * + * @param {string} workflow The workflow file name. + */ +const API = (workflow = 'plugin-check.yml') => { + const runs = `repos/${REPO}/actions/workflows/${workflow}/runs` + + return { + forSha: (sha) => `${runs}?status=completed&head_sha=${sha}&per_page=10`, + goingForSha: (sha) => `${runs}?head_sha=${sha}&per_page=20`, + onBranch: (branch) => `${runs}?status=completed&branch=${branch}&per_page=50`, + artifacts: (id) => `repos/${REPO}/actions/runs/${id}/artifacts?name=plugin-check-report`, + } +} + +/** + * A run as the workflow runs API lists it. + * + * @param {number} id Run id. + * @param {string} sha Head commit. + * + * @returns {object} The run. + */ +function run(id, sha = OLD_SHA) { + return { id, html_url: `https://github.com/${REPO}/actions/runs/${id}`, head_sha: sha, status: 'completed' } +} + +/** + * Sets up the stand-in's answers and runs fetch-baseline.sh against them. + * + * @param {string[]} args Arguments for the script. + * @param {object} answers Each API path mapped to the JSON body it answers with. + * @param {object} baselines Each run id mapped to the baseline.json its artifact holds. + * + * @returns {{status: number, stdout: string, stderr: string, calls: string[], output: string}} + * The exit status, both streams, the gh calls made and the baseline file it wrote. + */ +function fetchBaseline(args, answers = {}, baselines = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-fetch-')) + const bin = path.join(dir, 'bin') + const data = path.join(dir, 'data') + const log = path.join(dir, 'gh.log') + const output = path.join(dir, 'out', 'baseline.json') + + fs.mkdirSync(bin) + fs.mkdirSync(data) + fs.writeFileSync(path.join(bin, 'gh'), FAKE_GH, { mode: 0o755 }) + fs.writeFileSync(log, '') + + for (const [apiPath, body] of Object.entries(answers)) { + fs.writeFileSync(path.join(data, keyFor(apiPath)), JSON.stringify(body)) + } + for (const [id, baseline] of Object.entries(baselines)) { + fs.mkdirSync(path.join(data, `artifact-${id}`)) + fs.writeFileSync(path.join(data, `artifact-${id}`, 'baseline.json'), JSON.stringify(baseline)) + } + + const result = spawnSync('bash', [FETCH, ...args, output], { + encoding: 'utf8', + env: { + PATH: `${bin}:${process.env.PATH}`, + HOME: process.env.HOME, + GITHUB_REPOSITORY: REPO, + FAKE_GH_DIR: data, + FAKE_GH_LOG: log, + }, + }) + + return { + status: result.status, + stdout: result.stdout.trim(), + stderr: result.stderr, + calls: fs.readFileSync(log, 'utf8').trim().split('\n').filter(Boolean), + output: fs.existsSync(output) ? fs.readFileSync(output, 'utf8') : '', + } +} + +const BASELINE = { errors: { 'Some.Rule': 2 }, warnings: {} } +const ONE_ARTIFACT = { artifacts: [{ name: 'plugin-check-report', expired: false }] } +const EXPIRED = { artifacts: [{ name: 'plugin-check-report', expired: true }] } + +describe('fetch-baseline.sh', () => { + it('takes the run for the base commit and calls it exact', () => { + const api = API() + const result = fetchBaseline(['--sha', SHA, 'develop'], { + [api.forSha(SHA)]: { workflow_runs: [run(200, SHA)] }, + [api.onBranch('develop')]: { workflow_runs: [run(150)] }, + [api.artifacts(200)]: ONE_ARTIFACT, + }, { 200: BASELINE }) + + assert.equal(result.status, 0, result.stderr) + assert.equal(result.stdout, `200 https://github.com/${REPO}/actions/runs/200 exact`) + assert.deepEqual(JSON.parse(result.output), BASELINE) + }) + + it('falls back to an older run on the branch, skips an expired artifact, and calls it older', () => { + const api = API() + const result = fetchBaseline(['--sha', SHA, 'develop'], { + [api.forSha(SHA)]: { workflow_runs: [] }, + [api.onBranch('develop')]: { workflow_runs: [run(120), run(150)] }, + [api.artifacts(150)]: EXPIRED, + [api.artifacts(120)]: ONE_ARTIFACT, + }, { 120: BASELINE }) + + assert.equal(result.status, 0, result.stderr) + assert.equal(result.stdout, `120 https://github.com/${REPO}/actions/runs/120 older`) + // Newest first by run id, whatever order the API answered in. + assert.ok( + result.calls.findIndex((call) => call.includes('runs/150/')) < + result.calls.findIndex((call) => call.includes('runs/120/')) + ) + }) + + it('never compares a run with itself or with a newer run', () => { + const api = API() + const result = fetchBaseline(['--before-run', '150', 'develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(160), run(150), run(140)] }, + [api.artifacts(140)]: ONE_ARTIFACT, + }, { 140: BASELINE }) + + assert.equal(result.status, 0, result.stderr) + assert.match(result.stdout, /^140 /) + assert.ok(!result.calls.some((call) => /runs\/1[56]0\//.test(call))) + }) + + it('does not wait for a run for the base commit that has already finished', () => { + const api = API() + const result = fetchBaseline(['--sha', SHA, '--wait', '1200', 'develop'], { + [api.goingForSha(SHA)]: { workflow_runs: [run(200, SHA)] }, + [api.forSha(SHA)]: { workflow_runs: [run(200, SHA)] }, + [api.onBranch('develop')]: { workflow_runs: [] }, + [api.artifacts(200)]: ONE_ARTIFACT, + }, { 200: BASELINE }) + + assert.equal(result.status, 0, result.stderr) + assert.match(result.stdout, /^200 .* exact$/) + }) + + it('exits 1 when no run has a baseline', () => { + const api = API() + const result = fetchBaseline(['develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(150)] }, + [api.artifacts(150)]: { artifacts: [] }, + }) + + assert.equal(result.status, 1) + assert.match(result.stderr, /No Plugin Check run on develop has a baseline/) + assert.equal(result.output, '') + }) + + it('exits 2, not 1, when the lookup itself fails', () => { + // No answer for the branch's runs: the stand-in fails the way gh does on an API error. + const result = fetchBaseline(['develop'], {}) + + assert.equal(result.status, 2) + assert.match(result.stderr, /Could not look up Plugin Check runs/) + }) + + it('exits 2 when the download fails', () => { + const api = API() + const result = fetchBaseline(['develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(150)] }, + [api.artifacts(150)]: ONE_ARTIFACT, + }) + + assert.equal(result.status, 2) + assert.match(result.stderr, /download of the artifact of run 150/) + }) + + it('reads the runs of plugin-check.yml by default', () => { + const result = fetchBaseline(['develop'], {}) + + assert.ok(result.calls[0].includes('/workflows/plugin-check.yml/runs'), result.calls[0]) + }) + + it('reads the runs of the workflow file --workflow names', () => { + const api = API('wporg-check.yml') + const result = fetchBaseline(['--workflow', 'wporg-check.yml', 'develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(150)] }, + [api.artifacts(150)]: ONE_ARTIFACT, + }, { 150: BASELINE }) + + assert.equal(result.status, 0, result.stderr) + assert.ok(result.calls[0].includes('/workflows/wporg-check.yml/runs'), result.calls[0]) + }) + + it('refuses a --workflow that is a path or a ref, before calling the API', () => { + for (const workflow of ['.github/workflows/plugin-check.yml', 'plugin-check.yml@refs/heads/develop']) { + const result = fetchBaseline(['--workflow', workflow, 'develop'], {}) + + assert.equal(result.status, 2, workflow) + assert.match(result.stderr, /--workflow must be a file name/) + assert.deepEqual(result.calls, []) + } + }) +}) diff --git a/tests/actions/plugin-check-report/fixtures/baseline.json b/tests/actions/plugin-check-report/fixtures/baseline.json new file mode 100644 index 0000000..532809e --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/baseline.json @@ -0,0 +1,9 @@ +{ + "errors": { + "WordPress.Security.EscapeOutput.OutputNotEscaped": 2, + "WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents": 1 + }, + "warnings": { + "WordPress.DB.DirectDatabaseQuery.DirectQuery": 1 + } +} diff --git a/tests/actions/plugin-check-report/fixtures/clean.txt b/tests/actions/plugin-check-report/fixtures/clean.txt new file mode 100644 index 0000000..ad6c4f9 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/clean.txt @@ -0,0 +1 @@ +Success: Checks complete. No errors found. diff --git a/tests/actions/plugin-check-report/fixtures/empty.txt b/tests/actions/plugin-check-report/fixtures/empty.txt new file mode 100644 index 0000000..e69de29 diff --git a/tests/actions/plugin-check-report/fixtures/errors-down.txt b/tests/actions/plugin-check-report/fixtures/errors-down.txt new file mode 100644 index 0000000..69b539e --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/errors-down.txt @@ -0,0 +1,4 @@ +FILE: give.php +[{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""}] +FILE: src/Forms/Form.php +[{"line":40,"column":5,"type":"ERROR","code":"WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents","message":"file_get_contents() is discouraged.","docs":""},{"line":88,"column":9,"type":"WARNING","code":"WordPress.DB.DirectDatabaseQuery.DirectQuery","message":"Use of a direct database call is discouraged.","docs":""}] diff --git a/tests/actions/plugin-check-report/fixtures/errors-up.txt b/tests/actions/plugin-check-report/fixtures/errors-up.txt new file mode 100644 index 0000000..119911f --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/errors-up.txt @@ -0,0 +1,4 @@ +FILE: give.php +[{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""},{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""},{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""}] +FILE: src/Forms/Form.php +[{"line":40,"column":5,"type":"ERROR","code":"WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents","message":"file_get_contents() is discouraged.","docs":""},{"line":88,"column":9,"type":"WARNING","code":"WordPress.DB.DirectDatabaseQuery.DirectQuery","message":"Use of a direct database call is discouraged.","docs":""}] diff --git a/tests/actions/plugin-check-report/fixtures/garbage.txt b/tests/actions/plugin-check-report/fixtures/garbage.txt new file mode 100644 index 0000000..c31d675 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/garbage.txt @@ -0,0 +1,3 @@ + +Database Error +

Error establishing a database connection

diff --git a/tests/actions/plugin-check-report/fixtures/rule-up-total-same.txt b/tests/actions/plugin-check-report/fixtures/rule-up-total-same.txt new file mode 100644 index 0000000..6e001fe --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/rule-up-total-same.txt @@ -0,0 +1,4 @@ +FILE: give.php +[{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""},{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""},{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""}] +FILE: src/Forms/Form.php +[{"line":88,"column":9,"type":"WARNING","code":"WordPress.DB.DirectDatabaseQuery.DirectQuery","message":"Use of a direct database call is discouraged.","docs":""}] diff --git a/tests/actions/plugin-check-report/fixtures/same-as-baseline.txt b/tests/actions/plugin-check-report/fixtures/same-as-baseline.txt new file mode 100644 index 0000000..49384e3 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/same-as-baseline.txt @@ -0,0 +1,4 @@ +FILE: give.php +[{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""},{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""}] +FILE: src/Forms/Form.php +[{"line":40,"column":5,"type":"ERROR","code":"WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents","message":"file_get_contents() is discouraged.","docs":""},{"line":88,"column":9,"type":"WARNING","code":"WordPress.DB.DirectDatabaseQuery.DirectQuery","message":"Use of a direct database call is discouraged.","docs":""}] diff --git a/tests/actions/plugin-check-report/fixtures/warnings-up.txt b/tests/actions/plugin-check-report/fixtures/warnings-up.txt new file mode 100644 index 0000000..91737b2 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/warnings-up.txt @@ -0,0 +1,4 @@ +FILE: give.php +[{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""},{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""}] +FILE: src/Forms/Form.php +[{"line":40,"column":5,"type":"ERROR","code":"WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents","message":"file_get_contents() is discouraged.","docs":""},{"line":88,"column":9,"type":"WARNING","code":"WordPress.DB.DirectDatabaseQuery.DirectQuery","message":"Use of a direct database call is discouraged.","docs":""},{"line":88,"column":9,"type":"WARNING","code":"WordPress.DB.DirectDatabaseQuery.DirectQuery","message":"Use of a direct database call is discouraged.","docs":""},{"line":88,"column":9,"type":"WARNING","code":"WordPress.DB.DirectDatabaseQuery.DirectQuery","message":"Use of a direct database call is discouraged.","docs":""}] diff --git a/tests/actions/plugin-check-report/ratchet.test.js b/tests/actions/plugin-check-report/ratchet.test.js new file mode 100644 index 0000000..5cbd281 --- /dev/null +++ b/tests/actions/plugin-check-report/ratchet.test.js @@ -0,0 +1,297 @@ +'use strict' + +const { describe, it } = require('node:test') +const assert = require('node:assert') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') +const { spawnSync } = require('node:child_process') + +const RATCHET = path.join(__dirname, '../../../.github/actions/plugin-check-report/ratchet.sh') +const FIXTURES = path.join(__dirname, 'fixtures') +const BASELINE = path.join(FIXTURES, 'baseline.json') + +const ESCAPE = 'WordPress.Security.EscapeOutput.OutputNotEscaped' +const DIRECT_QUERY = 'WordPress.DB.DirectDatabaseQuery.DirectQuery' + +/** + * Runs ratchet.sh the way the action does, and returns what it printed and wrote. + * + * The environment is built from scratch rather than copied from process.env. In CI the test runner + * has GITHUB_STEP_SUMMARY, GITHUB_RUN_ID and friends set, and the script would append its summary + * to this repository's own job and link to this run. Only PATH and HOME are kept, so bash and jq + * are found. The script's `::error::` lines go to the captured stdout, never to the runner's log, + * so a passing test run does not annotate its own workflow. + * + * @param {string[]} args Arguments after the script, usually a results file. + * @param {object} env Extra environment, such as PLUGIN_CHECK_BASELINE. + * + * @returns {{status: number, stdout: string, stderr: string, comment: string, summary: string}} + * The exit status, both streams, the comment file and the job summary it wrote. + */ +function ratchet(args, env = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-ratchet-')) + const comment = path.join(dir, 'comment.md') + const summary = path.join(dir, 'summary.md') + + const result = spawnSync('bash', [RATCHET, '--comment-file', comment, ...args], { + encoding: 'utf8', + env: { + PATH: process.env.PATH, + HOME: process.env.HOME, + GITHUB_STEP_SUMMARY: summary, + // A path that does not exist, so a test that wants no baseline gets none, instead of a + // baseline.json some earlier local run left next to the script. + PLUGIN_CHECK_BASELINE: path.join(dir, 'no-baseline.json'), + ...env, + }, + }) + + const read = (file) => (fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : '') + + return { + status: result.status, + stdout: result.stdout, + stderr: result.stderr, + comment: read(comment), + summary: read(summary), + } +} + +/** + * The path of one of the sample results files. + * + * @param {string} name File name under fixtures/. + * + * @returns {string} The absolute path. + */ +function fixture(name) { + return path.join(FIXTURES, name) +} + +/** + * A results file with one new ERROR code per entry, for the tests about long lists. + * + * @param {number} codes How many different codes to write. + * + * @returns {string} The path of the file. + */ +function resultsWithNewCodes(codes) { + const results = Array.from({ length: codes }, (_, index) => ({ + line: index + 1, + column: 1, + type: 'ERROR', + code: `Test.Rule.Number${index + 1}`, + message: 'A rule added for the test.', + docs: '', + })) + const file = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-results-')), 'results.txt') + fs.writeFileSync(file, `FILE: plugin.php\n${JSON.stringify(results)}\n`) + + return file +} + +describe('ratchet.sh', () => { + describe('results it refuses', () => { + it('fails when the results file is missing', () => { + const run = ratchet([fixture('does-not-exist.txt')]) + + assert.equal(run.status, 1) + assert.match(run.stdout, /::error title=Plugin Check::Results file not found/) + assert.equal(run.comment, '', 'no comment body, so the workflow writes "could not run"') + }) + + it('fails when the results file is not Plugin Check output', () => { + const run = ratchet([fixture('garbage.txt')]) + + assert.equal(run.status, 1) + assert.match(run.stdout, /is not Plugin Check output.*First unexpected line: /) + }) + + it('fails when the results file is empty', () => { + const run = ratchet([fixture('empty.txt')]) + + assert.equal(run.status, 1) + assert.match(run.stdout, /has no Plugin Check output/) + }) + + it('fails on a mode it does not know, before reading anything', () => { + const run = ratchet([fixture('clean.txt')], { PLUGIN_CHECK_MODE: 'strcit' }) + + assert.equal(run.status, 1) + assert.match(run.stdout, /Unknown mode 'strcit'/) + }) + }) + + describe('--update', () => { + it('writes the counts per rule code', () => { + const out = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-update-')), 'baseline.json') + const run = ratchet(['--update', fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: out }) + + assert.equal(run.status, 0) + assert.deepEqual(JSON.parse(fs.readFileSync(out, 'utf8')), { + errors: { + [ESCAPE]: 3, + 'WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents': 1, + }, + warnings: { [DIRECT_QUERY]: 1 }, + }) + }) + + it('writes empty counts for a clean run', () => { + const out = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-update-')), 'baseline.json') + const run = ratchet(['--update', fixture('clean.txt')], { PLUGIN_CHECK_BASELINE: out }) + + assert.equal(run.status, 0) + assert.deepEqual(JSON.parse(fs.readFileSync(out, 'utf8')), { errors: {}, warnings: {} }) + }) + }) + + describe('compare in ratchet mode', () => { + it('passes a clean run with no baseline, with a warning and the one-column table', () => { + const run = ratchet([fixture('clean.txt')]) + + assert.equal(run.status, 0) + assert.match(run.stdout, /::warning title=Plugin Check::No baseline to compare with/) + assert.match(run.comment, /⚠️ Plugin Check ran, but there was no baseline to compare with/) + assert.match(run.comment, /\| Errors {3}\| 0 \|/) + assert.doesNotMatch(run.comment, /Change/) + }) + + it('passes when every count matches the baseline', () => { + const run = ratchet([fixture('same-as-baseline.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + + assert.equal(run.status, 0) + assert.match(run.comment, /✅ Plugin Check passed/) + assert.match(run.comment, /\| Errors {3}\| 3 \| 3 \| ±0 \|/) + }) + + it('fails when a rule has more errors than the baseline', () => { + const run = ratchet([fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + + assert.equal(run.status, 1) + assert.match(run.stdout, new RegExp(`::error title=Plugin Check::${ESCAPE} has 3 errors, the baseline has 2`)) + assert.match(run.comment, /❌ Plugin Check failed: 1 rule has more errors than the baseline allows/) + assert.match(run.comment, new RegExp(`Went up: \`${ESCAPE}\` \\(\\+1\\)`)) + }) + + it('fails when a rule goes up even though the total stays the same', () => { + const run = ratchet([fixture('rule-up-total-same.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + + assert.equal(run.status, 1) + assert.match(run.comment, /\| Errors {3}\| 3 \| 3 \| ±0 \|/) + assert.match(run.comment, /Rules that went down: 1\./) + }) + + it('passes with a notice when errors go down', () => { + const run = ratchet([fixture('errors-down.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + + assert.equal(run.status, 0) + assert.match(run.stdout, new RegExp(`::notice title=Plugin Check::${ESCAPE} \\(errors\\) went down from 2 to 1`)) + assert.match(run.comment, /\| Errors {3}\| 2 \| 3 \| -1 \|/) + }) + + it('passes with a notice when only warnings go up', () => { + const run = ratchet([fixture('warnings-up.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + + assert.equal(run.status, 0) + assert.match(run.stdout, /Warnings do not fail the check/) + assert.match(run.comment, new RegExp(`Went up: \`${DIRECT_QUERY}\` warning \\(\\+2\\)`)) + }) + + it('only warns when the baseline is from an older run of the target branch', () => { + const run = ratchet([fixture('errors-up.txt')], { + PLUGIN_CHECK_BASELINE: BASELINE, + PLUGIN_CHECK_ADVISORY: 'true', + }) + + assert.equal(run.status, 0) + assert.match(run.stdout, /::warning title=Plugin Check::.*Not enforced/) + assert.match(run.comment, /⚠️ Plugin Check passed, not enforced/) + }) + }) + + describe('compare in strict mode', () => { + it('fails on any error, even with no baseline', () => { + const run = ratchet([fixture('errors-down.txt')], { PLUGIN_CHECK_MODE: 'strict' }) + + assert.equal(run.status, 1) + assert.match(run.stdout, /2 errors found\. Strict mode fails on any error/) + assert.match(run.comment, /❌ Plugin Check failed: 2 errors, and strict mode allows none/) + }) + + it('fails on errors that went down, which ratchet mode would pass', () => { + const run = ratchet([fixture('errors-down.txt')], { + PLUGIN_CHECK_MODE: 'strict', + PLUGIN_CHECK_BASELINE: BASELINE, + }) + + assert.equal(run.status, 1) + }) + + it('passes a clean run', () => { + const run = ratchet([fixture('clean.txt')], { PLUGIN_CHECK_MODE: 'strict' }) + + assert.equal(run.status, 0) + assert.match(run.comment, /✅ Plugin Check passed: no errors/) + }) + + it('passes a run with warnings only', () => { + const results = fixture('warnings-up.txt') + const onlyWarnings = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-warn-')), 'results.txt') + // The warnings-up sample without its ERROR results. + const lines = fs.readFileSync(results, 'utf8').split('\n').map((line) => { + if (!line.startsWith('[')) { + return line + } + const kept = JSON.parse(line).filter((result) => result.type === 'WARNING') + return JSON.stringify(kept) + }) + fs.writeFileSync(onlyWarnings, lines.join('\n')) + + const run = ratchet([onlyWarnings], { PLUGIN_CHECK_MODE: 'strict' }) + + assert.equal(run.status, 0) + }) + }) + + describe('the comment', () => { + it('starts with the default marker', () => { + const run = ratchet([fixture('clean.txt')]) + + assert.equal(run.comment.split('\n')[0], '') + }) + + it('starts with the marker the workflow passes', () => { + const marker = '' + const run = ratchet([fixture('clean.txt')], { COMMENT_MARKER: marker }) + + assert.equal(run.comment.split('\n')[0], marker) + }) + + it('names the baseline column after where it came from, with a link', () => { + const run = ratchet([fixture('same-as-baseline.txt')], { + PLUGIN_CHECK_BASELINE: BASELINE, + BASELINE_LABEL: 'Target branch', + BASELINE_URL: 'https://github.com/o/r/actions/runs/1', + }) + + assert.match(run.comment, /\| Now \| \[Target branch\]\(https:\/\/github\.com\/o\/r\/actions\/runs\/1\) \| Change \|/) + }) + + it('lists at most five rules and never grows past fifteen lines', () => { + const run = ratchet([resultsWithNewCodes(7)], { PLUGIN_CHECK_BASELINE: BASELINE }) + + assert.equal(run.status, 1) + assert.match(run.comment, /Went up \(top 5 of 7\)/) + assert.doesNotMatch(run.comment, /Number6/) + assert.ok(run.comment.trimEnd().split('\n').length <= 15) + }) + + it('writes the same text to the job summary', () => { + const run = ratchet([fixture('errors-down.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + + assert.equal(run.summary, run.comment) + }) + }) +}) From b196b3ac3d5c863de623d5d291340d31765d4615 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Fri, 30 Oct 2026 19:36:47 -0300 Subject: [PATCH 03/31] docs(plugin-check): use neutral example names --- .github/actions/plugin-check-report/README.md | 4 ++-- .github/actions/plugin-check-report/action.yml | 4 ++-- .github/workflows/plugin-check.yml | 6 +++--- tests/actions/plugin-check-report/fetch-baseline.test.js | 2 +- tests/actions/plugin-check-report/fixtures/errors-down.txt | 2 +- tests/actions/plugin-check-report/fixtures/errors-up.txt | 2 +- .../plugin-check-report/fixtures/rule-up-total-same.txt | 2 +- .../plugin-check-report/fixtures/same-as-baseline.txt | 2 +- tests/actions/plugin-check-report/fixtures/warnings-up.txt | 2 +- tests/actions/plugin-check-report/ratchet.test.js | 2 +- 10 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/actions/plugin-check-report/README.md b/.github/actions/plugin-check-report/README.md index b63148c..c842f4e 100644 --- a/.github/actions/plugin-check-report/README.md +++ b/.github/actions/plugin-check-report/README.md @@ -28,8 +28,8 @@ jobs: - A `.puprc` whose `build` makes the plugin as it ships, and whose `zip_name` equals `plugin-slug`. The zip's top folder is named after `zip_name`, and Plugin Check takes the slug from the folder name. -- A `pup` Composer script that puts `pup.phar` in `bin/` and runs it, as GiveWP and Kadence Blocks - have: +- A `pup` Composer script that puts `pup.phar` in `bin/` and runs it, as Kadence Blocks + has: ```json "pup": [ diff --git a/.github/actions/plugin-check-report/action.yml b/.github/actions/plugin-check-report/action.yml index 343802e..09a0a56 100644 --- a/.github/actions/plugin-check-report/action.yml +++ b/.github/actions/plugin-check-report/action.yml @@ -84,7 +84,7 @@ runs: mkdir -p "$dir/report" "$dir/baseline" # In a called workflow GITHUB_WORKFLOW_REF is the caller's, like - # impress-org/givewp/.github/workflows/plugin-check.yml@refs/pull/12/merge. The runs that + # example-org/my-plugin/.github/workflows/plugin-check.yml@refs/pull/12/merge. The runs that # hold the baseline belong to that file, not to the toolbox's workflow. workflow=$INPUT_WORKFLOW_FILE if [ -z "$workflow" ]; then @@ -114,7 +114,7 @@ runs: fi # The name and the baseline.json inside it are what fetch-baseline.sh looks for. Keep both: - # they are also what GiveWP's runs from before the shared workflow uploaded, so its old + # repositories that ran an earlier copy of this check upload the same names, so their old # baselines keep working. - name: Upload the report id: artifact diff --git a/.github/workflows/plugin-check.yml b/.github/workflows/plugin-check.yml index 32f32fb..8bc1b70 100644 --- a/.github/workflows/plugin-check.yml +++ b/.github/workflows/plugin-check.yml @@ -4,7 +4,7 @@ name: Plugin Check # so tests, dotfiles, node_modules and other dev-only files cannot cause false positives. # # A reusable workflow. A plugin repository calls it from a thin workflow of its own; copy -# templates/workflows/plugin-check.yml. Ported from GiveWP's plugin-check.yml. +# templates/workflows/plugin-check.yml. # # The plugin is built with pup by default (`composer -- pup build`, then `pup package`), so the # repository's .puprc and .distignore decide what is checked, the same as the release. A repository @@ -32,7 +32,7 @@ on: inputs: plugin-slug: description: >- - The plugin's folder name, e.g. `give`. It has to equal the top folder of the zip pup + The plugin's folder name, e.g. `my-plugin`. It has to equal the top folder of the zip pup builds (`zip_name` in .puprc), because Plugin Check takes the slug from the folder name. required: true type: string @@ -154,7 +154,7 @@ jobs: name: Plugin Check run runs-on: ${{ inputs.runs-on }} - # Build, WordPress start and the check together take about 15 minutes for GiveWP. The cap is + # Build, WordPress start and the check together take about 15 minutes for a large plugin. The cap is # for a container that stops answering. timeout-minutes: 30 diff --git a/tests/actions/plugin-check-report/fetch-baseline.test.js b/tests/actions/plugin-check-report/fetch-baseline.test.js index 8b5e228..8879718 100644 --- a/tests/actions/plugin-check-report/fetch-baseline.test.js +++ b/tests/actions/plugin-check-report/fetch-baseline.test.js @@ -9,7 +9,7 @@ const { spawnSync } = require('node:child_process') const FETCH = path.join(__dirname, '../../../.github/actions/plugin-check-report/fetch-baseline.sh') -const REPO = 'impress-org/givewp' +const REPO = 'example-org/my-plugin' const SHA = 'a'.repeat(40) const OLD_SHA = 'b'.repeat(40) diff --git a/tests/actions/plugin-check-report/fixtures/errors-down.txt b/tests/actions/plugin-check-report/fixtures/errors-down.txt index 69b539e..fe81472 100644 --- a/tests/actions/plugin-check-report/fixtures/errors-down.txt +++ b/tests/actions/plugin-check-report/fixtures/errors-down.txt @@ -1,4 +1,4 @@ -FILE: give.php +FILE: my-plugin.php [{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""}] FILE: src/Forms/Form.php [{"line":40,"column":5,"type":"ERROR","code":"WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents","message":"file_get_contents() is discouraged.","docs":""},{"line":88,"column":9,"type":"WARNING","code":"WordPress.DB.DirectDatabaseQuery.DirectQuery","message":"Use of a direct database call is discouraged.","docs":""}] diff --git a/tests/actions/plugin-check-report/fixtures/errors-up.txt b/tests/actions/plugin-check-report/fixtures/errors-up.txt index 119911f..49ae2cc 100644 --- a/tests/actions/plugin-check-report/fixtures/errors-up.txt +++ b/tests/actions/plugin-check-report/fixtures/errors-up.txt @@ -1,4 +1,4 @@ -FILE: give.php +FILE: my-plugin.php [{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""},{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""},{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""}] FILE: src/Forms/Form.php [{"line":40,"column":5,"type":"ERROR","code":"WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents","message":"file_get_contents() is discouraged.","docs":""},{"line":88,"column":9,"type":"WARNING","code":"WordPress.DB.DirectDatabaseQuery.DirectQuery","message":"Use of a direct database call is discouraged.","docs":""}] diff --git a/tests/actions/plugin-check-report/fixtures/rule-up-total-same.txt b/tests/actions/plugin-check-report/fixtures/rule-up-total-same.txt index 6e001fe..2efabff 100644 --- a/tests/actions/plugin-check-report/fixtures/rule-up-total-same.txt +++ b/tests/actions/plugin-check-report/fixtures/rule-up-total-same.txt @@ -1,4 +1,4 @@ -FILE: give.php +FILE: my-plugin.php [{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""},{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""},{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""}] FILE: src/Forms/Form.php [{"line":88,"column":9,"type":"WARNING","code":"WordPress.DB.DirectDatabaseQuery.DirectQuery","message":"Use of a direct database call is discouraged.","docs":""}] diff --git a/tests/actions/plugin-check-report/fixtures/same-as-baseline.txt b/tests/actions/plugin-check-report/fixtures/same-as-baseline.txt index 49384e3..b2ba8a2 100644 --- a/tests/actions/plugin-check-report/fixtures/same-as-baseline.txt +++ b/tests/actions/plugin-check-report/fixtures/same-as-baseline.txt @@ -1,4 +1,4 @@ -FILE: give.php +FILE: my-plugin.php [{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""},{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""}] FILE: src/Forms/Form.php [{"line":40,"column":5,"type":"ERROR","code":"WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents","message":"file_get_contents() is discouraged.","docs":""},{"line":88,"column":9,"type":"WARNING","code":"WordPress.DB.DirectDatabaseQuery.DirectQuery","message":"Use of a direct database call is discouraged.","docs":""}] diff --git a/tests/actions/plugin-check-report/fixtures/warnings-up.txt b/tests/actions/plugin-check-report/fixtures/warnings-up.txt index 91737b2..4b9f1cc 100644 --- a/tests/actions/plugin-check-report/fixtures/warnings-up.txt +++ b/tests/actions/plugin-check-report/fixtures/warnings-up.txt @@ -1,4 +1,4 @@ -FILE: give.php +FILE: my-plugin.php [{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""},{"line":12,"column":10,"type":"ERROR","code":"WordPress.Security.EscapeOutput.OutputNotEscaped","message":"All output should be run through an escaping function.","docs":""}] FILE: src/Forms/Form.php [{"line":40,"column":5,"type":"ERROR","code":"WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents","message":"file_get_contents() is discouraged.","docs":""},{"line":88,"column":9,"type":"WARNING","code":"WordPress.DB.DirectDatabaseQuery.DirectQuery","message":"Use of a direct database call is discouraged.","docs":""},{"line":88,"column":9,"type":"WARNING","code":"WordPress.DB.DirectDatabaseQuery.DirectQuery","message":"Use of a direct database call is discouraged.","docs":""},{"line":88,"column":9,"type":"WARNING","code":"WordPress.DB.DirectDatabaseQuery.DirectQuery","message":"Use of a direct database call is discouraged.","docs":""}] diff --git a/tests/actions/plugin-check-report/ratchet.test.js b/tests/actions/plugin-check-report/ratchet.test.js index 5cbd281..07d28c9 100644 --- a/tests/actions/plugin-check-report/ratchet.test.js +++ b/tests/actions/plugin-check-report/ratchet.test.js @@ -263,7 +263,7 @@ describe('ratchet.sh', () => { }) it('starts with the marker the workflow passes', () => { - const marker = '' + const marker = '' const run = ratchet([fixture('clean.txt')], { COMMENT_MARKER: marker }) assert.equal(run.comment.split('\n')[0], marker) From 498329d6075504c760a56c9e968edc5738fe51e2 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Fri, 30 Oct 2026 19:51:40 -0300 Subject: [PATCH 04/31] docs(plugin-check): reference the workflow and the action at main --- .github/actions/plugin-check-report/README.md | 2 +- .github/workflows/plugin-check.yml | 6 ++++-- templates/workflows/plugin-check.yml | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/actions/plugin-check-report/README.md b/.github/actions/plugin-check-report/README.md index c842f4e..24badcb 100644 --- a/.github/actions/plugin-check-report/README.md +++ b/.github/actions/plugin-check-report/README.md @@ -18,7 +18,7 @@ jobs: contents: read issues: write pull-requests: write - uses: stellarwp/plugin-toolbox/.github/workflows/plugin-check.yml@v1 + uses: stellarwp/plugin-toolbox/.github/workflows/plugin-check.yml@main with: plugin-slug: my-plugin ``` diff --git a/.github/workflows/plugin-check.yml b/.github/workflows/plugin-check.yml index 8bc1b70..f1d4026 100644 --- a/.github/workflows/plugin-check.yml +++ b/.github/workflows/plugin-check.yml @@ -398,9 +398,11 @@ jobs: echo "url=https://img.shields.io/badge/Check%20Finished-${stamp}-darkgreen" >> "$GITHUB_OUTPUT" # Full path, never ./: in a called workflow a relative path resolves against the caller's - # repository. + # repository. @main, like the callers: this repository publishes no releases, and a branch + # is the only ref the workflow and the action can share. A caller that tests another branch + # of this repository still gets the action from main. - name: Compare with the baseline - uses: stellarwp/plugin-toolbox/.github/actions/plugin-check-report@v1 + uses: stellarwp/plugin-toolbox/.github/actions/plugin-check-report@main with: results: ${{ env.RESULTS }} comment-file: ${{ env.COMMENT_BODY }} diff --git a/templates/workflows/plugin-check.yml b/templates/workflows/plugin-check.yml index 01e6133..987d216 100644 --- a/templates/workflows/plugin-check.yml +++ b/templates/workflows/plugin-check.yml @@ -35,6 +35,6 @@ jobs: contents: read # check out the repository issues: write # the sticky pull request comment pull-requests: write # the sticky pull request comment - uses: stellarwp/plugin-toolbox/.github/workflows/plugin-check.yml@v1 + uses: stellarwp/plugin-toolbox/.github/workflows/plugin-check.yml@main with: plugin-slug: my-plugin From ae432cf4e1f83f0d2243ab5441e3cb80c686d2b9 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Fri, 30 Oct 2026 22:39:58 -0300 Subject: [PATCH 05/31] fix(plugin-check): bootstrap pup without forwarding arguments --- .github/actions/plugin-check-report/README.md | 2 +- .github/workflows/plugin-check.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/plugin-check-report/README.md b/.github/actions/plugin-check-report/README.md index 24badcb..6c448c9 100644 --- a/.github/actions/plugin-check-report/README.md +++ b/.github/actions/plugin-check-report/README.md @@ -49,7 +49,7 @@ The JavaScript toolchain comes from files the repository already has: `.nvmrc` s By default, with pup, the same steps the release zip uses, from the root of the checkout: ```bash -composer run -q pup -- --version > /dev/null # downloads bin/pup.phar +composer run -q pup > /dev/null # downloads bin/pup.phar composer -- pup build version=$(php -d display_errors=0 bin/pup.phar get-version --dev) composer -- pup package "$version" diff --git a/.github/workflows/plugin-check.yml b/.github/workflows/plugin-check.yml index f1d4026..6a776b4 100644 --- a/.github/workflows/plugin-check.yml +++ b/.github/workflows/plugin-check.yml @@ -293,7 +293,7 @@ jobs: exit 0 fi - composer run -q pup -- --version > /dev/null + composer run -q pup > /dev/null composer -- pup build version=$(php -d display_errors=0 bin/pup.phar get-version --dev) composer -- pup package "$version" From d86f2550bdbee4252875606fedca3530e0d223ec Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Fri, 30 Oct 2026 22:45:33 -0300 Subject: [PATCH 06/31] fix(plugin-check): count only this repository's runs and validate the wait options --- .github/actions/plugin-check-report/README.md | 7 +- .../plugin-check-report/fetch-baseline.sh | 58 +++++++-- .../fetch-baseline.test.js | 122 +++++++++++++++++- 3 files changed, 169 insertions(+), 18 deletions(-) diff --git a/.github/actions/plugin-check-report/README.md b/.github/actions/plugin-check-report/README.md index 6c448c9..f727f1d 100644 --- a/.github/actions/plugin-check-report/README.md +++ b/.github/actions/plugin-check-report/README.md @@ -101,6 +101,11 @@ artifact of an earlier run of the same workflow file: - With no earlier run at all, nothing is compared and the check passes with a warning. A failed lookup (an API or download error) fails instead, so an outage never passes as "no baseline". +Only runs of the repository's own branches count. A pull request from a fork has a run of the same +workflow, and the runs list matches it by branch name and commit, so a fork's run on a branch called +`develop` would look like a run of `develop`. Its artifact is whatever the fork's workflow uploaded, +so runs whose head commit is in another repository are skipped. + The earlier runs are found by the caller's workflow file name, read from `GITHUB_WORKFLOW_REF`. Renaming the caller file starts again with no baseline. Artifacts expire after the repository's retention period (90 days by default), so a branch with no run for that long starts with none too. @@ -172,7 +177,7 @@ not start the run. The job that runs the Plugin Check action lowers itself to `c | `mode` | no | `ratchet` | `ratchet` or `strict` | | `comment-marker` | no | `` | First line of the comment body | | `workflow-file` | no | from `GITHUB_WORKFLOW_REF` | Whose runs hold the baseline | -| `baseline-wait` | no | `1200` | Seconds to wait for the base commit's run | +| `baseline-wait` | no | `1200` | Seconds to wait for the base commit's run. A whole number, or the check fails | | `badge-url` | no | none | Image under the status line | | `token` | no | `github.token` | Needs `actions: read` | diff --git a/.github/actions/plugin-check-report/fetch-baseline.sh b/.github/actions/plugin-check-report/fetch-baseline.sh index c4b4718..379b11a 100755 --- a/.github/actions/plugin-check-report/fetch-baseline.sh +++ b/.github/actions/plugin-check-report/fetch-baseline.sh @@ -42,24 +42,43 @@ wait_for=0 before_run="" # How long to wait for a run for --sha to show up at all. GRACE=120 +usage="Usage: $0 [--workflow ] [--sha ] [--wait ] [--before-run ] []" + +lookup_failed() { + echo "Could not look up Plugin Check runs: $1" >&2 + exit 2 +} + +# A missing or malformed argument exits 2, like any other lookup that cannot be trusted. Exit 1 is +# kept for "no run has a baseline", so a typo must never read as that. +need_value() { + [ "$#" -ge 2 ] && [ -n "$2" ] || lookup_failed "$1 needs a value. $usage" +} + +# A plain count of seconds or a run id. At most 18 digits keeps it inside bash's 64-bit integers, +# and 10# keeps a leading zero from being read as octal later. +number() { + case "$2" in + '' | *[!0-9]*) lookup_failed "$1 must be a whole number, not '$2'" ;; + esac + [ "${#2}" -le 18 ] || lookup_failed "$1 is too large: '$2'" + echo $((10#$2)) +} + while [ $# -gt 0 ]; do case "$1" in - --workflow) workflow=${2:?"--workflow needs a file name"}; shift 2 ;; - --sha) sha=${2:?"--sha needs a commit"}; shift 2 ;; - --wait) wait_for=${2:?"--wait needs seconds"}; shift 2 ;; - --before-run) before_run=${2:?"--before-run needs a run id"}; shift 2 ;; + --workflow) need_value "$@"; workflow=$2; shift 2 ;; + --sha) need_value "$@"; sha=$2; shift 2 ;; + --wait) need_value "$@"; wait_for=$(number --wait "$2"); shift 2 ;; + --before-run) need_value "$@"; before_run=$(number --before-run "$2"); shift 2 ;; *) break ;; esac done -branch=${1:?"Usage: $0 [--workflow ] [--sha ] [--wait ] [--before-run ] []"} +[ "$#" -ge 1 ] && [ -n "$1" ] || lookup_failed "a branch is needed. $usage" +branch=$1 output=${2:-"$(cd "$(dirname "$0")" && pwd)/baseline.json"} -lookup_failed() { - echo "Could not look up Plugin Check runs: $1" >&2 - exit 2 -} - # Any other command that fails (mktemp, mkdir, cp, ...) is a failure too, never "no baseline": # exit 1 is kept for the one place below that means it. trap 'echo "fetch-baseline.sh failed at line $LINENO." >&2; exit 2' ERR @@ -72,6 +91,19 @@ case "$workflow" in esac runs="repos/$repo/actions/workflows/$workflow/runs" +# Only runs of this repository's own branches count. The runs API matches `branch` and `head_sha` +# against the head of a pull request too, so a run for a pull request from a fork, from a branch +# named like the target, would otherwise be taken for a run of the target, and its artifact is +# whatever that fork's workflow chose to upload. +# +# jq reads $repo, so the filters stay in single quotes. +# shellcheck disable=SC2016 +own='select((.head_repository.full_name // "" | ascii_downcase) == ($repo | ascii_downcase))' +# shellcheck disable=SC2016 +own_runs='.workflow_runs[] | '"$own"' | "\(.id) \(.html_url) \(.head_sha)"' +# shellcheck disable=SC2016 +own_counts='[.workflow_runs[] | '"$own"'] | "\(length) \([.[] | select(.status != "completed")] | length)"' + # The run for the base commit may still be going: a pull request opened right after a merge starts # at the same time as the push run on the target branch. Waiting for it beats comparing with an # older run. @@ -81,7 +113,7 @@ if [ -n "$sha" ] && [ "$wait_for" -gt 0 ]; then while :; do # " " for the commit. counts=$(gh api "$runs?head_sha=$sha&per_page=20" \ - --jq '"\(.workflow_runs | length) \([.workflow_runs[] | select(.status != "completed")] | length)"') \ + | jq -r --arg repo "$repo" "$own_counts") \ || lookup_failed "runs for $sha" read -r total going <<< "$counts" # A run can take a moment to show up after its push. After GRACE seconds with none, the @@ -103,12 +135,12 @@ fi candidates="" if [ -n "$sha" ]; then candidates=$(gh api "$runs?status=completed&head_sha=$sha&per_page=10" \ - --jq '.workflow_runs[] | "\(.id) \(.html_url) \(.head_sha)"') || lookup_failed "runs for $sha" + | jq -r --arg repo "$repo" "$own_runs") || lookup_failed "runs for $sha" fi # Newest first by run id (ids only grow), not by the order the API happens to return. The 50 # newest runs are plenty: if none of them has a baseline, an older one would be stale anyway. on_branch=$(gh api "$runs?status=completed&branch=$branch&per_page=50" \ - --jq '.workflow_runs[] | "\(.id) \(.html_url) \(.head_sha)"' | sort -k1,1nr) || lookup_failed "runs on $branch" + | jq -r --arg repo "$repo" "$own_runs" | sort -k1,1nr) || lookup_failed "runs on $branch" candidates+=${candidates:+$'\n'}$on_branch tried=" " diff --git a/tests/actions/plugin-check-report/fetch-baseline.test.js b/tests/actions/plugin-check-report/fetch-baseline.test.js index 8879718..7c8831e 100644 --- a/tests/actions/plugin-check-report/fetch-baseline.test.js +++ b/tests/actions/plugin-check-report/fetch-baseline.test.js @@ -85,13 +85,21 @@ const API = (workflow = 'plugin-check.yml') => { /** * A run as the workflow runs API lists it. * - * @param {number} id Run id. - * @param {string} sha Head commit. + * @param {number} id Run id. + * @param {string} sha Head commit. + * @param {string} headRepo Repository the run's head commit belongs to. A pull request from a + * fork has the fork's name here. * * @returns {object} The run. */ -function run(id, sha = OLD_SHA) { - return { id, html_url: `https://github.com/${REPO}/actions/runs/${id}`, head_sha: sha, status: 'completed' } +function run(id, sha = OLD_SHA, headRepo = REPO) { + return { + id, + html_url: `https://github.com/${REPO}/actions/runs/${id}`, + head_sha: sha, + status: 'completed', + head_repository: { full_name: headRepo }, + } } /** @@ -253,6 +261,112 @@ describe('fetch-baseline.sh', () => { assert.ok(result.calls[0].includes('/workflows/wporg-check.yml/runs'), result.calls[0]) }) + it('ignores a run whose head commit is in a fork, even on a branch named like the target', () => { + const api = API() + const result = fetchBaseline(['--sha', SHA, 'develop'], { + // A fork's pull request run for the base commit, and a newer one on a fork branch "develop". + [api.forSha(SHA)]: { workflow_runs: [run(300, SHA, 'someone/my-plugin')] }, + [api.onBranch('develop')]: { workflow_runs: [run(250, OLD_SHA, 'someone/my-plugin'), run(150)] }, + [api.artifacts(150)]: ONE_ARTIFACT, + [api.artifacts(250)]: ONE_ARTIFACT, + [api.artifacts(300)]: ONE_ARTIFACT, + }, { 150: BASELINE, 250: { errors: { 'Some.Rule': 9999 }, warnings: {} }, 300: { errors: { 'Some.Rule': 9999 }, warnings: {} } }) + + assert.equal(result.status, 0, result.stderr) + assert.match(result.stdout, /^150 .* older$/) + assert.deepEqual(JSON.parse(result.output), BASELINE) + assert.ok(!result.calls.some((call) => call.includes('/runs/250/') || call.includes('/runs/300/')), 'no fork run was looked at') + }) + + it('exits 1 when only fork runs exist', () => { + const api = API() + const result = fetchBaseline(['develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(250, OLD_SHA, 'someone/my-plugin')] }, + }) + + assert.equal(result.status, 1) + assert.match(result.stderr, /has a baseline/) + }) + + it('matches the repository name without regard to case', () => { + const api = API() + const result = fetchBaseline(['develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(150, OLD_SHA, 'Example-Org/My-Plugin')] }, + [api.artifacts(150)]: ONE_ARTIFACT, + }, { 150: BASELINE }) + + assert.equal(result.status, 0, result.stderr) + }) + + it('does not wait for a fork run that is still going', () => { + const api = API() + const result = fetchBaseline(['--sha', SHA, '--wait', '1', 'develop'], { + // Our run for the commit is done, and a fork's run for the same commit is not. Counted, the + // fork's run would make the script sleep 30 seconds and then give up waiting. + [api.goingForSha(SHA)]: { + workflow_runs: [run(200, SHA), { ...run(300, SHA, 'someone/my-plugin'), status: 'in_progress' }], + }, + [api.forSha(SHA)]: { workflow_runs: [run(200, SHA)] }, + [api.onBranch('develop')]: { workflow_runs: [] }, + [api.artifacts(200)]: ONE_ARTIFACT, + }, { 200: BASELINE }) + + assert.equal(result.status, 0, result.stderr) + assert.match(result.stdout, /^200 .* exact$/) + assert.doesNotMatch(result.stderr, /still going/) + }) + + it('exits 2, not 1, for an option value that is empty or not a whole number', () => { + const cases = [ + [['--wait', '20m', 'develop'], /--wait must be a whole number, not '20m'/], + [['--wait', '', 'develop'], /--wait needs a value/], + [['--before-run', 'abc', 'develop'], /--before-run must be a whole number, not 'abc'/], + [['--before-run', '-5', 'develop'], /--before-run must be a whole number/], + [['--before-run', '9'.repeat(19), 'develop'], /--before-run is too large/], + [['--sha', '', 'develop'], /--sha needs a value/], + [['', 'baseline.json'], /a branch is needed/], + ] + + for (const [args, message] of cases) { + const result = fetchBaseline(args, {}) + + assert.equal(result.status, 2, args.join(' ')) + assert.match(result.stderr, message, args.join(' ')) + assert.deepEqual(result.calls, [], `${args.join(' ')} asked the API nothing`) + } + }) + + it('exits 2 when an option or the branch is missing altogether', () => { + // fetchBaseline() always adds an output path after the arguments, which a flag at the end of + // the line would take for its value, so these run the script as it is. + const cases = [ + [['--wait'], /--wait needs a value/], + [['--workflow'], /--workflow needs a value/], + [[], /a branch is needed/], + ] + + for (const [args, message] of cases) { + const result = spawnSync('bash', [FETCH, ...args], { + encoding: 'utf8', + env: { PATH: process.env.PATH, HOME: process.env.HOME, GITHUB_REPOSITORY: REPO }, + }) + + assert.equal(result.status, 2, args.join(' ')) + assert.match(result.stderr, message, args.join(' ')) + } + }) + + it('reads a run id with a leading zero as a decimal number', () => { + const api = API() + const result = fetchBaseline(['--before-run', '0150', 'develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(150), run(120)] }, + [api.artifacts(120)]: ONE_ARTIFACT, + }, { 120: BASELINE }) + + assert.equal(result.status, 0, result.stderr) + assert.match(result.stdout, /^120 /) + }) + it('refuses a --workflow that is a path or a ref, before calling the API', () => { for (const workflow of ['.github/workflows/plugin-check.yml', 'plugin-check.yml@refs/heads/develop']) { const result = fetchBaseline(['--workflow', workflow, 'develop'], {}) From 21a5833e553dd21f39c2421bb6c8bab23394f3e7 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Fri, 30 Oct 2026 23:07:58 -0300 Subject: [PATCH 07/31] fix(plugin-check): read the 100 newest runs on the branch, the most one page holds --- .github/actions/plugin-check-report/fetch-baseline.sh | 9 ++++++--- tests/actions/plugin-check-report/fetch-baseline.test.js | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/actions/plugin-check-report/fetch-baseline.sh b/.github/actions/plugin-check-report/fetch-baseline.sh index 379b11a..a268971 100755 --- a/.github/actions/plugin-check-report/fetch-baseline.sh +++ b/.github/actions/plugin-check-report/fetch-baseline.sh @@ -137,9 +137,12 @@ if [ -n "$sha" ]; then candidates=$(gh api "$runs?status=completed&head_sha=$sha&per_page=10" \ | jq -r --arg repo "$repo" "$own_runs") || lookup_failed "runs for $sha" fi -# Newest first by run id (ids only grow), not by the order the API happens to return. The 50 -# newest runs are plenty: if none of them has a baseline, an older one would be stale anyway. -on_branch=$(gh api "$runs?status=completed&branch=$branch&per_page=50" \ +# Newest first by run id (ids only grow), not by the order the API happens to return. One page of +# 100, the most the API gives, is plenty: if none of them has a baseline, an older one would be +# stale anyway. Fork runs are dropped after the page is read, so a flood of them could push ours +# off it; the result is then "no baseline", which passes with a warning, and no fork baseline is +# taken. +on_branch=$(gh api "$runs?status=completed&branch=$branch&per_page=100" \ | jq -r --arg repo "$repo" "$own_runs" | sort -k1,1nr) || lookup_failed "runs on $branch" candidates+=${candidates:+$'\n'}$on_branch diff --git a/tests/actions/plugin-check-report/fetch-baseline.test.js b/tests/actions/plugin-check-report/fetch-baseline.test.js index 7c8831e..d8eae06 100644 --- a/tests/actions/plugin-check-report/fetch-baseline.test.js +++ b/tests/actions/plugin-check-report/fetch-baseline.test.js @@ -77,7 +77,7 @@ const API = (workflow = 'plugin-check.yml') => { return { forSha: (sha) => `${runs}?status=completed&head_sha=${sha}&per_page=10`, goingForSha: (sha) => `${runs}?head_sha=${sha}&per_page=20`, - onBranch: (branch) => `${runs}?status=completed&branch=${branch}&per_page=50`, + onBranch: (branch) => `${runs}?status=completed&branch=${branch}&per_page=100`, artifacts: (id) => `repos/${REPO}/actions/runs/${id}/artifacts?name=plugin-check-report`, } } From 672612b0f7fffe606c26f21814d30cb915a92de4 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Fri, 30 Oct 2026 23:11:05 -0300 Subject: [PATCH 08/31] fix(plugin-check): call the report action with the self-repository reference --- .github/workflows/plugin-check.yml | 10 +++++----- .github/workflows/test.yml | 7 +++++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/plugin-check.yml b/.github/workflows/plugin-check.yml index 6a776b4..1a73db4 100644 --- a/.github/workflows/plugin-check.yml +++ b/.github/workflows/plugin-check.yml @@ -397,12 +397,12 @@ jobs: stamp=${stamp//\//%2F} echo "url=https://img.shields.io/badge/Check%20Finished-${stamp}-darkgreen" >> "$GITHUB_OUTPUT" - # Full path, never ./: in a called workflow a relative path resolves against the caller's - # repository. @main, like the callers: this repository publishes no releases, and a branch - # is the only ref the workflow and the action can share. A caller that tests another branch - # of this repository still gets the action from main. + # `$/` is this repository at the commit that runs this workflow, so the action is always the + # same revision as the workflow, whichever ref the caller used, and it needs no checkout. It + # needs a runner of version 2.336.0 or newer; this job runs on ubuntu-latest. Not ./, which + # in a called workflow resolves against the caller's repository. - name: Compare with the baseline - uses: stellarwp/plugin-toolbox/.github/actions/plugin-check-report@main + uses: $/.github/actions/plugin-check-report with: results: ${{ env.RESULTS }} comment-file: ${{ env.COMMENT_BODY }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ed1acb0..08c2003 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,12 +53,15 @@ jobs: persist-credentials: false # Pinned, so a new actionlint rule cannot fail a pull request that did not change anything. - # `queue` is ignored because GitHub accepts it in `concurrency` (release-tags.yml uses it) - # and actionlint 1.7.12 does not know it yet. + # Two things are ignored because GitHub accepts them and actionlint 1.7.12 does not know them + # yet: `queue` in `concurrency` (release-tags.yml uses it), and a `uses: $/path` reference to + # this repository (the reusable workflows use it for their own actions). Only that exact shape + # is ignored, so any other action reference that lacks a ref still fails. - name: Run actionlint run: | bash <(curl -sSfL https://raw.githubusercontent.com/rhysd/actionlint/v1.7.12/scripts/download-actionlint.bash) 1.7.12 ./actionlint -ignore 'unexpected key "queue" for "concurrency" section' \ + -ignore 'specifying action "\$/[^"]*" in invalid format because ref is missing' \ .github/workflows/*.yml templates/workflows/*.yml # actionlint does not read composite actions or standalone scripts. shellcheck comes with the From 887aad8d5e50f94d4d8cc3c165d2e485ac1b9056 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Tue, 3 Nov 2026 14:26:22 -0300 Subject: [PATCH 09/31] Post the Plugin Check comment with the pr-comment action, not peter-evans --- .github/workflows/plugin-check.yml | 39 +++++++----------------------- 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/.github/workflows/plugin-check.yml b/.github/workflows/plugin-check.yml index 1a73db4..63c6b97 100644 --- a/.github/workflows/plugin-check.yml +++ b/.github/workflows/plugin-check.yml @@ -10,7 +10,8 @@ name: Plugin Check # repository's .puprc and .distignore decide what is checked, the same as the release. A repository # without pup passes `build-command` instead; see the plugin-check-report README. # -# Three jobs, so the third-party action never holds a token that can write: +# Three jobs, so the third-party action never holds a token that can write. The comments are written +# by .github/actions/pr-comment, which calls the GitHub API itself: # start posts "in progress" on the pull request. # check builds the plugin and runs wordpress/plugin-check-action. Read-only token. # report compares the results with the baseline and replaces the comment with the result. @@ -102,7 +103,7 @@ on: env: # Hidden line that marks the sticky pull request comment. The report action writes it as the - # first line of the body, and both comment steps search for it. Named after the repository, so + # first line of the body, and the pr-comment action searches for it. Named after the repository, so # two plugin repositories never share a marker. COMMENT_MARKER: '' # Only pull requests from this repository get a comment: a fork's token is read-only. @@ -128,22 +129,12 @@ jobs: stamp=${stamp//\//%2F} echo "url=https://img.shields.io/badge/Check%20Started-${stamp}-blue" >> "$GITHUB_OUTPUT" - - name: Find the Plugin Check comment - id: find_comment - continue-on-error: true - uses: peter-evans/find-comment@v4 - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-includes: ${{ env.COMMENT_MARKER }} - + # A failed API call here must not stop the check. - name: Post "in progress" continue-on-error: true - uses: peter-evans/create-or-update-comment@v5 + uses: $/.github/actions/pr-comment with: - comment-id: ${{ steps.find_comment.outputs.comment-id }} - issue-number: ${{ github.event.pull_request.number }} - edit-mode: replace + marker: ${{ env.COMMENT_MARKER }} body: | ${{ env.COMMENT_MARKER }} ⏳ Plugin Check in progress @@ -422,22 +413,10 @@ jobs: printf '%s\n❌ Plugin Check could not run\n\n![Plugin Check finished](%s)\n\n📎 [Run](%s)\n' "$COMMENT_MARKER" "$BADGE" "$RUN_URL" > "$COMMENT_BODY" fi - - name: Find the Plugin Check comment - id: find_comment - if: ${{ !cancelled() && env.COMMENT == 'true' }} - continue-on-error: true - uses: peter-evans/find-comment@v4 - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-includes: ${{ env.COMMENT_MARKER }} - - name: Post the result if: ${{ !cancelled() && env.COMMENT == 'true' }} continue-on-error: true - uses: peter-evans/create-or-update-comment@v5 + uses: $/.github/actions/pr-comment with: - comment-id: ${{ steps.find_comment.outputs.comment-id }} - issue-number: ${{ github.event.pull_request.number }} - edit-mode: replace - body-path: ${{ env.COMMENT_BODY }} + marker: ${{ env.COMMENT_MARKER }} + body-file: ${{ env.COMMENT_BODY }} From 58b83a4f3508e070d5fb79d5fdd7cd3b78f4fa66 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Tue, 3 Nov 2026 14:26:22 -0300 Subject: [PATCH 10/31] Say that the Plugin Check comment comes from pr-comment --- .github/actions/plugin-check-report/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/plugin-check-report/README.md b/.github/actions/plugin-check-report/README.md index f727f1d..9c6e25a 100644 --- a/.github/actions/plugin-check-report/README.md +++ b/.github/actions/plugin-check-report/README.md @@ -128,6 +128,9 @@ One comment per pull request, replaced on every run, found by a hidden first lin result, the totals, up to five rules that went up, and links to the run and the artifact. It never grows past 15 lines. Pull requests from forks get no comment: their token cannot write. +The workflow writes it with the [pr-comment](../pr-comment/) action. The comment is found by its +first line, not by its author. + ## The check name A job in a called workflow reports as ` / `. With the template's job id, From b717d34a98a40e73ffb248d186c14cbc066e9a18 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Tue, 3 Nov 2026 17:35:51 -0300 Subject: [PATCH 11/31] Say that pup reads .gitattributes too, and why bin/pup.phar still matters --- .github/actions/plugin-check-report/README.md | 13 +++++++++---- .github/workflows/plugin-check.yml | 6 +++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/actions/plugin-check-report/README.md b/.github/actions/plugin-check-report/README.md index 9c6e25a..1051500 100644 --- a/.github/actions/plugin-check-report/README.md +++ b/.github/actions/plugin-check-report/README.md @@ -28,8 +28,8 @@ jobs: - A `.puprc` whose `build` makes the plugin as it ships, and whose `zip_name` equals `plugin-slug`. The zip's top folder is named after `zip_name`, and Plugin Check takes the slug from the folder name. -- A `pup` Composer script that puts `pup.phar` in `bin/` and runs it, as Kadence Blocks - has: +- A `pup` Composer script that downloads `pup.phar` when it is missing and then runs it, so one + `composer pup` both installs and runs pup. Kadence Blocks has: ```json "pup": [ @@ -38,7 +38,12 @@ jobs: ] ``` -- A `.distignore` (or pup's default ignore list) that keeps dev-only files out of the zip. + The script is what matters, but the path is not free yet: the workflow also runs + `php bin/pup.phar` itself, to ask pup for the version and the zip name, so keep the phar at + `bin/pup.phar`. + +- Ignore rules that keep dev-only files out of the zip: a `.distignore`, `export-ignore` lines in + `.gitattributes` (pup reads both), or pup's default ignore list. - The caller file named `plugin-check.yml`. See [The baseline](#the-baseline). The JavaScript toolchain comes from files the repository already has: `.nvmrc` sets up Node, @@ -57,7 +62,7 @@ name=$(php -d display_errors=0 bin/pup.phar zip-name "$version") unzip -q "$name.zip" -d "$PLUGIN_CHECK_BUILD_DIR" ``` -So `.puprc` and `.distignore` decide what is checked. If the release needs a step the check should +So `.puprc` and the ignore files decide what is checked. If the release needs a step the check should see too, such as making the POT file, put it in `.puprc`'s `build`, not in the workflow. ### `build-command`, for a repository without pup diff --git a/.github/workflows/plugin-check.yml b/.github/workflows/plugin-check.yml index 63c6b97..d11a221 100644 --- a/.github/workflows/plugin-check.yml +++ b/.github/workflows/plugin-check.yml @@ -7,7 +7,7 @@ name: Plugin Check # templates/workflows/plugin-check.yml. # # The plugin is built with pup by default (`composer -- pup build`, then `pup package`), so the -# repository's .puprc and .distignore decide what is checked, the same as the release. A repository +# repository's .puprc and ignore files decide what is checked, the same as the release. A repository # without pup passes `build-command` instead; see the plugin-check-report README. # # Three jobs, so the third-party action never holds a token that can write. The comments are written @@ -258,7 +258,7 @@ jobs: restore-keys: | plugin-check-${{ runner.os }}-${{ hashFiles('source/package-lock.json', 'source/bun.lock', 'source/bun.lockb') }}- - # The same pup steps the release zip uses, so .puprc and .distignore decide what is checked. + # The same pup steps the release zip uses, so .puprc and the ignore files decide what is checked. # The secrets reach this step only, never the job, so the third-party action later in this # job does not see them in its environment. - name: Build the plugin @@ -308,7 +308,7 @@ jobs: line=$(echo "$line" | xargs) [ -n "$line" ] || continue if [ -e "$dir/$line" ]; then - echo "::error title=Plugin Check::The cache path $line is inside the built plugin. Add it to .distignore." + echo "::error title=Plugin Check::The cache path $line is inside the built plugin. Add it to .distignore or to .gitattributes with export-ignore." exit 1 fi done <<< "$CACHE_PATHS" From 60b91503b64fa39fab085cff7a5117a33738cede Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Tue, 3 Nov 2026 18:32:42 -0300 Subject: [PATCH 12/31] Pin the exact output of the Plugin Check scripts as golden files --- .../fixtures/expected/artifact-only.md | 9 + .../fixtures/expected/artifact-only.out | 2 + .../fixtures/expected/artifact-only.summary | 9 + .../fixtures/expected/badge-and-artifact.md | 13 + .../fixtures/expected/badge-and-artifact.out | 2 + .../expected/badge-and-artifact.summary | 13 + .../fixtures/expected/clean-no-baseline.md | 9 + .../fixtures/expected/clean-no-baseline.out | 2 + .../expected/clean-no-baseline.summary | 9 + .../fixtures/expected/errors-down.md | 11 + .../fixtures/expected/errors-down.out | 2 + .../fixtures/expected/errors-down.summary | 11 + .../expected/errors-up-advisory-uppercase.md | 11 + .../expected/errors-up-advisory-uppercase.out | 2 + .../errors-up-advisory-uppercase.summary | 11 + .../fixtures/expected/errors-up-advisory.md | 11 + .../fixtures/expected/errors-up-advisory.out | 2 + .../expected/errors-up-advisory.summary | 11 + .../fixtures/expected/errors-up.md | 11 + .../fixtures/expected/errors-up.out | 2 + .../fixtures/expected/errors-up.summary | 11 + .../fixtures/expected/label-and-url.md | 9 + .../fixtures/expected/label-and-url.out | 1 + .../fixtures/expected/label-and-url.summary | 9 + .../expected/marker-with-line-breaks.md | 15 ++ .../expected/marker-with-line-breaks.out | 2 + .../expected/marker-with-line-breaks.summary | 15 ++ .../fixtures/expected/marker.md | 9 + .../fixtures/expected/marker.out | 2 + .../fixtures/expected/marker.summary | 9 + .../fixtures/expected/results-empty.out | 1 + .../fixtures/expected/results-garbage.out | 1 + .../fixtures/expected/results-missing.out | 1 + .../fixtures/expected/rule-up-total-same.md | 12 + .../fixtures/expected/rule-up-total-same.out | 3 + .../expected/rule-up-total-same.summary | 12 + .../fixtures/expected/same-as-baseline.md | 9 + .../fixtures/expected/same-as-baseline.out | 1 + .../expected/same-as-baseline.summary | 9 + .../fixtures/expected/seven-new-codes.md | 12 + .../fixtures/expected/seven-new-codes.out | 11 + .../fixtures/expected/seven-new-codes.summary | 12 + .../fixtures/expected/six-new-codes.md | 12 + .../fixtures/expected/six-new-codes.out | 10 + .../fixtures/expected/six-new-codes.summary | 12 + .../fixtures/expected/status.json | 28 +++ .../fixtures/expected/strict-clean.md | 9 + .../fixtures/expected/strict-clean.out | 2 + .../fixtures/expected/strict-clean.summary | 9 + .../fixtures/expected/strict-errors-down.md | 11 + .../fixtures/expected/strict-errors-down.out | 3 + .../expected/strict-errors-down.summary | 11 + .../fixtures/expected/strict-no-baseline.md | 9 + .../fixtures/expected/strict-no-baseline.out | 3 + .../expected/strict-no-baseline.summary | 9 + .../fixtures/expected/strict-warnings-only.md | 9 + .../expected/strict-warnings-only.out | 2 + .../expected/strict-warnings-only.summary | 9 + .../fixtures/expected/unknown-mode.out | 1 + .../fixtures/expected/update-clean.json | 4 + .../fixtures/expected/update-clean.out | 1 + .../fixtures/expected/update-errors-up.json | 9 + .../fixtures/expected/update-errors-up.out | 1 + .../fixtures/expected/update-garbage.out | 1 + .../fixtures/expected/warnings-up.md | 11 + .../fixtures/expected/warnings-up.out | 2 + .../fixtures/expected/warnings-up.summary | 11 + tests/actions/plugin-check-report/golden.js | 234 ++++++++++++++++++ .../ratchet-golden.test.js | 42 ++++ 69 files changed, 784 insertions(+) create mode 100644 tests/actions/plugin-check-report/fixtures/expected/artifact-only.md create mode 100644 tests/actions/plugin-check-report/fixtures/expected/artifact-only.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/artifact-only.summary create mode 100644 tests/actions/plugin-check-report/fixtures/expected/badge-and-artifact.md create mode 100644 tests/actions/plugin-check-report/fixtures/expected/badge-and-artifact.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/badge-and-artifact.summary create mode 100644 tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.md create mode 100644 tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.summary create mode 100644 tests/actions/plugin-check-report/fixtures/expected/errors-down.md create mode 100644 tests/actions/plugin-check-report/fixtures/expected/errors-down.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/errors-down.summary create mode 100644 tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.md create mode 100644 tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.summary create mode 100644 tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.md create mode 100644 tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.summary create mode 100644 tests/actions/plugin-check-report/fixtures/expected/errors-up.md create mode 100644 tests/actions/plugin-check-report/fixtures/expected/errors-up.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/errors-up.summary create mode 100644 tests/actions/plugin-check-report/fixtures/expected/label-and-url.md create mode 100644 tests/actions/plugin-check-report/fixtures/expected/label-and-url.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/label-and-url.summary create mode 100644 tests/actions/plugin-check-report/fixtures/expected/marker-with-line-breaks.md create mode 100644 tests/actions/plugin-check-report/fixtures/expected/marker-with-line-breaks.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/marker-with-line-breaks.summary create mode 100644 tests/actions/plugin-check-report/fixtures/expected/marker.md create mode 100644 tests/actions/plugin-check-report/fixtures/expected/marker.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/marker.summary create mode 100644 tests/actions/plugin-check-report/fixtures/expected/results-empty.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/results-garbage.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/results-missing.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.md create mode 100644 tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.summary create mode 100644 tests/actions/plugin-check-report/fixtures/expected/same-as-baseline.md create mode 100644 tests/actions/plugin-check-report/fixtures/expected/same-as-baseline.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/same-as-baseline.summary create mode 100644 tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.md create mode 100644 tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.summary create mode 100644 tests/actions/plugin-check-report/fixtures/expected/six-new-codes.md create mode 100644 tests/actions/plugin-check-report/fixtures/expected/six-new-codes.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/six-new-codes.summary create mode 100644 tests/actions/plugin-check-report/fixtures/expected/status.json create mode 100644 tests/actions/plugin-check-report/fixtures/expected/strict-clean.md create mode 100644 tests/actions/plugin-check-report/fixtures/expected/strict-clean.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/strict-clean.summary create mode 100644 tests/actions/plugin-check-report/fixtures/expected/strict-errors-down.md create mode 100644 tests/actions/plugin-check-report/fixtures/expected/strict-errors-down.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/strict-errors-down.summary create mode 100644 tests/actions/plugin-check-report/fixtures/expected/strict-no-baseline.md create mode 100644 tests/actions/plugin-check-report/fixtures/expected/strict-no-baseline.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/strict-no-baseline.summary create mode 100644 tests/actions/plugin-check-report/fixtures/expected/strict-warnings-only.md create mode 100644 tests/actions/plugin-check-report/fixtures/expected/strict-warnings-only.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/strict-warnings-only.summary create mode 100644 tests/actions/plugin-check-report/fixtures/expected/unknown-mode.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/update-clean.json create mode 100644 tests/actions/plugin-check-report/fixtures/expected/update-clean.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/update-errors-up.json create mode 100644 tests/actions/plugin-check-report/fixtures/expected/update-errors-up.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/update-garbage.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/warnings-up.md create mode 100644 tests/actions/plugin-check-report/fixtures/expected/warnings-up.out create mode 100644 tests/actions/plugin-check-report/fixtures/expected/warnings-up.summary create mode 100644 tests/actions/plugin-check-report/golden.js create mode 100644 tests/actions/plugin-check-report/ratchet-golden.test.js diff --git a/tests/actions/plugin-check-report/fixtures/expected/artifact-only.md b/tests/actions/plugin-check-report/fixtures/expected/artifact-only.md new file mode 100644 index 0000000..f6e1a8c --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/artifact-only.md @@ -0,0 +1,9 @@ + +⚠️ Plugin Check ran, but there was no baseline to compare with + +| | Now | +|:--|--:| +| Errors | 0 | +| Warnings | 0 | + +📎 [Full results and new baseline](https://github.com/o/r/actions/runs/1/artifacts/2) diff --git a/tests/actions/plugin-check-report/fixtures/expected/artifact-only.out b/tests/actions/plugin-check-report/fixtures/expected/artifact-only.out new file mode 100644 index 0000000..ffed42c --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/artifact-only.out @@ -0,0 +1,2 @@ +::warning title=Plugin Check::No baseline to compare with (/nonexistent/baseline.json), so nothing was compared. Run the workflow on the target branch to make one. +Plugin Check ran with no baseline, so nothing was compared. diff --git a/tests/actions/plugin-check-report/fixtures/expected/artifact-only.summary b/tests/actions/plugin-check-report/fixtures/expected/artifact-only.summary new file mode 100644 index 0000000..f6e1a8c --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/artifact-only.summary @@ -0,0 +1,9 @@ + +⚠️ Plugin Check ran, but there was no baseline to compare with + +| | Now | +|:--|--:| +| Errors | 0 | +| Warnings | 0 | + +📎 [Full results and new baseline](https://github.com/o/r/actions/runs/1/artifacts/2) diff --git a/tests/actions/plugin-check-report/fixtures/expected/badge-and-artifact.md b/tests/actions/plugin-check-report/fixtures/expected/badge-and-artifact.md new file mode 100644 index 0000000..c9247b3 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/badge-and-artifact.md @@ -0,0 +1,13 @@ + +❌ Plugin Check failed: 1 rule has more errors than the baseline allows + +![Plugin Check finished](https://img.shields.io/badge/Check%20Finished-now-darkgreen) + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 4 | 3 | +1 | +| Warnings | 1 | 1 | ±0 | + +Went up: `WordPress.Security.EscapeOutput.OutputNotEscaped` (+1) + +📎 [Full results and new baseline](https://github.com/o/r/actions/runs/1/artifacts/2) · [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/badge-and-artifact.out b/tests/actions/plugin-check-report/fixtures/expected/badge-and-artifact.out new file mode 100644 index 0000000..90bf791 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/badge-and-artifact.out @@ -0,0 +1,2 @@ +::error title=Plugin Check::WordPress.Security.EscapeOutput.OutputNotEscaped has 3 errors, the baseline has 2. The full list is in the plugin-check-results artifact. +Plugin Check found more errors than the baseline allows. diff --git a/tests/actions/plugin-check-report/fixtures/expected/badge-and-artifact.summary b/tests/actions/plugin-check-report/fixtures/expected/badge-and-artifact.summary new file mode 100644 index 0000000..c9247b3 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/badge-and-artifact.summary @@ -0,0 +1,13 @@ + +❌ Plugin Check failed: 1 rule has more errors than the baseline allows + +![Plugin Check finished](https://img.shields.io/badge/Check%20Finished-now-darkgreen) + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 4 | 3 | +1 | +| Warnings | 1 | 1 | ±0 | + +Went up: `WordPress.Security.EscapeOutput.OutputNotEscaped` (+1) + +📎 [Full results and new baseline](https://github.com/o/r/actions/runs/1/artifacts/2) · [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.md b/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.md new file mode 100644 index 0000000..40ace2c --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.md @@ -0,0 +1,9 @@ + +⚠️ Plugin Check ran, but there was no baseline to compare with + +| | Now | +|:--|--:| +| Errors | 0 | +| Warnings | 0 | + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.out b/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.out new file mode 100644 index 0000000..ffed42c --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.out @@ -0,0 +1,2 @@ +::warning title=Plugin Check::No baseline to compare with (/nonexistent/baseline.json), so nothing was compared. Run the workflow on the target branch to make one. +Plugin Check ran with no baseline, so nothing was compared. diff --git a/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.summary b/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.summary new file mode 100644 index 0000000..40ace2c --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.summary @@ -0,0 +1,9 @@ + +⚠️ Plugin Check ran, but there was no baseline to compare with + +| | Now | +|:--|--:| +| Errors | 0 | +| Warnings | 0 | + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-down.md b/tests/actions/plugin-check-report/fixtures/expected/errors-down.md new file mode 100644 index 0000000..00b7a4c --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/errors-down.md @@ -0,0 +1,11 @@ + +✅ Plugin Check passed + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 2 | 3 | -1 | +| Warnings | 1 | 1 | ±0 | + +Rules that went down: 1. + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-down.out b/tests/actions/plugin-check-report/fixtures/expected/errors-down.out new file mode 100644 index 0000000..df39f8a --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/errors-down.out @@ -0,0 +1,2 @@ +::notice title=Plugin Check::WordPress.Security.EscapeOutput.OutputNotEscaped (errors) went down from 2 to 1. +Plugin Check error counts are within the baseline. diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-down.summary b/tests/actions/plugin-check-report/fixtures/expected/errors-down.summary new file mode 100644 index 0000000..00b7a4c --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/errors-down.summary @@ -0,0 +1,11 @@ + +✅ Plugin Check passed + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 2 | 3 | -1 | +| Warnings | 1 | 1 | ±0 | + +Rules that went down: 1. + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.md b/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.md new file mode 100644 index 0000000..c5e77f1 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.md @@ -0,0 +1,11 @@ + +❌ Plugin Check failed: 1 rule has more errors than the baseline allows + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 4 | 3 | +1 | +| Warnings | 1 | 1 | ±0 | + +Went up: `WordPress.Security.EscapeOutput.OutputNotEscaped` (+1) + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.out b/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.out new file mode 100644 index 0000000..90bf791 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.out @@ -0,0 +1,2 @@ +::error title=Plugin Check::WordPress.Security.EscapeOutput.OutputNotEscaped has 3 errors, the baseline has 2. The full list is in the plugin-check-results artifact. +Plugin Check found more errors than the baseline allows. diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.summary b/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.summary new file mode 100644 index 0000000..c5e77f1 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.summary @@ -0,0 +1,11 @@ + +❌ Plugin Check failed: 1 rule has more errors than the baseline allows + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 4 | 3 | +1 | +| Warnings | 1 | 1 | ±0 | + +Went up: `WordPress.Security.EscapeOutput.OutputNotEscaped` (+1) + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.md b/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.md new file mode 100644 index 0000000..be36055 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.md @@ -0,0 +1,11 @@ + +⚠️ Plugin Check passed, not enforced: 1 rule(s) have more errors than an older run of the target branch + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 4 | 3 | +1 | +| Warnings | 1 | 1 | ±0 | + +Went up: `WordPress.Security.EscapeOutput.OutputNotEscaped` (+1) + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.out b/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.out new file mode 100644 index 0000000..2418fe5 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.out @@ -0,0 +1,2 @@ +::warning title=Plugin Check::WordPress.Security.EscapeOutput.OutputNotEscaped has 3 errors, an older run of the target branch has 2. Not enforced, because the target may have changed since. +Plugin Check error counts are within the baseline. diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.summary b/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.summary new file mode 100644 index 0000000..be36055 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.summary @@ -0,0 +1,11 @@ + +⚠️ Plugin Check passed, not enforced: 1 rule(s) have more errors than an older run of the target branch + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 4 | 3 | +1 | +| Warnings | 1 | 1 | ±0 | + +Went up: `WordPress.Security.EscapeOutput.OutputNotEscaped` (+1) + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-up.md b/tests/actions/plugin-check-report/fixtures/expected/errors-up.md new file mode 100644 index 0000000..c5e77f1 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/errors-up.md @@ -0,0 +1,11 @@ + +❌ Plugin Check failed: 1 rule has more errors than the baseline allows + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 4 | 3 | +1 | +| Warnings | 1 | 1 | ±0 | + +Went up: `WordPress.Security.EscapeOutput.OutputNotEscaped` (+1) + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-up.out b/tests/actions/plugin-check-report/fixtures/expected/errors-up.out new file mode 100644 index 0000000..90bf791 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/errors-up.out @@ -0,0 +1,2 @@ +::error title=Plugin Check::WordPress.Security.EscapeOutput.OutputNotEscaped has 3 errors, the baseline has 2. The full list is in the plugin-check-results artifact. +Plugin Check found more errors than the baseline allows. diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-up.summary b/tests/actions/plugin-check-report/fixtures/expected/errors-up.summary new file mode 100644 index 0000000..c5e77f1 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/errors-up.summary @@ -0,0 +1,11 @@ + +❌ Plugin Check failed: 1 rule has more errors than the baseline allows + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 4 | 3 | +1 | +| Warnings | 1 | 1 | ±0 | + +Went up: `WordPress.Security.EscapeOutput.OutputNotEscaped` (+1) + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/label-and-url.md b/tests/actions/plugin-check-report/fixtures/expected/label-and-url.md new file mode 100644 index 0000000..1a85ac8 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/label-and-url.md @@ -0,0 +1,9 @@ + +✅ Plugin Check passed + +| | Now | [Target branch](https://github.com/o/r/actions/runs/1) | Change | +|:--|--:|--:|:--| +| Errors | 3 | 3 | ±0 | +| Warnings | 1 | 1 | ±0 | + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/label-and-url.out b/tests/actions/plugin-check-report/fixtures/expected/label-and-url.out new file mode 100644 index 0000000..0d8c877 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/label-and-url.out @@ -0,0 +1 @@ +Plugin Check error counts are within the baseline. diff --git a/tests/actions/plugin-check-report/fixtures/expected/label-and-url.summary b/tests/actions/plugin-check-report/fixtures/expected/label-and-url.summary new file mode 100644 index 0000000..1a85ac8 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/label-and-url.summary @@ -0,0 +1,9 @@ + +✅ Plugin Check passed + +| | Now | [Target branch](https://github.com/o/r/actions/runs/1) | Change | +|:--|--:|--:|:--| +| Errors | 3 | 3 | ±0 | +| Warnings | 1 | 1 | ±0 | + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/marker-with-line-breaks.md b/tests/actions/plugin-check-report/fixtures/expected/marker-with-line-breaks.md new file mode 100644 index 0000000..fa5abf2 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/marker-with-line-breaks.md @@ -0,0 +1,15 @@ +line 1 +line 2 +line 3 +line 4 +line 5 +line 6 +line 7 +line 8 +line 9 +line 10 +line 11 +line 12 +line 13 +line 14 +line 15 diff --git a/tests/actions/plugin-check-report/fixtures/expected/marker-with-line-breaks.out b/tests/actions/plugin-check-report/fixtures/expected/marker-with-line-breaks.out new file mode 100644 index 0000000..90bf791 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/marker-with-line-breaks.out @@ -0,0 +1,2 @@ +::error title=Plugin Check::WordPress.Security.EscapeOutput.OutputNotEscaped has 3 errors, the baseline has 2. The full list is in the plugin-check-results artifact. +Plugin Check found more errors than the baseline allows. diff --git a/tests/actions/plugin-check-report/fixtures/expected/marker-with-line-breaks.summary b/tests/actions/plugin-check-report/fixtures/expected/marker-with-line-breaks.summary new file mode 100644 index 0000000..fa5abf2 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/marker-with-line-breaks.summary @@ -0,0 +1,15 @@ +line 1 +line 2 +line 3 +line 4 +line 5 +line 6 +line 7 +line 8 +line 9 +line 10 +line 11 +line 12 +line 13 +line 14 +line 15 diff --git a/tests/actions/plugin-check-report/fixtures/expected/marker.md b/tests/actions/plugin-check-report/fixtures/expected/marker.md new file mode 100644 index 0000000..df1c76e --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/marker.md @@ -0,0 +1,9 @@ + +⚠️ Plugin Check ran, but there was no baseline to compare with + +| | Now | +|:--|--:| +| Errors | 0 | +| Warnings | 0 | + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/marker.out b/tests/actions/plugin-check-report/fixtures/expected/marker.out new file mode 100644 index 0000000..ffed42c --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/marker.out @@ -0,0 +1,2 @@ +::warning title=Plugin Check::No baseline to compare with (/nonexistent/baseline.json), so nothing was compared. Run the workflow on the target branch to make one. +Plugin Check ran with no baseline, so nothing was compared. diff --git a/tests/actions/plugin-check-report/fixtures/expected/marker.summary b/tests/actions/plugin-check-report/fixtures/expected/marker.summary new file mode 100644 index 0000000..df1c76e --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/marker.summary @@ -0,0 +1,9 @@ + +⚠️ Plugin Check ran, but there was no baseline to compare with + +| | Now | +|:--|--:| +| Errors | 0 | +| Warnings | 0 | + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/results-empty.out b/tests/actions/plugin-check-report/fixtures/expected/results-empty.out new file mode 100644 index 0000000..b79b7db --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/results-empty.out @@ -0,0 +1 @@ +::error title=Plugin Check::The results file has no Plugin Check output, so the check did not run. diff --git a/tests/actions/plugin-check-report/fixtures/expected/results-garbage.out b/tests/actions/plugin-check-report/fixtures/expected/results-garbage.out new file mode 100644 index 0000000..ec39073 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/results-garbage.out @@ -0,0 +1 @@ +::error title=Plugin Check::The results file is not Plugin Check output, so the check did not run. First unexpected line: diff --git a/tests/actions/plugin-check-report/fixtures/expected/results-missing.out b/tests/actions/plugin-check-report/fixtures/expected/results-missing.out new file mode 100644 index 0000000..1286ae1 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/results-missing.out @@ -0,0 +1 @@ +::error title=Plugin Check::Results file not found: /nonexistent/results.txt diff --git a/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.md b/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.md new file mode 100644 index 0000000..04294cc --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.md @@ -0,0 +1,12 @@ + +❌ Plugin Check failed: 1 rule has more errors than the baseline allows + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 3 | 3 | ±0 | +| Warnings | 1 | 1 | ±0 | + +Went up: `WordPress.Security.EscapeOutput.OutputNotEscaped` (+1) +Rules that went down: 1. + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.out b/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.out new file mode 100644 index 0000000..68830be --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.out @@ -0,0 +1,3 @@ +::error title=Plugin Check::WordPress.Security.EscapeOutput.OutputNotEscaped has 3 errors, the baseline has 2. The full list is in the plugin-check-results artifact. +::notice title=Plugin Check::WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents (errors) went down from 1 to 0. +Plugin Check found more errors than the baseline allows. diff --git a/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.summary b/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.summary new file mode 100644 index 0000000..04294cc --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.summary @@ -0,0 +1,12 @@ + +❌ Plugin Check failed: 1 rule has more errors than the baseline allows + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 3 | 3 | ±0 | +| Warnings | 1 | 1 | ±0 | + +Went up: `WordPress.Security.EscapeOutput.OutputNotEscaped` (+1) +Rules that went down: 1. + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/same-as-baseline.md b/tests/actions/plugin-check-report/fixtures/expected/same-as-baseline.md new file mode 100644 index 0000000..c965444 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/same-as-baseline.md @@ -0,0 +1,9 @@ + +✅ Plugin Check passed + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 3 | 3 | ±0 | +| Warnings | 1 | 1 | ±0 | + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/same-as-baseline.out b/tests/actions/plugin-check-report/fixtures/expected/same-as-baseline.out new file mode 100644 index 0000000..0d8c877 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/same-as-baseline.out @@ -0,0 +1 @@ +Plugin Check error counts are within the baseline. diff --git a/tests/actions/plugin-check-report/fixtures/expected/same-as-baseline.summary b/tests/actions/plugin-check-report/fixtures/expected/same-as-baseline.summary new file mode 100644 index 0000000..c965444 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/same-as-baseline.summary @@ -0,0 +1,9 @@ + +✅ Plugin Check passed + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 3 | 3 | ±0 | +| Warnings | 1 | 1 | ±0 | + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.md b/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.md new file mode 100644 index 0000000..0a97eea --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.md @@ -0,0 +1,12 @@ + +❌ Plugin Check failed: 7 rules have more errors than the baseline allows + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 7 | 3 | +4 | +| Warnings | 0 | 1 | -1 | + +Went up (top 5 of 7): `Test.Rule.Number1` (+1), `Test.Rule.Number2` (+1), `Test.Rule.Number3` (+1), `Test.Rule.Number4` (+1), `Test.Rule.Number5` (+1) +Rules that went down: 3. + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.out b/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.out new file mode 100644 index 0000000..c8c976c --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.out @@ -0,0 +1,11 @@ +::error title=Plugin Check::Test.Rule.Number1 has 1 errors, the baseline has 0. The full list is in the plugin-check-results artifact. +::error title=Plugin Check::Test.Rule.Number2 has 1 errors, the baseline has 0. The full list is in the plugin-check-results artifact. +::error title=Plugin Check::Test.Rule.Number3 has 1 errors, the baseline has 0. The full list is in the plugin-check-results artifact. +::error title=Plugin Check::Test.Rule.Number4 has 1 errors, the baseline has 0. The full list is in the plugin-check-results artifact. +::error title=Plugin Check::Test.Rule.Number5 has 1 errors, the baseline has 0. The full list is in the plugin-check-results artifact. +::error title=Plugin Check::Test.Rule.Number6 has 1 errors, the baseline has 0. The full list is in the plugin-check-results artifact. +::error title=Plugin Check::Test.Rule.Number7 has 1 errors, the baseline has 0. The full list is in the plugin-check-results artifact. +::notice title=Plugin Check::WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents (errors) went down from 1 to 0. +::notice title=Plugin Check::WordPress.Security.EscapeOutput.OutputNotEscaped (errors) went down from 2 to 0. +::notice title=Plugin Check::WordPress.DB.DirectDatabaseQuery.DirectQuery (warnings) went down from 1 to 0. +Plugin Check found more errors than the baseline allows. diff --git a/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.summary b/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.summary new file mode 100644 index 0000000..0a97eea --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.summary @@ -0,0 +1,12 @@ + +❌ Plugin Check failed: 7 rules have more errors than the baseline allows + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 7 | 3 | +4 | +| Warnings | 0 | 1 | -1 | + +Went up (top 5 of 7): `Test.Rule.Number1` (+1), `Test.Rule.Number2` (+1), `Test.Rule.Number3` (+1), `Test.Rule.Number4` (+1), `Test.Rule.Number5` (+1) +Rules that went down: 3. + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.md b/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.md new file mode 100644 index 0000000..9960632 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.md @@ -0,0 +1,12 @@ + +❌ Plugin Check failed: 6 rules have more errors than the baseline allows + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 6 | 3 | +3 | +| Warnings | 0 | 1 | -1 | + +Went up (top 5 of 6): `Test.Rule.Number1` (+1), `Test.Rule.Number2` (+1), `Test.Rule.Number3` (+1), `Test.Rule.Number4` (+1), `Test.Rule.Number5` (+1) +Rules that went down: 3. + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.out b/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.out new file mode 100644 index 0000000..733d898 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.out @@ -0,0 +1,10 @@ +::error title=Plugin Check::Test.Rule.Number1 has 1 errors, the baseline has 0. The full list is in the plugin-check-results artifact. +::error title=Plugin Check::Test.Rule.Number2 has 1 errors, the baseline has 0. The full list is in the plugin-check-results artifact. +::error title=Plugin Check::Test.Rule.Number3 has 1 errors, the baseline has 0. The full list is in the plugin-check-results artifact. +::error title=Plugin Check::Test.Rule.Number4 has 1 errors, the baseline has 0. The full list is in the plugin-check-results artifact. +::error title=Plugin Check::Test.Rule.Number5 has 1 errors, the baseline has 0. The full list is in the plugin-check-results artifact. +::error title=Plugin Check::Test.Rule.Number6 has 1 errors, the baseline has 0. The full list is in the plugin-check-results artifact. +::notice title=Plugin Check::WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents (errors) went down from 1 to 0. +::notice title=Plugin Check::WordPress.Security.EscapeOutput.OutputNotEscaped (errors) went down from 2 to 0. +::notice title=Plugin Check::WordPress.DB.DirectDatabaseQuery.DirectQuery (warnings) went down from 1 to 0. +Plugin Check found more errors than the baseline allows. diff --git a/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.summary b/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.summary new file mode 100644 index 0000000..9960632 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.summary @@ -0,0 +1,12 @@ + +❌ Plugin Check failed: 6 rules have more errors than the baseline allows + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 6 | 3 | +3 | +| Warnings | 0 | 1 | -1 | + +Went up (top 5 of 6): `Test.Rule.Number1` (+1), `Test.Rule.Number2` (+1), `Test.Rule.Number3` (+1), `Test.Rule.Number4` (+1), `Test.Rule.Number5` (+1) +Rules that went down: 3. + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/status.json b/tests/actions/plugin-check-report/fixtures/expected/status.json new file mode 100644 index 0000000..ad302bd --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/status.json @@ -0,0 +1,28 @@ +{ + "clean-no-baseline": 0, + "same-as-baseline": 0, + "errors-up": 1, + "rule-up-total-same": 1, + "errors-down": 0, + "warnings-up": 0, + "errors-up-advisory": 0, + "errors-up-advisory-uppercase": 1, + "strict-no-baseline": 1, + "strict-errors-down": 1, + "strict-clean": 0, + "strict-warnings-only": 0, + "label-and-url": 0, + "seven-new-codes": 1, + "six-new-codes": 1, + "badge-and-artifact": 1, + "artifact-only": 0, + "marker": 0, + "marker-with-line-breaks": 1, + "results-missing": 1, + "results-garbage": 1, + "results-empty": 1, + "unknown-mode": 1, + "update-errors-up": 0, + "update-clean": 0, + "update-garbage": 1 +} diff --git a/tests/actions/plugin-check-report/fixtures/expected/strict-clean.md b/tests/actions/plugin-check-report/fixtures/expected/strict-clean.md new file mode 100644 index 0000000..f6d8e1f --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/strict-clean.md @@ -0,0 +1,9 @@ + +✅ Plugin Check passed: no errors + +| | Now | +|:--|--:| +| Errors | 0 | +| Warnings | 0 | + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/strict-clean.out b/tests/actions/plugin-check-report/fixtures/expected/strict-clean.out new file mode 100644 index 0000000..ffed42c --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/strict-clean.out @@ -0,0 +1,2 @@ +::warning title=Plugin Check::No baseline to compare with (/nonexistent/baseline.json), so nothing was compared. Run the workflow on the target branch to make one. +Plugin Check ran with no baseline, so nothing was compared. diff --git a/tests/actions/plugin-check-report/fixtures/expected/strict-clean.summary b/tests/actions/plugin-check-report/fixtures/expected/strict-clean.summary new file mode 100644 index 0000000..f6d8e1f --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/strict-clean.summary @@ -0,0 +1,9 @@ + +✅ Plugin Check passed: no errors + +| | Now | +|:--|--:| +| Errors | 0 | +| Warnings | 0 | + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/strict-errors-down.md b/tests/actions/plugin-check-report/fixtures/expected/strict-errors-down.md new file mode 100644 index 0000000..a8f2c38 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/strict-errors-down.md @@ -0,0 +1,11 @@ + +❌ Plugin Check failed: 2 errors, and strict mode allows none + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 2 | 3 | -1 | +| Warnings | 1 | 1 | ±0 | + +Rules that went down: 1. + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/strict-errors-down.out b/tests/actions/plugin-check-report/fixtures/expected/strict-errors-down.out new file mode 100644 index 0000000..46ba4e0 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/strict-errors-down.out @@ -0,0 +1,3 @@ +::notice title=Plugin Check::WordPress.Security.EscapeOutput.OutputNotEscaped (errors) went down from 2 to 1. +::error title=Plugin Check::2 errors found. Strict mode fails on any error. +Plugin Check found errors, and strict mode allows none. diff --git a/tests/actions/plugin-check-report/fixtures/expected/strict-errors-down.summary b/tests/actions/plugin-check-report/fixtures/expected/strict-errors-down.summary new file mode 100644 index 0000000..a8f2c38 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/strict-errors-down.summary @@ -0,0 +1,11 @@ + +❌ Plugin Check failed: 2 errors, and strict mode allows none + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 2 | 3 | -1 | +| Warnings | 1 | 1 | ±0 | + +Rules that went down: 1. + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/strict-no-baseline.md b/tests/actions/plugin-check-report/fixtures/expected/strict-no-baseline.md new file mode 100644 index 0000000..4fd50c5 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/strict-no-baseline.md @@ -0,0 +1,9 @@ + +❌ Plugin Check failed: 2 errors, and strict mode allows none + +| | Now | +|:--|--:| +| Errors | 2 | +| Warnings | 1 | + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/strict-no-baseline.out b/tests/actions/plugin-check-report/fixtures/expected/strict-no-baseline.out new file mode 100644 index 0000000..bc6ddf6 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/strict-no-baseline.out @@ -0,0 +1,3 @@ +::warning title=Plugin Check::No baseline to compare with (/nonexistent/baseline.json), so nothing was compared. Run the workflow on the target branch to make one. +::error title=Plugin Check::2 errors found. Strict mode fails on any error. +Plugin Check found errors, and strict mode allows none. diff --git a/tests/actions/plugin-check-report/fixtures/expected/strict-no-baseline.summary b/tests/actions/plugin-check-report/fixtures/expected/strict-no-baseline.summary new file mode 100644 index 0000000..4fd50c5 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/strict-no-baseline.summary @@ -0,0 +1,9 @@ + +❌ Plugin Check failed: 2 errors, and strict mode allows none + +| | Now | +|:--|--:| +| Errors | 2 | +| Warnings | 1 | + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/strict-warnings-only.md b/tests/actions/plugin-check-report/fixtures/expected/strict-warnings-only.md new file mode 100644 index 0000000..d2368c1 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/strict-warnings-only.md @@ -0,0 +1,9 @@ + +✅ Plugin Check passed: no errors + +| | Now | +|:--|--:| +| Errors | 0 | +| Warnings | 3 | + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/strict-warnings-only.out b/tests/actions/plugin-check-report/fixtures/expected/strict-warnings-only.out new file mode 100644 index 0000000..ffed42c --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/strict-warnings-only.out @@ -0,0 +1,2 @@ +::warning title=Plugin Check::No baseline to compare with (/nonexistent/baseline.json), so nothing was compared. Run the workflow on the target branch to make one. +Plugin Check ran with no baseline, so nothing was compared. diff --git a/tests/actions/plugin-check-report/fixtures/expected/strict-warnings-only.summary b/tests/actions/plugin-check-report/fixtures/expected/strict-warnings-only.summary new file mode 100644 index 0000000..d2368c1 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/strict-warnings-only.summary @@ -0,0 +1,9 @@ + +✅ Plugin Check passed: no errors + +| | Now | +|:--|--:| +| Errors | 0 | +| Warnings | 3 | + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/unknown-mode.out b/tests/actions/plugin-check-report/fixtures/expected/unknown-mode.out new file mode 100644 index 0000000..45bf6be --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/unknown-mode.out @@ -0,0 +1 @@ +::error title=Plugin Check::Unknown mode 'strcit'. Use 'ratchet' or 'strict'. diff --git a/tests/actions/plugin-check-report/fixtures/expected/update-clean.json b/tests/actions/plugin-check-report/fixtures/expected/update-clean.json new file mode 100644 index 0000000..75fdfd5 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/update-clean.json @@ -0,0 +1,4 @@ +{ + "errors": {}, + "warnings": {} +} diff --git a/tests/actions/plugin-check-report/fixtures/expected/update-clean.out b/tests/actions/plugin-check-report/fixtures/expected/update-clean.out new file mode 100644 index 0000000..1c5ea8d --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/update-clean.out @@ -0,0 +1 @@ +Wrote /baseline.json: 0 error codes, 0 warning codes diff --git a/tests/actions/plugin-check-report/fixtures/expected/update-errors-up.json b/tests/actions/plugin-check-report/fixtures/expected/update-errors-up.json new file mode 100644 index 0000000..5344a3d --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/update-errors-up.json @@ -0,0 +1,9 @@ +{ + "errors": { + "WordPress.Security.EscapeOutput.OutputNotEscaped": 3, + "WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents": 1 + }, + "warnings": { + "WordPress.DB.DirectDatabaseQuery.DirectQuery": 1 + } +} diff --git a/tests/actions/plugin-check-report/fixtures/expected/update-errors-up.out b/tests/actions/plugin-check-report/fixtures/expected/update-errors-up.out new file mode 100644 index 0000000..4240312 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/update-errors-up.out @@ -0,0 +1 @@ +Wrote /baseline.json: 2 error codes, 1 warning codes diff --git a/tests/actions/plugin-check-report/fixtures/expected/update-garbage.out b/tests/actions/plugin-check-report/fixtures/expected/update-garbage.out new file mode 100644 index 0000000..ec39073 --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/update-garbage.out @@ -0,0 +1 @@ +::error title=Plugin Check::The results file is not Plugin Check output, so the check did not run. First unexpected line: diff --git a/tests/actions/plugin-check-report/fixtures/expected/warnings-up.md b/tests/actions/plugin-check-report/fixtures/expected/warnings-up.md new file mode 100644 index 0000000..5d8ef4f --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/warnings-up.md @@ -0,0 +1,11 @@ + +✅ Plugin Check passed + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 3 | 3 | ±0 | +| Warnings | 3 | 1 | +2 | + +Went up: `WordPress.DB.DirectDatabaseQuery.DirectQuery` warning (+2) + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/warnings-up.out b/tests/actions/plugin-check-report/fixtures/expected/warnings-up.out new file mode 100644 index 0000000..2211d5d --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/warnings-up.out @@ -0,0 +1,2 @@ +::notice title=Plugin Check::WordPress.DB.DirectDatabaseQuery.DirectQuery has 3 warnings, the baseline has 1. Warnings do not fail the check. +Plugin Check error counts are within the baseline. diff --git a/tests/actions/plugin-check-report/fixtures/expected/warnings-up.summary b/tests/actions/plugin-check-report/fixtures/expected/warnings-up.summary new file mode 100644 index 0000000..5d8ef4f --- /dev/null +++ b/tests/actions/plugin-check-report/fixtures/expected/warnings-up.summary @@ -0,0 +1,11 @@ + +✅ Plugin Check passed + +| | Now | Baseline | Change | +|:--|--:|--:|:--| +| Errors | 3 | 3 | ±0 | +| Warnings | 3 | 1 | +2 | + +Went up: `WordPress.DB.DirectDatabaseQuery.DirectQuery` warning (+2) + +📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/golden.js b/tests/actions/plugin-check-report/golden.js new file mode 100644 index 0000000..4b19b12 --- /dev/null +++ b/tests/actions/plugin-check-report/golden.js @@ -0,0 +1,234 @@ +'use strict' + +/** + * The cases whose exact output is pinned in fixtures/expected/. + * + * The files there were written by the original ratchet.sh. Every implementation of the compare has + * to print the same status, the same stdout, the same comment, the same job summary and, for + * --update, the same baseline file, byte for byte. A case is described once here and run by a + * `execute` function, so the same list checks the bash script and the JavaScript that replaces it. + */ + +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') + +const FIXTURES = path.join(__dirname, 'fixtures') +const EXPECTED = path.join(FIXTURES, 'expected') + +/** A path that never exists, so a case without a baseline gets none and prints a fixed path. */ +const NO_BASELINE = '/nonexistent/baseline.json' + +/** + * A results file with one new ERROR code per entry. + * + * @param {number} codes How many different codes to write. + * + * @returns {string} The file's text. + */ +function newCodes(codes) { + const results = Array.from({ length: codes }, (_, index) => ({ + line: index + 1, + column: 1, + type: 'ERROR', + code: `Test.Rule.Number${index + 1}`, + message: 'A rule added for the test.', + docs: '', + })) + + return `FILE: plugin.php\n${JSON.stringify(results)}\n` +} + +/** + * The warnings-up sample without its ERROR results. + * + * @returns {string} The file's text. + */ +function warningsOnly() { + const text = fs.readFileSync(path.join(FIXTURES, 'warnings-up.txt'), 'utf8') + + return text + .split('\n') + .map((line) => { + if (!line.startsWith('[')) { + return line + } + + return JSON.stringify(JSON.parse(line).filter((result) => result.type === 'WARNING')) + }) + .join('\n') +} + +const RUN = { RUN_URL: 'https://example.test/actions/runs/1' } +const BASELINE = 'baseline.json' + +/** + * Each case: a name, the results (a fixture name, or `text` to write), the baseline fixture to use, + * extra arguments, and the environment. + */ +const CASES = [ + { name: 'clean-no-baseline', results: 'clean.txt', env: RUN }, + { name: 'same-as-baseline', results: 'same-as-baseline.txt', baseline: BASELINE, env: RUN }, + { name: 'errors-up', results: 'errors-up.txt', baseline: BASELINE, env: RUN }, + { name: 'rule-up-total-same', results: 'rule-up-total-same.txt', baseline: BASELINE, env: RUN }, + { name: 'errors-down', results: 'errors-down.txt', baseline: BASELINE, env: RUN }, + { name: 'warnings-up', results: 'warnings-up.txt', baseline: BASELINE, env: RUN }, + { + name: 'errors-up-advisory', + results: 'errors-up.txt', + baseline: BASELINE, + env: { ...RUN, PLUGIN_CHECK_ADVISORY: 'true' }, + }, + { + name: 'errors-up-advisory-uppercase', + results: 'errors-up.txt', + baseline: BASELINE, + env: { ...RUN, PLUGIN_CHECK_ADVISORY: 'TRUE' }, + }, + { name: 'strict-no-baseline', results: 'errors-down.txt', env: { ...RUN, PLUGIN_CHECK_MODE: 'strict' } }, + { + name: 'strict-errors-down', + results: 'errors-down.txt', + baseline: BASELINE, + env: { ...RUN, PLUGIN_CHECK_MODE: 'strict' }, + }, + { name: 'strict-clean', results: 'clean.txt', env: { ...RUN, PLUGIN_CHECK_MODE: 'strict' } }, + { name: 'strict-warnings-only', text: warningsOnly, env: { ...RUN, PLUGIN_CHECK_MODE: 'strict' } }, + { + name: 'label-and-url', + results: 'same-as-baseline.txt', + baseline: BASELINE, + env: { ...RUN, BASELINE_LABEL: 'Target branch', BASELINE_URL: 'https://github.com/o/r/actions/runs/1' }, + }, + { name: 'seven-new-codes', text: () => newCodes(7), baseline: BASELINE, env: RUN }, + { name: 'six-new-codes', text: () => newCodes(6), baseline: BASELINE, env: RUN }, + { + name: 'badge-and-artifact', + results: 'errors-up.txt', + baseline: BASELINE, + env: { + ...RUN, + COMMENT_BADGE: 'https://img.shields.io/badge/Check%20Finished-now-darkgreen', + ARTIFACT_URL: 'https://github.com/o/r/actions/runs/1/artifacts/2', + }, + }, + { name: 'artifact-only', results: 'clean.txt', env: { ARTIFACT_URL: 'https://github.com/o/r/actions/runs/1/artifacts/2' } }, + { + name: 'marker', + results: 'clean.txt', + env: { ...RUN, COMMENT_MARKER: '' }, + }, + { + name: 'marker-with-line-breaks', + results: 'errors-up.txt', + baseline: BASELINE, + env: { ...RUN, COMMENT_MARKER: Array.from({ length: 20 }, (_, index) => `line ${index + 1}`).join('\n') }, + }, + { name: 'results-missing', results: false, args: ['/nonexistent/results.txt'], env: RUN }, + { name: 'results-garbage', results: 'garbage.txt', env: RUN }, + { name: 'results-empty', results: 'empty.txt', env: RUN }, + { name: 'unknown-mode', results: 'clean.txt', env: { ...RUN, PLUGIN_CHECK_MODE: 'strcit' } }, + { name: 'update-errors-up', results: 'errors-up.txt', update: true, env: RUN }, + { name: 'update-clean', results: 'clean.txt', update: true, env: RUN }, + { name: 'update-garbage', results: 'garbage.txt', update: true, env: RUN }, +] + +/** + * Runs one case. + * + * @param {object} spec A case from CASES. + * @param {Function} execute `(args, env) => {status, stdout}`. Runs the script under test with the + * arguments and environment given. + * + * @returns {{status: number, stdout: string, comment: string, summary: string, baseline: string}} + * What the script did. The temporary folder in stdout is written ``. + */ +function runCase(spec, execute) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-golden-')) + const comment = path.join(dir, 'comment.md') + const summary = path.join(dir, 'summary.md') + const written = path.join(dir, 'baseline.json') + + let results + if (spec.text) { + results = path.join(dir, 'results.txt') + fs.writeFileSync(results, spec.text()) + } else { + results = spec.args ? spec.args[0] : path.join(FIXTURES, spec.results) + } + + const env = { + GITHUB_STEP_SUMMARY: summary, + PLUGIN_CHECK_BASELINE: spec.update ? written : spec.baseline ? path.join(FIXTURES, spec.baseline) : NO_BASELINE, + ...spec.env, + } + const args = spec.update ? ['--update', results] : ['--comment-file', comment, results] + const result = execute(args, env) + + const read = (file) => (fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : '') + + return { + status: result.status, + stdout: result.stdout.split(dir).join(''), + comment: read(comment), + summary: read(summary), + baseline: read(written), + } +} + +/** + * The pinned files of a case, as written by the original script. + * + * @param {string} name A case name. + * + * @returns {{status: number, stdout: string, comment: string, summary: string, baseline: string}} + * The expected result. + */ +function expectedFor(name) { + const read = (extension) => { + const file = path.join(EXPECTED, `${name}.${extension}`) + + return fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : '' + } + const statuses = JSON.parse(fs.readFileSync(path.join(EXPECTED, 'status.json'), 'utf8')) + + return { + status: statuses[name], + stdout: read('out'), + comment: read('md'), + summary: read('summary'), + baseline: read('json'), + } +} + +/** + * Writes the pinned files from a run of the script under test. Only the original bash script is + * allowed to make them: run it once, review the diff, and commit. + * + * @param {object[]} outcomes `{name, result}` for each case. + */ +function writeExpected(outcomes) { + fs.mkdirSync(EXPECTED, { recursive: true }) + const statuses = {} + + for (const { name, result } of outcomes) { + statuses[name] = result.status + for (const [extension, text] of [ + ['out', result.stdout], + ['md', result.comment], + ['summary', result.summary], + ['json', result.baseline], + ]) { + const file = path.join(EXPECTED, `${name}.${extension}`) + if (text === '') { + fs.rmSync(file, { force: true }) + } else { + fs.writeFileSync(file, text) + } + } + } + + fs.writeFileSync(path.join(EXPECTED, 'status.json'), `${JSON.stringify(statuses, null, 2)}\n`) +} + +module.exports = { CASES, FIXTURES, expectedFor, runCase, writeExpected } diff --git a/tests/actions/plugin-check-report/ratchet-golden.test.js b/tests/actions/plugin-check-report/ratchet-golden.test.js new file mode 100644 index 0000000..6752f92 --- /dev/null +++ b/tests/actions/plugin-check-report/ratchet-golden.test.js @@ -0,0 +1,42 @@ +'use strict' + +const { describe, it } = require('node:test') +const assert = require('node:assert') +const path = require('node:path') +const { spawnSync } = require('node:child_process') + +const { CASES, expectedFor, runCase, writeExpected } = require('./golden.js') + +const RATCHET = path.join(__dirname, '../../../.github/actions/plugin-check-report/ratchet.sh') + +/** + * Runs ratchet.sh with a built environment, like ratchet.test.js does: only PATH and HOME are kept, + * so the runner's own GITHUB_* variables never reach the script. + * + * @param {string[]} args Arguments after the script. + * @param {object} env The environment to add. + * + * @returns {{status: number, stdout: string}} The exit status and what the script printed. + */ +function execute(args, env) { + const result = spawnSync('bash', [RATCHET, ...args], { + encoding: 'utf8', + env: { PATH: process.env.PATH, HOME: process.env.HOME, ...env }, + }) + + return { status: result.status, stdout: result.stdout } +} + +// UPDATE_GOLDEN=1 rewrites fixtures/expected/ from this script. The files are the specification the +// JavaScript has to match, so rewrite them only from the original script, and read the diff. +if (process.env.UPDATE_GOLDEN === '1') { + writeExpected(CASES.map((spec) => ({ name: spec.name, result: runCase(spec, execute) }))) +} + +describe('ratchet.sh, pinned output', () => { + for (const spec of CASES) { + it(`prints the pinned output for ${spec.name}`, () => { + assert.deepStrictEqual(runCase(spec, execute), expectedFor(spec.name)) + }) + } +}) From 207af3434454699804f066b84a58589577d764f6 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Tue, 3 Nov 2026 18:36:29 -0300 Subject: [PATCH 13/31] Add the Plugin Check compare in JavaScript, next to the bash script --- .../plugin-check-report.js | 906 ++++++++++++++++++ .../plugin-check-report/compare.test.js | 572 +++++++++++ 2 files changed, 1478 insertions(+) create mode 100644 .github/actions/plugin-check-report/plugin-check-report.js create mode 100644 tests/actions/plugin-check-report/compare.test.js diff --git a/.github/actions/plugin-check-report/plugin-check-report.js b/.github/actions/plugin-check-report/plugin-check-report.js new file mode 100644 index 0000000..be5a69f --- /dev/null +++ b/.github/actions/plugin-check-report/plugin-check-report.js @@ -0,0 +1,906 @@ +'use strict' + +/** + * Compares WordPress.org Plugin Check results with a baseline, and finds the baseline. + * + * The baseline is the count of errors and warnings per rule code of an earlier run of the same + * workflow, kept as the `plugin-check-report` artifact of that run. Nothing is committed. + * + * Two jobs: + * compare Counts the results, compares them with a baseline file, writes the pull request + * comment and the job summary, and says whether the check passes. With `update` it only + * writes the baseline file for this run. + * fetch Finds the earlier run that has a baseline and downloads its baseline.json. + * + * The small functions are pure, so a test can call them with plain values. The functions that talk + * to GitHub take their client as an argument, so a test can pass a real Octokit with the network + * replaced, as the other actions do. The same code runs in three places: the action's steps + * (`runPaths`, `runUpdate`, `runFetch`, `runCompare`, through actions/github-script), a command line + * (`node plugin-check-report.js compare|fetch ...`), and the tests. + * + * Every function that decides an exit status returns it: 0 passes, 1 is a failed check or a missing + * baseline, 2 is a lookup that could not be trusted. A step sets `process.exitCode` from it, so the + * step fails without github-script adding a second error. + */ + +const fs = require('node:fs') +const path = require('node:path') + +/** Title of every annotation. */ +const TITLE = 'Plugin Check' + +/** The pull request comment must stay short. Everything else is behind the links. */ +const MAX_COMMENT_LINES = 15 +const MAX_RULES_LISTED = 5 + +/** The first unexpected line of a results file is shown cut to this many bytes. */ +const MAX_UNEXPECTED_BYTES = 200 + +const MODES = ['ratchet', 'strict'] +const DEFAULT_MODE = 'ratchet' + +/** A line of Plugin Check output: a file name, a JSON array of its results, or the clean message. */ +const KNOWN_LINE = /^(FILE: |\[|Success:)/ +const OUTPUT_LINE = /^(FILE: |Success:)/ +/** Blank as `grep '[[:space:]]'` reads it: ASCII white space only. */ +const BLANK_LINE = /^[ \t\n\v\f\r]*$/ + +/** + * A failure the person running the check has to fix: the message is shown, the step fails with the + * exit code, and no stack trace helps. + */ +class CheckFailure extends Error { + /** + * @param {string} message What went wrong, in plain words. + * @param {number} exitCode The exit status for the step. + */ + constructor(message, exitCode = 1) { + super(message) + this.name = 'CheckFailure' + this.exitCode = exitCode + } +} + +/** + * Orders two strings by Unicode code point, as `jq` sorts them. JavaScript's default order compares + * UTF-16 units, which puts an emoji before U+FFFF; UTF-8 bytes sort by code point. + * + * @param {string} a First string. + * @param {string} b Second string. + * + * @returns {number} Negative, zero or positive. + */ +function compareCodePoints(a, b) { + return Buffer.compare(Buffer.from(a), Buffer.from(b)) +} + +/** + * Cuts a string to a number of UTF-8 bytes. A character cut in the middle shows as U+FFFD. + * + * @param {string} text The text. + * @param {number} bytes The most bytes to keep. + * + * @returns {string} The text, or its first bytes. + */ +function cutBytes(text, bytes) { + const buffer = Buffer.from(text) + + return buffer.length <= bytes ? text : buffer.subarray(0, bytes).toString('utf8') +} + +/** + * Splits a run of JSON values written one after another, as `jq` reads its input: whitespace + * between values, any number of them. + * + * @param {string} text The text. + * + * @throws {CheckFailure} When it is not JSON. + * + * @returns {Array} The values, in order. + */ +function readJsonValues(text) { + const values = [] + let position = 0 + + while (true) { + while (position < text.length && /\s/.test(text[position])) { + position++ + } + if (position >= text.length) { + return values + } + + const end = endOfJsonValue(text, position) + try { + values.push(JSON.parse(text.slice(position, end))) + } catch (error) { + throw new CheckFailure(`jq: parse error: ${error.message}`, 2) + } + position = end + } +} + +/** + * Finds where the JSON value that starts at a position ends. + * + * @param {string} text The text. + * @param {number} start Index of the value's first character. + * + * @throws {CheckFailure} When the value never ends. + * + * @returns {number} Index just after the value. + */ +function endOfJsonValue(text, start) { + const first = text[start] + + if (first === '[' || first === '{') { + let depth = 0 + let inString = false + + for (let index = start; index < text.length; index++) { + const char = text[index] + + if (inString) { + if (char === '\\') { + index++ + } else if (char === '"') { + inString = false + } + } else if (char === '"') { + inString = true + } else if (char === '[' || char === '{') { + depth++ + } else if ((char === ']' || char === '}') && --depth === 0) { + return index + 1 + } + } + + throw new CheckFailure('jq: parse error: Unfinished JSON term at EOF', 2) + } + + const scalar = /^("(?:[^"\\]|\\.)*"|[^\s[\]{}",:]+)/.exec(text.slice(start)) + if (!scalar) { + throw new CheckFailure('jq: parse error: Invalid JSON', 2) + } + + return start + scalar[0].length +} + +/** + * Why a results file cannot be Plugin Check's output, or null when it can. + * + * Anything else in the file (an HTML database error, a PHP fatal) means the check itself broke. + * Without this it would count as "no results" and pass. A clean run prints "Success: ..." and a run + * with results prints "FILE: ..." lines; an empty file means the command failed with its error on + * stderr, which the results file does not hold. + * + * @param {string} text The results file. + * + * @returns {string|null} The message, or null. + */ +function checkResultsText(text) { + const lines = text.split('\n') + const unexpected = lines.find((line) => !KNOWN_LINE.test(line) && !BLANK_LINE.test(line)) + + if (unexpected !== undefined) { + return `The results file is not Plugin Check output, so the check did not run. First unexpected line: ${cutBytes(unexpected, MAX_UNEXPECTED_BYTES)}` + } + + if (!lines.some((line) => OUTPUT_LINE.test(line))) { + return 'The results file has no Plugin Check output, so the check did not run.' + } + + return null +} + +/** + * The results of a file: every line that starts with `[` holds a JSON array of one file's results. + * + * @param {string} text The results file. + * + * @throws {CheckFailure} When a line is not JSON, or does not hold what Plugin Check prints. + * + * @returns {object[]} All results, in file order. + */ +function parseResults(text) { + const results = [] + + for (const line of text.split('\n')) { + if (!line.startsWith('[')) { + continue + } + + let values + try { + values = readJsonValues(line) + } catch (error) { + throw new CheckFailure(error.message, 5) + } + + for (const value of values) { + if (!Array.isArray(value)) { + throw new CheckFailure('A line of the results file does not hold a list of results.', 5) + } + for (const item of value) { + results.push(item) + } + } + } + + return results +} + +/** + * Counts the ERROR and WARNING results per rule code. + * + * A `null` item is skipped, as jq skips it. Any other item that is not an object, and any counted + * result whose code is not a string, fails: a rule with no name cannot be compared. + * + * @param {object[]} results From parseResults. + * + * @throws {CheckFailure} For an item or a code of the wrong type. + * + * @returns {{errors: Map, warnings: Map}} Counts, codes sorted by + * Unicode code point. + */ +function countPerCode(results) { + for (const item of results) { + if (item !== null && (typeof item !== 'object' || Array.isArray(item))) { + throw new CheckFailure(`A Plugin Check result is not an object: ${JSON.stringify(item)}`, 5) + } + } + + const count = (type) => { + const counts = new Map() + + for (const item of results) { + if (item === null || item.type !== type) { + continue + } + if (typeof item.code !== 'string') { + throw new CheckFailure(`A Plugin Check ${type} result has no rule code: ${JSON.stringify(item)}`, 5) + } + counts.set(item.code, (counts.get(item.code) ?? 0) + 1) + } + + return new Map([...counts].sort(([a], [b]) => compareCodePoints(a, b))) + } + + return { errors: count('ERROR'), warnings: count('WARNING') } +} + +/** + * The text of a baseline file: sorted keys, two-space indent, one final newline, and DEL written as + * \u007f, exactly as `jq -S` prints it. Built by hand, because an object puts keys such as "10" + * before "9" whatever order they were added in. + * + * @param {{errors: Map, warnings: Map}} counts From countPerCode. + * + * @returns {string} The file's text. + */ +function formatBaseline(counts) { + const kind = (map) => { + if (map.size === 0) { + return '{}' + } + + const lines = [...map].map(([code, count]) => ` ${JSON.stringify(code).replace(/\x7f/g, '\\u007f')}: ${count}`) + + return `{\n${lines.join(',\n')}\n }` + } + + return `{\n "errors": ${kind(counts.errors)},\n "warnings": ${kind(counts.warnings)}\n}\n` +} + +/** + * Reads a baseline file. + * + * An empty file, or one that holds `null`, is an empty baseline. A kind that is `null` or missing + * is empty. A count that is `null` or `false` is zero. Anything that cannot be compared fails. + * + * @param {string} text The baseline file. + * + * @throws {CheckFailure} When it is not a baseline. + * + * @returns {{errors: Map, warnings: Map}} Counts. + */ +function readBaseline(text) { + const empty = { errors: new Map(), warnings: new Map() } + const [value] = readJsonValues(text) + + if (value === undefined || value === null) { + return empty + } + + const invalid = (reason) => new CheckFailure(`The baseline is not valid: ${reason}`, 5) + + if (typeof value !== 'object' || Array.isArray(value)) { + throw invalid('it must be an object with "errors" and "warnings".') + } + + const counts = {} + for (const kind of ['errors', 'warnings']) { + const entries = value[kind] + counts[kind] = new Map() + + if (entries === undefined || entries === null) { + continue + } + if (typeof entries !== 'object' || Array.isArray(entries)) { + throw invalid(`"${kind}" must be an object of counts per rule code.`) + } + + for (const [code, count] of Object.entries(entries)) { + if (count === null || count === false) { + counts[kind].set(code, 0) + } else if (typeof count === 'number' && Number.isInteger(count)) { + counts[kind].set(code, count) + } else { + throw new CheckFailure( + `The baseline is not valid: the count of ${displayCode(code)} in "${kind}" must be a whole number.`, + typeof count === 'number' ? 1 : 5 + ) + } + } + } + + return counts +} + +/** + * A rule code as it is shown in a message: a backslash, a tab, a newline and a carriage return are + * written out, as jq's @tsv writes them. + * + * @param {string} code The rule code. + * + * @returns {string} The code, safe to show on one line. + */ +function displayCode(code) { + return code.replace(/\\/g, '\\\\').replace(/\t/g, '\\t').replace(/\n/g, '\\n').replace(/\r/g, '\\r') +} + +/** + * One row for each rule code found in either set of counts, for errors and then warnings. + * + * Rules that went up come first, errors before warnings, the biggest increase first; the rest keep + * their order by kind and code. The sort is stable, as jq's is, so the order is the same every time. + * + * @param {object} now This run's counts. + * @param {object} base The baseline's counts. + * + * @returns {{kind: string, code: string, base: number, now: number}[]} The rows. + */ +function buildRows(now, base) { + const rows = [] + + for (const kind of ['errors', 'warnings']) { + const codes = [...new Set([...now[kind].keys(), ...base[kind].keys()])].sort(compareCodePoints) + + for (const code of codes) { + rows.push({ kind, code, base: base[kind].get(code) ?? 0, now: now[kind].get(code) ?? 0 }) + } + } + + return rows.sort( + (a, b) => + (a.now > a.base ? 0 : 1) - (b.now > b.base ? 0 : 1) || + (a.kind === 'errors' ? 0 : 1) - (b.kind === 'errors' ? 0 : 1) || + -(a.now - a.base) - -(b.now - b.base) + ) +} + +/** + * Judges the rows: which messages to print, the totals, and whether the check fails. + * + * @param {object} input + * @param {object[]} input.rows From buildRows. + * @param {string} input.mode `ratchet` or `strict`. + * @param {boolean} input.advisory The baseline is from an older commit than the target, so more + * errors are only reported. + * @param {boolean} input.noBaseline There was nothing to compare with: only the totals count. + * + * @returns {object} `messages` (`{level, text}` in print order), `totals`, `wentUp`, `wentDown`, + * `errorsUp`, `upList` and `failed`. + */ +function evaluate({ rows, mode, advisory, noBaseline }) { + const totals = { errorsBase: 0, errorsNow: 0, warningsBase: 0, warningsNow: 0 } + const messages = [] + let failed = false + let wentDown = 0 + let wentUp = 0 + let errorsUp = 0 + let upList = '' + + for (const { kind, code, base: allowed, now: found } of rows) { + let label = '' + + if (kind === 'errors') { + totals.errorsBase += allowed + totals.errorsNow += found + } else { + totals.warningsBase += allowed + totals.warningsNow += found + label = ' warning' + } + + // With no baseline, only the totals mean anything. + if (noBaseline) { + continue + } + + const shown = displayCode(code) + + if (found > allowed) { + wentUp++ + if (wentUp <= MAX_RULES_LISTED) { + upList += `${upList ? ', ' : ''}\`${shown}\`${label} (+${found - allowed})` + } + + if (kind !== 'errors') { + messages.push({ + level: 'notice', + text: `${shown} has ${found} warnings, the baseline has ${allowed}. Warnings do not fail the check.`, + }) + continue + } + + errorsUp++ + if (advisory) { + messages.push({ + level: 'warning', + text: `${shown} has ${found} errors, an older run of the target branch has ${allowed}. Not enforced, because the target may have changed since.`, + }) + } else { + failed = true + messages.push({ + level: 'error', + text: `${shown} has ${found} errors, the baseline has ${allowed}. The full list is in the plugin-check-results artifact.`, + }) + } + } else if (found < allowed) { + wentDown++ + messages.push({ level: 'notice', text: `${shown} (${kind}) went down from ${allowed} to ${found}.` }) + } + } + + // Strict mode ignores the baseline for pass or fail: any error fails. The counts and the table + // still compare with the baseline, so the comment shows what changed. + if (mode === 'strict') { + failed = totals.errorsNow > 0 + if (failed) { + messages.push({ level: 'error', text: `${totals.errorsNow} errors found. Strict mode fails on any error.` }) + } + } + + return { messages, totals, wentUp, wentDown, errorsUp, upList, failed } +} + +/** + * "+3", "-2" or "±0". + * + * @param {number} now This run's count. + * @param {number} base The baseline's count. + * + * @returns {string} The change. + */ +function change(now, base) { + const diff = now - base + + if (diff > 0) { + return `+${diff}` + } + + return diff < 0 ? String(diff) : '±0' +} + +/** + * The status line at the top of the comment. + * + * The totals can go down while a rule goes up, so the status names the rules, not the totals. + * + * @param {object} input + * @param {string} input.mode `ratchet` or `strict`. + * @param {boolean} input.failed The check fails. + * @param {boolean} input.noBaseline Nothing was compared. + * @param {boolean} input.advisory The baseline is from an older run of the target branch. + * @param {number} input.errorsUp How many rules have more errors. + * @param {number} input.errorsNow This run's error total. + * + * @returns {string} The line. + */ +function statusLine({ mode, failed, noBaseline, advisory, errorsUp, errorsNow }) { + if (mode === 'strict' && failed) { + return `❌ Plugin Check failed: ${errorsNow} errors, and strict mode allows none` + } + if (mode === 'strict') { + return '✅ Plugin Check passed: no errors' + } + if (noBaseline) { + return '⚠️ Plugin Check ran, but there was no baseline to compare with' + } + if (errorsUp > 0 && advisory) { + return `⚠️ Plugin Check passed, not enforced: ${errorsUp} rule(s) have more errors than an older run of the target branch` + } + if (failed) { + return errorsUp === 1 + ? '❌ Plugin Check failed: 1 rule has more errors than the baseline allows' + : `❌ Plugin Check failed: ${errorsUp} rules have more errors than the baseline allows` + } + + return '✅ Plugin Check passed' +} + +/** + * The line of links at the bottom of the comment. + * + * @param {object} input + * @param {string} input.artifactUrl Link to the uploaded results and the fresh baseline, or ''. + * @param {string} input.runUrl Link to the workflow run, or ''. + * @param {string} input.resultsPath The results file, shown for a run on a developer's machine. + * + * @returns {string} The line. + */ +function linksLine({ artifactUrl, runUrl, resultsPath }) { + if (artifactUrl) { + return `📎 [Full results and new baseline](${artifactUrl})${runUrl ? ` · [Run](${runUrl})` : ''}` + } + + return runUrl ? `📎 [Run](${runUrl})` : `📎 Local run, see ${resultsPath}` +} + +/** + * The body of the pull request comment. + * + * @param {object} input + * @param {string} input.marker Hidden first line. + * @param {string} input.status From statusLine. + * @param {string} input.badge Image URL shown under the status line, or ''. + * @param {string} input.column Name of the baseline column, with its link when it has one. + * @param {boolean} input.noBaseline Leave the baseline and change columns out. + * @param {object} input.totals From evaluate. + * @param {number} input.wentUp How many rules went up. + * @param {number} input.wentDown How many rules went down. + * @param {string} input.upList The first rules that went up, ready to print. + * @param {string} input.links From linksLine. + * + * @returns {string} The body, cut to MAX_COMMENT_LINES lines, with no final line break. + */ +function buildComment({ marker, status, badge, column, noBaseline, totals, wentUp, wentDown, upList, links }) { + let body = `${marker}\n${status}\n\n` + + if (badge) { + body += `![Plugin Check finished](${badge})\n\n` + } + + if (noBaseline) { + body += '| | Now |\n|:--|--:|\n' + body += `| Errors | ${totals.errorsNow} |\n` + body += `| Warnings | ${totals.warningsNow} |\n\n` + } else { + body += `| | Now | ${column} | Change |\n|:--|--:|--:|:--|\n` + body += `| Errors | ${totals.errorsNow} | ${totals.errorsBase} | ${change(totals.errorsNow, totals.errorsBase)} |\n` + body += `| Warnings | ${totals.warningsNow} | ${totals.warningsBase} | ${change(totals.warningsNow, totals.warningsBase)} |\n\n` + } + + if (wentUp > MAX_RULES_LISTED) { + body += `Went up (top ${MAX_RULES_LISTED} of ${wentUp}): ${upList}\n` + } else if (wentUp > 0) { + body += `Went up: ${upList}\n` + } + if (wentDown > 0) { + body += `Rules that went down: ${wentDown}.\n` + } + // A blank line separates the notes above from the links, when there are notes. + if (wentUp > 0 || wentDown > 0) { + body += '\n' + } + body += links + + // Hard cap: whatever happens above, the comment never grows past MAX_COMMENT_LINES. + return body.split('\n').slice(0, MAX_COMMENT_LINES).join('\n').replace(/\n+$/, '') +} + +/** + * Compares a results file with a baseline, or writes the baseline for the results. + * + * @param {object} input + * @param {string[]} input.args Arguments: `--update` and `--comment-file ` in any order, then + * the results file. + * @param {object} input.env Environment. PLUGIN_CHECK_MODE (`ratchet`, the default, or `strict`), + * COMMENT_MARKER, PLUGIN_CHECK_BASELINE (default: baseline.json next + * to this file), BASELINE_LABEL, BASELINE_URL, PLUGIN_CHECK_ADVISORY + * (`true` when the baseline is from an older commit than the target), + * COMMENT_BADGE, RUN_URL, ARTIFACT_URL, GITHUB_STEP_SUMMARY. + * @param {object} input.log Where messages go: `out`, `err`, `error`, `warning`, `notice`. + * + * @returns {number} The exit status: 0 passes, 1 fails the check, another number is a file that + * could not be read as it should be. + */ +function compare({ args, env, log }) { + try { + return compareOrThrow({ args, env, log }) + } catch (error) { + if (error instanceof CheckFailure) { + log.err(error.message) + + return error.exitCode + } + + throw error + } +} + +/** + * The body of compare(), which lets a CheckFailure through. + * + * @see compare + * + * @param {object} input The same input as compare(). + * + * @returns {number} The exit status. + */ +function compareOrThrow({ args, env, log }) { + // Checked before anything else, so a typo in the caller never runs as the default. + const mode = env.PLUGIN_CHECK_MODE || DEFAULT_MODE + if (!MODES.includes(mode)) { + log.error(`Unknown mode '${mode}'. Use 'ratchet' or 'strict'.`) + + return 1 + } + + let update = false + let commentFile = '' + let rest = [...args] + while (rest.length > 0) { + if (rest[0] === '--update') { + update = true + rest = rest.slice(1) + } else if (rest[0] === '--comment-file') { + if (!rest[1]) { + log.err('--comment-file needs a path') + + return 1 + } + commentFile = rest[1] + rest = rest.slice(2) + } else { + break + } + } + + const resultsPath = rest[0] + if (!resultsPath) { + log.err('Usage: compare [--update | --comment-file ] ') + + return 1 + } + + const baselinePath = env.PLUGIN_CHECK_BASELINE || path.join(__dirname, 'baseline.json') + + if (!isFile(resultsPath)) { + log.error(`Results file not found: ${resultsPath}`) + + return 1 + } + + const text = fs.readFileSync(resultsPath, 'utf8') + const refusal = checkResultsText(text) + if (refusal) { + log.error(refusal) + + return 1 + } + + const now = countPerCode(parseResults(text)) + + if (update) { + writeText(baselinePath, formatBaseline(now), 'write') + log.out(`Wrote ${baselinePath}: ${now.errors.size} error codes, ${now.warnings.size} warning codes`) + + return 0 + } + + // No earlier run to compare with: report the counts and pass. Failing here would block every pull + // request whenever the target branch has no finished run (its first run, or expired artifacts). + const noBaseline = !isFile(baselinePath) + let base = { errors: new Map(), warnings: new Map() } + if (noBaseline) { + log.warning( + `No baseline to compare with (${baselinePath}), so nothing was compared. Run the workflow on the target branch to make one.` + ) + } else { + base = readBaseline(fs.readFileSync(baselinePath, 'utf8')) + } + + const advisory = env.PLUGIN_CHECK_ADVISORY === 'true' + const verdict = evaluate({ rows: buildRows(now, base), mode, advisory, noBaseline }) + for (const { level, text: message } of verdict.messages) { + log[level](message) + } + + let runUrl = env.RUN_URL || '' + if (!runUrl && env.GITHUB_RUN_ID) { + for (const name of ['GITHUB_SERVER_URL', 'GITHUB_REPOSITORY']) { + if (env[name] === undefined) { + log.err(`${name}: unbound variable`) + + return 1 + } + } + runUrl = `${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}` + } + + // The column is named after where the baseline came from ("Target branch", "Previous run"), and + // links to that run when known. + let column = env.BASELINE_LABEL || 'Baseline' + if (env.BASELINE_URL) { + column = `[${column}](${env.BASELINE_URL})` + } + + const body = buildComment({ + marker: env.COMMENT_MARKER || '', + status: statusLine({ mode, advisory, noBaseline, ...verdict, errorsNow: verdict.totals.errorsNow }), + badge: env.COMMENT_BADGE || '', + column, + noBaseline, + totals: verdict.totals, + wentUp: verdict.wentUp, + wentDown: verdict.wentDown, + upList: verdict.upList, + links: linksLine({ artifactUrl: env.ARTIFACT_URL || '', runUrl, resultsPath }), + }) + + if (commentFile) { + writeText(commentFile, `${body}\n`, 'write') + } + if (env.GITHUB_STEP_SUMMARY) { + writeText(env.GITHUB_STEP_SUMMARY, `${body}\n`, 'append') + } + + if (verdict.failed) { + log.out( + mode === 'strict' + ? 'Plugin Check found errors, and strict mode allows none.' + : 'Plugin Check found more errors than the baseline allows.' + ) + + return 1 + } + + log.out( + noBaseline + ? 'Plugin Check ran with no baseline, so nothing was compared.' + : 'Plugin Check error counts are within the baseline.' + ) + + return 0 +} + +/** + * Writes a file, or adds to it. A folder that does not exist fails the step with the reason. + * + * @param {string} file The path. + * @param {string} text What to write. + * @param {string} how `write` replaces the file, `append` adds to it. + * + * @throws {CheckFailure} When the file cannot be written. + */ +function writeText(file, text, how) { + try { + if (how === 'append') { + fs.appendFileSync(file, text) + } else { + fs.writeFileSync(file, text) + } + } catch (error) { + throw new CheckFailure(`Could not write ${file}: ${error.message}`, 1) + } +} + +/** + * Whether a path is a file, following links, as `[ -f ]` does. + * + * @param {string} file The path. + * + * @returns {boolean} True for a file. + */ +function isFile(file) { + try { + return fs.statSync(file).isFile() + } catch { + return false + } +} + +/** + * Where messages go when the code runs from a command line: annotations as the `::error` lines a + * runner would turn into annotations, and plain text on the two streams. + * + * @param {{out: Function, err: Function}} io Writes one line to standard output or standard error. + * + * @returns {object} The log. + */ +function commandLog(io) { + const annotation = (level) => (message) => io.out(`::${level} title=${TITLE}::${message}`) + + return { + out: io.out, + err: io.err, + error: annotation('error'), + warning: annotation('warning'), + notice: annotation('notice'), + } +} + +/** + * Where messages go inside an actions/github-script step. + * + * @param {object} core The toolkit github-script passes in. + * + * @returns {object} The log. + */ +function actionLog(core) { + const annotate = (method) => (message) => core[method](message, { title: TITLE }) + + return { + out: (line) => core.info(line), + err: (line) => core.info(line), + error: annotate('error'), + warning: annotate('warning'), + notice: annotate('notice'), + } +} + +/** + * The command line. `main(['compare', ...args])` is `compare()` with the real environment. + * + * @param {string[]} argv The arguments after the script name. + * @param {object} env The environment, default process.env. + * @param {object} io `out` and `err`, each writing one line. Default: the process streams. + * + * @returns {Promise} The exit status. + */ +async function main(argv, env = process.env, io = { + out: (line) => process.stdout.write(`${line}\n`), + err: (line) => process.stderr.write(`${line}\n`), +}) { + const [command, ...args] = argv + + if (command === 'compare') { + return compare({ args, env, log: commandLog(io) }) + } + + io.err('Usage: plugin-check-report.js compare [--update | --comment-file ] ') + + return 2 +} + +if (require.main === module) { + main(process.argv.slice(2)).then((status) => { + process.exitCode = status + }) +} + +module.exports = { + CheckFailure, + MODES, + actionLog, + buildComment, + buildRows, + change, + checkResultsText, + commandLog, + compare, + compareCodePoints, + countPerCode, + cutBytes, + displayCode, + evaluate, + formatBaseline, + linksLine, + main, + parseResults, + readBaseline, + readJsonValues, + statusLine, +} diff --git a/tests/actions/plugin-check-report/compare.test.js b/tests/actions/plugin-check-report/compare.test.js new file mode 100644 index 0000000..ec4c02b --- /dev/null +++ b/tests/actions/plugin-check-report/compare.test.js @@ -0,0 +1,572 @@ +'use strict' + +const { describe, it } = require('node:test') +const assert = require('node:assert') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') + +const report = require('../../../.github/actions/plugin-check-report/plugin-check-report.js') +const { CASES, expectedFor, runCase } = require('./golden.js') + +const FIXTURES = path.join(__dirname, 'fixtures') +const BASELINE = path.join(FIXTURES, 'baseline.json') + +const ESCAPE = 'WordPress.Security.EscapeOutput.OutputNotEscaped' +const DIRECT_QUERY = 'WordPress.DB.DirectDatabaseQuery.DirectQuery' + +/** + * Runs the compare in this process, with an environment built from scratch rather than copied from + * process.env. In CI the test runner has GITHUB_STEP_SUMMARY, GITHUB_RUN_ID and friends set, and the + * compare would append its summary to this repository's own job and link to this run. Messages go + * to captured lines, never to the runner's log, so a passing test run does not annotate itself. + * + * @param {string[]} args Arguments of the compare, usually a results file. + * @param {object} env The environment to add. + * + * @returns {{status: number, stdout: string, stderr: string}} The exit status and both streams. + */ +function execute(args, env) { + const stdout = [] + const stderr = [] + const log = report.commandLog({ out: (line) => stdout.push(`${line}\n`), err: (line) => stderr.push(`${line}\n`) }) + + return { status: report.compare({ args, env, log }), stdout: stdout.join(''), stderr: stderr.join('') } +} + +/** + * Runs the compare the way the action does, and returns what it printed and wrote. + * + * @param {string[]} args Arguments after `--comment-file`, usually a results file. + * @param {object} env Extra environment, such as PLUGIN_CHECK_BASELINE. + * + * @returns {{status: number, stdout: string, stderr: string, comment: string, summary: string}} + * The exit status, both streams, the comment file and the job summary it wrote. + */ +function ratchet(args, env = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-compare-')) + const comment = path.join(dir, 'comment.md') + const summary = path.join(dir, 'summary.md') + + const result = execute(['--comment-file', comment, ...args], { + GITHUB_STEP_SUMMARY: summary, + // A path that does not exist, so a test that wants no baseline gets none, instead of a + // baseline.json some earlier local run left next to the script. + PLUGIN_CHECK_BASELINE: path.join(dir, 'no-baseline.json'), + ...env, + }) + + const read = (file) => (fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : '') + + return { ...result, comment: read(comment), summary: read(summary) } +} + +/** + * The path of one of the sample results files. + * + * @param {string} name File name under fixtures/. + * + * @returns {string} The absolute path. + */ +function fixture(name) { + return path.join(FIXTURES, name) +} + +/** + * A results file with the lines given, in a temporary folder. + * + * @param {string} text The file's text. + * + * @returns {string} The path of the file. + */ +function resultsFile(text) { + const file = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-results-')), 'results.txt') + fs.writeFileSync(file, text) + + return file +} + +/** + * A results file with the results given as one file's JSON line. + * + * @param {object[]} results The results. + * + * @returns {string} The path of the file. + */ +function resultsOf(results) { + return resultsFile(`FILE: plugin.php\n${JSON.stringify(results)}\n`) +} + +/** + * A results file with one new ERROR code per entry, for the tests about long lists. + * + * @param {number} codes How many different codes to write. + * + * @returns {string} The path of the file. + */ +function resultsWithNewCodes(codes) { + return resultsOf( + Array.from({ length: codes }, (_, index) => ({ + line: index + 1, + column: 1, + type: 'ERROR', + code: `Test.Rule.Number${index + 1}`, + message: 'A rule added for the test.', + docs: '', + })) + ) +} + +/** + * A baseline file with the text given, in a temporary folder. + * + * @param {string|object} content The file's text, or a value to write as JSON. + * + * @returns {string} The path of the file. + */ +function baselineFile(content) { + const file = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-baseline-')), 'baseline.json') + fs.writeFileSync(file, typeof content === 'string' ? content : JSON.stringify(content)) + + return file +} + +describe('plugin-check-report compare, pinned output', () => { + for (const spec of CASES) { + it(`prints what the original script printed for ${spec.name}`, () => { + assert.deepStrictEqual(runCase(spec, execute), expectedFor(spec.name)) + }) + } +}) + +describe('plugin-check-report compare', () => { + describe('results it refuses', () => { + it('fails when the results file is missing', () => { + const run = ratchet([fixture('does-not-exist.txt')]) + + assert.equal(run.status, 1) + assert.match(run.stdout, /::error title=Plugin Check::Results file not found/) + assert.equal(run.comment, '', 'no comment body, so the workflow writes "could not run"') + }) + + it('fails when the results file is not Plugin Check output', () => { + const run = ratchet([fixture('garbage.txt')]) + + assert.equal(run.status, 1) + assert.match(run.stdout, /is not Plugin Check output.*First unexpected line: /) + }) + + it('fails when the results file is empty', () => { + const run = ratchet([fixture('empty.txt')]) + + assert.equal(run.status, 1) + assert.match(run.stdout, /has no Plugin Check output/) + }) + + it('fails on a mode it does not know, before reading anything', () => { + const run = ratchet([fixture('clean.txt')], { PLUGIN_CHECK_MODE: 'strcit' }) + + assert.equal(run.status, 1) + assert.match(run.stdout, /Unknown mode 'strcit'/) + }) + + it('fails on a mode it does not know, even with a missing results file', () => { + const run = ratchet([fixture('does-not-exist.txt')], { PLUGIN_CHECK_MODE: 'strcit' }) + + assert.match(run.stdout, /Unknown mode/) + assert.doesNotMatch(run.stdout, /Results file not found/) + }) + + it('fails with a usage message when no results file is given', () => { + const run = execute([], {}) + + assert.equal(run.status, 1) + assert.match(run.stderr, /Usage/) + }) + + it('fails when --comment-file has no path', () => { + for (const args of [['--comment-file'], ['--comment-file', '']]) { + const run = execute(args, {}) + + assert.equal(run.status, 1, args.join(' ')) + assert.match(run.stderr, /--comment-file needs a path/) + } + }) + + it('cuts the first unexpected line to 200 bytes', () => { + const run = ratchet([resultsFile(`${'x'.repeat(300)}\n`)]) + + assert.equal(run.status, 1) + assert.match(run.stdout, new RegExp(`First unexpected line: x{200}\\n`)) + }) + + it('does not count a line of white space as unexpected', () => { + const run = ratchet([resultsFile('FILE: plugin.php\n \t \n[]\n\n')]) + + assert.equal(run.status, 0) + }) + + it('fails when a results line is not JSON', () => { + const run = ratchet([resultsFile('FILE: plugin.php\n[{"type":"ERROR"\n')]) + + assert.equal(run.status, 5) + assert.match(run.stderr, /parse error/) + assert.equal(run.comment, '') + }) + + it('fails when a counted result has no rule code', () => { + for (const code of [undefined, null, 5, true, { a: 1 }]) { + const run = ratchet([resultsOf([{ type: 'ERROR', code }])]) + + assert.equal(run.status, 5, String(code)) + assert.match(run.stderr, /has no rule code/) + } + }) + + it('fails when a result is not an object, but skips null', () => { + for (const item of [1, 'x', [1], true]) { + assert.equal(ratchet([resultsOf([item])]).status, 5, JSON.stringify(item)) + } + + assert.equal(ratchet([resultsOf([null])]).status, 0) + }) + + it('ignores results that are neither ERROR nor WARNING', () => { + const run = ratchet([resultsOf([{ type: 'INFO' }, { type: 'error', code: 'a' }])]) + + assert.equal(run.status, 0) + assert.match(run.comment, /\| Errors {3}\| 0 \|/) + }) + + it('reads several lists written on one line', () => { + const run = ratchet([resultsFile('FILE: a.php\n[{"type":"ERROR","code":"a"}] [{"type":"ERROR","code":"b"}]\n')], { + PLUGIN_CHECK_MODE: 'strict', + }) + + assert.match(run.stdout, /2 errors found/) + }) + }) + + describe('--update', () => { + it('writes the counts per rule code', () => { + const out = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-update-')), 'baseline.json') + const run = ratchet(['--update', fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: out }) + + assert.equal(run.status, 0) + assert.deepEqual(JSON.parse(fs.readFileSync(out, 'utf8')), { + errors: { + [ESCAPE]: 3, + 'WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents': 1, + }, + warnings: { [DIRECT_QUERY]: 1 }, + }) + }) + + it('writes empty counts for a clean run', () => { + const out = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-update-')), 'baseline.json') + const run = ratchet(['--update', fixture('clean.txt')], { PLUGIN_CHECK_BASELINE: out }) + + assert.equal(run.status, 0) + assert.deepEqual(JSON.parse(fs.readFileSync(out, 'utf8')), { errors: {}, warnings: {} }) + }) + + it('writes no comment and no summary', () => { + const out = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-update-')), 'baseline.json') + const run = ratchet(['--update', fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: out }) + + assert.equal(run.comment, '') + assert.equal(run.summary, '') + }) + + it('fails, with the reason, when the folder of the file does not exist', () => { + const run = ratchet(['--update', fixture('clean.txt')], { PLUGIN_CHECK_BASELINE: '/nonexistent/folder/baseline.json' }) + + assert.equal(run.status, 1) + assert.match(run.stderr, /Could not write \/nonexistent\/folder\/baseline\.json/) + }) + }) + + describe('compare in ratchet mode', () => { + it('passes a clean run with no baseline, with a warning and the one-column table', () => { + const run = ratchet([fixture('clean.txt')]) + + assert.equal(run.status, 0) + assert.match(run.stdout, /::warning title=Plugin Check::No baseline to compare with/) + assert.match(run.comment, /⚠️ Plugin Check ran, but there was no baseline to compare with/) + assert.match(run.comment, /\| Errors {3}\| 0 \|/) + assert.doesNotMatch(run.comment, /Change/) + }) + + it('passes when every count matches the baseline', () => { + const run = ratchet([fixture('same-as-baseline.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + + assert.equal(run.status, 0) + assert.match(run.comment, /✅ Plugin Check passed/) + assert.match(run.comment, /\| Errors {3}\| 3 \| 3 \| ±0 \|/) + }) + + it('fails when a rule has more errors than the baseline', () => { + const run = ratchet([fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + + assert.equal(run.status, 1) + assert.match(run.stdout, new RegExp(`::error title=Plugin Check::${ESCAPE} has 3 errors, the baseline has 2`)) + assert.match(run.comment, /❌ Plugin Check failed: 1 rule has more errors than the baseline allows/) + assert.match(run.comment, new RegExp(`Went up: \`${ESCAPE}\` \\(\\+1\\)`)) + }) + + it('fails when a rule goes up even though the total stays the same', () => { + const run = ratchet([fixture('rule-up-total-same.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + + assert.equal(run.status, 1) + assert.match(run.comment, /\| Errors {3}\| 3 \| 3 \| ±0 \|/) + assert.match(run.comment, /Rules that went down: 1\./) + }) + + it('passes with a notice when errors go down', () => { + const run = ratchet([fixture('errors-down.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + + assert.equal(run.status, 0) + assert.match(run.stdout, new RegExp(`::notice title=Plugin Check::${ESCAPE} \\(errors\\) went down from 2 to 1`)) + assert.match(run.comment, /\| Errors {3}\| 2 \| 3 \| -1 \|/) + }) + + it('passes with a notice when only warnings go up', () => { + const run = ratchet([fixture('warnings-up.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + + assert.equal(run.status, 0) + assert.match(run.stdout, /Warnings do not fail the check/) + assert.match(run.comment, new RegExp(`Went up: \`${DIRECT_QUERY}\` warning \\(\\+2\\)`)) + }) + + it('only warns when the baseline is from an older run of the target branch', () => { + const run = ratchet([fixture('errors-up.txt')], { + PLUGIN_CHECK_BASELINE: BASELINE, + PLUGIN_CHECK_ADVISORY: 'true', + }) + + assert.equal(run.status, 0) + assert.match(run.stdout, /::warning title=Plugin Check::.*Not enforced/) + assert.match(run.comment, /⚠️ Plugin Check passed, not enforced/) + }) + + it('takes only the exact word true as advisory', () => { + const run = ratchet([fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: BASELINE, PLUGIN_CHECK_ADVISORY: 'TRUE' }) + + assert.equal(run.status, 1) + }) + + it('treats an empty baseline file, and null, as no counts at all', () => { + for (const content of ['', 'null', ' \n']) { + const run = ratchet([fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: baselineFile(content) }) + + assert.equal(run.status, 1, JSON.stringify(content)) + assert.match(run.stdout, /has 3 errors, the baseline has 0/) + } + }) + + it('treats a missing kind, and a null or false count, as zero', () => { + const run = ratchet([fixture('errors-up.txt')], { + PLUGIN_CHECK_BASELINE: baselineFile({ errors: { [ESCAPE]: null } }), + }) + + assert.match(run.stdout, /has 3 errors, the baseline has 0/) + }) + + it('reads only the first value of a baseline file', () => { + const run = ratchet([fixture('same-as-baseline.txt')], { + PLUGIN_CHECK_BASELINE: baselineFile(`${fs.readFileSync(BASELINE, 'utf8')}\n{"errors":{"x":9}}`), + }) + + assert.equal(run.status, 0) + }) + + it('fails, with a reason, on a baseline that cannot be compared', () => { + const cases = [ + ['not json', 2], + ['[]', 5], + ['5', 5], + ['{"errors":5}', 5], + ['{"errors":[]}', 5], + ['{"errors":false}', 5], + ['{"errors":{"a":"2"}}', 5], + ['{"errors":{"a":true}}', 5], + ['{"errors":{"a":1.5}}', 1], + ] + + for (const [content, status] of cases) { + const run = ratchet([resultsOf([{ type: 'ERROR', code: 'a' }])], { PLUGIN_CHECK_BASELINE: baselineFile(content) }) + + assert.equal(run.status, status, content) + assert.equal(run.comment, '', content) + } + }) + + it('counts a rule with an empty code like any other', () => { + const run = ratchet([resultsOf([{ type: 'ERROR', code: '' }, { type: 'ERROR', code: '' }])], { + PLUGIN_CHECK_BASELINE: baselineFile({ errors: { '': 1 }, warnings: {} }), + }) + + assert.equal(run.status, 1) + assert.match(run.comment, /\| Errors {3}\| 2 \| 1 \| \+1 \|/) + }) + + it('shows a rule code on one line, with a tab or a new line written out', () => { + const run = ratchet([resultsOf([{ type: 'ERROR', code: 'a\tb\\c' }])], { PLUGIN_CHECK_BASELINE: baselineFile({}) }) + + assert.match(run.stdout, /::error title=Plugin Check::a\\tb\\\\c has 1 errors/) + }) + + it('takes the links from GITHUB_RUN_ID when RUN_URL is not set', () => { + const run = ratchet([fixture('clean.txt')], { + GITHUB_RUN_ID: '42', + GITHUB_SERVER_URL: 'https://github.com', + GITHUB_REPOSITORY: 'example-org/my-plugin', + }) + + assert.match(run.comment, /📎 \[Run\]\(https:\/\/github\.com\/example-org\/my-plugin\/actions\/runs\/42\)/) + }) + + it('shows the results file for a run on a developer machine', () => { + const run = ratchet([fixture('clean.txt')]) + + assert.match(run.comment, /📎 Local run, see .*clean\.txt/) + }) + }) + + describe('compare in strict mode', () => { + it('fails on any error, even with no baseline', () => { + const run = ratchet([fixture('errors-down.txt')], { PLUGIN_CHECK_MODE: 'strict' }) + + assert.equal(run.status, 1) + assert.match(run.stdout, /2 errors found\. Strict mode fails on any error/) + assert.match(run.comment, /❌ Plugin Check failed: 2 errors, and strict mode allows none/) + }) + + it('fails on errors that went down, which ratchet mode would pass', () => { + const run = ratchet([fixture('errors-down.txt')], { + PLUGIN_CHECK_MODE: 'strict', + PLUGIN_CHECK_BASELINE: BASELINE, + }) + + assert.equal(run.status, 1) + }) + + it('passes a clean run', () => { + const run = ratchet([fixture('clean.txt')], { PLUGIN_CHECK_MODE: 'strict' }) + + assert.equal(run.status, 0) + assert.match(run.comment, /✅ Plugin Check passed: no errors/) + }) + + it('passes a run with warnings only', () => { + const lines = fs + .readFileSync(fixture('warnings-up.txt'), 'utf8') + .split('\n') + .map((line) => { + if (!line.startsWith('[')) { + return line + } + + return JSON.stringify(JSON.parse(line).filter((result) => result.type === 'WARNING')) + }) + + const run = ratchet([resultsFile(lines.join('\n'))], { PLUGIN_CHECK_MODE: 'strict' }) + + assert.equal(run.status, 0) + }) + + it('prints the per-rule messages before the strict one', () => { + const run = ratchet([fixture('errors-up.txt')], { PLUGIN_CHECK_MODE: 'strict', PLUGIN_CHECK_BASELINE: BASELINE }) + const lines = run.stdout.split('\n') + + assert.match(lines[0], /has 3 errors, the baseline has 2/) + assert.match(lines[1], /4 errors found\. Strict mode/) + }) + }) + + describe('the comment', () => { + it('starts with the default marker', () => { + const run = ratchet([fixture('clean.txt')]) + + assert.equal(run.comment.split('\n')[0], '') + }) + + it('starts with the marker the workflow passes', () => { + const marker = '' + const run = ratchet([fixture('clean.txt')], { COMMENT_MARKER: marker }) + + assert.equal(run.comment.split('\n')[0], marker) + }) + + it('names the baseline column after where it came from, with a link', () => { + const run = ratchet([fixture('same-as-baseline.txt')], { + PLUGIN_CHECK_BASELINE: BASELINE, + BASELINE_LABEL: 'Target branch', + BASELINE_URL: 'https://github.com/o/r/actions/runs/1', + }) + + assert.match(run.comment, /\| Now \| \[Target branch\]\(https:\/\/github\.com\/o\/r\/actions\/runs\/1\) \| Change \|/) + }) + + it('lists at most five rules and never grows past fifteen lines', () => { + const run = ratchet([resultsWithNewCodes(7)], { PLUGIN_CHECK_BASELINE: BASELINE }) + + assert.equal(run.status, 1) + assert.match(run.comment, /Went up \(top 5 of 7\)/) + assert.doesNotMatch(run.comment, /Number6/) + assert.ok(run.comment.trimEnd().split('\n').length <= 15) + }) + + it('cuts a comment to fifteen lines when a value holds line breaks, and drops the empty end', () => { + const marker = `${'\n'.repeat(20)}end` + const run = ratchet([fixture('clean.txt')], { COMMENT_MARKER: marker }) + + // The first fifteen lines are all empty, so nothing is left once the empty end is dropped. + assert.equal(run.comment, '\n') + }) + + it('writes the same text to the job summary', () => { + const run = ratchet([fixture('errors-down.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + + assert.equal(run.summary, run.comment) + }) + + it('writes a badge under the status line', () => { + const run = ratchet([fixture('clean.txt')], { COMMENT_BADGE: 'https://example.test/badge.svg' }) + + assert.match(run.comment, /\n!\[Plugin Check finished\]\(https:\/\/example\.test\/badge\.svg\)\n/) + }) + + it('fails, with the reason, when the comment file cannot be written', () => { + const run = execute(['--comment-file', '/nonexistent/folder/comment.md', fixture('clean.txt')], { RUN_URL: 'u' }) + + assert.equal(run.status, 1) + assert.match(run.stderr, /Could not write \/nonexistent\/folder\/comment\.md/) + }) + }) +}) + +describe('the baseline file text', () => { + it('sorts codes by Unicode code point, not by UTF-16 unit', () => { + const counts = report.countPerCode([ + { type: 'ERROR', code: '😀' }, + { type: 'ERROR', code: '￿' }, + { type: 'ERROR', code: '10' }, + { type: 'ERROR', code: '9' }, + ]) + + assert.deepEqual([...counts.errors.keys()], ['10', '9', '￿', '😀']) + }) + + it('writes DEL as \\u007f and ends with one new line', () => { + const text = report.formatBaseline(report.countPerCode([{ type: 'ERROR', code: 'a\x7fb' }])) + + assert.equal(text, '{\n "errors": {\n "a\\u007fb": 1\n },\n "warnings": {}\n}\n') + }) + + it('keeps a code such as __proto__ as a code', () => { + const counts = report.countPerCode([{ type: 'WARNING', code: '__proto__' }]) + + assert.equal(report.formatBaseline(counts), '{\n "errors": {},\n "warnings": {\n "__proto__": 1\n }\n}\n') + }) +}) From c63159817a830cc16c5f232ae0f91e2707c34f89 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Tue, 3 Nov 2026 18:39:34 -0300 Subject: [PATCH 14/31] Add the baseline lookup in JavaScript, next to the bash script --- .../plugin-check-report.js | 699 +++++++++++++++++- .../actions/plugin-check-report/fetch.test.js | 614 +++++++++++++++ 2 files changed, 1306 insertions(+), 7 deletions(-) create mode 100644 tests/actions/plugin-check-report/fetch.test.js diff --git a/.github/actions/plugin-check-report/plugin-check-report.js b/.github/actions/plugin-check-report/plugin-check-report.js index be5a69f..5b3294b 100644 --- a/.github/actions/plugin-check-report/plugin-check-report.js +++ b/.github/actions/plugin-check-report/plugin-check-report.js @@ -24,6 +24,7 @@ */ const fs = require('node:fs') +const os = require('node:os') const path = require('node:path') /** Title of every annotation. */ @@ -851,26 +852,695 @@ function actionLog(core) { } } +/** The artifact every run of the workflow uploads, with the baseline.json inside it. */ +const ARTIFACT = 'plugin-check-report' + +/** How long a pull request waits for the run of its base commit to show up at all, in seconds. */ +const GRACE_SECONDS = 120 +const POLL_SECONDS = 30 + +const FETCH_USAGE = + 'Usage: plugin-check-report.js fetch [--workflow ] [--sha ] [--wait ] [--before-run ] []' + +/** + * A lookup that cannot be trusted. It exits with 2, never 1, so "no baseline" is never mistaken for + * a typo or an outage. + * + * @param {string} reason What failed. + * + * @returns {CheckFailure} The failure to throw. + */ +function lookupFailed(reason) { + return new CheckFailure(`Could not look up Plugin Check runs: ${reason}`, 2) +} + +/** + * Checks a whole number from the command line. + * + * At most 18 digits, which keeps it inside a 64-bit integer, and a leading zero is not octal. + * + * @param {string} flag The option it belongs to, for the message. + * @param {string} value The text. + * + * @throws {CheckFailure} When it is not a whole number, or is too large. + * + * @returns {bigint} The number. + */ +function readWholeNumber(flag, value) { + if (!/^[0-9]+$/.test(value)) { + throw lookupFailed(`${flag} must be a whole number, not '${value}'`) + } + if (value.length > 18) { + throw lookupFailed(`${flag} is too large: '${value}'`) + } + + return BigInt(value) +} + +/** + * Checks the name of the workflow file. It goes into an API path, and a path or a ref there would + * ask for some other resource. + * + * @param {string} workflow The file name, such as `plugin-check.yml`. + * + * @throws {CheckFailure} When it is empty, or holds a `/` or an `@`. + * + * @returns {string} The name. + */ +function checkWorkflowFile(workflow) { + if (workflow === '' || /[/@]/.test(workflow)) { + throw lookupFailed(`--workflow must be a file name such as plugin-check.yml, not '${workflow}'`) + } + + return workflow +} + +/** + * The file name of the calling workflow. In a called workflow GITHUB_WORKFLOW_REF is the caller's, like + * `example-org/my-plugin/.github/workflows/plugin-check.yml@refs/pull/12/merge`. The runs that + * hold the baseline belong to that file, not to the toolbox's workflow. + * + * @param {string} ref The value of GITHUB_WORKFLOW_REF. + * + * @returns {string} The file name, or '' when there is no ref. + */ +function workflowFromRef(ref) { + const withoutRef = String(ref ?? '').split('@')[0] + + return withoutRef.slice(withoutRef.lastIndexOf('/') + 1) +} + +/** + * Reads the command line of the lookup. + * + * Options come first, in any order, then the branch and an optional output file. An option with no + * value, or an empty one, fails here, so a typo never reads as "no baseline". + * + * @param {string[]} args The arguments. + * + * @throws {CheckFailure} For anything the lookup cannot use. + * + * @returns {{workflow: string, sha: string, wait: number, beforeRun: bigint|null, branch: string, + * output: string}} What to look up. + */ +function parseFetchArgs(args) { + const options = { workflow: 'plugin-check.yml', sha: '', wait: 0, beforeRun: null } + let rest = [...args] + + const needValue = () => { + if (rest.length < 2 || rest[1] === '') { + throw lookupFailed(`${rest[0]} needs a value. ${FETCH_USAGE}`) + } + } + + while (rest.length > 0) { + switch (rest[0]) { + case '--workflow': + needValue() + options.workflow = rest[1] + break + case '--sha': + needValue() + options.sha = rest[1] + break + case '--wait': + needValue() + options.wait = Number(readWholeNumber('--wait', rest[1])) + break + case '--before-run': + needValue() + options.beforeRun = readWholeNumber('--before-run', rest[1]) + break + default: + return finishFetchArgs(options, rest) + } + rest = rest.slice(2) + } + + return finishFetchArgs(options, rest) +} + +/** + * The branch and the output file, after the options. + * + * @param {object} options What the options said. + * @param {string[]} rest The arguments left. + * + * @throws {CheckFailure} When there is no branch. + * + * @returns {object} The options, the branch and the output file. + */ +function finishFetchArgs(options, rest) { + if (rest.length < 1 || rest[0] === '') { + throw lookupFailed(`a branch is needed. ${FETCH_USAGE}`) + } + + return { ...options, branch: rest[0], output: rest[1] || path.join(__dirname, 'baseline.json') } +} + +/** + * Lower-cases the letters A to Z only, as `ascii_downcase` does. + * + * @param {string} text The text. + * + * @returns {string} The text. + */ +function asciiLower(text) { + return text.replace(/[A-Z]/g, (letter) => letter.toLowerCase()) +} + +/** + * Whether a run belongs to this repository's own branches. + * + * The runs API matches `branch` and `head_sha` against the head of a pull request too, so a run for + * a pull request from a fork, from a branch named like the target, would otherwise be taken for a + * run of the target, and its artifact is whatever that fork's workflow chose to upload. + * + * @param {object} run A workflow run from the API. + * @param {string} repo `owner/name`. + * + * @returns {boolean} True when the run's head commit is in this repository. + */ +function isOwnRun(run, repo) { + return asciiLower(run.head_repository?.full_name || '') === asciiLower(repo) +} + +/** + * How many of this repository's own runs there are, and how many of them are not finished. + * + * @param {object[]} runs The runs from the API. + * @param {string} repo `owner/name`. + * + * @returns {{total: number, going: number}} The counts. + */ +function countRuns(runs, repo) { + const own = runs.filter((run) => isOwnRun(run, repo)) + + return { total: own.length, going: own.filter((run) => run.status !== 'completed').length } +} + +/** + * The runs to try, best first. + * + * Runs for the base commit come first, in the order the API gave them. Then the latest runs on the + * branch, newest first by run id. A run is tried once, and a run that is not older than + * `beforeRun` is skipped, so a run never compares with itself, and a re-run of an older push never + * compares with a newer one. + * + * @param {object} input + * @param {object[]} input.shaRuns Own runs for the base commit. + * @param {object[]} input.branchRuns Own runs on the branch. + * @param {bigint|null} input.beforeRun Skip runs with this id or a higher one. + * + * @returns {object[]} The runs. + */ +function orderCandidates({ shaRuns, branchRuns, beforeRun }) { + const newestFirst = [...branchRuns].sort((a, b) => (a.id < b.id ? 1 : a.id > b.id ? -1 : 0)) + const tried = new Set() + const candidates = [] + + for (const run of [...shaRuns, ...newestFirst]) { + if (beforeRun !== null && BigInt(run.id) >= beforeRun) { + continue + } + if (tried.has(String(run.id))) { + continue + } + tried.add(String(run.id)) + candidates.push(run) + } + + return candidates +} + +/** + * The runs of a workflow, from the API. + * + * @param {object} github A GitHub client. + * @param {object} query `owner`, `repo`, `workflow`, then the filters in the order they are sent. + * @param {object} log The log, for the reason a request failed. + * @param {string} reason What the caller was looking for, for the message. + * + * @throws {CheckFailure} When the request fails or the answer is not a list of runs. + * + * @returns {Promise} The runs. + */ +async function listRuns(github, { owner, repo, workflow, ...filters }, log, reason) { + let data + try { + ;({ data } = await github.request('GET /repos/{owner}/{repo}/actions/workflows/{workflow_id}/runs', { + owner, + repo, + workflow_id: workflow, + ...filters, + })) + } catch (error) { + log.err(`${error.message}${error.status ? ` (HTTP ${error.status})` : ''}`) + throw lookupFailed(reason) + } + + if (!Array.isArray(data?.workflow_runs)) { + throw lookupFailed(reason) + } + + return data.workflow_runs +} + +/** + * Waits for the run of the base commit to finish, when it is still going. + * + * A pull request opened right after a merge starts at the same time as the push run on the target + * branch. Waiting for it beats comparing with an older run. A run that does not show up at all is + * waited for only a short time, since the commit may never have had one. + * + * @param {object} input + * @param {object} input.github A GitHub client. + * @param {object} input.query `owner`, `repo`, `workflow`. + * @param {string} input.sha The base commit. + * @param {number} input.wait The most seconds to wait. + * @param {Function} input.now The time in whole seconds. + * @param {Function} input.sleep Waits for a number of seconds. + * @param {object} input.log The log. + */ +async function waitForBaseRun({ github, query, sha, wait, now, sleep, log }) { + const repo = `${query.owner}/${query.repo}` + const started = now() + const deadline = started + wait + + while (true) { + const runs = await listRuns(github, { ...query, head_sha: sha, per_page: 20 }, log, `runs for ${sha}`) + const { total, going } = countRuns(runs, repo) + + if (total === 0 && now() >= started + GRACE_SECONDS) { + return + } + if (total !== 0 && going === 0) { + return + } + if (now() >= deadline) { + log.err(`The run for ${sha} is still going after ${wait} seconds; trying older runs.`) + + return + } + + await sleep(POLL_SECONDS) + } +} + +/** + * Finds the earlier run that has a baseline, and downloads its baseline.json. + * + * Runs are tried in this order, and the first one with the artifact wins: runs for the base commit + * when `--sha` is given, then the latest runs on the branch. The artifact is downloaded with the + * GitHub CLI. A run that broke before the report job, whose artifact expired, or whose artifact has + * no baseline.json has nothing to give, and the next run is tried. + * + * @param {object} input + * @param {string[]} input.args The command line, see parseFetchArgs. + * @param {object} input.env The environment. GITHUB_REPOSITORY names the repository. + * @param {object} input.github A GitHub client. + * @param {object} input.exec `getExecOutput(command, args, options)`, as in @actions/exec. + * @param {object} input.log The log. + * @param {Function} input.now The time in whole seconds. Default: the clock. + * @param {Function} input.sleep Waits for a number of seconds. Default: a real wait. + * + * @returns {Promise<{status: number, id?: number, url?: string, match?: string}>} Status 0 with the + * run found and `exact` or `older` (the run is not for the base commit); 1 when no run has + * a baseline; 2 when the lookup itself failed, so "no baseline" is not known. + */ +async function fetchBaseline({ + args, + env, + github, + exec, + log, + now = () => Math.floor(Date.now() / 1000), + sleep = (seconds) => new Promise((resolve) => setTimeout(resolve, seconds * 1000)), +}) { + let temp = null + + try { + const { workflow, sha, wait, beforeRun, branch, output } = parseFetchArgs(args) + + const repo = env.GITHUB_REPOSITORY || (await repositoryFromGh(exec)) + checkWorkflowFile(workflow) + const [owner, name] = repo.split('/') + const query = { owner, repo: name, workflow } + + if (sha && wait > 0) { + await waitForBaseRun({ github, query, sha, wait, now, sleep, log }) + } + + const shaRuns = sha + ? await listRuns(github, { ...query, status: 'completed', head_sha: sha, per_page: 10 }, log, `runs for ${sha}`) + : [] + const branchRuns = await listRuns( + github, + { ...query, status: 'completed', branch, per_page: 100 }, + log, + `runs on ${branch}` + ) + + const candidates = orderCandidates({ + shaRuns: shaRuns.filter((run) => isOwnRun(run, repo)), + branchRuns: branchRuns.filter((run) => isOwnRun(run, repo)), + beforeRun, + }) + + temp = fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-fetch-')) + + for (const run of candidates) { + const file = await downloadBaseline({ github, exec, log, owner, repo: name, id: run.id, dir: path.join(temp, 'report') }) + if (file === null) { + continue + } + + fs.mkdirSync(path.dirname(output), { recursive: true }) + fs.copyFileSync(file, output) + + return { status: 0, id: run.id, url: run.html_url, match: sha && run.head_sha !== sha ? 'older' : 'exact' } + } + + log.err(`No Plugin Check run on ${branch}${sha ? ` or ${sha}` : ''} has a baseline.`) + + return { status: 1 } + } catch (error) { + const failure = error instanceof CheckFailure ? error : lookupFailed(error.message) + log.err(failure.message) + + return { status: failure.exitCode } + } finally { + if (temp) { + fs.rmSync(temp, { recursive: true, force: true }) + } + } +} + +/** + * Downloads the baseline.json of one run's artifact. + * + * @param {object} input + * @param {object} input.github A GitHub client. + * @param {object} input.exec `getExecOutput`. + * @param {object} input.log The log. + * @param {string} input.owner Repository owner. + * @param {string} input.repo Repository name. + * @param {number} input.id The run. + * @param {string} input.dir A folder to download into. + * + * @throws {CheckFailure} When the artifacts cannot be listed or the download fails. + * + * @returns {Promise} The path of the baseline.json, or null when the run has none. + */ +async function downloadBaseline({ github, exec, log, owner, repo, id, dir }) { + let data + try { + ;({ data } = await github.request('GET /repos/{owner}/{repo}/actions/runs/{run_id}/artifacts', { + owner, + repo, + run_id: id, + name: ARTIFACT, + })) + } catch (error) { + log.err(`${error.message}${error.status ? ` (HTTP ${error.status})` : ''}`) + throw lookupFailed(`artifacts of run ${id}`) + } + + if (!Array.isArray(data?.artifacts)) { + throw lookupFailed(`artifacts of run ${id}`) + } + if (!data.artifacts.some((artifact) => !artifact.expired)) { + return null + } + + fs.rmSync(dir, { recursive: true, force: true }) + const download = await exec.getExecOutput( + 'gh', + ['run', 'download', String(id), '--repo', `${owner}/${repo}`, '--name', ARTIFACT, '--dir', dir], + { silent: true, ignoreReturnCode: true } + ) + if (download.exitCode !== 0) { + if (download.stderr) { + log.err(download.stderr.trimEnd()) + } + throw lookupFailed(`download of the artifact of run ${id}`) + } + + const file = path.join(dir, 'baseline.json') + + return isFile(file) && fs.statSync(file).size > 0 ? file : null +} + +/** + * The repository of the current folder, from the GitHub CLI, for a run on a developer's machine. + * + * @param {object} exec `getExecOutput`. + * + * @throws {CheckFailure} When there is no repository. + * + * @returns {Promise} `owner/name`. + */ +async function repositoryFromGh(exec) { + const result = await exec.getExecOutput('gh', ['repo', 'view', '--json', 'nameWithOwner', '--jq', '.nameWithOwner'], { + silent: true, + ignoreReturnCode: true, + }) + const repo = result.stdout.trim() + + if (result.exitCode !== 0 || !repo) { + throw lookupFailed('no repository') + } + + return repo +} + /** - * The command line. `main(['compare', ...args])` is `compare()` with the real environment. + * The label and the advisory flag for a baseline that was found. + * + * A baseline from an older run than the base commit's cannot fail the check, because the target + * may have changed since. + * + * @param {string} event `pull_request`, or any other event name. + * @param {string} match `exact` or `older`. + * + * @returns {{label: string, advisory: boolean}} How the comment names the baseline. + */ +function describeBaseline(event, match) { + if (match === 'older') { + return { label: 'Target branch, older run', advisory: true } + } + + return { label: event === 'pull_request' ? 'Target branch' : 'Previous run', advisory: false } +} + +/** + * The first step: where this run keeps its files, and which workflow's runs hold the baseline. + * + * One folder per run under RUNNER_TEMP, cleared first, so a second use of the action in the same + * job never reads the first one's files. + * + * @param {object} input + * @param {object} input.core The toolkit github-script passes in. + * @param {object} input.env The environment: RUNNER_TEMP, GITHUB_WORKFLOW_REF, GITHUB_REPOSITORY and + * INPUT_COMMENT_MARKER. + * + * @returns {number} 0. + */ +function runPaths({ core, env }) { + const dir = path.join(env.RUNNER_TEMP, 'plugin-check-report') + fs.rmSync(dir, { recursive: true, force: true }) + fs.mkdirSync(path.join(dir, 'report'), { recursive: true }) + fs.mkdirSync(path.join(dir, 'baseline'), { recursive: true }) + + const workflow = workflowFromRef(env.GITHUB_WORKFLOW_REF) + + core.setOutput('report', path.join(dir, 'report')) + core.setOutput('baseline', path.join(dir, 'baseline', 'baseline.json')) + core.setOutput('workflow', workflow) + core.setOutput('marker', env.INPUT_COMMENT_MARKER || ``) + core.info(`Baselines are read from the runs of ${workflow}.`) + + return 0 +} + +/** + * The second step: a baseline made from this run, kept even when the compare fails. Later runs + * download it as their baseline. + * + * @param {object} input + * @param {object} input.core The toolkit. + * @param {object} input.env RESULTS and REPORT_DIR. + * + * @returns {number} The exit status. A results file that is not Plugin Check output fails here too, + * with the same message, and then nothing is uploaded. + */ +function runUpdate({ core, env }) { + if (!isFile(env.RESULTS ?? '')) { + return 0 + } + + fs.copyFileSync(env.RESULTS, path.join(env.REPORT_DIR, path.basename(env.RESULTS))) + + return compare({ + args: ['--update', env.RESULTS], + env: { ...env, PLUGIN_CHECK_BASELINE: path.join(env.REPORT_DIR, 'baseline.json') }, + log: actionLog(core), + }) +} + +/** + * The step that finds the baseline. A pull request takes the run for its base commit; for a stacked + * pull request that is the run of the pull request below it. When that run is still going, it is + * waited for. When it has no baseline, the latest run on the base branch is used instead, and + * more errors than it are only reported, not failed. Any other event takes the run before it on + * the same branch, so a merge that adds errors fails there, where the whole team sees it. + * + * Sets the outputs `label`, `url` and `advisory` when it finds one. + * + * @param {object} input + * @param {object} input.github A GitHub client. + * @param {object} input.core The toolkit. + * @param {object} input.exec `exec` from github-script. + * @param {object} input.env EVENT, BASE_REF, BASE_SHA, WAIT, WORKFLOW, BASELINE, GITHUB_REPOSITORY, + * GITHUB_RUN_ID and GITHUB_REF_NAME. + * @param {Function} input.now Optional clock. + * @param {Function} input.sleep Optional wait. + * + * @returns {Promise} 0 when a baseline was found or none exists, 2 when the lookup failed. + */ +async function runFetch({ github, core, exec, env, now, sleep }) { + const pullRequest = env.EVENT === 'pull_request' + const args = pullRequest + ? ['--workflow', env.WORKFLOW ?? '', '--sha', env.BASE_SHA ?? '', '--wait', env.WAIT ?? '', env.BASE_REF ?? '', env.BASELINE ?? ''] + : ['--workflow', env.WORKFLOW ?? '', '--before-run', env.GITHUB_RUN_ID ?? '', env.GITHUB_REF_NAME ?? '', env.BASELINE ?? ''] + + const found = await fetchBaseline({ args, env, github, exec, log: actionLog(core), now, sleep }) + + if (found.status === 1) { + core.info('No earlier run has a baseline.') + + return 0 + } + if (found.status !== 0) { + return found.status + } + + const { label, advisory } = describeBaseline(env.EVENT, found.match) + core.setOutput('label', label) + core.setOutput('url', found.url) + core.setOutput('advisory', String(advisory)) + core.info(`Baseline from ${found.url} (${found.match})`) + + return 0 +} + +/** + * The last step: fails the step when the mode says so. Also writes the comment body and the same + * text to the job summary. + * + * @param {object} input + * @param {object} input.core The toolkit. + * @param {object} input.env RESULTS, COMMENT_FILE, and the variables compare() reads. + * + * @returns {number} The exit status. + */ +function runCompare({ core, env }) { + const args = env.COMMENT_FILE ? ['--comment-file', env.COMMENT_FILE] : [] + + return compare({ args: [...args, env.RESULTS ?? ''], env, log: actionLog(core) }) +} + +/** + * A GitHub client for a run on a developer's machine. The token comes from GITHUB_TOKEN, else + * GH_TOKEN, else the GitHub CLI. + * + * @param {object} env The environment. + * @param {object} exec `getExecOutput`. + * + * @returns {Promise} A client. + */ +async function clientFromEnv(env, exec) { + let token = env.GITHUB_TOKEN || env.GH_TOKEN + + if (!token) { + const result = await exec.getExecOutput('gh', ['auth', 'token'], { silent: true, ignoreReturnCode: true }) + token = result.stdout.trim() + } + if (!token) { + throw lookupFailed('no token. Set GITHUB_TOKEN, or sign in with `gh auth login`') + } + + const { getOctokit } = await import('@actions/github') + + return getOctokit(token) +} + +/** + * `getExecOutput` for a command line, built on child_process. + * + * @param {object} env The environment the commands run with. + * + * @returns {{getExecOutput: Function}} The same shape as @actions/exec. + */ +function commandExec(env) { + const { spawnSync } = require('node:child_process') + + return { + getExecOutput: async (command, args = []) => { + const result = spawnSync(command, args, { encoding: 'utf8', env }) + + return { exitCode: result.status ?? 127, stdout: result.stdout ?? '', stderr: result.stderr ?? '' } + }, + } +} + +/** + * The command line: `compare` and `fetch`, with the arguments and the environment variables of the + * two bash scripts they replace. * * @param {string[]} argv The arguments after the script name. * @param {object} env The environment, default process.env. * @param {object} io `out` and `err`, each writing one line. Default: the process streams. + * @param {object} deps Stand-ins for a test: `github`, `exec`, `now`, `sleep`. * * @returns {Promise} The exit status. */ -async function main(argv, env = process.env, io = { - out: (line) => process.stdout.write(`${line}\n`), - err: (line) => process.stderr.write(`${line}\n`), -}) { +async function main( + argv, + env = process.env, + io = { + out: (line) => process.stdout.write(`${line}\n`), + err: (line) => process.stderr.write(`${line}\n`), + }, + deps = {} +) { const [command, ...args] = argv + const log = commandLog(io) if (command === 'compare') { - return compare({ args, env, log: commandLog(io) }) + return compare({ args, env, log }) + } + + if (command === 'fetch') { + const exec = deps.exec ?? commandExec(env) + let github = deps.github + try { + github ??= await clientFromEnv(env, exec) + } catch (error) { + log.err(error.message) + + return 2 + } + + const found = await fetchBaseline({ args, env, github, exec, log, now: deps.now, sleep: deps.sleep }) + if (found.status === 0) { + log.out(`${found.id} ${found.url} ${found.match}`) + } + + return found.status } - io.err('Usage: plugin-check-report.js compare [--update | --comment-file ] ') + io.err('Usage: plugin-check-report.js compare|fetch ...') return 2 } @@ -882,25 +1552,40 @@ if (require.main === module) { } module.exports = { + ARTIFACT, CheckFailure, MODES, actionLog, + asciiLower, buildComment, buildRows, change, checkResultsText, + checkWorkflowFile, commandLog, compare, compareCodePoints, countPerCode, + countRuns, cutBytes, + describeBaseline, displayCode, evaluate, + fetchBaseline, formatBaseline, + isOwnRun, linksLine, main, + orderCandidates, + parseFetchArgs, parseResults, readBaseline, readJsonValues, + readWholeNumber, + runCompare, + runFetch, + runPaths, + runUpdate, statusLine, + workflowFromRef, } diff --git a/tests/actions/plugin-check-report/fetch.test.js b/tests/actions/plugin-check-report/fetch.test.js new file mode 100644 index 0000000..dbd6aef --- /dev/null +++ b/tests/actions/plugin-check-report/fetch.test.js @@ -0,0 +1,614 @@ +'use strict' + +const { describe, it } = require('node:test') +const assert = require('node:assert') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') + +const report = require('../../../.github/actions/plugin-check-report/plugin-check-report.js') +const { octokitFor, recordingExec } = require('../../support/actions.js') + +const REPO = 'example-org/my-plugin' +const SHA = 'a'.repeat(40) +const OLD_SHA = 'b'.repeat(40) + +/** + * The API paths the lookup asks for, as they go over the wire. + * + * @param {string} workflow The workflow file name. + */ +const API = (workflow = 'plugin-check.yml') => { + const runs = `/repos/${REPO}/actions/workflows/${workflow}/runs` + + return { + forSha: (sha) => `${runs}?status=completed&head_sha=${sha}&per_page=10`, + goingForSha: (sha) => `${runs}?head_sha=${sha}&per_page=20`, + onBranch: (branch) => `${runs}?status=completed&branch=${encodeURIComponent(branch)}&per_page=100`, + artifacts: (id) => `/repos/${REPO}/actions/runs/${id}/artifacts?name=plugin-check-report`, + } +} + +/** + * A run as the workflow runs API lists it. + * + * @param {number} id Run id. + * @param {string} sha Head commit. + * @param {string} headRepo Repository the run's head commit belongs to. A pull request from a + * fork has the fork's name here. + * + * @returns {object} The run. + */ +function run(id, sha = OLD_SHA, headRepo = REPO) { + return { + id, + html_url: `https://github.com/${REPO}/actions/runs/${id}`, + head_sha: sha, + status: 'completed', + head_repository: { full_name: headRepo }, + } +} + +/** + * A clock that only moves when the code sleeps, so the wait loop runs in no time. + * + * @returns {{now: Function, sleep: Function, slept: number[]}} The clock and the sleeps it saw. + */ +function fakeClock() { + const clock = { time: 1000, slept: [] } + + return { + now: () => clock.time, + sleep: async (seconds) => { + clock.slept.push(seconds) + clock.time += seconds + }, + slept: clock.slept, + } +} + +/** + * Runs the lookup against answers written down, the way the old tests ran the script against a + * stand-in `gh`. + * + * @param {string[]} args Arguments of the lookup. An output file is added unless `bare` is set. + * @param {object} answers Each API path mapped to the JSON body it answers with. A path with no + * answer is a 404, like a failed request. + * @param {object} baselines Each run id mapped to the baseline.json its artifact holds. A run + * with no entry has an artifact that cannot be downloaded. + * @param {object} options `bare` (no output file), `env`, `clock`, `files` (run id mapped to the + * exact text of the downloaded baseline.json). + * + * @returns {Promise<{status: number, stdout: string, stderr: string, paths: string[], calls: object[], + * output: string}>} The exit status, both streams, the API paths asked for in order, the + * commands run and the baseline file written. + */ +async function fetchBaseline(args, answers = {}, baselines = {}, options = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-fetch-')) + const output = path.join(dir, 'out', 'baseline.json') + + const rules = Object.entries(answers).map(([apiPath, body]) => ({ path: apiPath, body })) + const { github, requests } = await octokitFor(rules) + const { exec, calls } = recordingExec( + Object.entries(baselines).map(([id, baseline]) => ({ + command: 'gh', + args: new RegExp(`^run download ${id} `), + effect: (callArgs) => { + const target = callArgs[callArgs.indexOf('--dir') + 1] + fs.mkdirSync(target, { recursive: true }) + fs.writeFileSync(path.join(target, 'baseline.json'), options.files?.[id] ?? JSON.stringify(baseline)) + }, + })) + ) + + const stdout = [] + const stderr = [] + const clock = options.clock ?? fakeClock() + const status = await report.main( + ['fetch', ...args, ...(options.bare ? [] : [output])], + { GITHUB_REPOSITORY: REPO, ...options.env }, + { out: (line) => stdout.push(line), err: (line) => stderr.push(line) }, + { github, exec, now: clock.now, sleep: clock.sleep } + ) + + return { + status, + stdout: stdout.join('\n'), + stderr: stderr.join('\n'), + paths: requests.map((request) => request.path), + calls, + output: fs.existsSync(output) ? fs.readFileSync(output, 'utf8') : '', + } +} + +const BASELINE = { errors: { 'Some.Rule': 2 }, warnings: {} } +const ONE_ARTIFACT = { artifacts: [{ name: 'plugin-check-report', expired: false }] } +const EXPIRED = { artifacts: [{ name: 'plugin-check-report', expired: true }] } + +describe('plugin-check-report fetch', () => { + it('takes the run for the base commit and calls it exact', async () => { + const api = API() + const result = await fetchBaseline(['--sha', SHA, 'develop'], { + [api.forSha(SHA)]: { workflow_runs: [run(200, SHA)] }, + [api.onBranch('develop')]: { workflow_runs: [run(150)] }, + [api.artifacts(200)]: ONE_ARTIFACT, + }, { 200: BASELINE }) + + assert.equal(result.status, 0, result.stderr) + assert.equal(result.stdout, `200 https://github.com/${REPO}/actions/runs/200 exact`) + assert.deepEqual(JSON.parse(result.output), BASELINE) + }) + + it('falls back to an older run on the branch, skips an expired artifact, and calls it older', async () => { + const api = API() + const result = await fetchBaseline(['--sha', SHA, 'develop'], { + [api.forSha(SHA)]: { workflow_runs: [] }, + [api.onBranch('develop')]: { workflow_runs: [run(120), run(150)] }, + [api.artifacts(150)]: EXPIRED, + [api.artifacts(120)]: ONE_ARTIFACT, + }, { 120: BASELINE }) + + assert.equal(result.status, 0, result.stderr) + assert.equal(result.stdout, `120 https://github.com/${REPO}/actions/runs/120 older`) + // Newest first by run id, whatever order the API answered in. + assert.ok( + result.paths.findIndex((request) => request.includes('runs/150/')) < + result.paths.findIndex((request) => request.includes('runs/120/')) + ) + }) + + it('never compares a run with itself or with a newer run', async () => { + const api = API() + const result = await fetchBaseline(['--before-run', '150', 'develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(160), run(150), run(140)] }, + [api.artifacts(140)]: ONE_ARTIFACT, + }, { 140: BASELINE }) + + assert.equal(result.status, 0, result.stderr) + assert.match(result.stdout, /^140 /) + assert.ok(!result.paths.some((request) => /runs\/1[56]0\//.test(request))) + }) + + it('does not wait for a run for the base commit that has already finished', async () => { + const api = API() + const clock = fakeClock() + const result = await fetchBaseline(['--sha', SHA, '--wait', '1200', 'develop'], { + [api.goingForSha(SHA)]: { workflow_runs: [run(200, SHA)] }, + [api.forSha(SHA)]: { workflow_runs: [run(200, SHA)] }, + [api.onBranch('develop')]: { workflow_runs: [] }, + [api.artifacts(200)]: ONE_ARTIFACT, + }, { 200: BASELINE }, { clock }) + + assert.equal(result.status, 0, result.stderr) + assert.match(result.stdout, /^200 .* exact$/) + assert.deepEqual(clock.slept, []) + }) + + it('exits 1 when no run has a baseline', async () => { + const api = API() + const result = await fetchBaseline(['develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(150)] }, + [api.artifacts(150)]: { artifacts: [] }, + }) + + assert.equal(result.status, 1) + assert.match(result.stderr, /No Plugin Check run on develop has a baseline/) + assert.equal(result.output, '') + }) + + it('exits 2, not 1, when the lookup itself fails', async () => { + // No answer for the branch's runs: the request is a 404, like an API error. + const result = await fetchBaseline(['develop'], {}) + + assert.equal(result.status, 2) + assert.match(result.stderr, /Could not look up Plugin Check runs: runs on develop/) + }) + + it('exits 2 when the download fails', async () => { + const api = API() + const result = await fetchBaseline(['develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(150)] }, + [api.artifacts(150)]: ONE_ARTIFACT, + }) + + assert.equal(result.status, 2) + assert.match(result.stderr, /download of the artifact of run 150/) + }) + + it('exits 2 when the list of artifacts cannot be read', async () => { + const api = API() + const result = await fetchBaseline(['develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(150)] }, + }) + + assert.equal(result.status, 2) + assert.match(result.stderr, /artifacts of run 150/) + }) + + it('stops at a failed lookup even when an older run could have worked', async () => { + const api = API() + const result = await fetchBaseline(['develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(150), run(120)] }, + [api.artifacts(150)]: ONE_ARTIFACT, + [api.artifacts(120)]: ONE_ARTIFACT, + }, { 120: BASELINE }) + + assert.equal(result.status, 2) + assert.ok(!result.paths.some((request) => request.includes('runs/120/'))) + }) + + it('reads the runs of plugin-check.yml by default', async () => { + const result = await fetchBaseline(['develop'], {}) + + assert.ok(result.paths[0].includes('/workflows/plugin-check.yml/runs'), result.paths[0]) + }) + + it('reads the runs of the workflow file --workflow names', async () => { + const api = API('wporg-check.yml') + const result = await fetchBaseline(['--workflow', 'wporg-check.yml', 'develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(150)] }, + [api.artifacts(150)]: ONE_ARTIFACT, + }, { 150: BASELINE }) + + assert.equal(result.status, 0, result.stderr) + assert.ok(result.paths[0].includes('/workflows/wporg-check.yml/runs'), result.paths[0]) + }) + + it('ignores a run whose head commit is in a fork, even on a branch named like the target', async () => { + const api = API() + const result = await fetchBaseline(['--sha', SHA, 'develop'], { + // A fork's pull request run for the base commit, and a newer one on a fork branch "develop". + [api.forSha(SHA)]: { workflow_runs: [run(300, SHA, 'someone/my-plugin')] }, + [api.onBranch('develop')]: { workflow_runs: [run(250, OLD_SHA, 'someone/my-plugin'), run(150)] }, + [api.artifacts(150)]: ONE_ARTIFACT, + [api.artifacts(250)]: ONE_ARTIFACT, + [api.artifacts(300)]: ONE_ARTIFACT, + }, { 150: BASELINE, 250: { errors: { 'Some.Rule': 9999 }, warnings: {} }, 300: { errors: { 'Some.Rule': 9999 }, warnings: {} } }) + + assert.equal(result.status, 0, result.stderr) + assert.match(result.stdout, /^150 .* older$/) + assert.deepEqual(JSON.parse(result.output), BASELINE) + assert.ok( + !result.paths.some((request) => request.includes('/runs/250/') || request.includes('/runs/300/')), + 'no fork run was looked at' + ) + }) + + it('exits 1 when only fork runs exist', async () => { + const api = API() + const result = await fetchBaseline(['develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(250, OLD_SHA, 'someone/my-plugin')] }, + }) + + assert.equal(result.status, 1) + assert.match(result.stderr, /has a baseline/) + }) + + it('matches the repository name without regard to case', async () => { + const api = API() + const result = await fetchBaseline(['develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(150, OLD_SHA, 'Example-Org/My-Plugin')] }, + [api.artifacts(150)]: ONE_ARTIFACT, + }, { 150: BASELINE }) + + assert.equal(result.status, 0, result.stderr) + }) + + it('treats a run with no head repository as not ours', async () => { + const api = API() + const result = await fetchBaseline(['develop'], { + [api.onBranch('develop')]: { workflow_runs: [{ ...run(150), head_repository: null }] }, + }) + + assert.equal(result.status, 1) + }) + + it('does not wait for a fork run that is still going', async () => { + const api = API() + const clock = fakeClock() + const result = await fetchBaseline(['--sha', SHA, '--wait', '1', 'develop'], { + // Our run for the commit is done, and a fork's run for the same commit is not. Counted, the + // fork's run would make the lookup sleep 30 seconds and then give up waiting. + [api.goingForSha(SHA)]: { + workflow_runs: [run(200, SHA), { ...run(300, SHA, 'someone/my-plugin'), status: 'in_progress' }], + }, + [api.forSha(SHA)]: { workflow_runs: [run(200, SHA)] }, + [api.onBranch('develop')]: { workflow_runs: [] }, + [api.artifacts(200)]: ONE_ARTIFACT, + }, { 200: BASELINE }, { clock }) + + assert.equal(result.status, 0, result.stderr) + assert.match(result.stdout, /^200 .* exact$/) + assert.doesNotMatch(result.stderr, /still going/) + assert.deepEqual(clock.slept, []) + }) + + it('exits 2, not 1, for an option value that is empty or not a whole number', async () => { + const cases = [ + [['--wait', '20m', 'develop'], /--wait must be a whole number, not '20m'/], + [['--wait', '', 'develop'], /--wait needs a value/], + [['--before-run', 'abc', 'develop'], /--before-run must be a whole number, not 'abc'/], + [['--before-run', '-5', 'develop'], /--before-run must be a whole number/], + [['--before-run', '9'.repeat(19), 'develop'], /--before-run is too large/], + [['--sha', '', 'develop'], /--sha needs a value/], + [['', 'baseline.json'], /a branch is needed/], + ] + + for (const [args, message] of cases) { + const result = await fetchBaseline(args, {}, {}, { bare: true }) + + assert.equal(result.status, 2, args.join(' ')) + assert.match(result.stderr, message, args.join(' ')) + assert.deepEqual(result.paths, [], `${args.join(' ')} asked the API nothing`) + } + }) + + it('exits 2 when an option or the branch is missing altogether', async () => { + const cases = [ + [['--wait'], /--wait needs a value/], + [['--workflow'], /--workflow needs a value/], + [[], /a branch is needed/], + ] + + for (const [args, message] of cases) { + const result = await fetchBaseline(args, {}, {}, { bare: true }) + + assert.equal(result.status, 2, args.join(' ')) + assert.match(result.stderr, message, args.join(' ')) + } + }) + + it('reads a run id with a leading zero as a decimal number', async () => { + const api = API() + const result = await fetchBaseline(['--before-run', '0150', 'develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(150), run(120)] }, + [api.artifacts(120)]: ONE_ARTIFACT, + }, { 120: BASELINE }) + + assert.equal(result.status, 0, result.stderr) + assert.match(result.stdout, /^120 /) + }) + + it('takes a run id of 18 digits for --before-run', async () => { + const api = API() + const result = await fetchBaseline(['--before-run', '999999999999999999', 'develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(150)] }, + [api.artifacts(150)]: ONE_ARTIFACT, + }, { 150: BASELINE }) + + assert.equal(result.status, 0, result.stderr) + assert.match(result.stdout, /^150 /) + }) + + it('refuses a --workflow that is a path or a ref, before calling the API', async () => { + for (const workflow of ['.github/workflows/plugin-check.yml', 'plugin-check.yml@refs/heads/develop']) { + const result = await fetchBaseline(['--workflow', workflow, 'develop'], {}) + + assert.equal(result.status, 2, workflow) + assert.match(result.stderr, /--workflow must be a file name/) + assert.deepEqual(result.paths, []) + } + }) + + it('asks the GitHub CLI for the repository when GITHUB_REPOSITORY is not set', async () => { + const { github, requests } = await octokitFor([]) + const { exec, calls } = recordingExec([{ command: 'gh', args: /^repo view/, stdout: `${REPO}\n` }]) + const found = await report.fetchBaseline({ + args: ['develop'], + env: {}, + github, + exec, + log: report.commandLog({ out: () => {}, err: () => {} }), + }) + + assert.equal(calls[0].command, 'gh') + assert.match(calls[0].args.join(' '), /^repo view/) + assert.equal(requests[0].path.startsWith(`/repos/${REPO}/`), true) + assert.equal(found.status, 2, 'the runs are not answered, so the lookup fails after the repository') + }) + + it('exits 2 when there is no repository at all', async () => { + const result = await fetchBaseline(['develop'], {}, {}, { env: { GITHUB_REPOSITORY: '' } }) + + assert.equal(result.status, 2) + assert.match(result.stderr, /no repository/) + }) + + it('takes the next run when the artifact has no baseline.json, or an empty one', async () => { + const api = API() + const result = await fetchBaseline(['develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(150), run(140), run(130)] }, + [api.artifacts(150)]: ONE_ARTIFACT, + [api.artifacts(140)]: ONE_ARTIFACT, + [api.artifacts(130)]: ONE_ARTIFACT, + }, { 150: {}, 140: {}, 130: BASELINE }, { files: { 150: '' } }) + + assert.equal(result.status, 0, result.stderr) + assert.match(result.stdout, /^140 /, 'an empty file is skipped, and a non-empty one is taken') + }) + + it('creates the folder of the output file', async () => { + const api = API() + const result = await fetchBaseline(['develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(150)] }, + [api.artifacts(150)]: ONE_ARTIFACT, + }, { 150: BASELINE }) + + assert.equal(result.status, 0) + assert.deepEqual(JSON.parse(result.output), BASELINE) + }) + + it('asks for the pages of runs in the order the old script did, with the filters in the query', async () => { + const api = API() + const result = await fetchBaseline(['--sha', SHA, '--wait', '5', 'develop'], { + [api.goingForSha(SHA)]: { workflow_runs: [run(200, SHA)] }, + [api.forSha(SHA)]: { workflow_runs: [run(200, SHA)] }, + [api.onBranch('develop')]: { workflow_runs: [] }, + [api.artifacts(200)]: ONE_ARTIFACT, + }, { 200: BASELINE }) + + assert.deepEqual(result.paths, [ + api.goingForSha(SHA), + api.forSha(SHA), + api.onBranch('develop'), + api.artifacts(200), + ]) + }) + + it('encodes a branch name with a slash in the query', async () => { + const api = API() + const result = await fetchBaseline(['feature/x y'], {}) + + assert.deepEqual(result.paths, [api.onBranch('feature/x y')]) + assert.match(result.paths[0], /branch=feature%2Fx%20y&/) + }) + + it('downloads with the GitHub CLI, for the repository and the artifact name', async () => { + const api = API() + const result = await fetchBaseline(['develop'], { + [api.onBranch('develop')]: { workflow_runs: [run(150)] }, + [api.artifacts(150)]: ONE_ARTIFACT, + }, { 150: BASELINE }) + + assert.equal(result.calls.length, 1) + assert.equal(result.calls[0].command, 'gh') + assert.deepEqual(result.calls[0].args.slice(0, 7), ['run', 'download', '150', '--repo', REPO, '--name', 'plugin-check-report']) + }) + + describe('the wait for the run of the base commit', () => { + it('polls every 30 seconds until the run is done', async () => { + const api = API() + const clock = fakeClock() + const { github, requests } = await octokitFor([ + { + path: api.goingForSha(SHA), + responses: [ + { body: { workflow_runs: [{ ...run(200, SHA), status: 'in_progress' }] } }, + { body: { workflow_runs: [{ ...run(200, SHA), status: 'in_progress' }] } }, + { body: { workflow_runs: [run(200, SHA)] } }, + ], + }, + { path: api.forSha(SHA), body: { workflow_runs: [run(200, SHA)] } }, + { path: api.onBranch('develop'), body: { workflow_runs: [] } }, + { path: api.artifacts(200), body: ONE_ARTIFACT }, + ]) + const { exec } = recordingExec([ + { + command: 'gh', + args: /^run download 200 /, + effect: (args) => { + const target = args[args.indexOf('--dir') + 1] + fs.mkdirSync(target, { recursive: true }) + fs.writeFileSync(path.join(target, 'baseline.json'), JSON.stringify(BASELINE)) + }, + }, + ]) + const output = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-wait-')), 'baseline.json') + + const status = await report.main( + ['fetch', '--sha', SHA, '--wait', '1200', 'develop', output], + { GITHUB_REPOSITORY: REPO }, + { out: () => {}, err: () => {} }, + { github, exec, now: clock.now, sleep: clock.sleep } + ) + + assert.equal(status, 0) + assert.deepEqual(clock.slept, [30, 30]) + assert.equal(requests.filter((request) => request.path === api.goingForSha(SHA)).length, 3) + }) + + it('gives up after the wait, says so, and goes on with older runs', async () => { + const api = API() + const clock = fakeClock() + const stillGoing = { workflow_runs: [{ ...run(200, SHA), status: 'in_progress' }] } + const result = await fetchBaseline(['--sha', SHA, '--wait', '60', 'develop'], { + [api.goingForSha(SHA)]: stillGoing, + [api.forSha(SHA)]: { workflow_runs: [] }, + [api.onBranch('develop')]: { workflow_runs: [run(150)] }, + [api.artifacts(150)]: ONE_ARTIFACT, + }, { 150: BASELINE }, { clock }) + + assert.equal(result.status, 0, result.stderr) + assert.match(result.stdout, /^150 .* older$/) + assert.match(result.stderr, new RegExp(`The run for ${SHA} is still going after 60 seconds; trying older runs\\.`)) + assert.deepEqual(clock.slept, [30, 30]) + }) + + it('stops waiting for a run that never shows up after 120 seconds', async () => { + const api = API() + const clock = fakeClock() + const result = await fetchBaseline(['--sha', SHA, '--wait', '1200', 'develop'], { + [api.goingForSha(SHA)]: { workflow_runs: [] }, + [api.forSha(SHA)]: { workflow_runs: [] }, + [api.onBranch('develop')]: { workflow_runs: [run(150)] }, + [api.artifacts(150)]: ONE_ARTIFACT, + }, { 150: BASELINE }, { clock }) + + assert.equal(result.status, 0, result.stderr) + assert.deepEqual(clock.slept, [30, 30, 30, 30]) + assert.doesNotMatch(result.stderr, /still going/) + }) + + it('says it is still going, not that the run never showed up, when the wait is short', async () => { + const api = API() + const clock = fakeClock() + const result = await fetchBaseline(['--sha', SHA, '--wait', '20', 'develop'], { + [api.goingForSha(SHA)]: { workflow_runs: [] }, + [api.forSha(SHA)]: { workflow_runs: [] }, + [api.onBranch('develop')]: { workflow_runs: [] }, + }, {}, { clock }) + + assert.equal(result.status, 1) + assert.match(result.stderr, /still going after 20 seconds/) + }) + + it('does not wait at all with --wait 0, or without --sha', async () => { + const api = API() + const clock = fakeClock() + const result = await fetchBaseline(['--wait', '300', 'develop'], { + [api.onBranch('develop')]: { workflow_runs: [] }, + }, {}, { clock }) + + assert.equal(result.status, 1) + assert.deepEqual(result.paths, [api.onBranch('develop')]) + }) + + it('exits 2 when polling fails', async () => { + const result = await fetchBaseline(['--sha', SHA, '--wait', '60', 'develop'], {}) + + assert.equal(result.status, 2) + assert.match(result.stderr, new RegExp(`runs for ${SHA}`)) + }) + }) +}) + +describe('plugin-check-report fetch helpers', () => { + it('reads the workflow file from the workflow ref', () => { + assert.equal( + report.workflowFromRef('example-org/my-plugin/.github/workflows/plugin-check.yml@refs/pull/12/merge'), + 'plugin-check.yml' + ) + assert.equal(report.workflowFromRef(''), '') + assert.equal(report.workflowFromRef(undefined), '') + }) + + it('lower-cases only A to Z', () => { + assert.equal(report.asciiLower('Example-Org/ÄB'), 'example-org/Äb') + }) + + it('names the baseline for the event and the match', () => { + assert.deepEqual(report.describeBaseline('pull_request', 'exact'), { label: 'Target branch', advisory: false }) + assert.deepEqual(report.describeBaseline('pull_request', 'older'), { label: 'Target branch, older run', advisory: true }) + assert.deepEqual(report.describeBaseline('push', 'exact'), { label: 'Previous run', advisory: false }) + }) + + it('orders runs: the base commit first, then the branch, newest first, each once', () => { + const ordered = report.orderCandidates({ + shaRuns: [run(5), run(9)], + branchRuns: [run(7), run(9), run(11)], + beforeRun: null, + }) + + assert.deepEqual(ordered.map((candidate) => candidate.id), [5, 9, 11, 7]) + }) +}) From 6144039126203af3d5434339d29613c9c18f47b3 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Tue, 3 Nov 2026 18:54:58 -0300 Subject: [PATCH 15/31] Fail on a Plugin Check baseline count too large to compare exactly --- .github/actions/plugin-check-report/README.md | 10 ++++++++++ .../plugin-check-report/plugin-check-report.js | 8 +++++++- .../actions/plugin-check-report/compare.test.js | 16 ++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/.github/actions/plugin-check-report/README.md b/.github/actions/plugin-check-report/README.md index 1051500..61eb8d3 100644 --- a/.github/actions/plugin-check-report/README.md +++ b/.github/actions/plugin-check-report/README.md @@ -226,6 +226,16 @@ You need `bash`, `jq`, and for the baseline the GitHub CLI signed in (`gh auth l `PLUGIN_CHECK_MODE=strict` to see the strict result. Skip step 2 to see the counts with no baseline. +## Differences from the earlier bash scripts + +The first version of this action was two bash scripts. The JavaScript that replaced them gives the +same results on every input Plugin Check can produce. These differences are on purpose, and only +show with a hand-edited baseline or an odd results file: + +- A baseline count above 9007199254740991 (2^53 - 1) fails the compare with exit status 5 and says + so. JavaScript cannot hold a larger number exactly, so a comment would show another number. Bash + compared them exactly. + ## How it is tested `tests/actions/plugin-check-report/` runs both scripts with bash from `node --test`. `ratchet.sh` diff --git a/.github/actions/plugin-check-report/plugin-check-report.js b/.github/actions/plugin-check-report/plugin-check-report.js index 5b3294b..3bf7c29 100644 --- a/.github/actions/plugin-check-report/plugin-check-report.js +++ b/.github/actions/plugin-check-report/plugin-check-report.js @@ -334,8 +334,14 @@ function readBaseline(text) { for (const [code, count] of Object.entries(entries)) { if (count === null || count === false) { counts[kind].set(code, 0) - } else if (typeof count === 'number' && Number.isInteger(count)) { + } else if (Number.isSafeInteger(count)) { counts[kind].set(code, count) + } else if (Number.isInteger(count)) { + // JavaScript cannot hold it exactly, so a table or a message would show another number. + throw new CheckFailure( + `The baseline is not valid: the count of ${displayCode(code)} in "${kind}" is more than ${Number.MAX_SAFE_INTEGER}, which cannot be compared exactly.`, + 5 + ) } else { throw new CheckFailure( `The baseline is not valid: the count of ${displayCode(code)} in "${kind}" must be a whole number.`, diff --git a/tests/actions/plugin-check-report/compare.test.js b/tests/actions/plugin-check-report/compare.test.js index ec4c02b..de1d8fe 100644 --- a/tests/actions/plugin-check-report/compare.test.js +++ b/tests/actions/plugin-check-report/compare.test.js @@ -401,6 +401,22 @@ describe('plugin-check-report compare', () => { } }) + it('fails on a count too large to compare exactly, instead of printing another number', () => { + for (const content of ['{"errors":{"a":9007199254740993}}', '{"warnings":{"a":1e300}}', '{"errors":{"a":-9007199254740993}}']) { + const run = ratchet([resultsOf([{ type: 'ERROR', code: 'a' }])], { PLUGIN_CHECK_BASELINE: baselineFile(content) }) + + assert.equal(run.status, 5, content) + assert.match(run.stderr, /more than 9007199254740991, which cannot be compared exactly/) + assert.equal(run.comment, '', content) + } + + const largest = ratchet([resultsOf([{ type: 'ERROR', code: 'a' }])], { + PLUGIN_CHECK_BASELINE: baselineFile('{"errors":{"a":9007199254740991}}'), + }) + assert.equal(largest.status, 0, 'the largest exact number still works') + assert.match(largest.comment, /\| 1 \| 9007199254740991 \| -9007199254740990 \|/) + }) + it('counts a rule with an empty code like any other', () => { const run = ratchet([resultsOf([{ type: 'ERROR', code: '' }, { type: 'ERROR', code: '' }])], { PLUGIN_CHECK_BASELINE: baselineFile({ errors: { '': 1 }, warnings: {} }), From bf3b903c4fe1bc8c8a53a982e5115425e458d370 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Tue, 3 Nov 2026 18:56:58 -0300 Subject: [PATCH 16/31] Run the Plugin Check report with github-script instead of bash --- .github/actions/plugin-check-report/README.md | 32 +- .../actions/plugin-check-report/action.yml | 120 +++---- .github/workflows/plugin-check.yml | 2 +- .gitignore | 2 +- .../plugin-check-report/action.test.js | 300 ++++++++++++++++++ 5 files changed, 366 insertions(+), 90 deletions(-) create mode 100644 tests/actions/plugin-check-report/action.test.js diff --git a/.github/actions/plugin-check-report/README.md b/.github/actions/plugin-check-report/README.md index 61eb8d3..ff4ad7d 100644 --- a/.github/actions/plugin-check-report/README.md +++ b/.github/actions/plugin-check-report/README.md @@ -184,7 +184,6 @@ not start the run. The job that runs the Plugin Check action lowers itself to `c | `comment-file` | no | none | Where to write the comment body | | `mode` | no | `ratchet` | `ratchet` or `strict` | | `comment-marker` | no | `` | First line of the comment body | -| `workflow-file` | no | from `GITHUB_WORKFLOW_REF` | Whose runs hold the baseline | | `baseline-wait` | no | `1200` | Seconds to wait for the base commit's run. A whole number, or the check fails | | `badge-url` | no | none | Image under the status line | | `token` | no | `github.token` | Needs `actions: read` | @@ -196,7 +195,9 @@ not start the run. The job that runs the Plugin Check action lowers itself to `c ## Running it locally -You need `bash`, `jq`, and for the baseline the GitHub CLI signed in (`gh auth login`). +You need Node 20 or newer, and for the baseline the GitHub CLI signed in (`gh auth login`). Run +`npm install` once in the toolbox checkout, which installs the GitHub client the lookup uses. The +command line takes the same arguments and environment variables as the two bash scripts did. 1. Build the plugin and run Plugin Check in a local WordPress, for example with wp-env: @@ -205,26 +206,29 @@ You need `bash`, `jq`, and for the baseline the GitHub CLI signed in (`gh auth l npx wp-env run cli wp plugin check my-plugin --format=json > results.txt ``` - Keep only the command's output in the file. A notice from wp-env at the top fails the ratchet + Keep only the command's output in the file. A notice from wp-env at the top fails the compare with "not Plugin Check output", on purpose. 2. Get a baseline from the branch you compare with: ```bash GITHUB_REPOSITORY=my-org/my-plugin \ - bash .github/actions/plugin-check-report/fetch-baseline.sh develop baseline.json + node .github/actions/plugin-check-report/plugin-check-report.js fetch develop baseline.json ``` + The token comes from `GITHUB_TOKEN`, else `GH_TOKEN`, else `gh auth token`. The exit status is 0 + when a baseline was written, 1 when no run has one, and 2 when the lookup itself failed. + 3. Compare: ```bash PLUGIN_CHECK_BASELINE=baseline.json \ - bash .github/actions/plugin-check-report/ratchet.sh --comment-file comment.md results.txt + node .github/actions/plugin-check-report/plugin-check-report.js compare --comment-file comment.md results.txt ``` Exit 0 means the check passes. `comment.md` holds what the pull request would show. Add `PLUGIN_CHECK_MODE=strict` to see the strict result. Skip step 2 to see the counts with no - baseline. + baseline. `compare --update results.txt` writes the baseline for a results file instead. ## Differences from the earlier bash scripts @@ -238,7 +242,15 @@ show with a hand-edited baseline or an odd results file: ## How it is tested -`tests/actions/plugin-check-report/` runs both scripts with bash from `node --test`. `ratchet.sh` -reads the sample results files in `fixtures/`; `fetch-baseline.sh` talks to a stand-in `gh` that -answers from JSON files and records each call. Run `npm install` once, then `node --test`. They need -`jq`, which GitHub's Ubuntu runners have. +The work is in `plugin-check-report.js`: small functions that take plain values, so a test calls +them directly. `tests/actions/plugin-check-report/` has three files, run with `node --test`: + +- `compare.test.js` runs the compare in-process on the sample results files in `fixtures/`, and + checks the comment, the job summary and the annotations against the exact files in + `fixtures/expected/`. Those files were written by the original bash script, so they pin that the + JavaScript prints what it printed. +- `fetch.test.js` runs the lookup against a real GitHub client with the network replaced by written + answers, a fake clock for the wait, and a stand-in for the `gh` download. +- `action.test.js` calls the four step functions the way `actions/github-script` does. + +No `jq` or `bash` is needed. diff --git a/.github/actions/plugin-check-report/action.yml b/.github/actions/plugin-check-report/action.yml index 09a0a56..9815095 100644 --- a/.github/actions/plugin-check-report/action.yml +++ b/.github/actions/plugin-check-report/action.yml @@ -7,8 +7,9 @@ # .github/workflows/plugin-check.yml does both, so this action stays usable with a results file # from anywhere. # -# The work is in two bash scripts next to this file, ratchet.sh and fetch-baseline.sh. They are -# run by path, so there is nothing to build: the file in the repository is the file that runs. +# The work is in one JavaScript file next to this file, plugin-check-report.js. Each step below is +# an actions/github-script step that requires it by path, so there is nothing to build: the file in +# the repository is the file that runs. The same file runs from a command line; see the README. name: Plugin Check report description: Compares Plugin Check results with an earlier run's baseline, and saves a new baseline @@ -36,17 +37,11 @@ inputs: comment. Empty uses ``. required: false default: '' - workflow-file: - description: >- - File name of the workflow whose earlier runs hold the baseline, e.g. `plugin-check.yml`. - Empty reads it from GITHUB_WORKFLOW_REF, which names the calling workflow's file when this - runs inside a reusable workflow. - required: false - default: '' baseline-wait: description: >- Seconds a pull request waits for the run of its base commit to finish, when that run is - still going. + still going. The reusable workflow lets its report job run for 30 minutes, so the default + leaves room for the work around the wait. required: false default: '1200' badge-url: @@ -71,51 +66,38 @@ runs: using: 'composite' steps: # One folder per run under RUNNER_TEMP, cleared first, so a second use of the action in the same - # job never reads the first one's files. + # job never reads the first one's files. Also finds out which workflow's runs hold the baseline: + # in a called workflow GITHUB_WORKFLOW_REF is the caller's, and the runs belong to that file, not + # to the toolbox's workflow. - name: Set file paths id: paths - shell: bash + uses: actions/github-script@v9 env: - INPUT_WORKFLOW_FILE: ${{ inputs.workflow-file }} + # github-script's require() resolves against the workspace of the repository being built, + # not this action's directory, so the path has to be passed in. + ACTION_PATH: ${{ github.action_path }} INPUT_COMMENT_MARKER: ${{ inputs.comment-marker }} - run: | - dir="$RUNNER_TEMP/plugin-check-report" - rm -rf "$dir" - mkdir -p "$dir/report" "$dir/baseline" - - # In a called workflow GITHUB_WORKFLOW_REF is the caller's, like - # example-org/my-plugin/.github/workflows/plugin-check.yml@refs/pull/12/merge. The runs that - # hold the baseline belong to that file, not to the toolbox's workflow. - workflow=$INPUT_WORKFLOW_FILE - if [ -z "$workflow" ]; then - ref=${GITHUB_WORKFLOW_REF%%@*} - workflow=${ref##*/} - fi - - { - echo "report=$dir/report" - echo "baseline=$dir/baseline/baseline.json" - echo "workflow=$workflow" - echo "marker=${INPUT_COMMENT_MARKER:-}" - } >> "$GITHUB_OUTPUT" - echo "Baselines are read from the runs of $workflow." + with: + script: | + const { runPaths } = require(`${process.env.ACTION_PATH}/plugin-check-report.js`) + process.exitCode = runPaths({ core, env: process.env }) # A baseline made from this run, uploaded even when the compare below fails. Later runs # download it as their baseline: pull requests into this branch, and the next push. - name: Write a new baseline - shell: bash + uses: actions/github-script@v9 env: + ACTION_PATH: ${{ github.action_path }} RESULTS: ${{ inputs.results }} REPORT_DIR: ${{ steps.paths.outputs.report }} - run: | - if [ -f "$RESULTS" ]; then - cp "$RESULTS" "$REPORT_DIR/" - PLUGIN_CHECK_BASELINE="$REPORT_DIR/baseline.json" bash "$GITHUB_ACTION_PATH/ratchet.sh" --update "$RESULTS" - fi + with: + script: | + const { runUpdate } = require(`${process.env.ACTION_PATH}/plugin-check-report.js`) + process.exitCode = runUpdate({ core, env: process.env }) - # The name and the baseline.json inside it are what fetch-baseline.sh looks for. Keep both: - # repositories that ran an earlier copy of this check upload the same names, so their old - # baselines keep working. + # The name and the baseline.json inside it are what the lookup in plugin-check-report.js looks + # for. Keep both: repositories that ran an earlier copy of this check upload the same names, so + # their old baselines keep working. - name: Upload the report id: artifact uses: actions/upload-artifact@v7 @@ -131,13 +113,15 @@ runs: # the base branch is used instead, and more errors than it are only reported, not failed: that # run is older than the target. A push takes the run before it on the same branch, so a merge # that adds errors fails there, where the whole team sees it. - # With no run to take it from, ratchet.sh compares nothing and passes with a warning. A failed + # With no run to take it from, the compare checks nothing and passes with a warning. A failed # lookup (GitHub API or download error) fails the step instead, so an outage never passes as # "no baseline". - name: Get the baseline id: baseline - shell: bash + uses: actions/github-script@v9 env: + ACTION_PATH: ${{ github.action_path }} + # The GitHub CLI downloads the artifact, and reads its token from here. GH_TOKEN: ${{ inputs.token }} EVENT: ${{ github.event_name }} BASE_REF: ${{ github.event.pull_request.base.ref }} @@ -145,39 +129,19 @@ runs: WAIT: ${{ inputs.baseline-wait }} WORKFLOW: ${{ steps.paths.outputs.workflow }} BASELINE: ${{ steps.paths.outputs.baseline }} - run: | - fetch="$GITHUB_ACTION_PATH/fetch-baseline.sh" - status=0 - if [ "$EVENT" = pull_request ]; then - label="Target branch" - found=$(bash "$fetch" --workflow "$WORKFLOW" --sha "$BASE_SHA" --wait "$WAIT" "$BASE_REF" "$BASELINE") || status=$? - else - label="Previous run" - found=$(bash "$fetch" --workflow "$WORKFLOW" --before-run "$GITHUB_RUN_ID" "$GITHUB_REF_NAME" "$BASELINE") || status=$? - fi - case "$status" in - 0) ;; - 1) echo "No earlier run has a baseline."; exit 0 ;; - *) exit "$status" ;; - esac - read -r _ url match <<< "$found" - advisory=false - if [ "$match" = older ]; then - label="Target branch, older run" - advisory=true - fi - { - echo "label=$label" - echo "url=$url" - echo "advisory=$advisory" - } >> "$GITHUB_OUTPUT" - echo "Baseline from $url ($match)" + with: + github-token: ${{ inputs.token }} + script: | + const { runFetch } = require(`${process.env.ACTION_PATH}/plugin-check-report.js`) + process.exitCode = await runFetch({ github, core, exec, env: process.env }) # Fails the step when the mode says so. Also writes the comment body and the same text to the - # job summary. + # job summary. The step ends with an exit status, not an extra error annotation: the messages + # above it already say what is wrong. - name: Compare with the baseline - shell: bash + uses: actions/github-script@v9 env: + ACTION_PATH: ${{ github.action_path }} RESULTS: ${{ inputs.results }} COMMENT_FILE: ${{ inputs.comment-file }} PLUGIN_CHECK_MODE: ${{ inputs.mode }} @@ -189,7 +153,7 @@ runs: COMMENT_BADGE: ${{ inputs.badge-url }} ARTIFACT_URL: ${{ steps.artifact.outputs.artifact-url }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: | - args=() - [ -z "$COMMENT_FILE" ] || args=(--comment-file "$COMMENT_FILE") - bash "$GITHUB_ACTION_PATH/ratchet.sh" "${args[@]}" "$RESULTS" + with: + script: | + const { runCompare } = require(`${process.env.ACTION_PATH}/plugin-check-report.js`) + process.exitCode = runCompare({ core, env: process.env }) diff --git a/.github/workflows/plugin-check.yml b/.github/workflows/plugin-check.yml index d11a221..972bf81 100644 --- a/.github/workflows/plugin-check.yml +++ b/.github/workflows/plugin-check.yml @@ -26,7 +26,7 @@ name: Plugin Check # # Run it locally: see "Running it locally" in .github/actions/plugin-check-report/README.md. In # short, `wp plugin check --format=json > results.txt` in a local WordPress, then -# `bash .github/actions/plugin-check-report/ratchet.sh results.txt`. +# `node .github/actions/plugin-check-report/plugin-check-report.js compare results.txt`. on: workflow_call: diff --git a/.gitignore b/.gitignore index b58b9f1..ef9bf5d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,5 @@ bun.lock bun.lockb yarn.lock -# fetch-baseline.sh writes the baseline here when run by hand (see the plugin-check-report README). +# plugin-check-report.js fetch writes the baseline here when run by hand (see the plugin-check-report README). .github/actions/plugin-check-report/baseline.json diff --git a/tests/actions/plugin-check-report/action.test.js b/tests/actions/plugin-check-report/action.test.js new file mode 100644 index 0000000..c1575b3 --- /dev/null +++ b/tests/actions/plugin-check-report/action.test.js @@ -0,0 +1,300 @@ +'use strict' + +const { describe, it } = require('node:test') +const assert = require('node:assert') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') + +const report = require('../../../.github/actions/plugin-check-report/plugin-check-report.js') +const { actionsFor, recordingExec } = require('../../support/actions.js') + +const FIXTURES = path.join(__dirname, 'fixtures') +const REPO = 'example-org/my-plugin' +const SHA = 'a'.repeat(40) + +const temp = (prefix) => fs.mkdtempSync(path.join(os.tmpdir(), `plugin-check-${prefix}-`)) + +const RUNS = `/repos/${REPO}/actions/workflows/plugin-check.yml/runs` +const run = (id, sha, headRepo = REPO) => ({ + id, + html_url: `https://github.com/${REPO}/actions/runs/${id}`, + head_sha: sha, + status: 'completed', + head_repository: { full_name: headRepo }, +}) + +/** + * Exec rules for a baseline that downloads. + * + * @param {number} id The run. + * @param {string} baseline What its baseline.json holds. + * + * @returns {object[]} The rule. + */ +function downloads(id, baseline = '{"errors":{"Some.Rule":2},"warnings":{}}') { + return [ + { + command: 'gh', + args: new RegExp(`^run download ${id} `), + effect: (args) => { + const target = args[args.indexOf('--dir') + 1] + fs.mkdirSync(target, { recursive: true }) + fs.writeFileSync(path.join(target, 'baseline.json'), baseline) + }, + }, + ] +} + +const ARTIFACT = { body: { artifacts: [{ name: 'plugin-check-report', expired: false }] } } + +describe('plugin-check-report action steps', () => { + describe('runPaths', () => { + it('makes a clean folder for the run and names the workflow of the caller', async (t) => { + const { core, outputs, logged } = await actionsFor(t) + const tempDir = temp('paths') + fs.mkdirSync(path.join(tempDir, 'plugin-check-report', 'report'), { recursive: true }) + fs.writeFileSync(path.join(tempDir, 'plugin-check-report', 'report', 'old.txt'), 'from an earlier use') + + const status = report.runPaths({ + core, + env: { + RUNNER_TEMP: tempDir, + GITHUB_REPOSITORY: REPO, + GITHUB_WORKFLOW_REF: `${REPO}/.github/workflows/plugin-check.yml@refs/pull/12/merge`, + }, + }) + + assert.equal(status, 0) + assert.deepEqual(outputs(), { + report: path.join(tempDir, 'plugin-check-report', 'report'), + baseline: path.join(tempDir, 'plugin-check-report', 'baseline', 'baseline.json'), + workflow: 'plugin-check.yml', + marker: ``, + }) + assert.deepEqual(fs.readdirSync(path.join(tempDir, 'plugin-check-report', 'report')), []) + assert.match(logged(), /Baselines are read from the runs of plugin-check\.yml\./) + }) + + it('uses the marker the caller gives', async (t) => { + const { core, outputs } = await actionsFor(t) + + report.runPaths({ + core, + env: { RUNNER_TEMP: temp('marker'), GITHUB_REPOSITORY: REPO, GITHUB_WORKFLOW_REF: 'a/b/.github/workflows/x.yml@r', INPUT_COMMENT_MARKER: '' }, + }) + + assert.equal(outputs().marker, '') + }) + }) + + describe('runUpdate', () => { + it('writes the baseline of this run next to a copy of the results', async (t) => { + const { core, logged } = await actionsFor(t) + const reportDir = temp('report') + + const status = report.runUpdate({ + core, + env: { RESULTS: path.join(FIXTURES, 'errors-up.txt'), REPORT_DIR: reportDir }, + }) + + assert.equal(status, 0) + assert.ok(fs.existsSync(path.join(reportDir, 'errors-up.txt'))) + assert.deepEqual(JSON.parse(fs.readFileSync(path.join(reportDir, 'baseline.json'), 'utf8')).warnings, { + 'WordPress.DB.DirectDatabaseQuery.DirectQuery': 1, + }) + assert.match(logged(), /Wrote .*baseline\.json: 2 error codes, 1 warning codes/) + }) + + it('does nothing when there is no results file', async (t) => { + const { core } = await actionsFor(t) + const reportDir = temp('report') + + assert.equal(report.runUpdate({ core, env: { RESULTS: '/nonexistent/results.txt', REPORT_DIR: reportDir } }), 0) + assert.deepEqual(fs.readdirSync(reportDir), []) + }) + + it('fails on a results file that is not Plugin Check output, as the compare does', async (t) => { + const { core, logged } = await actionsFor(t) + + const status = report.runUpdate({ + core, + env: { RESULTS: path.join(FIXTURES, 'garbage.txt'), REPORT_DIR: temp('report') }, + }) + + assert.equal(status, 1) + assert.match(logged(), /::error title=Plugin Check::The results file is not Plugin Check output/) + }) + }) + + describe('runFetch', () => { + it('takes the run for the base commit of a pull request and labels it', async (t) => { + const baseline = path.join(temp('fetch'), 'baseline.json') + const { github, core, outputs, logged } = await actionsFor(t, [ + { path: `${RUNS}?head_sha=${SHA}&per_page=20`, body: { workflow_runs: [run(200, SHA)] } }, + { path: `${RUNS}?status=completed&head_sha=${SHA}&per_page=10`, body: { workflow_runs: [run(200, SHA)] } }, + { path: `${RUNS}?status=completed&branch=develop&per_page=100`, body: { workflow_runs: [] } }, + { path: `/repos/${REPO}/actions/runs/200/artifacts?name=plugin-check-report`, ...ARTIFACT }, + ]) + const { exec } = recordingExec(downloads(200)) + + const status = await report.runFetch({ + github, + core, + exec, + env: { GITHUB_REPOSITORY: REPO, EVENT: 'pull_request', BASE_REF: 'develop', BASE_SHA: SHA, WAIT: '1200', WORKFLOW: 'plugin-check.yml', BASELINE: baseline }, + now: () => 1000, + sleep: async () => {}, + }) + + assert.equal(status, 0) + assert.deepEqual(outputs(), { + label: 'Target branch', + url: `https://github.com/${REPO}/actions/runs/200`, + advisory: 'false', + }) + assert.match(logged(), /Baseline from https:\/\/github\.com\/example-org\/my-plugin\/actions\/runs\/200 \(exact\)/) + assert.ok(fs.existsSync(baseline)) + }) + + it('marks a baseline from an older run as advisory', async (t) => { + const { github, core, outputs } = await actionsFor(t, [ + { path: `${RUNS}?head_sha=${SHA}&per_page=20`, body: { workflow_runs: [] } }, + { path: `${RUNS}?status=completed&head_sha=${SHA}&per_page=10`, body: { workflow_runs: [] } }, + { path: `${RUNS}?status=completed&branch=develop&per_page=100`, body: { workflow_runs: [run(150, 'b'.repeat(40))] } }, + { path: `/repos/${REPO}/actions/runs/150/artifacts?name=plugin-check-report`, ...ARTIFACT }, + ]) + const { exec } = recordingExec(downloads(150)) + + await report.runFetch({ + github, + core, + exec, + env: { GITHUB_REPOSITORY: REPO, EVENT: 'pull_request', BASE_REF: 'develop', BASE_SHA: SHA, WAIT: '0', WORKFLOW: 'plugin-check.yml', BASELINE: path.join(temp('fetch'), 'baseline.json') }, + }) + + assert.deepEqual({ label: outputs().label, advisory: outputs().advisory }, { label: 'Target branch, older run', advisory: 'true' }) + }) + + it('takes the run before this one on the same branch for a push', async (t) => { + const { github, core, outputs } = await actionsFor(t, [ + { path: `${RUNS}?status=completed&branch=develop&per_page=100`, body: { workflow_runs: [run(300, SHA), run(150, SHA)] } }, + { path: `/repos/${REPO}/actions/runs/150/artifacts?name=plugin-check-report`, ...ARTIFACT }, + ]) + const { exec } = recordingExec(downloads(150)) + + const status = await report.runFetch({ + github, + core, + exec, + env: { GITHUB_REPOSITORY: REPO, EVENT: 'push', GITHUB_RUN_ID: '200', GITHUB_REF_NAME: 'develop', WORKFLOW: 'plugin-check.yml', BASELINE: path.join(temp('fetch'), 'baseline.json') }, + }) + + assert.equal(status, 0) + assert.equal(outputs().label, 'Previous run') + assert.match(outputs().url, /runs\/150$/) + }) + + it('passes with a message, and no outputs, when no run has a baseline', async (t) => { + const { github, core, outputs, logged } = await actionsFor(t, [ + { path: `${RUNS}?status=completed&branch=develop&per_page=100`, body: { workflow_runs: [] } }, + ]) + const { exec } = recordingExec() + + const status = await report.runFetch({ + github, + core, + exec, + env: { GITHUB_REPOSITORY: REPO, EVENT: 'push', GITHUB_RUN_ID: '200', GITHUB_REF_NAME: 'develop', WORKFLOW: 'plugin-check.yml', BASELINE: path.join(temp('fetch'), 'baseline.json') }, + }) + + assert.equal(status, 0) + assert.deepEqual(outputs(), {}) + assert.match(logged(), /No earlier run has a baseline\./) + }) + + it('fails with exit status 2 when the lookup fails, so an outage never passes as no baseline', async (t) => { + const { github, core, logged } = await actionsFor(t, []) + const { exec } = recordingExec() + + const status = await report.runFetch({ + github, + core, + exec, + env: { GITHUB_REPOSITORY: REPO, EVENT: 'push', GITHUB_RUN_ID: '200', GITHUB_REF_NAME: 'develop', WORKFLOW: 'plugin-check.yml', BASELINE: path.join(temp('fetch'), 'baseline.json') }, + }) + + assert.equal(status, 2) + assert.match(logged(), /Could not look up Plugin Check runs: runs on develop/) + }) + + it('fails with exit status 2 when the workflow file is unknown', async (t) => { + const { github, core } = await actionsFor(t, []) + const { exec } = recordingExec() + + const status = await report.runFetch({ + github, + core, + exec, + env: { GITHUB_REPOSITORY: REPO, EVENT: 'push', GITHUB_RUN_ID: '200', GITHUB_REF_NAME: 'develop', WORKFLOW: '', BASELINE: 'x' }, + }) + + assert.equal(status, 2) + }) + }) + + describe('runCompare', () => { + const base = () => ({ + RESULTS: path.join(FIXTURES, 'errors-up.txt'), + PLUGIN_CHECK_BASELINE: path.join(FIXTURES, 'baseline.json'), + RUN_URL: 'https://example.test/run/1', + }) + + it('fails the step with the annotation, the comment and the same text in the job summary', async (t) => { + const { core, logged, summary } = await actionsFor(t) + const commentFile = path.join(temp('compare'), 'comment.md') + + const status = report.runCompare({ core, env: { ...base(), COMMENT_FILE: commentFile, GITHUB_STEP_SUMMARY: process.env.GITHUB_STEP_SUMMARY } }) + + assert.equal(status, 1) + assert.match(logged(), /::error title=Plugin Check::WordPress\.Security\.EscapeOutput\.OutputNotEscaped has 3 errors, the baseline has 2\./) + assert.match(logged(), /Plugin Check found more errors than the baseline allows\./) + const comment = fs.readFileSync(commentFile, 'utf8') + assert.match(comment, /❌ Plugin Check failed: 1 rule has more errors than the baseline allows/) + assert.equal(summary(), comment) + }) + + it('passes, and writes no comment file when none is asked for', async (t) => { + const { core, logged } = await actionsFor(t) + + const status = report.runCompare({ core, env: { ...base(), RESULTS: path.join(FIXTURES, 'same-as-baseline.txt'), COMMENT_FILE: '' } }) + + assert.equal(status, 0) + assert.match(logged(), /Plugin Check error counts are within the baseline\./) + }) + + it('fails when the results file is missing, with no comment', async (t) => { + const { core, logged } = await actionsFor(t) + const commentFile = path.join(temp('compare'), 'comment.md') + + const status = report.runCompare({ core, env: { ...base(), RESULTS: '/nonexistent/results.txt', COMMENT_FILE: commentFile } }) + + assert.equal(status, 1) + assert.match(logged(), /::error title=Plugin Check::Results file not found: \/nonexistent\/results\.txt/) + assert.equal(fs.existsSync(commentFile), false) + }) + + it('escapes a percent sign in a message, as the runner reads annotations', async (t) => { + const { core, logged } = await actionsFor(t) + const results = path.join(temp('compare'), 'results.txt') + fs.writeFileSync(results, 'FILE: a.php\n[{"type":"ERROR","code":"100%done"}]\n') + + report.runCompare({ core, env: { ...base(), RESULTS: results, PLUGIN_CHECK_BASELINE: '/nonexistent/baseline.json' } }) + + assert.match(logged(), /::warning title=Plugin Check::No baseline to compare with/) + const failing = report.runCompare({ core, env: { ...base(), RESULTS: results, PLUGIN_CHECK_BASELINE: path.join(FIXTURES, 'baseline.json') } }) + assert.equal(failing, 1) + assert.match(logged(), /::error title=Plugin Check::100%25done has 1 errors/) + }) + }) +}) From 42b2118e190eec46c3d0bf6fff59cd0b7987c68d Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Tue, 3 Nov 2026 18:57:34 -0300 Subject: [PATCH 17/31] Remove the Plugin Check bash scripts and their tests --- .../plugin-check-report/fetch-baseline.sh | 183 --------- .../actions/plugin-check-report/ratchet.sh | 286 ------------- .github/workflows/test.yml | 8 +- .../fetch-baseline.test.js | 379 ------------------ tests/actions/plugin-check-report/golden.js | 32 +- .../ratchet-golden.test.js | 42 -- .../plugin-check-report/ratchet.test.js | 297 -------------- 7 files changed, 2 insertions(+), 1225 deletions(-) delete mode 100755 .github/actions/plugin-check-report/fetch-baseline.sh delete mode 100755 .github/actions/plugin-check-report/ratchet.sh delete mode 100644 tests/actions/plugin-check-report/fetch-baseline.test.js delete mode 100644 tests/actions/plugin-check-report/ratchet-golden.test.js delete mode 100644 tests/actions/plugin-check-report/ratchet.test.js diff --git a/.github/actions/plugin-check-report/fetch-baseline.sh b/.github/actions/plugin-check-report/fetch-baseline.sh deleted file mode 100755 index a268971..0000000 --- a/.github/actions/plugin-check-report/fetch-baseline.sh +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env bash -# -# Downloads the baseline.json that the Plugin Check workflow wrote for a branch: the -# plugin-check-report artifact of an earlier run. That run checked the branch as it really is, so -# the baseline always matches it. Nothing is committed, so two branches worked on at the same time -# can never leave a stale baseline behind. See this action's README. -# -# Usage: -# fetch-baseline.sh [--workflow ] [--sha ] [--wait ] [--before-run ] [] -# -# --workflow is the file name of the workflow whose runs hold the baseline (default: -# plugin-check.yml). When the check runs as a reusable workflow, the runs belong to the CALLER's -# file, so the action passes that name, read from GITHUB_WORKFLOW_REF. -# -# Runs are tried in this order, and the first one with the artifact wins: -# 1. runs for --sha (a pull request's base commit), when given. With --wait, a run for that -# commit that is still going is waited for, up to that many seconds. A run that does not -# show up at all is waited for only a short time (GRACE seconds), since the commit may never -# have had one; -# 2. the latest runs on , newest first. -# --before-run keeps only runs started before that one (run ids only grow), so a run never -# compares with itself, and a re-run of an older push never compares with a newer one. -# -# defaults to baseline.json next to this script, which ratchet.sh reads by default -# and git ignores. -# -# Prints " ". is "exact" when the run is for --sha (or no --sha -# was given), and "older" when --sha was given but only an older run on the branch had a baseline. -# -# Exit status: -# 0 baseline written -# 1 no run has the artifact (none finished yet, or all expired) -# 2 the lookup itself failed (GitHub API or download error), so "no baseline" is not known -# -# Needs the GitHub CLI, signed in (GH_TOKEN in CI) with read access to Actions. - -set -euo pipefail - -workflow=plugin-check.yml -sha="" -wait_for=0 -before_run="" -# How long to wait for a run for --sha to show up at all. -GRACE=120 -usage="Usage: $0 [--workflow ] [--sha ] [--wait ] [--before-run ] []" - -lookup_failed() { - echo "Could not look up Plugin Check runs: $1" >&2 - exit 2 -} - -# A missing or malformed argument exits 2, like any other lookup that cannot be trusted. Exit 1 is -# kept for "no run has a baseline", so a typo must never read as that. -need_value() { - [ "$#" -ge 2 ] && [ -n "$2" ] || lookup_failed "$1 needs a value. $usage" -} - -# A plain count of seconds or a run id. At most 18 digits keeps it inside bash's 64-bit integers, -# and 10# keeps a leading zero from being read as octal later. -number() { - case "$2" in - '' | *[!0-9]*) lookup_failed "$1 must be a whole number, not '$2'" ;; - esac - [ "${#2}" -le 18 ] || lookup_failed "$1 is too large: '$2'" - echo $((10#$2)) -} - -while [ $# -gt 0 ]; do - case "$1" in - --workflow) need_value "$@"; workflow=$2; shift 2 ;; - --sha) need_value "$@"; sha=$2; shift 2 ;; - --wait) need_value "$@"; wait_for=$(number --wait "$2"); shift 2 ;; - --before-run) need_value "$@"; before_run=$(number --before-run "$2"); shift 2 ;; - *) break ;; - esac -done - -[ "$#" -ge 1 ] && [ -n "$1" ] || lookup_failed "a branch is needed. $usage" -branch=$1 -output=${2:-"$(cd "$(dirname "$0")" && pwd)/baseline.json"} - -# Any other command that fails (mktemp, mkdir, cp, ...) is a failure too, never "no baseline": -# exit 1 is kept for the one place below that means it. -trap 'echo "fetch-baseline.sh failed at line $LINENO." >&2; exit 2' ERR - -repo=${GITHUB_REPOSITORY:-$(gh repo view --json nameWithOwner --jq .nameWithOwner)} || lookup_failed "no repository" -# Only a bare file name: it goes into an API path, and a path or a ref there would ask for some -# other resource. -case "$workflow" in - */* | *@* | "") lookup_failed "--workflow must be a file name such as plugin-check.yml, not '$workflow'" ;; -esac -runs="repos/$repo/actions/workflows/$workflow/runs" - -# Only runs of this repository's own branches count. The runs API matches `branch` and `head_sha` -# against the head of a pull request too, so a run for a pull request from a fork, from a branch -# named like the target, would otherwise be taken for a run of the target, and its artifact is -# whatever that fork's workflow chose to upload. -# -# jq reads $repo, so the filters stay in single quotes. -# shellcheck disable=SC2016 -own='select((.head_repository.full_name // "" | ascii_downcase) == ($repo | ascii_downcase))' -# shellcheck disable=SC2016 -own_runs='.workflow_runs[] | '"$own"' | "\(.id) \(.html_url) \(.head_sha)"' -# shellcheck disable=SC2016 -own_counts='[.workflow_runs[] | '"$own"'] | "\(length) \([.[] | select(.status != "completed")] | length)"' - -# The run for the base commit may still be going: a pull request opened right after a merge starts -# at the same time as the push run on the target branch. Waiting for it beats comparing with an -# older run. -if [ -n "$sha" ] && [ "$wait_for" -gt 0 ]; then - started=$(date +%s) - deadline=$((started + wait_for)) - while :; do - # " " for the commit. - counts=$(gh api "$runs?head_sha=$sha&per_page=20" \ - | jq -r --arg repo "$repo" "$own_counts") \ - || lookup_failed "runs for $sha" - read -r total going <<< "$counts" - # A run can take a moment to show up after its push. After GRACE seconds with none, the - # commit never had one. - if [ "$total" -eq 0 ] && [ "$(date +%s)" -ge $((started + GRACE)) ]; then - break - fi - [ "$total" -eq 0 ] || [ "$going" -gt 0 ] || break - if [ "$(date +%s)" -ge "$deadline" ]; then - echo "The run for $sha is still going after $wait_for seconds; trying older runs." >&2 - break - fi - sleep 30 - done -fi - -# " " lines, best first. Captured, not streamed, so a failed API call -# stops here instead of looking like an empty list. -candidates="" -if [ -n "$sha" ]; then - candidates=$(gh api "$runs?status=completed&head_sha=$sha&per_page=10" \ - | jq -r --arg repo "$repo" "$own_runs") || lookup_failed "runs for $sha" -fi -# Newest first by run id (ids only grow), not by the order the API happens to return. One page of -# 100, the most the API gives, is plenty: if none of them has a baseline, an older one would be -# stale anyway. Fork runs are dropped after the page is read, so a flood of them could push ours -# off it; the result is then "no baseline", which passes with a warning, and no fork baseline is -# taken. -on_branch=$(gh api "$runs?status=completed&branch=$branch&per_page=100" \ - | jq -r --arg repo "$repo" "$own_runs" | sort -k1,1nr) || lookup_failed "runs on $branch" -candidates+=${candidates:+$'\n'}$on_branch - -tried=" " -tmp=$(mktemp -d) -trap 'rm -rf "$tmp"' EXIT - -while read -r id url head; do - [ -n "$id" ] || continue - if [ -n "$before_run" ] && [ "$id" -ge "$before_run" ]; then - continue - fi - case "$tried" in *" $id "*) continue ;; esac - tried+="$id " - - # A run that broke before the report job, or whose artifact expired, has nothing to give. - found=$(gh api "repos/$repo/actions/runs/$id/artifacts?name=plugin-check-report" \ - --jq '[.artifacts[] | select(.expired | not)] | length') || lookup_failed "artifacts of run $id" - [ "$found" -gt 0 ] || continue - - rm -rf "$tmp/report" - gh run download "$id" --repo "$repo" --name plugin-check-report --dir "$tmp/report" > /dev/null \ - || lookup_failed "download of the artifact of run $id" - [ -s "$tmp/report/baseline.json" ] || continue - - match=exact - if [ -n "$sha" ] && [ "$head" != "$sha" ]; then - match=older - fi - - mkdir -p "$(dirname "$output")" - cp "$tmp/report/baseline.json" "$output" - echo "$id $url $match" - exit 0 -done <<< "$candidates" - -echo "No Plugin Check run on $branch${sha:+ or $sha} has a baseline." >&2 -exit 1 diff --git a/.github/actions/plugin-check-report/ratchet.sh b/.github/actions/plugin-check-report/ratchet.sh deleted file mode 100755 index be260c2..0000000 --- a/.github/actions/plugin-check-report/ratchet.sh +++ /dev/null @@ -1,286 +0,0 @@ -#!/usr/bin/env bash -# -# Plugin Check ratchet. Counts ERROR and WARNING results per rule code and compares them with a -# baseline: the counts of an earlier run, fetched with fetch-baseline.sh. Only a rule with more -# ERRORS than the baseline fails. More warnings only add a notice. Fewer of either passes. With no -# baseline at all, nothing is compared and the check passes with a warning. In strict mode any -# error fails, baseline or not. See this action's README. -# -# Usage: -# ratchet.sh [--comment-file ] compare the results with the baseline -# ratchet.sh --update write the baseline from the results -# -# is the output of `wp plugin check --format=json`: for each file, a -# "FILE: " line and then a line holding a JSON array of that file's results. -# -# --comment-file writes a short Markdown summary for the pull request comment. The same summary -# goes to the job summary when GITHUB_STEP_SUMMARY is set. -# -# Environment (all optional): -# PLUGIN_CHECK_MODE "ratchet" (default): fail only when a rule has more errors than the -# baseline. "strict": fail when there is any error at all. -# COMMENT_MARKER hidden first line of the comment (default: "") -# PLUGIN_CHECK_BASELINE baseline path (default: baseline.json next to this script, where -# fetch-baseline.sh puts it) -# BASELINE_LABEL what the baseline is, for the comment's column (default: "Baseline") -# BASELINE_URL link to the run the baseline came from -# PLUGIN_CHECK_ADVISORY "true" when the baseline is from an older commit than the one the pull -# request targets. Rules with more errors are then reported as warnings -# and do not fail the check: the target may have changed since. -# COMMENT_BADGE image URL shown under the status line -# RUN_URL link to the workflow run -# ARTIFACT_URL link to the uploaded results and fresh baseline - -set -euo pipefail - -# The pull request comment must stay short. Everything else is behind the links. -MAX_COMMENT_LINES=15 -MAX_RULES_LISTED=5 - -# Hidden line that marks the sticky comment. The workflow searches for it, so no other comment -# is ever replaced. The workflow passes the same value it searches for. -COMMENT_MARKER=${COMMENT_MARKER:-''} - -# Checked before anything else, so a typo in the caller never runs as the default. -mode=${PLUGIN_CHECK_MODE:-ratchet} -case "$mode" in - ratchet | strict) ;; - *) - echo "::error title=Plugin Check::Unknown mode '$mode'. Use 'ratchet' or 'strict'." - exit 1 - ;; -esac - -update=false -comment_file="" -while [ $# -gt 0 ]; do - case "$1" in - --update) update=true; shift ;; - --comment-file) comment_file=${2:?"--comment-file needs a path"}; shift 2 ;; - *) break ;; - esac -done - -results=${1:?"Usage: $0 [--update | --comment-file ] "} -baseline=${PLUGIN_CHECK_BASELINE:-"$(cd "$(dirname "$0")" && pwd)/baseline.json"} - -if [ ! -f "$results" ]; then - echo "::error title=Plugin Check::Results file not found: $results" - exit 1 -fi - -# A results file that holds anything else (an HTML database error, a PHP fatal) means the check -# itself broke. Without this guard that file would count as "no results" and pass. -unexpected=$({ grep -v -E '^(FILE: |\[|Success:)' "$results" || true; } | grep -v -E '^[[:space:]]*$' | head -n 1 | cut -c 1-200 || true) -if [ -n "$unexpected" ]; then - echo "::error title=Plugin Check::The results file is not Plugin Check output, so the check did not run. First unexpected line: $unexpected" - exit 1 -fi - -# A clean run prints "Success: ..." and a run with results prints "FILE: ..." lines. An empty -# file means the command failed with its error on stderr, which the results file does not hold. -if ! grep -q -E '^(FILE: |Success:)' "$results"; then - echo "::error title=Plugin Check::The results file has no Plugin Check output, so the check did not run." - exit 1 -fi - -# Only the JSON lines matter. grep exits 1 when there are none (a clean run), which is fine. -counts=$({ grep '^\[' "$results" || true; } | jq -s -S ' - def per_code($type): - map(select(.type == $type)) - | group_by(.code) - | map({ key: .[0].code, value: length }) - | from_entries; - (add // []) as $all - | { errors: ($all | per_code("ERROR")), warnings: ($all | per_code("WARNING")) } -') - -if [ "$update" = true ]; then - printf '%s\n' "$counts" > "$baseline" - echo "Wrote $baseline: $(jq '.errors | length' <<< "$counts") error codes, $(jq '.warnings | length' <<< "$counts") warning codes" - exit 0 -fi - -# No earlier run to compare with: report the counts and pass. Failing here would block every pull -# request whenever the target branch has no finished run (its first run, or expired artifacts). -if [ ! -f "$baseline" ]; then - echo "::warning title=Plugin Check::No baseline to compare with ($baseline), so nothing was compared. Run the workflow on the target branch to make one." - baseline=$(mktemp) - jq -n '{errors: {}, warnings: {}}' > "$baseline" - no_baseline=true -else - no_baseline=false -fi - -# One tab-separated row per kind and rule code found in either file: -# kind, code, baseline count, current count. Rules that went up come first, errors before -# warnings, biggest increase first, so the comment can show the top few. -rows=$(jq -r -n --argjson now "$counts" --slurpfile base "$baseline" ' - $base[0] as $base - | [ - ("errors", "warnings") as $kind - | ((($now[$kind] // {}) + ($base[$kind] // {})) | keys[]) as $code - | { kind: $kind, code: $code, base: ($base[$kind][$code] // 0), now: ($now[$kind][$code] // 0) } - ] - | sort_by((if .now > .base then 0 else 1 end), (if .kind == "errors" then 0 else 1 end), -(.now - .base)) - | .[] - | [.kind, .code, .base, .now] - | @tsv -') - -failed=false -errors_base=0 -errors_now=0 -warnings_base=0 -warnings_now=0 -went_down=0 -went_up=0 -errors_up=0 -up_list="" - -while IFS=$'\t' read -r kind code allowed found; do - [ -n "$code" ] || continue - - if [ "$kind" = errors ]; then - errors_base=$((errors_base + allowed)) - errors_now=$((errors_now + found)) - label="" - else - warnings_base=$((warnings_base + allowed)) - warnings_now=$((warnings_now + found)) - label=" warning" - fi - - # With no baseline, only the totals mean anything. - [ "$no_baseline" = false ] || continue - - if [ "$found" -gt "$allowed" ]; then - went_up=$((went_up + 1)) - if [ "$went_up" -le "$MAX_RULES_LISTED" ]; then - up_list+="${up_list:+, }\`$code\`$label (+$((found - allowed)))" - fi - if [ "$kind" = errors ]; then - errors_up=$((errors_up + 1)) - if [ "${PLUGIN_CHECK_ADVISORY:-}" = true ]; then - echo "::warning title=Plugin Check::$code has $found errors, an older run of the target branch has $allowed. Not enforced, because the target may have changed since." - else - failed=true - echo "::error title=Plugin Check::$code has $found errors, the baseline has $allowed. The full list is in the plugin-check-results artifact." - fi - else - echo "::notice title=Plugin Check::$code has $found warnings, the baseline has $allowed. Warnings do not fail the check." - fi - elif [ "$found" -lt "$allowed" ]; then - went_down=$((went_down + 1)) - echo "::notice title=Plugin Check::$code ($kind) went down from $allowed to $found." - fi -done <<< "$rows" - -# Strict mode ignores the baseline for pass or fail: any error fails. The counts and the table -# still compare with the baseline, so the comment shows what changed. -if [ "$mode" = strict ]; then - if [ "$errors_now" -gt 0 ]; then - failed=true - echo "::error title=Plugin Check::$errors_now errors found. Strict mode fails on any error." - else - failed=false - fi -fi - -# "+3", "-2" or "±0". -change() { - local diff=$(($1 - $2)) - if [ "$diff" -gt 0 ]; then echo "+$diff"; elif [ "$diff" -lt 0 ]; then echo "$diff"; else echo "±0"; fi -} - -# The totals can go down while a rule goes up, so the status names the rules, not the totals. -if [ "$mode" = strict ] && [ "$failed" = true ]; then - status="❌ Plugin Check failed: $errors_now errors, and strict mode allows none" -elif [ "$mode" = strict ]; then - status="✅ Plugin Check passed: no errors" -elif [ "$no_baseline" = true ]; then - status="⚠️ Plugin Check ran, but there was no baseline to compare with" -elif [ "$errors_up" -gt 0 ] && [ "${PLUGIN_CHECK_ADVISORY:-}" = true ]; then - status="⚠️ Plugin Check passed, not enforced: $errors_up rule(s) have more errors than an older run of the target branch" -elif [ "$failed" = true ]; then - if [ "$errors_up" -eq 1 ]; then - status="❌ Plugin Check failed: 1 rule has more errors than the baseline allows" - else - status="❌ Plugin Check failed: $errors_up rules have more errors than the baseline allows" - fi -else - status="✅ Plugin Check passed" -fi - -run_url=${RUN_URL:-} -if [ -z "$run_url" ] && [ -n "${GITHUB_RUN_ID:-}" ]; then - run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" -fi -if [ -n "${ARTIFACT_URL:-}" ]; then - links="📎 [Full results and new baseline](${ARTIFACT_URL})" - [ -z "$run_url" ] || links+=" · [Run]($run_url)" -elif [ -n "$run_url" ]; then - links="📎 [Run]($run_url)" -else - links="📎 Local run, see $results" -fi - -body="$COMMENT_MARKER"$'\n'"$status"$'\n\n' -if [ -n "${COMMENT_BADGE:-}" ]; then - body+="![Plugin Check finished](${COMMENT_BADGE})"$'\n\n' -fi -# The column is named after where the baseline came from ("Target branch", "Previous run"), and -# links to that run when known. -column=${BASELINE_LABEL:-Baseline} -if [ -n "${BASELINE_URL:-}" ]; then - column="[$column](${BASELINE_URL})" -fi -if [ "$no_baseline" = true ]; then - body+="| | Now |"$'\n' - body+="|:--|--:|"$'\n' - body+="| Errors | $errors_now |"$'\n' - body+="| Warnings | $warnings_now |"$'\n\n' -else - body+="| | Now | $column | Change |"$'\n' - body+="|:--|--:|--:|:--|"$'\n' - body+="| Errors | $errors_now | $errors_base | $(change "$errors_now" "$errors_base") |"$'\n' - body+="| Warnings | $warnings_now | $warnings_base | $(change "$warnings_now" "$warnings_base") |"$'\n\n' -fi -if [ "$went_up" -gt "$MAX_RULES_LISTED" ]; then - body+="Went up (top $MAX_RULES_LISTED of $went_up): $up_list"$'\n' -elif [ "$went_up" -gt 0 ]; then - body+="Went up: $up_list"$'\n' -fi -if [ "$went_down" -gt 0 ]; then - body+="Rules that went down: $went_down."$'\n' -fi -# A blank line separates the notes above from the links, when there are notes. -if [ "$went_up" -gt 0 ] || [ "$went_down" -gt 0 ]; then - body+=$'\n' -fi -body+="$links" - -# Hard cap: whatever happens above, the comment never grows past MAX_COMMENT_LINES. -body=$(head -n "$MAX_COMMENT_LINES" <<< "$body") - -if [ -n "$comment_file" ]; then - printf '%s\n' "$body" > "$comment_file" -fi -if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then - printf '%s\n' "$body" >> "$GITHUB_STEP_SUMMARY" -fi - -if [ "$failed" = true ]; then - if [ "$mode" = strict ]; then - echo "Plugin Check found errors, and strict mode allows none." - else - echo "Plugin Check found more errors than the baseline allows." - fi - exit 1 -fi - -if [ "$no_baseline" = true ]; then - echo "Plugin Check ran with no baseline, so nothing was compared." -else - echo "Plugin Check error counts are within the baseline." -fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 08c2003..c0870c1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,8 +3,7 @@ name: Tests # This repo's own checks. A bug here reaches every repo that pins a tag from it, so the logic in the # actions is unit tested rather than only exercised by a real release. # -# Run it locally: `npm install` once, then `node --test`. For the lint job, `actionlint` and -# `shellcheck .github/actions/*/*.sh`. +# Run it locally: `npm install` once, then `node --test`. For the lint job, `actionlint`. on: push: @@ -63,8 +62,3 @@ jobs: ./actionlint -ignore 'unexpected key "queue" for "concurrency" section' \ -ignore 'specifying action "\$/[^"]*" in invalid format because ref is missing' \ .github/workflows/*.yml templates/workflows/*.yml - - # actionlint does not read composite actions or standalone scripts. shellcheck comes with the - # runner image. - - name: Run shellcheck on the action scripts - run: shellcheck .github/actions/*/*.sh diff --git a/tests/actions/plugin-check-report/fetch-baseline.test.js b/tests/actions/plugin-check-report/fetch-baseline.test.js deleted file mode 100644 index d8eae06..0000000 --- a/tests/actions/plugin-check-report/fetch-baseline.test.js +++ /dev/null @@ -1,379 +0,0 @@ -'use strict' - -const { describe, it } = require('node:test') -const assert = require('node:assert') -const fs = require('node:fs') -const os = require('node:os') -const path = require('node:path') -const { spawnSync } = require('node:child_process') - -const FETCH = path.join(__dirname, '../../../.github/actions/plugin-check-report/fetch-baseline.sh') - -const REPO = 'example-org/my-plugin' -const SHA = 'a'.repeat(40) -const OLD_SHA = 'b'.repeat(40) - -/** - * A stand-in for the GitHub CLI, put first on PATH. - * - * `gh api --jq ` answers from a JSON file named after the path, and applies the - * filter with the real jq, so the script's own filters run as they would against GitHub. A path - * with no file answers the way gh does for a 404: a message on stderr and exit 1. That is how a - * test makes the lookup fail. - * - * `gh run download --dir ` copies the folder artifact-, standing in for the - * plugin-check-report artifact of that run. - * - * Every call is appended to a log, so a test can assert which requests were sent. - */ -const FAKE_GH = `#!/usr/bin/env bash -printf '%s\\n' "$*" >> "$FAKE_GH_LOG" -if [ "$1" = api ]; then - key=$(printf '%s' "$2" | tr -c 'A-Za-z0-9' '_') - file="$FAKE_GH_DIR/$key.json" - if [ ! -f "$file" ]; then - echo "gh: Not Found (HTTP 404)" >&2 - exit 1 - fi - if [ "$3" = --jq ]; then - jq -r "$4" "$file" - else - cat "$file" - fi -elif [ "$1" = run ] && [ "$2" = download ]; then - id=$3 - while [ $# -gt 0 ]; do - [ "$1" = --dir ] && dir=$2 - shift - done - [ -d "$FAKE_GH_DIR/artifact-$id" ] || { echo "no artifact for run $id" >&2; exit 1; } - mkdir -p "$dir" - cp -R "$FAKE_GH_DIR/artifact-$id/." "$dir/" -else - echo "fake gh: no answer for: $*" >&2 - exit 1 -fi -` - -/** - * The file name the stand-in reads for an API path. - * - * @param {string} apiPath The path as the script passes it to `gh api`. - * - * @returns {string} The file name, every character other than a letter or digit made `_`. - */ -function keyFor(apiPath) { - return `${apiPath.replace(/[^A-Za-z0-9]/g, '_')}.json` -} - -/** - * The API paths fetch-baseline.sh asks for. - * - * @param {string} workflow The workflow file name. - */ -const API = (workflow = 'plugin-check.yml') => { - const runs = `repos/${REPO}/actions/workflows/${workflow}/runs` - - return { - forSha: (sha) => `${runs}?status=completed&head_sha=${sha}&per_page=10`, - goingForSha: (sha) => `${runs}?head_sha=${sha}&per_page=20`, - onBranch: (branch) => `${runs}?status=completed&branch=${branch}&per_page=100`, - artifacts: (id) => `repos/${REPO}/actions/runs/${id}/artifacts?name=plugin-check-report`, - } -} - -/** - * A run as the workflow runs API lists it. - * - * @param {number} id Run id. - * @param {string} sha Head commit. - * @param {string} headRepo Repository the run's head commit belongs to. A pull request from a - * fork has the fork's name here. - * - * @returns {object} The run. - */ -function run(id, sha = OLD_SHA, headRepo = REPO) { - return { - id, - html_url: `https://github.com/${REPO}/actions/runs/${id}`, - head_sha: sha, - status: 'completed', - head_repository: { full_name: headRepo }, - } -} - -/** - * Sets up the stand-in's answers and runs fetch-baseline.sh against them. - * - * @param {string[]} args Arguments for the script. - * @param {object} answers Each API path mapped to the JSON body it answers with. - * @param {object} baselines Each run id mapped to the baseline.json its artifact holds. - * - * @returns {{status: number, stdout: string, stderr: string, calls: string[], output: string}} - * The exit status, both streams, the gh calls made and the baseline file it wrote. - */ -function fetchBaseline(args, answers = {}, baselines = {}) { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-fetch-')) - const bin = path.join(dir, 'bin') - const data = path.join(dir, 'data') - const log = path.join(dir, 'gh.log') - const output = path.join(dir, 'out', 'baseline.json') - - fs.mkdirSync(bin) - fs.mkdirSync(data) - fs.writeFileSync(path.join(bin, 'gh'), FAKE_GH, { mode: 0o755 }) - fs.writeFileSync(log, '') - - for (const [apiPath, body] of Object.entries(answers)) { - fs.writeFileSync(path.join(data, keyFor(apiPath)), JSON.stringify(body)) - } - for (const [id, baseline] of Object.entries(baselines)) { - fs.mkdirSync(path.join(data, `artifact-${id}`)) - fs.writeFileSync(path.join(data, `artifact-${id}`, 'baseline.json'), JSON.stringify(baseline)) - } - - const result = spawnSync('bash', [FETCH, ...args, output], { - encoding: 'utf8', - env: { - PATH: `${bin}:${process.env.PATH}`, - HOME: process.env.HOME, - GITHUB_REPOSITORY: REPO, - FAKE_GH_DIR: data, - FAKE_GH_LOG: log, - }, - }) - - return { - status: result.status, - stdout: result.stdout.trim(), - stderr: result.stderr, - calls: fs.readFileSync(log, 'utf8').trim().split('\n').filter(Boolean), - output: fs.existsSync(output) ? fs.readFileSync(output, 'utf8') : '', - } -} - -const BASELINE = { errors: { 'Some.Rule': 2 }, warnings: {} } -const ONE_ARTIFACT = { artifacts: [{ name: 'plugin-check-report', expired: false }] } -const EXPIRED = { artifacts: [{ name: 'plugin-check-report', expired: true }] } - -describe('fetch-baseline.sh', () => { - it('takes the run for the base commit and calls it exact', () => { - const api = API() - const result = fetchBaseline(['--sha', SHA, 'develop'], { - [api.forSha(SHA)]: { workflow_runs: [run(200, SHA)] }, - [api.onBranch('develop')]: { workflow_runs: [run(150)] }, - [api.artifacts(200)]: ONE_ARTIFACT, - }, { 200: BASELINE }) - - assert.equal(result.status, 0, result.stderr) - assert.equal(result.stdout, `200 https://github.com/${REPO}/actions/runs/200 exact`) - assert.deepEqual(JSON.parse(result.output), BASELINE) - }) - - it('falls back to an older run on the branch, skips an expired artifact, and calls it older', () => { - const api = API() - const result = fetchBaseline(['--sha', SHA, 'develop'], { - [api.forSha(SHA)]: { workflow_runs: [] }, - [api.onBranch('develop')]: { workflow_runs: [run(120), run(150)] }, - [api.artifacts(150)]: EXPIRED, - [api.artifacts(120)]: ONE_ARTIFACT, - }, { 120: BASELINE }) - - assert.equal(result.status, 0, result.stderr) - assert.equal(result.stdout, `120 https://github.com/${REPO}/actions/runs/120 older`) - // Newest first by run id, whatever order the API answered in. - assert.ok( - result.calls.findIndex((call) => call.includes('runs/150/')) < - result.calls.findIndex((call) => call.includes('runs/120/')) - ) - }) - - it('never compares a run with itself or with a newer run', () => { - const api = API() - const result = fetchBaseline(['--before-run', '150', 'develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(160), run(150), run(140)] }, - [api.artifacts(140)]: ONE_ARTIFACT, - }, { 140: BASELINE }) - - assert.equal(result.status, 0, result.stderr) - assert.match(result.stdout, /^140 /) - assert.ok(!result.calls.some((call) => /runs\/1[56]0\//.test(call))) - }) - - it('does not wait for a run for the base commit that has already finished', () => { - const api = API() - const result = fetchBaseline(['--sha', SHA, '--wait', '1200', 'develop'], { - [api.goingForSha(SHA)]: { workflow_runs: [run(200, SHA)] }, - [api.forSha(SHA)]: { workflow_runs: [run(200, SHA)] }, - [api.onBranch('develop')]: { workflow_runs: [] }, - [api.artifacts(200)]: ONE_ARTIFACT, - }, { 200: BASELINE }) - - assert.equal(result.status, 0, result.stderr) - assert.match(result.stdout, /^200 .* exact$/) - }) - - it('exits 1 when no run has a baseline', () => { - const api = API() - const result = fetchBaseline(['develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(150)] }, - [api.artifacts(150)]: { artifacts: [] }, - }) - - assert.equal(result.status, 1) - assert.match(result.stderr, /No Plugin Check run on develop has a baseline/) - assert.equal(result.output, '') - }) - - it('exits 2, not 1, when the lookup itself fails', () => { - // No answer for the branch's runs: the stand-in fails the way gh does on an API error. - const result = fetchBaseline(['develop'], {}) - - assert.equal(result.status, 2) - assert.match(result.stderr, /Could not look up Plugin Check runs/) - }) - - it('exits 2 when the download fails', () => { - const api = API() - const result = fetchBaseline(['develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(150)] }, - [api.artifacts(150)]: ONE_ARTIFACT, - }) - - assert.equal(result.status, 2) - assert.match(result.stderr, /download of the artifact of run 150/) - }) - - it('reads the runs of plugin-check.yml by default', () => { - const result = fetchBaseline(['develop'], {}) - - assert.ok(result.calls[0].includes('/workflows/plugin-check.yml/runs'), result.calls[0]) - }) - - it('reads the runs of the workflow file --workflow names', () => { - const api = API('wporg-check.yml') - const result = fetchBaseline(['--workflow', 'wporg-check.yml', 'develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(150)] }, - [api.artifacts(150)]: ONE_ARTIFACT, - }, { 150: BASELINE }) - - assert.equal(result.status, 0, result.stderr) - assert.ok(result.calls[0].includes('/workflows/wporg-check.yml/runs'), result.calls[0]) - }) - - it('ignores a run whose head commit is in a fork, even on a branch named like the target', () => { - const api = API() - const result = fetchBaseline(['--sha', SHA, 'develop'], { - // A fork's pull request run for the base commit, and a newer one on a fork branch "develop". - [api.forSha(SHA)]: { workflow_runs: [run(300, SHA, 'someone/my-plugin')] }, - [api.onBranch('develop')]: { workflow_runs: [run(250, OLD_SHA, 'someone/my-plugin'), run(150)] }, - [api.artifacts(150)]: ONE_ARTIFACT, - [api.artifacts(250)]: ONE_ARTIFACT, - [api.artifacts(300)]: ONE_ARTIFACT, - }, { 150: BASELINE, 250: { errors: { 'Some.Rule': 9999 }, warnings: {} }, 300: { errors: { 'Some.Rule': 9999 }, warnings: {} } }) - - assert.equal(result.status, 0, result.stderr) - assert.match(result.stdout, /^150 .* older$/) - assert.deepEqual(JSON.parse(result.output), BASELINE) - assert.ok(!result.calls.some((call) => call.includes('/runs/250/') || call.includes('/runs/300/')), 'no fork run was looked at') - }) - - it('exits 1 when only fork runs exist', () => { - const api = API() - const result = fetchBaseline(['develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(250, OLD_SHA, 'someone/my-plugin')] }, - }) - - assert.equal(result.status, 1) - assert.match(result.stderr, /has a baseline/) - }) - - it('matches the repository name without regard to case', () => { - const api = API() - const result = fetchBaseline(['develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(150, OLD_SHA, 'Example-Org/My-Plugin')] }, - [api.artifacts(150)]: ONE_ARTIFACT, - }, { 150: BASELINE }) - - assert.equal(result.status, 0, result.stderr) - }) - - it('does not wait for a fork run that is still going', () => { - const api = API() - const result = fetchBaseline(['--sha', SHA, '--wait', '1', 'develop'], { - // Our run for the commit is done, and a fork's run for the same commit is not. Counted, the - // fork's run would make the script sleep 30 seconds and then give up waiting. - [api.goingForSha(SHA)]: { - workflow_runs: [run(200, SHA), { ...run(300, SHA, 'someone/my-plugin'), status: 'in_progress' }], - }, - [api.forSha(SHA)]: { workflow_runs: [run(200, SHA)] }, - [api.onBranch('develop')]: { workflow_runs: [] }, - [api.artifacts(200)]: ONE_ARTIFACT, - }, { 200: BASELINE }) - - assert.equal(result.status, 0, result.stderr) - assert.match(result.stdout, /^200 .* exact$/) - assert.doesNotMatch(result.stderr, /still going/) - }) - - it('exits 2, not 1, for an option value that is empty or not a whole number', () => { - const cases = [ - [['--wait', '20m', 'develop'], /--wait must be a whole number, not '20m'/], - [['--wait', '', 'develop'], /--wait needs a value/], - [['--before-run', 'abc', 'develop'], /--before-run must be a whole number, not 'abc'/], - [['--before-run', '-5', 'develop'], /--before-run must be a whole number/], - [['--before-run', '9'.repeat(19), 'develop'], /--before-run is too large/], - [['--sha', '', 'develop'], /--sha needs a value/], - [['', 'baseline.json'], /a branch is needed/], - ] - - for (const [args, message] of cases) { - const result = fetchBaseline(args, {}) - - assert.equal(result.status, 2, args.join(' ')) - assert.match(result.stderr, message, args.join(' ')) - assert.deepEqual(result.calls, [], `${args.join(' ')} asked the API nothing`) - } - }) - - it('exits 2 when an option or the branch is missing altogether', () => { - // fetchBaseline() always adds an output path after the arguments, which a flag at the end of - // the line would take for its value, so these run the script as it is. - const cases = [ - [['--wait'], /--wait needs a value/], - [['--workflow'], /--workflow needs a value/], - [[], /a branch is needed/], - ] - - for (const [args, message] of cases) { - const result = spawnSync('bash', [FETCH, ...args], { - encoding: 'utf8', - env: { PATH: process.env.PATH, HOME: process.env.HOME, GITHUB_REPOSITORY: REPO }, - }) - - assert.equal(result.status, 2, args.join(' ')) - assert.match(result.stderr, message, args.join(' ')) - } - }) - - it('reads a run id with a leading zero as a decimal number', () => { - const api = API() - const result = fetchBaseline(['--before-run', '0150', 'develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(150), run(120)] }, - [api.artifacts(120)]: ONE_ARTIFACT, - }, { 120: BASELINE }) - - assert.equal(result.status, 0, result.stderr) - assert.match(result.stdout, /^120 /) - }) - - it('refuses a --workflow that is a path or a ref, before calling the API', () => { - for (const workflow of ['.github/workflows/plugin-check.yml', 'plugin-check.yml@refs/heads/develop']) { - const result = fetchBaseline(['--workflow', workflow, 'develop'], {}) - - assert.equal(result.status, 2, workflow) - assert.match(result.stderr, /--workflow must be a file name/) - assert.deepEqual(result.calls, []) - } - }) -}) diff --git a/tests/actions/plugin-check-report/golden.js b/tests/actions/plugin-check-report/golden.js index 4b19b12..11bdda3 100644 --- a/tests/actions/plugin-check-report/golden.js +++ b/tests/actions/plugin-check-report/golden.js @@ -201,34 +201,4 @@ function expectedFor(name) { } } -/** - * Writes the pinned files from a run of the script under test. Only the original bash script is - * allowed to make them: run it once, review the diff, and commit. - * - * @param {object[]} outcomes `{name, result}` for each case. - */ -function writeExpected(outcomes) { - fs.mkdirSync(EXPECTED, { recursive: true }) - const statuses = {} - - for (const { name, result } of outcomes) { - statuses[name] = result.status - for (const [extension, text] of [ - ['out', result.stdout], - ['md', result.comment], - ['summary', result.summary], - ['json', result.baseline], - ]) { - const file = path.join(EXPECTED, `${name}.${extension}`) - if (text === '') { - fs.rmSync(file, { force: true }) - } else { - fs.writeFileSync(file, text) - } - } - } - - fs.writeFileSync(path.join(EXPECTED, 'status.json'), `${JSON.stringify(statuses, null, 2)}\n`) -} - -module.exports = { CASES, FIXTURES, expectedFor, runCase, writeExpected } +module.exports = { CASES, FIXTURES, expectedFor, runCase } diff --git a/tests/actions/plugin-check-report/ratchet-golden.test.js b/tests/actions/plugin-check-report/ratchet-golden.test.js deleted file mode 100644 index 6752f92..0000000 --- a/tests/actions/plugin-check-report/ratchet-golden.test.js +++ /dev/null @@ -1,42 +0,0 @@ -'use strict' - -const { describe, it } = require('node:test') -const assert = require('node:assert') -const path = require('node:path') -const { spawnSync } = require('node:child_process') - -const { CASES, expectedFor, runCase, writeExpected } = require('./golden.js') - -const RATCHET = path.join(__dirname, '../../../.github/actions/plugin-check-report/ratchet.sh') - -/** - * Runs ratchet.sh with a built environment, like ratchet.test.js does: only PATH and HOME are kept, - * so the runner's own GITHUB_* variables never reach the script. - * - * @param {string[]} args Arguments after the script. - * @param {object} env The environment to add. - * - * @returns {{status: number, stdout: string}} The exit status and what the script printed. - */ -function execute(args, env) { - const result = spawnSync('bash', [RATCHET, ...args], { - encoding: 'utf8', - env: { PATH: process.env.PATH, HOME: process.env.HOME, ...env }, - }) - - return { status: result.status, stdout: result.stdout } -} - -// UPDATE_GOLDEN=1 rewrites fixtures/expected/ from this script. The files are the specification the -// JavaScript has to match, so rewrite them only from the original script, and read the diff. -if (process.env.UPDATE_GOLDEN === '1') { - writeExpected(CASES.map((spec) => ({ name: spec.name, result: runCase(spec, execute) }))) -} - -describe('ratchet.sh, pinned output', () => { - for (const spec of CASES) { - it(`prints the pinned output for ${spec.name}`, () => { - assert.deepStrictEqual(runCase(spec, execute), expectedFor(spec.name)) - }) - } -}) diff --git a/tests/actions/plugin-check-report/ratchet.test.js b/tests/actions/plugin-check-report/ratchet.test.js deleted file mode 100644 index 07d28c9..0000000 --- a/tests/actions/plugin-check-report/ratchet.test.js +++ /dev/null @@ -1,297 +0,0 @@ -'use strict' - -const { describe, it } = require('node:test') -const assert = require('node:assert') -const fs = require('node:fs') -const os = require('node:os') -const path = require('node:path') -const { spawnSync } = require('node:child_process') - -const RATCHET = path.join(__dirname, '../../../.github/actions/plugin-check-report/ratchet.sh') -const FIXTURES = path.join(__dirname, 'fixtures') -const BASELINE = path.join(FIXTURES, 'baseline.json') - -const ESCAPE = 'WordPress.Security.EscapeOutput.OutputNotEscaped' -const DIRECT_QUERY = 'WordPress.DB.DirectDatabaseQuery.DirectQuery' - -/** - * Runs ratchet.sh the way the action does, and returns what it printed and wrote. - * - * The environment is built from scratch rather than copied from process.env. In CI the test runner - * has GITHUB_STEP_SUMMARY, GITHUB_RUN_ID and friends set, and the script would append its summary - * to this repository's own job and link to this run. Only PATH and HOME are kept, so bash and jq - * are found. The script's `::error::` lines go to the captured stdout, never to the runner's log, - * so a passing test run does not annotate its own workflow. - * - * @param {string[]} args Arguments after the script, usually a results file. - * @param {object} env Extra environment, such as PLUGIN_CHECK_BASELINE. - * - * @returns {{status: number, stdout: string, stderr: string, comment: string, summary: string}} - * The exit status, both streams, the comment file and the job summary it wrote. - */ -function ratchet(args, env = {}) { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-ratchet-')) - const comment = path.join(dir, 'comment.md') - const summary = path.join(dir, 'summary.md') - - const result = spawnSync('bash', [RATCHET, '--comment-file', comment, ...args], { - encoding: 'utf8', - env: { - PATH: process.env.PATH, - HOME: process.env.HOME, - GITHUB_STEP_SUMMARY: summary, - // A path that does not exist, so a test that wants no baseline gets none, instead of a - // baseline.json some earlier local run left next to the script. - PLUGIN_CHECK_BASELINE: path.join(dir, 'no-baseline.json'), - ...env, - }, - }) - - const read = (file) => (fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : '') - - return { - status: result.status, - stdout: result.stdout, - stderr: result.stderr, - comment: read(comment), - summary: read(summary), - } -} - -/** - * The path of one of the sample results files. - * - * @param {string} name File name under fixtures/. - * - * @returns {string} The absolute path. - */ -function fixture(name) { - return path.join(FIXTURES, name) -} - -/** - * A results file with one new ERROR code per entry, for the tests about long lists. - * - * @param {number} codes How many different codes to write. - * - * @returns {string} The path of the file. - */ -function resultsWithNewCodes(codes) { - const results = Array.from({ length: codes }, (_, index) => ({ - line: index + 1, - column: 1, - type: 'ERROR', - code: `Test.Rule.Number${index + 1}`, - message: 'A rule added for the test.', - docs: '', - })) - const file = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-results-')), 'results.txt') - fs.writeFileSync(file, `FILE: plugin.php\n${JSON.stringify(results)}\n`) - - return file -} - -describe('ratchet.sh', () => { - describe('results it refuses', () => { - it('fails when the results file is missing', () => { - const run = ratchet([fixture('does-not-exist.txt')]) - - assert.equal(run.status, 1) - assert.match(run.stdout, /::error title=Plugin Check::Results file not found/) - assert.equal(run.comment, '', 'no comment body, so the workflow writes "could not run"') - }) - - it('fails when the results file is not Plugin Check output', () => { - const run = ratchet([fixture('garbage.txt')]) - - assert.equal(run.status, 1) - assert.match(run.stdout, /is not Plugin Check output.*First unexpected line: /) - }) - - it('fails when the results file is empty', () => { - const run = ratchet([fixture('empty.txt')]) - - assert.equal(run.status, 1) - assert.match(run.stdout, /has no Plugin Check output/) - }) - - it('fails on a mode it does not know, before reading anything', () => { - const run = ratchet([fixture('clean.txt')], { PLUGIN_CHECK_MODE: 'strcit' }) - - assert.equal(run.status, 1) - assert.match(run.stdout, /Unknown mode 'strcit'/) - }) - }) - - describe('--update', () => { - it('writes the counts per rule code', () => { - const out = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-update-')), 'baseline.json') - const run = ratchet(['--update', fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: out }) - - assert.equal(run.status, 0) - assert.deepEqual(JSON.parse(fs.readFileSync(out, 'utf8')), { - errors: { - [ESCAPE]: 3, - 'WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents': 1, - }, - warnings: { [DIRECT_QUERY]: 1 }, - }) - }) - - it('writes empty counts for a clean run', () => { - const out = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-update-')), 'baseline.json') - const run = ratchet(['--update', fixture('clean.txt')], { PLUGIN_CHECK_BASELINE: out }) - - assert.equal(run.status, 0) - assert.deepEqual(JSON.parse(fs.readFileSync(out, 'utf8')), { errors: {}, warnings: {} }) - }) - }) - - describe('compare in ratchet mode', () => { - it('passes a clean run with no baseline, with a warning and the one-column table', () => { - const run = ratchet([fixture('clean.txt')]) - - assert.equal(run.status, 0) - assert.match(run.stdout, /::warning title=Plugin Check::No baseline to compare with/) - assert.match(run.comment, /⚠️ Plugin Check ran, but there was no baseline to compare with/) - assert.match(run.comment, /\| Errors {3}\| 0 \|/) - assert.doesNotMatch(run.comment, /Change/) - }) - - it('passes when every count matches the baseline', () => { - const run = ratchet([fixture('same-as-baseline.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) - - assert.equal(run.status, 0) - assert.match(run.comment, /✅ Plugin Check passed/) - assert.match(run.comment, /\| Errors {3}\| 3 \| 3 \| ±0 \|/) - }) - - it('fails when a rule has more errors than the baseline', () => { - const run = ratchet([fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) - - assert.equal(run.status, 1) - assert.match(run.stdout, new RegExp(`::error title=Plugin Check::${ESCAPE} has 3 errors, the baseline has 2`)) - assert.match(run.comment, /❌ Plugin Check failed: 1 rule has more errors than the baseline allows/) - assert.match(run.comment, new RegExp(`Went up: \`${ESCAPE}\` \\(\\+1\\)`)) - }) - - it('fails when a rule goes up even though the total stays the same', () => { - const run = ratchet([fixture('rule-up-total-same.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) - - assert.equal(run.status, 1) - assert.match(run.comment, /\| Errors {3}\| 3 \| 3 \| ±0 \|/) - assert.match(run.comment, /Rules that went down: 1\./) - }) - - it('passes with a notice when errors go down', () => { - const run = ratchet([fixture('errors-down.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) - - assert.equal(run.status, 0) - assert.match(run.stdout, new RegExp(`::notice title=Plugin Check::${ESCAPE} \\(errors\\) went down from 2 to 1`)) - assert.match(run.comment, /\| Errors {3}\| 2 \| 3 \| -1 \|/) - }) - - it('passes with a notice when only warnings go up', () => { - const run = ratchet([fixture('warnings-up.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) - - assert.equal(run.status, 0) - assert.match(run.stdout, /Warnings do not fail the check/) - assert.match(run.comment, new RegExp(`Went up: \`${DIRECT_QUERY}\` warning \\(\\+2\\)`)) - }) - - it('only warns when the baseline is from an older run of the target branch', () => { - const run = ratchet([fixture('errors-up.txt')], { - PLUGIN_CHECK_BASELINE: BASELINE, - PLUGIN_CHECK_ADVISORY: 'true', - }) - - assert.equal(run.status, 0) - assert.match(run.stdout, /::warning title=Plugin Check::.*Not enforced/) - assert.match(run.comment, /⚠️ Plugin Check passed, not enforced/) - }) - }) - - describe('compare in strict mode', () => { - it('fails on any error, even with no baseline', () => { - const run = ratchet([fixture('errors-down.txt')], { PLUGIN_CHECK_MODE: 'strict' }) - - assert.equal(run.status, 1) - assert.match(run.stdout, /2 errors found\. Strict mode fails on any error/) - assert.match(run.comment, /❌ Plugin Check failed: 2 errors, and strict mode allows none/) - }) - - it('fails on errors that went down, which ratchet mode would pass', () => { - const run = ratchet([fixture('errors-down.txt')], { - PLUGIN_CHECK_MODE: 'strict', - PLUGIN_CHECK_BASELINE: BASELINE, - }) - - assert.equal(run.status, 1) - }) - - it('passes a clean run', () => { - const run = ratchet([fixture('clean.txt')], { PLUGIN_CHECK_MODE: 'strict' }) - - assert.equal(run.status, 0) - assert.match(run.comment, /✅ Plugin Check passed: no errors/) - }) - - it('passes a run with warnings only', () => { - const results = fixture('warnings-up.txt') - const onlyWarnings = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-warn-')), 'results.txt') - // The warnings-up sample without its ERROR results. - const lines = fs.readFileSync(results, 'utf8').split('\n').map((line) => { - if (!line.startsWith('[')) { - return line - } - const kept = JSON.parse(line).filter((result) => result.type === 'WARNING') - return JSON.stringify(kept) - }) - fs.writeFileSync(onlyWarnings, lines.join('\n')) - - const run = ratchet([onlyWarnings], { PLUGIN_CHECK_MODE: 'strict' }) - - assert.equal(run.status, 0) - }) - }) - - describe('the comment', () => { - it('starts with the default marker', () => { - const run = ratchet([fixture('clean.txt')]) - - assert.equal(run.comment.split('\n')[0], '') - }) - - it('starts with the marker the workflow passes', () => { - const marker = '' - const run = ratchet([fixture('clean.txt')], { COMMENT_MARKER: marker }) - - assert.equal(run.comment.split('\n')[0], marker) - }) - - it('names the baseline column after where it came from, with a link', () => { - const run = ratchet([fixture('same-as-baseline.txt')], { - PLUGIN_CHECK_BASELINE: BASELINE, - BASELINE_LABEL: 'Target branch', - BASELINE_URL: 'https://github.com/o/r/actions/runs/1', - }) - - assert.match(run.comment, /\| Now \| \[Target branch\]\(https:\/\/github\.com\/o\/r\/actions\/runs\/1\) \| Change \|/) - }) - - it('lists at most five rules and never grows past fifteen lines', () => { - const run = ratchet([resultsWithNewCodes(7)], { PLUGIN_CHECK_BASELINE: BASELINE }) - - assert.equal(run.status, 1) - assert.match(run.comment, /Went up \(top 5 of 7\)/) - assert.doesNotMatch(run.comment, /Number6/) - assert.ok(run.comment.trimEnd().split('\n').length <= 15) - }) - - it('writes the same text to the job summary', () => { - const run = ratchet([fixture('errors-down.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) - - assert.equal(run.summary, run.comment) - }) - }) -}) From 0157eca31bcb50e4c7aff8170a350d855245bdba Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Tue, 3 Nov 2026 18:58:25 -0300 Subject: [PATCH 18/31] Rename the Plugin Check mode ratchet to baseline --- .github/actions/plugin-check-report/README.md | 16 ++- .../actions/plugin-check-report/action.yml | 6 +- .../plugin-check-report.js | 16 ++- .github/workflows/plugin-check.yml | 7 +- templates/workflows/plugin-check.yml | 2 +- .../plugin-check-report/action.test.js | 9 ++ .../plugin-check-report/compare.test.js | 120 ++++++++++-------- .../fixtures/expected/unknown-mode.out | 2 +- 8 files changed, 109 insertions(+), 69 deletions(-) diff --git a/.github/actions/plugin-check-report/README.md b/.github/actions/plugin-check-report/README.md index ff4ad7d..8b2fbb6 100644 --- a/.github/actions/plugin-check-report/README.md +++ b/.github/actions/plugin-check-report/README.md @@ -120,7 +120,7 @@ the caller. ## Modes -- `ratchet` (default): fail only when a rule has more errors than the baseline. More warnings, or +- `baseline` (default): fail only when a rule has more errors than the baseline. More warnings, or fewer of either, only add a notice. The totals can go down while one rule goes up, so the check looks at each rule. - `strict`: fail on any error. The comment still compares with the baseline. Switch to it once a @@ -152,7 +152,7 @@ the check to require in branch protection is `plugin-check / Plugin Check`. | `php-version` | `7.4` | PHP that builds `vendor/`. Use the oldest the plugin supports | | `exclude-checks` | empty | Checks to skip, one per line, passed to the Plugin Check action | | `cache-paths` | empty | Build caches kept between runs, one path per line, relative to the repository | -| `mode` | `ratchet` | `ratchet` or `strict`. See [Modes](#modes) | +| `mode` | `baseline` | `baseline` or `strict`. See [Modes](#modes) | | `submodules` | `false` | Passed to `actions/checkout` | | Secret | Becomes, in the build step only | @@ -182,7 +182,7 @@ not start the run. The job that runs the Plugin Check action lowers itself to `c |---|---|---|---| | `results` | yes | | The results file. A missing file fails the compare | | `comment-file` | no | none | Where to write the comment body | -| `mode` | no | `ratchet` | `ratchet` or `strict` | +| `mode` | no | `baseline` | `baseline` or `strict` | | `comment-marker` | no | `` | First line of the comment body | | `baseline-wait` | no | `1200` | Seconds to wait for the base commit's run. A whole number, or the check fails | | `badge-url` | no | none | Image under the status line | @@ -234,7 +234,15 @@ command line takes the same arguments and environment variables as the two bash The first version of this action was two bash scripts. The JavaScript that replaced them gives the same results on every input Plugin Check can produce. These differences are on purpose, and only -show with a hand-edited baseline or an odd results file: +show with a hand-edited baseline or an odd results file, except the first two: + +- The mode `ratchet` is now called `baseline`. The old word is refused, and the message says what + to use. +- The `workflow-file` input is gone. The runs that hold the baseline are the calling workflow's, + which the run itself knows. +- A rule with an empty code is counted like any other. Bash misread its row and showed wrong totals. +- A count in the baseline that is not a whole number (such as `1.5`) fails the compare with exit + status 1. Bash printed an arithmetic error and passed with zero totals. - A baseline count above 9007199254740991 (2^53 - 1) fails the compare with exit status 5 and says so. JavaScript cannot hold a larger number exactly, so a comment would show another number. Bash diff --git a/.github/actions/plugin-check-report/action.yml b/.github/actions/plugin-check-report/action.yml index 9815095..54c0b4f 100644 --- a/.github/actions/plugin-check-report/action.yml +++ b/.github/actions/plugin-check-report/action.yml @@ -1,6 +1,6 @@ # Compares a WordPress.org Plugin Check results file with a baseline: the counts of an earlier run # of the same workflow, never a committed file. Only a rule with more errors than the baseline -# fails (ratchet mode), or any error at all (strict mode). It also saves this run's counts as the +# fails (baseline mode), or any error at all (strict mode). It also saves this run's counts as the # next baseline and writes the short body for the pull request comment. # # It does not run Plugin Check and it does not post the comment. The reusable workflow @@ -27,10 +27,10 @@ inputs: default: '' mode: description: >- - `ratchet`: fail only when a rule has more errors than the baseline. `strict`: fail on any + `baseline`: fail only when a rule has more errors than the baseline. `strict`: fail on any error, baseline or not. required: false - default: 'ratchet' + default: 'baseline' comment-marker: description: >- Hidden first line of the comment body, which the caller searches for to find its own diff --git a/.github/actions/plugin-check-report/plugin-check-report.js b/.github/actions/plugin-check-report/plugin-check-report.js index 3bf7c29..5ecb4fd 100644 --- a/.github/actions/plugin-check-report/plugin-check-report.js +++ b/.github/actions/plugin-check-report/plugin-check-report.js @@ -37,8 +37,11 @@ const MAX_RULES_LISTED = 5 /** The first unexpected line of a results file is shown cut to this many bytes. */ const MAX_UNEXPECTED_BYTES = 200 -const MODES = ['ratchet', 'strict'] -const DEFAULT_MODE = 'ratchet' +const MODES = ['baseline', 'strict'] +const DEFAULT_MODE = 'baseline' + +/** Modes that were renamed, so the message can say what to use now. */ +const RENAMED_MODES = { ratchet: 'baseline' } /** A line of Plugin Check output: a file name, a JSON array of its results, or the clean message. */ const KNOWN_LINE = /^(FILE: |\[|Success:)/ @@ -401,7 +404,7 @@ function buildRows(now, base) { * * @param {object} input * @param {object[]} input.rows From buildRows. - * @param {string} input.mode `ratchet` or `strict`. + * @param {string} input.mode `baseline` or `strict`. * @param {boolean} input.advisory The baseline is from an older commit than the target, so more * errors are only reported. * @param {boolean} input.noBaseline There was nothing to compare with: only the totals count. @@ -506,7 +509,7 @@ function change(now, base) { * The totals can go down while a rule goes up, so the status names the rules, not the totals. * * @param {object} input - * @param {string} input.mode `ratchet` or `strict`. + * @param {string} input.mode `baseline` or `strict`. * @param {boolean} input.failed The check fails. * @param {boolean} input.noBaseline Nothing was compared. * @param {boolean} input.advisory The baseline is from an older run of the target branch. @@ -613,7 +616,7 @@ function buildComment({ marker, status, badge, column, noBaseline, totals, wentU * @param {object} input * @param {string[]} input.args Arguments: `--update` and `--comment-file ` in any order, then * the results file. - * @param {object} input.env Environment. PLUGIN_CHECK_MODE (`ratchet`, the default, or `strict`), + * @param {object} input.env Environment. PLUGIN_CHECK_MODE (`baseline`, the default, or `strict`), * COMMENT_MARKER, PLUGIN_CHECK_BASELINE (default: baseline.json next * to this file), BASELINE_LABEL, BASELINE_URL, PLUGIN_CHECK_ADVISORY * (`true` when the baseline is from an older commit than the target), @@ -650,7 +653,8 @@ function compareOrThrow({ args, env, log }) { // Checked before anything else, so a typo in the caller never runs as the default. const mode = env.PLUGIN_CHECK_MODE || DEFAULT_MODE if (!MODES.includes(mode)) { - log.error(`Unknown mode '${mode}'. Use 'ratchet' or 'strict'.`) + const renamed = RENAMED_MODES[mode] ? ` '${mode}' is now called '${RENAMED_MODES[mode]}'.` : '' + log.error(`Unknown mode '${mode}'. Use 'baseline' or 'strict'.${renamed}`) return 1 } diff --git a/.github/workflows/plugin-check.yml b/.github/workflows/plugin-check.yml index 972bf81..e8591ca 100644 --- a/.github/workflows/plugin-check.yml +++ b/.github/workflows/plugin-check.yml @@ -78,12 +78,11 @@ on: default: '' mode: description: >- - `ratchet`: fail only when a rule has more errors than the baseline. `strict`: fail on any - error. Not called `strict` because wordpress/plugin-check-action has a `strict` input - that means something else. + `baseline`: fail only when a rule has more errors than the baseline. `strict`: fail on any + error. Not the `strict` input of wordpress/plugin-check-action, which means something else. required: false type: string - default: 'ratchet' + default: 'baseline' submodules: description: 'Passed to actions/checkout: `false`, `true` or `recursive`.' required: false diff --git a/templates/workflows/plugin-check.yml b/templates/workflows/plugin-check.yml index 987d216..6fc5b22 100644 --- a/templates/workflows/plugin-check.yml +++ b/templates/workflows/plugin-check.yml @@ -10,7 +10,7 @@ name: Plugin Check # plugin-check-report action's README. # # Run it locally: `wp plugin check --format=json > results.txt` in a local WordPress, then -# the ratchet script on that file. See "Running it locally" in the action's README. +# the compare command on that file. See "Running it locally" in the action's README. on: pull_request: diff --git a/tests/actions/plugin-check-report/action.test.js b/tests/actions/plugin-check-report/action.test.js index c1575b3..712c1a6 100644 --- a/tests/actions/plugin-check-report/action.test.js +++ b/tests/actions/plugin-check-report/action.test.js @@ -284,6 +284,15 @@ describe('plugin-check-report action steps', () => { assert.equal(fs.existsSync(commentFile), false) }) + it('refuses the old mode name ratchet with a hint', async (t) => { + const { core, logged } = await actionsFor(t) + + const status = report.runCompare({ core, env: { ...base(), PLUGIN_CHECK_MODE: 'ratchet' } }) + + assert.equal(status, 1) + assert.match(logged(), /::error title=Plugin Check::Unknown mode 'ratchet'\. Use 'baseline' or 'strict'\. 'ratchet' is now called 'baseline'\./) + }) + it('escapes a percent sign in a message, as the runner reads annotations', async (t) => { const { core, logged } = await actionsFor(t) const results = path.join(temp('compare'), 'results.txt') diff --git a/tests/actions/plugin-check-report/compare.test.js b/tests/actions/plugin-check-report/compare.test.js index de1d8fe..2006b2d 100644 --- a/tests/actions/plugin-check-report/compare.test.js +++ b/tests/actions/plugin-check-report/compare.test.js @@ -43,7 +43,7 @@ function execute(args, env) { * @returns {{status: number, stdout: string, stderr: string, comment: string, summary: string}} * The exit status, both streams, the comment file and the job summary it wrote. */ -function ratchet(args, env = {}) { +function compareWith(args, env = {}) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-compare-')) const comment = path.join(dir, 'comment.md') const summary = path.join(dir, 'summary.md') @@ -142,7 +142,7 @@ describe('plugin-check-report compare, pinned output', () => { describe('plugin-check-report compare', () => { describe('results it refuses', () => { it('fails when the results file is missing', () => { - const run = ratchet([fixture('does-not-exist.txt')]) + const run = compareWith([fixture('does-not-exist.txt')]) assert.equal(run.status, 1) assert.match(run.stdout, /::error title=Plugin Check::Results file not found/) @@ -150,28 +150,48 @@ describe('plugin-check-report compare', () => { }) it('fails when the results file is not Plugin Check output', () => { - const run = ratchet([fixture('garbage.txt')]) + const run = compareWith([fixture('garbage.txt')]) assert.equal(run.status, 1) assert.match(run.stdout, /is not Plugin Check output.*First unexpected line: /) }) it('fails when the results file is empty', () => { - const run = ratchet([fixture('empty.txt')]) + const run = compareWith([fixture('empty.txt')]) assert.equal(run.status, 1) assert.match(run.stdout, /has no Plugin Check output/) }) it('fails on a mode it does not know, before reading anything', () => { - const run = ratchet([fixture('clean.txt')], { PLUGIN_CHECK_MODE: 'strcit' }) + const run = compareWith([fixture('clean.txt')], { PLUGIN_CHECK_MODE: 'strcit' }) assert.equal(run.status, 1) assert.match(run.stdout, /Unknown mode 'strcit'/) }) + it('refuses the old name ratchet, and says what to use', () => { + const run = compareWith([fixture('clean.txt')], { PLUGIN_CHECK_MODE: 'ratchet' }) + + assert.equal(run.status, 1) + assert.match( + run.stdout, + /::error title=Plugin Check::Unknown mode 'ratchet'\. Use 'baseline' or 'strict'\. 'ratchet' is now called 'baseline'\./ + ) + assert.equal(run.comment, '') + }) + + it('runs baseline mode when the mode is not set, and when it is empty', () => { + for (const env of [{}, { PLUGIN_CHECK_MODE: '' }, { PLUGIN_CHECK_MODE: 'baseline' }]) { + const run = compareWith([fixture('errors-up.txt')], { ...env, PLUGIN_CHECK_BASELINE: BASELINE }) + + assert.equal(run.status, 1, JSON.stringify(env)) + assert.match(run.comment, /rule has more errors than the baseline allows/) + } + }) + it('fails on a mode it does not know, even with a missing results file', () => { - const run = ratchet([fixture('does-not-exist.txt')], { PLUGIN_CHECK_MODE: 'strcit' }) + const run = compareWith([fixture('does-not-exist.txt')], { PLUGIN_CHECK_MODE: 'strcit' }) assert.match(run.stdout, /Unknown mode/) assert.doesNotMatch(run.stdout, /Results file not found/) @@ -194,20 +214,20 @@ describe('plugin-check-report compare', () => { }) it('cuts the first unexpected line to 200 bytes', () => { - const run = ratchet([resultsFile(`${'x'.repeat(300)}\n`)]) + const run = compareWith([resultsFile(`${'x'.repeat(300)}\n`)]) assert.equal(run.status, 1) assert.match(run.stdout, new RegExp(`First unexpected line: x{200}\\n`)) }) it('does not count a line of white space as unexpected', () => { - const run = ratchet([resultsFile('FILE: plugin.php\n \t \n[]\n\n')]) + const run = compareWith([resultsFile('FILE: plugin.php\n \t \n[]\n\n')]) assert.equal(run.status, 0) }) it('fails when a results line is not JSON', () => { - const run = ratchet([resultsFile('FILE: plugin.php\n[{"type":"ERROR"\n')]) + const run = compareWith([resultsFile('FILE: plugin.php\n[{"type":"ERROR"\n')]) assert.equal(run.status, 5) assert.match(run.stderr, /parse error/) @@ -216,7 +236,7 @@ describe('plugin-check-report compare', () => { it('fails when a counted result has no rule code', () => { for (const code of [undefined, null, 5, true, { a: 1 }]) { - const run = ratchet([resultsOf([{ type: 'ERROR', code }])]) + const run = compareWith([resultsOf([{ type: 'ERROR', code }])]) assert.equal(run.status, 5, String(code)) assert.match(run.stderr, /has no rule code/) @@ -225,21 +245,21 @@ describe('plugin-check-report compare', () => { it('fails when a result is not an object, but skips null', () => { for (const item of [1, 'x', [1], true]) { - assert.equal(ratchet([resultsOf([item])]).status, 5, JSON.stringify(item)) + assert.equal(compareWith([resultsOf([item])]).status, 5, JSON.stringify(item)) } - assert.equal(ratchet([resultsOf([null])]).status, 0) + assert.equal(compareWith([resultsOf([null])]).status, 0) }) it('ignores results that are neither ERROR nor WARNING', () => { - const run = ratchet([resultsOf([{ type: 'INFO' }, { type: 'error', code: 'a' }])]) + const run = compareWith([resultsOf([{ type: 'INFO' }, { type: 'error', code: 'a' }])]) assert.equal(run.status, 0) assert.match(run.comment, /\| Errors {3}\| 0 \|/) }) it('reads several lists written on one line', () => { - const run = ratchet([resultsFile('FILE: a.php\n[{"type":"ERROR","code":"a"}] [{"type":"ERROR","code":"b"}]\n')], { + const run = compareWith([resultsFile('FILE: a.php\n[{"type":"ERROR","code":"a"}] [{"type":"ERROR","code":"b"}]\n')], { PLUGIN_CHECK_MODE: 'strict', }) @@ -250,7 +270,7 @@ describe('plugin-check-report compare', () => { describe('--update', () => { it('writes the counts per rule code', () => { const out = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-update-')), 'baseline.json') - const run = ratchet(['--update', fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: out }) + const run = compareWith(['--update', fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: out }) assert.equal(run.status, 0) assert.deepEqual(JSON.parse(fs.readFileSync(out, 'utf8')), { @@ -264,7 +284,7 @@ describe('plugin-check-report compare', () => { it('writes empty counts for a clean run', () => { const out = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-update-')), 'baseline.json') - const run = ratchet(['--update', fixture('clean.txt')], { PLUGIN_CHECK_BASELINE: out }) + const run = compareWith(['--update', fixture('clean.txt')], { PLUGIN_CHECK_BASELINE: out }) assert.equal(run.status, 0) assert.deepEqual(JSON.parse(fs.readFileSync(out, 'utf8')), { errors: {}, warnings: {} }) @@ -272,23 +292,23 @@ describe('plugin-check-report compare', () => { it('writes no comment and no summary', () => { const out = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-update-')), 'baseline.json') - const run = ratchet(['--update', fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: out }) + const run = compareWith(['--update', fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: out }) assert.equal(run.comment, '') assert.equal(run.summary, '') }) it('fails, with the reason, when the folder of the file does not exist', () => { - const run = ratchet(['--update', fixture('clean.txt')], { PLUGIN_CHECK_BASELINE: '/nonexistent/folder/baseline.json' }) + const run = compareWith(['--update', fixture('clean.txt')], { PLUGIN_CHECK_BASELINE: '/nonexistent/folder/baseline.json' }) assert.equal(run.status, 1) assert.match(run.stderr, /Could not write \/nonexistent\/folder\/baseline\.json/) }) }) - describe('compare in ratchet mode', () => { + describe('compare in baseline mode', () => { it('passes a clean run with no baseline, with a warning and the one-column table', () => { - const run = ratchet([fixture('clean.txt')]) + const run = compareWith([fixture('clean.txt')]) assert.equal(run.status, 0) assert.match(run.stdout, /::warning title=Plugin Check::No baseline to compare with/) @@ -298,7 +318,7 @@ describe('plugin-check-report compare', () => { }) it('passes when every count matches the baseline', () => { - const run = ratchet([fixture('same-as-baseline.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + const run = compareWith([fixture('same-as-baseline.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) assert.equal(run.status, 0) assert.match(run.comment, /✅ Plugin Check passed/) @@ -306,7 +326,7 @@ describe('plugin-check-report compare', () => { }) it('fails when a rule has more errors than the baseline', () => { - const run = ratchet([fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + const run = compareWith([fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) assert.equal(run.status, 1) assert.match(run.stdout, new RegExp(`::error title=Plugin Check::${ESCAPE} has 3 errors, the baseline has 2`)) @@ -315,7 +335,7 @@ describe('plugin-check-report compare', () => { }) it('fails when a rule goes up even though the total stays the same', () => { - const run = ratchet([fixture('rule-up-total-same.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + const run = compareWith([fixture('rule-up-total-same.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) assert.equal(run.status, 1) assert.match(run.comment, /\| Errors {3}\| 3 \| 3 \| ±0 \|/) @@ -323,7 +343,7 @@ describe('plugin-check-report compare', () => { }) it('passes with a notice when errors go down', () => { - const run = ratchet([fixture('errors-down.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + const run = compareWith([fixture('errors-down.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) assert.equal(run.status, 0) assert.match(run.stdout, new RegExp(`::notice title=Plugin Check::${ESCAPE} \\(errors\\) went down from 2 to 1`)) @@ -331,7 +351,7 @@ describe('plugin-check-report compare', () => { }) it('passes with a notice when only warnings go up', () => { - const run = ratchet([fixture('warnings-up.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + const run = compareWith([fixture('warnings-up.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) assert.equal(run.status, 0) assert.match(run.stdout, /Warnings do not fail the check/) @@ -339,7 +359,7 @@ describe('plugin-check-report compare', () => { }) it('only warns when the baseline is from an older run of the target branch', () => { - const run = ratchet([fixture('errors-up.txt')], { + const run = compareWith([fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: BASELINE, PLUGIN_CHECK_ADVISORY: 'true', }) @@ -350,14 +370,14 @@ describe('plugin-check-report compare', () => { }) it('takes only the exact word true as advisory', () => { - const run = ratchet([fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: BASELINE, PLUGIN_CHECK_ADVISORY: 'TRUE' }) + const run = compareWith([fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: BASELINE, PLUGIN_CHECK_ADVISORY: 'TRUE' }) assert.equal(run.status, 1) }) it('treats an empty baseline file, and null, as no counts at all', () => { for (const content of ['', 'null', ' \n']) { - const run = ratchet([fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: baselineFile(content) }) + const run = compareWith([fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: baselineFile(content) }) assert.equal(run.status, 1, JSON.stringify(content)) assert.match(run.stdout, /has 3 errors, the baseline has 0/) @@ -365,7 +385,7 @@ describe('plugin-check-report compare', () => { }) it('treats a missing kind, and a null or false count, as zero', () => { - const run = ratchet([fixture('errors-up.txt')], { + const run = compareWith([fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: baselineFile({ errors: { [ESCAPE]: null } }), }) @@ -373,7 +393,7 @@ describe('plugin-check-report compare', () => { }) it('reads only the first value of a baseline file', () => { - const run = ratchet([fixture('same-as-baseline.txt')], { + const run = compareWith([fixture('same-as-baseline.txt')], { PLUGIN_CHECK_BASELINE: baselineFile(`${fs.readFileSync(BASELINE, 'utf8')}\n{"errors":{"x":9}}`), }) @@ -394,7 +414,7 @@ describe('plugin-check-report compare', () => { ] for (const [content, status] of cases) { - const run = ratchet([resultsOf([{ type: 'ERROR', code: 'a' }])], { PLUGIN_CHECK_BASELINE: baselineFile(content) }) + const run = compareWith([resultsOf([{ type: 'ERROR', code: 'a' }])], { PLUGIN_CHECK_BASELINE: baselineFile(content) }) assert.equal(run.status, status, content) assert.equal(run.comment, '', content) @@ -403,14 +423,14 @@ describe('plugin-check-report compare', () => { it('fails on a count too large to compare exactly, instead of printing another number', () => { for (const content of ['{"errors":{"a":9007199254740993}}', '{"warnings":{"a":1e300}}', '{"errors":{"a":-9007199254740993}}']) { - const run = ratchet([resultsOf([{ type: 'ERROR', code: 'a' }])], { PLUGIN_CHECK_BASELINE: baselineFile(content) }) + const run = compareWith([resultsOf([{ type: 'ERROR', code: 'a' }])], { PLUGIN_CHECK_BASELINE: baselineFile(content) }) assert.equal(run.status, 5, content) assert.match(run.stderr, /more than 9007199254740991, which cannot be compared exactly/) assert.equal(run.comment, '', content) } - const largest = ratchet([resultsOf([{ type: 'ERROR', code: 'a' }])], { + const largest = compareWith([resultsOf([{ type: 'ERROR', code: 'a' }])], { PLUGIN_CHECK_BASELINE: baselineFile('{"errors":{"a":9007199254740991}}'), }) assert.equal(largest.status, 0, 'the largest exact number still works') @@ -418,7 +438,7 @@ describe('plugin-check-report compare', () => { }) it('counts a rule with an empty code like any other', () => { - const run = ratchet([resultsOf([{ type: 'ERROR', code: '' }, { type: 'ERROR', code: '' }])], { + const run = compareWith([resultsOf([{ type: 'ERROR', code: '' }, { type: 'ERROR', code: '' }])], { PLUGIN_CHECK_BASELINE: baselineFile({ errors: { '': 1 }, warnings: {} }), }) @@ -427,13 +447,13 @@ describe('plugin-check-report compare', () => { }) it('shows a rule code on one line, with a tab or a new line written out', () => { - const run = ratchet([resultsOf([{ type: 'ERROR', code: 'a\tb\\c' }])], { PLUGIN_CHECK_BASELINE: baselineFile({}) }) + const run = compareWith([resultsOf([{ type: 'ERROR', code: 'a\tb\\c' }])], { PLUGIN_CHECK_BASELINE: baselineFile({}) }) assert.match(run.stdout, /::error title=Plugin Check::a\\tb\\\\c has 1 errors/) }) it('takes the links from GITHUB_RUN_ID when RUN_URL is not set', () => { - const run = ratchet([fixture('clean.txt')], { + const run = compareWith([fixture('clean.txt')], { GITHUB_RUN_ID: '42', GITHUB_SERVER_URL: 'https://github.com', GITHUB_REPOSITORY: 'example-org/my-plugin', @@ -443,7 +463,7 @@ describe('plugin-check-report compare', () => { }) it('shows the results file for a run on a developer machine', () => { - const run = ratchet([fixture('clean.txt')]) + const run = compareWith([fixture('clean.txt')]) assert.match(run.comment, /📎 Local run, see .*clean\.txt/) }) @@ -451,15 +471,15 @@ describe('plugin-check-report compare', () => { describe('compare in strict mode', () => { it('fails on any error, even with no baseline', () => { - const run = ratchet([fixture('errors-down.txt')], { PLUGIN_CHECK_MODE: 'strict' }) + const run = compareWith([fixture('errors-down.txt')], { PLUGIN_CHECK_MODE: 'strict' }) assert.equal(run.status, 1) assert.match(run.stdout, /2 errors found\. Strict mode fails on any error/) assert.match(run.comment, /❌ Plugin Check failed: 2 errors, and strict mode allows none/) }) - it('fails on errors that went down, which ratchet mode would pass', () => { - const run = ratchet([fixture('errors-down.txt')], { + it('fails on errors that went down, which baseline mode would pass', () => { + const run = compareWith([fixture('errors-down.txt')], { PLUGIN_CHECK_MODE: 'strict', PLUGIN_CHECK_BASELINE: BASELINE, }) @@ -468,7 +488,7 @@ describe('plugin-check-report compare', () => { }) it('passes a clean run', () => { - const run = ratchet([fixture('clean.txt')], { PLUGIN_CHECK_MODE: 'strict' }) + const run = compareWith([fixture('clean.txt')], { PLUGIN_CHECK_MODE: 'strict' }) assert.equal(run.status, 0) assert.match(run.comment, /✅ Plugin Check passed: no errors/) @@ -486,13 +506,13 @@ describe('plugin-check-report compare', () => { return JSON.stringify(JSON.parse(line).filter((result) => result.type === 'WARNING')) }) - const run = ratchet([resultsFile(lines.join('\n'))], { PLUGIN_CHECK_MODE: 'strict' }) + const run = compareWith([resultsFile(lines.join('\n'))], { PLUGIN_CHECK_MODE: 'strict' }) assert.equal(run.status, 0) }) it('prints the per-rule messages before the strict one', () => { - const run = ratchet([fixture('errors-up.txt')], { PLUGIN_CHECK_MODE: 'strict', PLUGIN_CHECK_BASELINE: BASELINE }) + const run = compareWith([fixture('errors-up.txt')], { PLUGIN_CHECK_MODE: 'strict', PLUGIN_CHECK_BASELINE: BASELINE }) const lines = run.stdout.split('\n') assert.match(lines[0], /has 3 errors, the baseline has 2/) @@ -502,20 +522,20 @@ describe('plugin-check-report compare', () => { describe('the comment', () => { it('starts with the default marker', () => { - const run = ratchet([fixture('clean.txt')]) + const run = compareWith([fixture('clean.txt')]) assert.equal(run.comment.split('\n')[0], '') }) it('starts with the marker the workflow passes', () => { const marker = '' - const run = ratchet([fixture('clean.txt')], { COMMENT_MARKER: marker }) + const run = compareWith([fixture('clean.txt')], { COMMENT_MARKER: marker }) assert.equal(run.comment.split('\n')[0], marker) }) it('names the baseline column after where it came from, with a link', () => { - const run = ratchet([fixture('same-as-baseline.txt')], { + const run = compareWith([fixture('same-as-baseline.txt')], { PLUGIN_CHECK_BASELINE: BASELINE, BASELINE_LABEL: 'Target branch', BASELINE_URL: 'https://github.com/o/r/actions/runs/1', @@ -525,7 +545,7 @@ describe('plugin-check-report compare', () => { }) it('lists at most five rules and never grows past fifteen lines', () => { - const run = ratchet([resultsWithNewCodes(7)], { PLUGIN_CHECK_BASELINE: BASELINE }) + const run = compareWith([resultsWithNewCodes(7)], { PLUGIN_CHECK_BASELINE: BASELINE }) assert.equal(run.status, 1) assert.match(run.comment, /Went up \(top 5 of 7\)/) @@ -535,20 +555,20 @@ describe('plugin-check-report compare', () => { it('cuts a comment to fifteen lines when a value holds line breaks, and drops the empty end', () => { const marker = `${'\n'.repeat(20)}end` - const run = ratchet([fixture('clean.txt')], { COMMENT_MARKER: marker }) + const run = compareWith([fixture('clean.txt')], { COMMENT_MARKER: marker }) // The first fifteen lines are all empty, so nothing is left once the empty end is dropped. assert.equal(run.comment, '\n') }) it('writes the same text to the job summary', () => { - const run = ratchet([fixture('errors-down.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + const run = compareWith([fixture('errors-down.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) assert.equal(run.summary, run.comment) }) it('writes a badge under the status line', () => { - const run = ratchet([fixture('clean.txt')], { COMMENT_BADGE: 'https://example.test/badge.svg' }) + const run = compareWith([fixture('clean.txt')], { COMMENT_BADGE: 'https://example.test/badge.svg' }) assert.match(run.comment, /\n!\[Plugin Check finished\]\(https:\/\/example\.test\/badge\.svg\)\n/) }) diff --git a/tests/actions/plugin-check-report/fixtures/expected/unknown-mode.out b/tests/actions/plugin-check-report/fixtures/expected/unknown-mode.out index 45bf6be..0ff583b 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/unknown-mode.out +++ b/tests/actions/plugin-check-report/fixtures/expected/unknown-mode.out @@ -1 +1 @@ -::error title=Plugin Check::Unknown mode 'strcit'. Use 'ratchet' or 'strict'. +::error title=Plugin Check::Unknown mode 'strcit'. Use 'baseline' or 'strict'. From 4d87f461caa21c5dcbf42d419ba893682239513d Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Tue, 3 Nov 2026 19:01:17 -0300 Subject: [PATCH 19/31] Say how the Plugin Check compare reads a count written as 1.0 --- .github/actions/plugin-check-report/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/plugin-check-report/README.md b/.github/actions/plugin-check-report/README.md index 8b2fbb6..8d67a94 100644 --- a/.github/actions/plugin-check-report/README.md +++ b/.github/actions/plugin-check-report/README.md @@ -242,7 +242,8 @@ show with a hand-edited baseline or an odd results file, except the first two: which the run itself knows. - A rule with an empty code is counted like any other. Bash misread its row and showed wrong totals. - A count in the baseline that is not a whole number (such as `1.5`) fails the compare with exit - status 1. Bash printed an arithmetic error and passed with zero totals. + status 1. Bash printed an arithmetic error and passed with zero totals. A count written as `1.0` + or `1e2` is read as the whole number it is. - A baseline count above 9007199254740991 (2^53 - 1) fails the compare with exit status 5 and says so. JavaScript cannot hold a larger number exactly, so a comment would show another number. Bash From 7f77356949d88c1c2d675777fd90b98fdc31c1fd Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Tue, 3 Nov 2026 20:40:19 -0300 Subject: [PATCH 20/31] Build the Plugin Check finish badge and the could-not-run comment in the action --- .github/actions/plugin-check-report/README.md | 17 +- .../actions/plugin-check-report/action.yml | 24 ++- .../plugin-check-report.js | 121 ++++++++++++- .github/workflows/plugin-check.yml | 25 +-- .../actions/plugin-check-report/badge.test.js | 162 ++++++++++++++++++ 5 files changed, 317 insertions(+), 32 deletions(-) create mode 100644 tests/actions/plugin-check-report/badge.test.js diff --git a/.github/actions/plugin-check-report/README.md b/.github/actions/plugin-check-report/README.md index 8d67a94..019faaf 100644 --- a/.github/actions/plugin-check-report/README.md +++ b/.github/actions/plugin-check-report/README.md @@ -136,6 +136,13 @@ grows past 15 lines. Pull requests from forks get no comment: their token cannot The workflow writes it with the [pr-comment](../pr-comment/) action. The comment is found by its first line, not by its author. +The action builds the body. Under the status line it puts a "Check Finished" badge with the time in +Eastern Time (`06:08 PM EDT` in summer, `EST` in winter), made by `finishBadge()` in +`plugin-check-report.js`, so no caller passes one. When the build or the check broke and there was +nothing to compare, the action writes a short body instead, "Plugin Check could not run" with the +same badge and a link to the run, so "in progress" never stays on the pull request. A body the +compare wrote is never replaced. + ## The check name A job in a called workflow reports as ` / `. With the template's job id, @@ -185,7 +192,6 @@ not start the run. The job that runs the Plugin Check action lowers itself to `c | `mode` | no | `baseline` | `baseline` or `strict` | | `comment-marker` | no | `` | First line of the comment body | | `baseline-wait` | no | `1200` | Seconds to wait for the base commit's run. A whole number, or the check fails | -| `badge-url` | no | none | Image under the status line | | `token` | no | `github.token` | Needs `actions: read` | | Output | What it is | @@ -240,6 +246,8 @@ show with a hand-edited baseline or an odd results file, except the first two: to use. - The `workflow-file` input is gone. The runs that hold the baseline are the calling workflow's, which the run itself knows. +- The `badge-url` input is gone. The action makes the "Check Finished" badge itself, and also writes + the "could not run" body that a shell step in the workflow wrote before. Both look as they did. - A rule with an empty code is counted like any other. Bash misread its row and showed wrong totals. - A count in the baseline that is not a whole number (such as `1.5`) fails the compare with exit status 1. Bash printed an arithmetic error and passed with zero totals. A count written as `1.0` @@ -252,7 +260,7 @@ show with a hand-edited baseline or an odd results file, except the first two: ## How it is tested The work is in `plugin-check-report.js`: small functions that take plain values, so a test calls -them directly. `tests/actions/plugin-check-report/` has three files, run with `node --test`: +them directly. `tests/actions/plugin-check-report/` has four files, run with `node --test`: - `compare.test.js` runs the compare in-process on the sample results files in `fixtures/`, and checks the comment, the job summary and the annotations against the exact files in @@ -260,6 +268,9 @@ them directly. `tests/actions/plugin-check-report/` has three files, run with `n JavaScript prints what it printed. - `fetch.test.js` runs the lookup against a real GitHub client with the network replaced by written answers, a fake clock for the wait, and a stand-in for the `gh` download. -- `action.test.js` calls the four step functions the way `actions/github-script` does. +- `action.test.js` calls the step functions the way `actions/github-script` does. +- `badge.test.js` checks the badge URL against what the old shell step printed, from the real + `date` command, for both sides of both daylight saving changes, midnight, noon and the end of a + year. It also covers the "could not run" body. No `jq` or `bash` is needed. diff --git a/.github/actions/plugin-check-report/action.yml b/.github/actions/plugin-check-report/action.yml index 54c0b4f..6112793 100644 --- a/.github/actions/plugin-check-report/action.yml +++ b/.github/actions/plugin-check-report/action.yml @@ -5,7 +5,8 @@ # # It does not run Plugin Check and it does not post the comment. The reusable workflow # .github/workflows/plugin-check.yml does both, so this action stays usable with a results file -# from anywhere. +# from anywhere. It does write the comment's body, with the "Check Finished" badge, and a body that +# says Plugin Check could not run when there was nothing to compare. # # The work is in one JavaScript file next to this file, plugin-check-report.js. Each step below is # an actions/github-script step that requires it by path, so there is nothing to build: the file in @@ -44,10 +45,6 @@ inputs: leaves room for the work around the wait. required: false default: '1200' - badge-url: - description: 'Image URL shown under the status line of the comment. Empty shows none.' - required: false - default: '' token: description: >- Token that lists earlier runs and downloads their artifacts. It needs `actions: read`. @@ -150,10 +147,25 @@ runs: COMMENT_MARKER: ${{ steps.paths.outputs.marker }} BASELINE_LABEL: ${{ steps.baseline.outputs.label }} BASELINE_URL: ${{ steps.baseline.outputs.url }} - COMMENT_BADGE: ${{ inputs.badge-url }} ARTIFACT_URL: ${{ steps.artifact.outputs.artifact-url }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} with: script: | const { runCompare } = require(`${process.env.ACTION_PATH}/plugin-check-report.js`) process.exitCode = runCompare({ core, env: process.env }) + + # When the build or the check broke, the compare wrote no body. Say so, instead of leaving "in + # progress" behind. Runs after a failed step too, but not when the run was cancelled. A body the + # compare wrote is left as it is. + - name: Write the comment if the compare could not run + if: ${{ !cancelled() && inputs.comment-file != '' }} + uses: actions/github-script@v9 + env: + ACTION_PATH: ${{ github.action_path }} + COMMENT_FILE: ${{ inputs.comment-file }} + COMMENT_MARKER: ${{ inputs.comment-marker || format('', github.repository) }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + with: + script: | + const { runCouldNotRun } = require(`${process.env.ACTION_PATH}/plugin-check-report.js`) + process.exitCode = runCouldNotRun({ core, env: process.env }) diff --git a/.github/actions/plugin-check-report/plugin-check-report.js b/.github/actions/plugin-check-report/plugin-check-report.js index 5ecb4fd..7d5111d 100644 --- a/.github/actions/plugin-check-report/plugin-check-report.js +++ b/.github/actions/plugin-check-report/plugin-check-report.js @@ -1454,10 +1454,122 @@ async function runFetch({ github, core, exec, env, now, sleep }) { * * @returns {number} The exit status. */ -function runCompare({ core, env }) { +function runCompare({ core, env, now }) { const args = env.COMMENT_FILE ? ['--comment-file', env.COMMENT_FILE] : [] - return compare({ args: [...args, env.RESULTS ?? ''], env, log: actionLog(core) }) + // The comment always shows the finish badge, so the action makes it: no caller passes one. + return compare({ + args: [...args, env.RESULTS ?? ''], + env: { ...env, COMMENT_BADGE: finishBadge(now) }, + log: actionLog(core), + }) +} + +/** The time zone of the time on the badges. */ +const BADGE_TIME_ZONE = 'America/New_York' + +/** + * The time on a badge, like `11/03 06:08 PM EST`: month and day, a 12-hour clock, and the zone's + * short name, which is EST in winter and EDT in summer. It is what `date '+%m/%d %I:%M %p %Z'` prints + * with `LC_ALL=C` and `TZ=America/New_York`. + * + * Tested for the default zone only. A zone whose short name Intl writes as `GMT+1` would need the + * `+` and the `-` of a shields.io badge escaped. + * + * @param {Date} now The moment. + * @param {string} timeZone An IANA time zone. + * + * @returns {string} The time, not yet escaped for a URL. + */ +function badgeStamp(now, timeZone) { + const parts = Object.fromEntries( + new Intl.DateTimeFormat('en-US', { + timeZone, + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + hour12: true, + timeZoneName: 'short', + }) + .formatToParts(now) + .map(({ type, value }) => [type, value]) + ) + + return `${parts.month}/${parts.day} ${parts.hour}:${parts.minute} ${parts.dayPeriod} ${parts.timeZoneName}` +} + +/** + * The "Check Finished" badge shown under the status line of the comment: a shields.io image with + * the time the check finished. Only a space and a slash are escaped, as the shell step this + * replaced did, so the colon of the time stays as it is. + * + * The workflow's "Check Started" badge in the `start` job is made by a shell step of the same + * shape. That job has no checkout to run this file from. + * + * @param {Date} now The moment. Default: now. + * @param {string} timeZone An IANA time zone. Default: America/New_York. + * + * @returns {string} The image URL. + */ +function finishBadge(now = new Date(), timeZone = BADGE_TIME_ZONE) { + const stamp = badgeStamp(now, timeZone).replace(/ /g, '%20').replace(/\//g, '%2F') + + return `https://img.shields.io/badge/Check%20Finished-${stamp}-darkgreen` +} + +/** + * The comment for a run that could not compare anything: the build or the check broke before the + * results existed, or the compare refused its settings. + * + * @param {object} input + * @param {string} input.marker The hidden first line. + * @param {string} input.badge The badge URL. + * @param {string} input.runUrl The link to the workflow run. + * + * @returns {string} The Markdown body, with a final line break. + */ +function couldNotRunComment({ marker, badge, runUrl }) { + return `${marker}\n❌ Plugin Check could not run\n\n![Plugin Check finished](${badge})\n\n📎 [Run](${runUrl})\n` +} + +/** + * The step after the compare: when the compare wrote no comment body, says so, instead of leaving + * "in progress" on the pull request. A body that exists and is not empty is left alone, whatever + * the verdict was. + * + * @param {object} input + * @param {object} input.core The toolkit. + * @param {object} input.env COMMENT_FILE, COMMENT_MARKER, RUN_URL and GITHUB_REPOSITORY. + * @param {Date} input.now Optional clock. + * + * @returns {number} 0, or 1 when the file cannot be written. + */ +function runCouldNotRun({ core, env, now }) { + const file = env.COMMENT_FILE ?? '' + if (!file) { + return 0 + } + + try { + if (fs.statSync(file).size > 0) { + return 0 + } + } catch { + // No file: write it. + } + + const marker = env.COMMENT_MARKER || `` + try { + writeText(file, couldNotRunComment({ marker, badge: finishBadge(now), runUrl: env.RUN_URL ?? '' }), 'write') + } catch (error) { + core.error(error.message, { title: TITLE }) + + return 1 + } + core.info('The compare wrote no comment, so the comment says Plugin Check could not run.') + + return 0 } /** @@ -1566,7 +1678,9 @@ module.exports = { CheckFailure, MODES, actionLog, + BADGE_TIME_ZONE, asciiLower, + badgeStamp, buildComment, buildRows, change, @@ -1577,11 +1691,13 @@ module.exports = { compareCodePoints, countPerCode, countRuns, + couldNotRunComment, cutBytes, describeBaseline, displayCode, evaluate, fetchBaseline, + finishBadge, formatBaseline, isOwnRun, linksLine, @@ -1593,6 +1709,7 @@ module.exports = { readJsonValues, readWholeNumber, runCompare, + runCouldNotRun, runFetch, runPaths, runUpdate, diff --git a/.github/workflows/plugin-check.yml b/.github/workflows/plugin-check.yml index e8591ca..7e472d8 100644 --- a/.github/workflows/plugin-check.yml +++ b/.github/workflows/plugin-check.yml @@ -379,18 +379,14 @@ jobs: name: plugin-check-results path: ${{ runner.temp }}/results - - name: Make the finish badge - id: finish_badge - run: | - stamp=$(LC_ALL=C TZ='America/New_York' date '+%m/%d %I:%M %p %Z') - stamp=${stamp// /%20} - stamp=${stamp//\//%2F} - echo "url=https://img.shields.io/badge/Check%20Finished-${stamp}-darkgreen" >> "$GITHUB_OUTPUT" - # `$/` is this repository at the commit that runs this workflow, so the action is always the # same revision as the workflow, whichever ref the caller used, and it needs no checkout. It # needs a runner of version 2.336.0 or newer; this job runs on ubuntu-latest. Not ./, which # in a called workflow resolves against the caller's repository. + # + # The action also writes the "Check Finished" badge into the body, and, when the build or the + # check broke and there was nothing to compare, a body that says Plugin Check could not run, + # so "in progress" never stays on the pull request. - name: Compare with the baseline uses: $/.github/actions/plugin-check-report with: @@ -398,19 +394,6 @@ jobs: comment-file: ${{ env.COMMENT_BODY }} comment-marker: ${{ env.COMMENT_MARKER }} mode: ${{ inputs.mode }} - badge-url: ${{ steps.finish_badge.outputs.url }} - - # When the build or the check broke, the action never wrote a body. Say so instead of leaving - # "in progress" behind. - - name: Prepare the comment - if: ${{ !cancelled() && env.COMMENT == 'true' }} - env: - BADGE: ${{ steps.finish_badge.outputs.url }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: | - if [ ! -s "$COMMENT_BODY" ]; then - printf '%s\n❌ Plugin Check could not run\n\n![Plugin Check finished](%s)\n\n📎 [Run](%s)\n' "$COMMENT_MARKER" "$BADGE" "$RUN_URL" > "$COMMENT_BODY" - fi - name: Post the result if: ${{ !cancelled() && env.COMMENT == 'true' }} diff --git a/tests/actions/plugin-check-report/badge.test.js b/tests/actions/plugin-check-report/badge.test.js new file mode 100644 index 0000000..f58ca14 --- /dev/null +++ b/tests/actions/plugin-check-report/badge.test.js @@ -0,0 +1,162 @@ +'use strict' + +const { describe, it } = require('node:test') +const assert = require('node:assert') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') + +const report = require('../../../.github/actions/plugin-check-report/plugin-check-report.js') +const { actionsFor } = require('../../support/actions.js') + +const URL_START = 'https://img.shields.io/badge/Check%20Finished-' + +/** + * What the shell step this replaced printed for each moment, from the real `date` command: + * + * stamp=$(LC_ALL=C TZ='America/New_York' date -r '+%m/%d %I:%M %p %Z') + * stamp=${stamp// /%20}; stamp=${stamp//\//%2F} + * echo "https://img.shields.io/badge/Check%20Finished-${stamp}-darkgreen" + * + * The moments are the ones where a clock formatter goes wrong: both sides of both daylight saving + * changes, the repeated hour of the fall change, midnight, noon and the end of a year. + */ +const GOLDEN = [ + ['winter, noon-ish', '2026-01-15T17:08:00Z', '01%2F15%2012:08%20PM%20EST'], + ['summer', '2026-07-04T22:05:00Z', '07%2F04%2006:05%20PM%20EDT'], + ['just before spring forward', '2026-03-08T06:59:59Z', '03%2F08%2001:59%20AM%20EST'], + ['spring forward', '2026-03-08T07:00:00Z', '03%2F08%2003:00%20AM%20EDT'], + ['repeated hour, first pass', '2026-11-01T05:30:00Z', '11%2F01%2001:30%20AM%20EDT'], + ['just before fall back', '2026-11-01T05:59:59Z', '11%2F01%2001:59%20AM%20EDT'], + ['fall back', '2026-11-01T06:00:00Z', '11%2F01%2001:00%20AM%20EST'], + ['repeated hour, second pass', '2026-11-01T06:30:00Z', '11%2F01%2001:30%20AM%20EST'], + ['midnight', '2026-01-15T05:00:00Z', '01%2F15%2012:00%20AM%20EST'], + ['noon', '2026-01-15T17:00:00Z', '01%2F15%2012:00%20PM%20EST'], + ['the last minute of a year', '2027-01-01T04:59:00Z', '12%2F31%2011:59%20PM%20EST'], +] + +describe('finishBadge', () => { + for (const [name, moment, stamp] of GOLDEN) { + it(`makes the URL the shell step made: ${name}`, () => { + assert.equal(report.finishBadge(new Date(moment)), `${URL_START}${stamp}-darkgreen`) + }) + } + + it('uses America/New_York unless told otherwise', () => { + assert.equal(report.BADGE_TIME_ZONE, 'America/New_York') + assert.equal(report.finishBadge(new Date('2026-07-04T22:05:00Z')), report.finishBadge(new Date('2026-07-04T22:05:00Z'), 'America/New_York')) + }) + + it('reads the clock when no moment is given', () => { + assert.match(report.finishBadge(), /^https:\/\/img\.shields\.io\/badge\/Check%20Finished-\d\d%2F\d\d%20\d\d:\d\d%20[AP]M%20E[SD]T-darkgreen$/) + }) + + it('escapes only the spaces and the slash, as the shell did', () => { + const url = report.finishBadge(new Date('2026-07-04T22:05:00Z')) + + assert.doesNotMatch(url, / |\/\d/) + assert.match(url, /06:05%20PM/) + }) +}) + +describe('the badge in the comment', () => { + const FIXTURES = path.join(__dirname, 'fixtures') + const env = (extra = {}) => ({ + RESULTS: path.join(FIXTURES, 'same-as-baseline.txt'), + PLUGIN_CHECK_BASELINE: path.join(FIXTURES, 'baseline.json'), + RUN_URL: 'https://example.test/run/1', + ...extra, + }) + + it('shows the finish badge under the status line, whatever the caller passed', async (t) => { + const { core } = await actionsFor(t) + const commentFile = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-badge-')), 'comment.md') + + const status = report.runCompare({ + core, + env: env({ COMMENT_FILE: commentFile, COMMENT_BADGE: 'https://example.test/other.svg' }), + now: new Date('2026-07-04T22:05:00Z'), + }) + + assert.equal(status, 0) + const lines = fs.readFileSync(commentFile, 'utf8').split('\n') + assert.match(lines[1], /^✅ Plugin Check passed/) + assert.equal(lines[3], `![Plugin Check finished](${URL_START}07%2F04%2006:05%20PM%20EDT-darkgreen)`) + }) +}) + +describe('runCouldNotRun', () => { + const temp = () => fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-fallback-')) + const env = (file, extra = {}) => ({ + COMMENT_FILE: file, + COMMENT_MARKER: '', + RUN_URL: 'https://github.com/example-org/my-plugin/actions/runs/7', + ...extra, + }) + + it('writes the comment when the compare wrote none', async (t) => { + const { core } = await actionsFor(t) + const file = path.join(temp(), 'comment.md') + + const status = report.runCouldNotRun({ core, env: env(file), now: new Date('2026-07-04T22:05:00Z') }) + + assert.equal(status, 0) + assert.equal( + fs.readFileSync(file, 'utf8'), + [ + '', + '❌ Plugin Check could not run', + '', + `![Plugin Check finished](${URL_START}07%2F04%2006:05%20PM%20EDT-darkgreen)`, + '', + '📎 [Run](https://github.com/example-org/my-plugin/actions/runs/7)', + '', + ].join('\n') + ) + }) + + it('writes it over an empty file, as `[ ! -s file ]` did', async (t) => { + const { core } = await actionsFor(t) + const file = path.join(temp(), 'comment.md') + fs.writeFileSync(file, '') + + report.runCouldNotRun({ core, env: env(file) }) + + assert.match(fs.readFileSync(file, 'utf8'), /❌ Plugin Check could not run/) + }) + + it('leaves a comment the compare wrote', async (t) => { + const { core } = await actionsFor(t) + const file = path.join(temp(), 'comment.md') + fs.writeFileSync(file, '\n❌ Plugin Check failed\n') + + const status = report.runCouldNotRun({ core, env: env(file) }) + + assert.equal(status, 0) + assert.equal(fs.readFileSync(file, 'utf8'), '\n❌ Plugin Check failed\n') + }) + + it('does nothing when no comment is asked for', async (t) => { + const { core } = await actionsFor(t) + + assert.equal(report.runCouldNotRun({ core, env: env('') }), 0) + }) + + it('names the marker after the repository when none is given', async (t) => { + const { core } = await actionsFor(t) + const file = path.join(temp(), 'comment.md') + + report.runCouldNotRun({ core, env: env(file, { COMMENT_MARKER: '', GITHUB_REPOSITORY: 'example-org/my-plugin' }) }) + + assert.match(fs.readFileSync(file, 'utf8'), /^\n/) + }) + + it('fails the step, naming the file, when it cannot be written', async (t) => { + const { core, logged } = await actionsFor(t) + + const status = report.runCouldNotRun({ core, env: env('/nonexistent-folder/comment.md') }) + + assert.equal(status, 1) + assert.match(logged(), /::error title=Plugin Check::Could not write \/nonexistent-folder\/comment\.md/) + }) +}) From 7d9367d4174c10300ecf70343bc66545fe24dca6 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Tue, 3 Nov 2026 20:40:39 -0300 Subject: [PATCH 21/31] Read the Plugin Check excluded checks from .plugin-check.json when the input is empty --- .github/actions/plugin-check-report/README.md | 44 +++- .../exclude-checks/action.yml | 46 ++++ .../plugin-check-report.js | 190 +++++++++++++- .github/workflows/plugin-check.yml | 22 +- templates/workflows/plugin-check.yml | 3 + .../plugin-check-report/exclude.test.js | 241 ++++++++++++++++++ 6 files changed, 538 insertions(+), 8 deletions(-) create mode 100644 .github/actions/plugin-check-report/exclude-checks/action.yml create mode 100644 tests/actions/plugin-check-report/exclude.test.js diff --git a/.github/actions/plugin-check-report/README.md b/.github/actions/plugin-check-report/README.md index 019faaf..31ca486 100644 --- a/.github/actions/plugin-check-report/README.md +++ b/.github/actions/plugin-check-report/README.md @@ -126,6 +126,39 @@ the caller. - `strict`: fail on any error. The comment still compares with the baseline. Switch to it once a plugin reaches zero errors; a new plugin with zero errors should start there. +## Excluding checks + +Some Plugin Check checks do not fit a plugin. Name them in a `.plugin-check.json` at the root of +the repository: + +```json +{ + "exclude-checks": ["late_escaping", "i18n_usage"] +} +``` + +One check name per entry, with no spaces or commas. Other keys are ignored, so the file can hold +more settings later. A missing file, a missing key or an empty list excludes nothing. A file that is +not valid JSON, or whose entries are not check names, fails the run in seconds, before the build starts, +with a message that says what to fix. + +The workflow's `exclude-checks` input does the same, one check per line. **When the input names any +check, it replaces the file** (the file is not even read). When it is empty, the file is used. + +The same list is available for a local run. This prints the flag `wp plugin check` takes, or +nothing when no check is excluded: + +```bash +node .github/actions/plugin-check-report/plugin-check-report.js exclude-checks +# --exclude-checks=late_escaping,i18n_usage + +npx wp-env run cli wp plugin check my-plugin --format=json \ + $(node .github/actions/plugin-check-report/plugin-check-report.js exclude-checks) > results.txt +``` + +It reads `.plugin-check.json` in the current folder, or the file named after it. The environment +variable `EXCLUDE_CHECKS` plays the part of the workflow input. + ## The comment One comment per pull request, replaced on every run, found by a hidden first line: @@ -155,9 +188,9 @@ the check to require in branch protection is `plugin-check / Plugin Check`. | `plugin-slug` | required | The plugin's folder name. Equals `zip_name` in `.puprc` | | `main-file` | `.php` | The main plugin file, checked after the build | | `build-command` | empty: pup | A bash command that builds the plugin instead of pup. See [`build-command`](#build-command-for-a-repository-without-pup) | -| `runs-on` | `ubuntu-latest` | Runner of the job that builds and checks | +| `runs-on` | `ubuntu-latest` | Runner of the job that builds and checks, of runner version 2.336.0 or newer (the job uses `$/`) | | `php-version` | `7.4` | PHP that builds `vendor/`. Use the oldest the plugin supports | -| `exclude-checks` | empty | Checks to skip, one per line, passed to the Plugin Check action | +| `exclude-checks` | empty | Checks to skip, one per line. Replaces `.plugin-check.json` when it names any. See [Excluding checks](#excluding-checks) | | `cache-paths` | empty | Build caches kept between runs, one path per line, relative to the repository | | `mode` | `baseline` | `baseline` or `strict`. See [Modes](#modes) | | `submodules` | `false` | Passed to `actions/checkout` | @@ -203,7 +236,8 @@ not start the run. The job that runs the Plugin Check action lowers itself to `c You need Node 20 or newer, and for the baseline the GitHub CLI signed in (`gh auth login`). Run `npm install` once in the toolbox checkout, which installs the GitHub client the lookup uses. The -command line takes the same arguments and environment variables as the two bash scripts did. +`compare` and `fetch` commands take the same arguments and environment variables as the two bash +scripts did. 1. Build the plugin and run Plugin Check in a local WordPress, for example with wp-env: @@ -260,7 +294,7 @@ show with a hand-edited baseline or an odd results file, except the first two: ## How it is tested The work is in `plugin-check-report.js`: small functions that take plain values, so a test calls -them directly. `tests/actions/plugin-check-report/` has four files, run with `node --test`: +them directly. `tests/actions/plugin-check-report/` has five files, run with `node --test`: - `compare.test.js` runs the compare in-process on the sample results files in `fixtures/`, and checks the comment, the job summary and the annotations against the exact files in @@ -272,5 +306,7 @@ them directly. `tests/actions/plugin-check-report/` has four files, run with `no - `badge.test.js` checks the badge URL against what the old shell step printed, from the real `date` command, for both sides of both daylight saving changes, midnight, noon and the end of a year. It also covers the "could not run" body. +- `exclude.test.js` covers the config file and the input, every way the file can be wrong, and the + command line. No `jq` or `bash` is needed. diff --git a/.github/actions/plugin-check-report/exclude-checks/action.yml b/.github/actions/plugin-check-report/exclude-checks/action.yml new file mode 100644 index 0000000..6986686 --- /dev/null +++ b/.github/actions/plugin-check-report/exclude-checks/action.yml @@ -0,0 +1,46 @@ +# Works out which Plugin Check checks to skip: the workflow's `exclude-checks` input when it names +# any, else the `exclude-checks` list in the repository's .plugin-check.json. The result is in the +# format wordpress/plugin-check-action takes for its own `exclude-checks` input: one check per +# line. A bad file fails the step with a message that says what to fix. +# +# The work is in plugin-check-report.js, one folder up, which also prints the same list as the +# `--exclude-checks=a,b` flag of `wp plugin check` for a local run. See the README of +# plugin-check-report. +name: Plugin Check excluded checks +description: Lists the checks to skip, from the exclude-checks input or the repository's .plugin-check.json + +inputs: + exclude-checks: + description: >- + Checks to skip, one per line. When it names any check, it replaces the file: the file is not + even read. + required: false + default: '' + config-file: + description: >- + Path of the repository's config file, relative to the workspace. A missing file means no + checks are skipped. + required: false + default: '.plugin-check.json' + +outputs: + checks: + description: 'The checks to skip, one per line. Empty when there are none.' + value: ${{ steps.list.outputs.checks }} + +runs: + using: 'composite' + steps: + - name: List the excluded checks + id: list + uses: actions/github-script@v9 + env: + # github-script's require() resolves against the workspace of the repository being built, + # not this action's directory, so the path has to be passed in. + ACTION_PATH: ${{ github.action_path }} + EXCLUDE_CHECKS: ${{ inputs.exclude-checks }} + CONFIG_FILE: ${{ inputs.config-file }} + with: + script: | + const { runExcludeChecks } = require(`${process.env.ACTION_PATH}/../plugin-check-report.js`) + process.exitCode = runExcludeChecks({ core, env: process.env }) diff --git a/.github/actions/plugin-check-report/plugin-check-report.js b/.github/actions/plugin-check-report/plugin-check-report.js index 7d5111d..7d04405 100644 --- a/.github/actions/plugin-check-report/plugin-check-report.js +++ b/.github/actions/plugin-check-report/plugin-check-report.js @@ -1572,6 +1572,159 @@ function runCouldNotRun({ core, env, now }) { return 0 } +/** The file a repository keeps its excluded checks in, at its root. */ +const EXCLUDE_FILE = '.plugin-check.json' + +/** + * The checks to skip, from the workflow input: one per line. A comma works too, because the Plugin + * Check action turns the lines into a comma list. Blank lines and repeats are dropped. + * + * @param {string} text The input. + * + * @returns {string[]} The check names, in order. + */ +function parseExcludeInput(text) { + return [...new Set(text.split(/[\r\n,]+/).map((line) => line.trim()).filter(Boolean))] +} + +/** + * The checks to skip, from the repository's `.plugin-check.json`: + * `{"exclude-checks": ["check_a", "check_b"]}`. Other keys are ignored, so the file can grow. A + * missing `exclude-checks` key means none. Repeats are dropped. + * + * @param {string} text The file's contents. + * @param {string} name What to call the file in a message. + * + * @returns {string[]} The check names, in order. + * + * @throws {CheckFailure} When the file cannot be used. The message says what to fix. + */ +function parseExcludeFile(text, name = EXCLUDE_FILE) { + let data + try { + data = JSON.parse(text) + } catch (error) { + throw new CheckFailure(`${name} is not valid JSON: ${error.message}`, 1) + } + + if (data === null || typeof data !== 'object' || Array.isArray(data)) { + throw new CheckFailure(`${name} must hold a JSON object, like {"exclude-checks": ["check_name"]}.`, 1) + } + + const list = data['exclude-checks'] + if (list === undefined) { + return [] + } + if (!Array.isArray(list)) { + throw new CheckFailure(`"exclude-checks" in ${name} must be a list of check names, like ["check_name"].`, 1) + } + + const checks = [] + for (const entry of list) { + const check = typeof entry === 'string' ? entry.trim() : '' + if (!check || /[\s,]/.test(check)) { + throw new CheckFailure( + `"exclude-checks" in ${name} has ${JSON.stringify(entry)}, which is not a check name. Each entry is one text without spaces or commas.`, + 1 + ) + } + checks.push(check) + } + + return [...new Set(checks)] +} + +/** + * Which checks to skip. The workflow input wins when it names any: then the file is not even read. + * Otherwise the file is used. With neither, none. + * + * @param {object} input + * @param {string} input.input The workflow input. + * @param {string|null} input.fileText The file's contents, or null when there is no file. + * @param {string} input.fileName What to call the file in a message. + * + * @returns {{checks: string[], source: string}} The check names, and where they came from: `input`, + * `file` or `none`. + * + * @throws {CheckFailure} When the file is used and cannot be. + */ +function excludedChecks({ input = '', fileText = null, fileName = EXCLUDE_FILE }) { + const fromInput = parseExcludeInput(input) + if (fromInput.length > 0) { + return { checks: fromInput, source: 'input' } + } + if (fileText === null) { + return { checks: [], source: 'none' } + } + + const checks = parseExcludeFile(fileText, fileName) + + return { checks, source: checks.length > 0 ? 'file' : 'none' } +} + +/** + * The file's contents, or null when there is no such file. + * + * @param {string} file The path. + * + * @returns {string|null} The text. + * + * @throws {CheckFailure} When the file exists but cannot be read. + */ +function readIfFile(file) { + if (!file || !isFile(file)) { + return null + } + + try { + return fs.readFileSync(file, 'utf8') + } catch (error) { + throw new CheckFailure(`Could not read ${file}: ${error.message}`, 1) + } +} + +/** + * The step before the Plugin Check action: works out which checks to skip, from the workflow input + * or else the repository's `.plugin-check.json`. + * + * Sets the output `checks`: one name per line, the format the Plugin Check action takes. + * + * @param {object} input + * @param {object} input.core The toolkit. + * @param {object} input.env EXCLUDE_CHECKS (the workflow input) and CONFIG_FILE (the path). + * + * @returns {number} 0, or 1 when the file cannot be used. + */ +function runExcludeChecks({ core, env }) { + const file = env.CONFIG_FILE ?? '' + + try { + const { checks, source } = excludedChecks({ + input: env.EXCLUDE_CHECKS ?? '', + // Not read when the input names checks: the input replaces the file. + fileText: parseExcludeInput(env.EXCLUDE_CHECKS ?? '').length > 0 ? null : readIfFile(file), + fileName: path.basename(file) || EXCLUDE_FILE, + }) + + core.setOutput('checks', checks.join('\n')) + core.info( + source === 'none' + ? 'No checks are excluded.' + : `Excluding ${checks.length} check(s) from the ${source === 'input' ? 'exclude-checks input' : path.basename(file)}: ${checks.join(', ')}` + ) + + return 0 + } catch (error) { + if (error instanceof CheckFailure) { + core.error(error.message, { title: TITLE }) + + return error.exitCode + } + + throw error + } +} + /** * A GitHub client for a run on a developer's machine. The token comes from GITHUB_TOKEN, else * GH_TOKEN, else the GitHub CLI. @@ -1662,7 +1815,36 @@ async function main( return found.status } - io.err('Usage: plugin-check-report.js compare|fetch ...') + // Prints the flag `wp plugin check` takes, so a local run skips the same checks as CI: + // `wp plugin check my-plugin $(node plugin-check-report.js exclude-checks)`. Prints nothing when + // no check is excluded. EXCLUDE_CHECKS, when set, replaces the file, as the workflow input does. + if (command === 'exclude-checks') { + const file = args[0] ?? EXCLUDE_FILE + + try { + const input = env.EXCLUDE_CHECKS ?? '' + const { checks } = excludedChecks({ + input, + fileText: parseExcludeInput(input).length > 0 ? null : readIfFile(file), + fileName: file, + }) + if (checks.length > 0) { + io.out(`--exclude-checks=${checks.join(',')}`) + } + + return 0 + } catch (error) { + if (error instanceof CheckFailure) { + log.err(error.message) + + return error.exitCode + } + + throw error + } + } + + io.err('Usage: plugin-check-report.js compare|fetch|exclude-checks ...') return 2 } @@ -1679,6 +1861,7 @@ module.exports = { MODES, actionLog, BADGE_TIME_ZONE, + EXCLUDE_FILE, asciiLower, badgeStamp, buildComment, @@ -1696,6 +1879,7 @@ module.exports = { describeBaseline, displayCode, evaluate, + excludedChecks, fetchBaseline, finishBadge, formatBaseline, @@ -1703,13 +1887,17 @@ module.exports = { linksLine, main, orderCandidates, + parseExcludeFile, + parseExcludeInput, parseFetchArgs, parseResults, readBaseline, + readIfFile, readJsonValues, readWholeNumber, runCompare, runCouldNotRun, + runExcludeChecks, runFetch, runPaths, runUpdate, diff --git a/.github/workflows/plugin-check.yml b/.github/workflows/plugin-check.yml index 7e472d8..7c8acbb 100644 --- a/.github/workflows/plugin-check.yml +++ b/.github/workflows/plugin-check.yml @@ -52,7 +52,9 @@ on: type: string default: '' runs-on: - description: 'Runner label for the job that builds and checks. The two small jobs run on ubuntu-latest.' + description: >- + Runner label for the job that builds and checks, of runner version 2.336.0 or newer: the + job uses the `$/` reference. The two small jobs run on ubuntu-latest. required: false type: string default: 'ubuntu-latest' @@ -64,7 +66,10 @@ on: type: string default: '7.4' exclude-checks: - description: 'Checks to skip, one per line. Passed to wordpress/plugin-check-action as it is.' + description: >- + Checks to skip, one per line. When it names any, it replaces the `exclude-checks` list in + the repository's `.plugin-check.json`; when empty, that file is used. See the + plugin-check-report README. required: false type: string default: '' @@ -168,6 +173,17 @@ jobs: # Nothing here pushes over git, and the job goes on to run third-party code. persist-credentials: false + # The checks to skip: the `exclude-checks` input when it names any, else the list in the + # repository's .plugin-check.json. First, so a bad file fails in seconds, before the build, with + # a message that says what to fix. Like the report job, this needs a runner of version 2.336.0 + # or newer for `$/`; the `runs-on` input says so. + - name: List the excluded checks + id: exclude + uses: $/.github/actions/plugin-check-report/exclude-checks + with: + exclude-checks: ${{ inputs.exclude-checks }} + config-file: ${{ github.workspace }}/source/.plugin-check.json + # The action's wp-env pulls these images when it starts, and pulling them is part of the # time that step takes. Started now, in the background, the download overlaps with the # build. The images are the ones wp-env's generated files name (FROM wordpress..., FROM @@ -344,7 +360,7 @@ jobs: uses: wordpress/plugin-check-action@v1 with: build-dir: ${{ github.workspace }}/build/${{ inputs.plugin-slug }} - exclude-checks: ${{ inputs.exclude-checks }} + exclude-checks: ${{ steps.exclude.outputs.checks }} report: name: Plugin Check diff --git a/templates/workflows/plugin-check.yml b/templates/workflows/plugin-check.yml index 6fc5b22..b19e7c3 100644 --- a/templates/workflows/plugin-check.yml +++ b/templates/workflows/plugin-check.yml @@ -9,6 +9,9 @@ name: Plugin Check # a .puprc whose zip_name equals plugin-slug, and a `pup` Composer script; see the # plugin-check-report action's README. # +# Checks that do not fit the plugin go in a .plugin-check.json at the repository root, as +# {"exclude-checks": ["check_name"]}; or pass the `exclude-checks` input, which replaces the file. +# # Run it locally: `wp plugin check --format=json > results.txt` in a local WordPress, then # the compare command on that file. See "Running it locally" in the action's README. diff --git a/tests/actions/plugin-check-report/exclude.test.js b/tests/actions/plugin-check-report/exclude.test.js new file mode 100644 index 0000000..ae2f3dd --- /dev/null +++ b/tests/actions/plugin-check-report/exclude.test.js @@ -0,0 +1,241 @@ +'use strict' + +const { describe, it } = require('node:test') +const assert = require('node:assert') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') + +const report = require('../../../.github/actions/plugin-check-report/plugin-check-report.js') +const { actionsFor } = require('../../support/actions.js') + +const temp = () => fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-exclude-')) + +/** + * A repository root holding a config file. + * + * @param {string} text What the file holds. + * + * @returns {string} The path of the file. + */ +function configFile(text) { + const file = path.join(temp(), '.plugin-check.json') + fs.writeFileSync(file, text) + + return file +} + +describe('parseExcludeInput', () => { + it('reads one check per line', () => { + assert.deepEqual(report.parseExcludeInput('late_escaping\ni18n_usage\n'), ['late_escaping', 'i18n_usage']) + }) + + it('trims, drops blank lines and repeats, and reads Windows line breaks', () => { + assert.deepEqual(report.parseExcludeInput(' a \r\n\r\n\n b\na\n'), ['a', 'b']) + }) + + it('reads a comma list too, which the Plugin Check action turns its lines into', () => { + assert.deepEqual(report.parseExcludeInput('a,b\nc'), ['a', 'b', 'c']) + }) + + it('is empty for an empty or blank input', () => { + assert.deepEqual(report.parseExcludeInput(''), []) + assert.deepEqual(report.parseExcludeInput(' \n\t\n'), []) + }) +}) + +describe('parseExcludeFile', () => { + it('reads the list', () => { + assert.deepEqual(report.parseExcludeFile('{"exclude-checks": ["late_escaping", "i18n_usage"]}'), ['late_escaping', 'i18n_usage']) + }) + + it('trims entries and drops repeats, keeping the first order', () => { + assert.deepEqual(report.parseExcludeFile('{"exclude-checks": [" b ", "a", "b"]}'), ['b', 'a']) + }) + + it('means none when the key is missing or the list is empty', () => { + assert.deepEqual(report.parseExcludeFile('{}'), []) + assert.deepEqual(report.parseExcludeFile('{"exclude-checks": []}'), []) + }) + + it('ignores other keys, so the file can grow', () => { + assert.deepEqual(report.parseExcludeFile('{"mode": "strict", "exclude-checks": ["a"]}'), ['a']) + }) + + const refused = [ + ['text that is not JSON', 'late_escaping', /^\.plugin-check\.json is not valid JSON: /], + ['an empty file', '', /^\.plugin-check\.json is not valid JSON: /], + ['a list at the top', '["a"]', /^\.plugin-check\.json must hold a JSON object, like \{"exclude-checks": \["check_name"\]\}\.$/], + ['null', 'null', /must hold a JSON object/], + ['a number', '3', /must hold a JSON object/], + ['a text instead of a list', '{"exclude-checks": "a,b"}', /^"exclude-checks" in \.plugin-check\.json must be a list of check names, like \["check_name"\]\.$/], + ['null instead of a list', '{"exclude-checks": null}', /must be a list of check names/], + ['a number in the list', '{"exclude-checks": [1]}', /has 1, which is not a check name/], + ['an empty entry', '{"exclude-checks": ["a", ""]}', /has "", which is not a check name/], + ['a blank entry', '{"exclude-checks": [" "]}', /has " ", which is not a check name/], + ['a space inside an entry', '{"exclude-checks": ["late escaping"]}', /has "late escaping", which is not a check name/], + ['a comma inside an entry', '{"exclude-checks": ["a,b"]}', /has "a,b", which is not a check name/], + ['a line break inside an entry', '{"exclude-checks": ["a\\nb"]}', /which is not a check name/], + ['a nested list', '{"exclude-checks": [["a"]]}', /which is not a check name/], + ] + for (const [name, text, message] of refused) { + it(`fails on ${name}`, () => { + assert.throws( + () => report.parseExcludeFile(text), + (error) => error instanceof report.CheckFailure && error.exitCode === 1 && message.test(error.message) + ) + }) + } + + it('names the file in the message', () => { + assert.throws( + () => report.parseExcludeFile('{', 'checks.json'), + (error) => error.message.startsWith('checks.json is not valid JSON: ') + ) + }) +}) + +describe('excludedChecks', () => { + it('uses the input when it names checks, and does not read the file', () => { + const result = report.excludedChecks({ input: 'a\nb', fileText: 'this is not JSON' }) + + assert.deepEqual(result, { checks: ['a', 'b'], source: 'input' }) + }) + + it('uses the file when the input is empty or blank', () => { + const fileText = '{"exclude-checks": ["from_file"]}' + + assert.deepEqual(report.excludedChecks({ input: '', fileText }), { checks: ['from_file'], source: 'file' }) + assert.deepEqual(report.excludedChecks({ input: ' \n', fileText }), { checks: ['from_file'], source: 'file' }) + }) + + it('excludes nothing with neither', () => { + assert.deepEqual(report.excludedChecks({}), { checks: [], source: 'none' }) + assert.deepEqual(report.excludedChecks({ input: '', fileText: null }), { checks: [], source: 'none' }) + }) + + it('says none for a file with an empty list', () => { + assert.deepEqual(report.excludedChecks({ fileText: '{"exclude-checks": []}' }), { checks: [], source: 'none' }) + }) + + it('fails on a bad file when the input is empty', () => { + assert.throws(() => report.excludedChecks({ input: '', fileText: '{' }), /is not valid JSON/) + }) +}) + +describe('runExcludeChecks', () => { + it('outputs the file\'s checks one per line', async (t) => { + const { core, outputs, logged } = await actionsFor(t) + const file = configFile('{"exclude-checks": ["late_escaping", "i18n_usage"]}') + + const status = report.runExcludeChecks({ core, env: { EXCLUDE_CHECKS: '', CONFIG_FILE: file } }) + + assert.equal(status, 0) + assert.equal(outputs().checks, 'late_escaping\ni18n_usage') + assert.match(logged(), /Excluding 2 check\(s\) from the \.plugin-check\.json: late_escaping, i18n_usage/) + }) + + it('lets the input replace the file, and leaves a bad file alone', async (t) => { + const { core, outputs, logged } = await actionsFor(t) + const file = configFile('this is not JSON') + + const status = report.runExcludeChecks({ core, env: { EXCLUDE_CHECKS: 'from_input\n', CONFIG_FILE: file } }) + + assert.equal(status, 0) + assert.equal(outputs().checks, 'from_input') + assert.match(logged(), /from the exclude-checks input: from_input/) + }) + + it('outputs nothing when there is no input and no file', async (t) => { + const { core, outputs, logged } = await actionsFor(t) + + const status = report.runExcludeChecks({ core, env: { EXCLUDE_CHECKS: '', CONFIG_FILE: path.join(temp(), '.plugin-check.json') } }) + + assert.equal(status, 0) + assert.equal(outputs().checks, '') + assert.match(logged(), /No checks are excluded\./) + }) + + it('fails the step with the message, as an annotation, when the file is bad', async (t) => { + const { core, logged } = await actionsFor(t) + const file = configFile('{"exclude-checks": "late_escaping"}') + + const status = report.runExcludeChecks({ core, env: { EXCLUDE_CHECKS: '', CONFIG_FILE: file } }) + + assert.equal(status, 1) + assert.match(logged(), /::error title=Plugin Check::"exclude-checks" in \.plugin-check\.json must be a list of check names/) + }) + + it('fails the step when the file cannot be read', async (t) => { + // Root reads a file with no permissions, so there is nothing to fail there. + if (process.getuid?.() === 0) { + return t.skip('root can read any file') + } + const { core, logged } = await actionsFor(t) + const file = configFile('{}') + fs.chmodSync(file, 0) + + const status = report.runExcludeChecks({ core, env: { EXCLUDE_CHECKS: '', CONFIG_FILE: file } }) + + fs.chmodSync(file, 0o600) + assert.equal(status, 1) + assert.match(logged(), /::error title=Plugin Check::Could not read /) + }) +}) + +describe('exclude-checks on the command line', () => { + const run = async (args, env = {}) => { + const out = [] + const err = [] + const status = await report.main(['exclude-checks', ...args], env, { out: (line) => out.push(line), err: (line) => err.push(line) }) + + return { status, out, err } + } + + it('prints the flag wp plugin check takes', async () => { + const file = configFile('{"exclude-checks": ["late_escaping", "i18n_usage"]}') + + assert.deepEqual(await run([file]), { status: 0, out: ['--exclude-checks=late_escaping,i18n_usage'], err: [] }) + }) + + it('prints nothing when no check is excluded', async () => { + assert.deepEqual(await run([path.join(temp(), '.plugin-check.json')]), { status: 0, out: [], err: [] }) + }) + + it('lets EXCLUDE_CHECKS replace the file, as the workflow input does', async () => { + const file = configFile('{"exclude-checks": ["from_file"]}') + + assert.deepEqual(await run([file], { EXCLUDE_CHECKS: 'a\nb' }), { status: 0, out: ['--exclude-checks=a,b'], err: [] }) + }) + + it('reads .plugin-check.json in the current folder by default', async () => { + const folder = temp() + fs.writeFileSync(path.join(folder, '.plugin-check.json'), '{"exclude-checks": ["from_cwd"]}') + const before = process.cwd() + process.chdir(folder) + try { + assert.deepEqual(await run([]), { status: 0, out: ['--exclude-checks=from_cwd'], err: [] }) + } finally { + process.chdir(before) + } + }) + + it('fails with the message on standard error when the file is bad', async () => { + const file = configFile('{') + + const { status, out, err } = await run([file]) + + assert.equal(status, 1) + assert.deepEqual(out, []) + assert.match(err[0], new RegExp(`^${file.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')} is not valid JSON`)) + }) + + it('lists the command in the usage line', async () => { + const err = [] + + const status = await report.main(['nope'], {}, { out: () => {}, err: (line) => err.push(line) }) + + assert.equal(status, 2) + assert.match(err[0], /compare\|fetch\|exclude-checks/) + }) +}) From ba4096d0be5b81c52d6bb77885619ba939e15ba0 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Wed, 4 Nov 2026 14:27:43 -0300 Subject: [PATCH 22/31] Install pup with the shared pup action in the Plugin Check workflow --- .github/actions/plugin-check-report/README.md | 33 +++++++--------- .github/workflows/plugin-check.yml | 38 +++++++++++++++---- templates/workflows/plugin-check.yml | 4 +- 3 files changed, 46 insertions(+), 29 deletions(-) diff --git a/.github/actions/plugin-check-report/README.md b/.github/actions/plugin-check-report/README.md index 31ca486..0826804 100644 --- a/.github/actions/plugin-check-report/README.md +++ b/.github/actions/plugin-check-report/README.md @@ -28,20 +28,9 @@ jobs: - A `.puprc` whose `build` makes the plugin as it ships, and whose `zip_name` equals `plugin-slug`. The zip's top folder is named after `zip_name`, and Plugin Check takes the slug from the folder name. -- A `pup` Composer script that downloads `pup.phar` when it is missing and then runs it, so one - `composer pup` both installs and runs pup. Kadence Blocks has: - - ```json - "pup": [ - "test -f ./bin/pup.phar || curl -o bin/pup.phar -L -C - https://github.com/stellarwp/pup/releases/download/2.0.0/pup.phar", - "@php bin/pup.phar" - ] - ``` - - The script is what matters, but the path is not free yet: the workflow also runs - `php bin/pup.phar` itself, to ask pup for the version and the zip name, so keep the phar at - `bin/pup.phar`. - +- Nothing for pup itself. The workflow installs it with the shared [pup](../pup/) action, so the + repository needs no copy of `pup.phar` and no `pup` Composer script. Set the `pup-version` input + to the release the repository's own release build uses, so the check sees the same files. - Ignore rules that keep dev-only files out of the zip: a `.distignore`, `export-ignore` lines in `.gitattributes` (pup reads both), or pup's default ignore list. - The caller file named `plugin-check.yml`. See [The baseline](#the-baseline). @@ -51,17 +40,20 @@ The JavaScript toolchain comes from files the repository already has: `.nvmrc` s ## How the plugin is built -By default, with pup, the same steps the release zip uses, from the root of the checkout: +By default, with pup, the same steps the release zip uses, from the root of the checkout. `pup` +here is `php pup.phar`, with the phar the shared pup action installed: ```bash -composer run -q pup > /dev/null # downloads bin/pup.phar -composer -- pup build -version=$(php -d display_errors=0 bin/pup.phar get-version --dev) -composer -- pup package "$version" -name=$(php -d display_errors=0 bin/pup.phar zip-name "$version") +pup build +version=$(pup get-version --dev | tail -n 1) +pup package "$version" +name=$(pup zip-name "$version" | tail -n 1) unzip -q "$name.zip" -d "$PLUGIN_CHECK_BUILD_DIR" ``` +One pup release builds, packages and names the zip, so the name looked for is always the name the +package step made. The last line pup prints is the value, as the pup action reads it. + So `.puprc` and the ignore files decide what is checked. If the release needs a step the check should see too, such as making the POT file, put it in `.puprc`'s `build`, not in the workflow. @@ -188,6 +180,7 @@ the check to require in branch protection is `plugin-check / Plugin Check`. | `plugin-slug` | required | The plugin's folder name. Equals `zip_name` in `.puprc` | | `main-file` | `.php` | The main plugin file, checked after the build | | `build-command` | empty: pup | A bash command that builds the plugin instead of pup. See [`build-command`](#build-command-for-a-repository-without-pup) | +| `pup-version` | `2.0.0` | The pup release that builds the plugin, or `latest`. Use the release the repository's release build uses | | `runs-on` | `ubuntu-latest` | Runner of the job that builds and checks, of runner version 2.336.0 or newer (the job uses `$/`) | | `php-version` | `7.4` | PHP that builds `vendor/`. Use the oldest the plugin supports | | `exclude-checks` | empty | Checks to skip, one per line. Replaces `.plugin-check.json` when it names any. See [Excluding checks](#excluding-checks) | diff --git a/.github/workflows/plugin-check.yml b/.github/workflows/plugin-check.yml index 7c8acbb..94c07f6 100644 --- a/.github/workflows/plugin-check.yml +++ b/.github/workflows/plugin-check.yml @@ -6,9 +6,10 @@ name: Plugin Check # A reusable workflow. A plugin repository calls it from a thin workflow of its own; copy # templates/workflows/plugin-check.yml. # -# The plugin is built with pup by default (`composer -- pup build`, then `pup package`), so the -# repository's .puprc and ignore files decide what is checked, the same as the release. A repository -# without pup passes `build-command` instead; see the plugin-check-report README. +# The plugin is built with pup by default (`pup build`, then `pup package`), so the repository's +# .puprc and ignore files decide what is checked, the same as the release. pup comes from the shared +# .github/actions/pup action, so the repository does not need a copy of it or a Composer script for +# it. A repository without pup passes `build-command` instead; see the plugin-check-report README. # # Three jobs, so the third-party action never holds a token that can write. The comments are written # by .github/actions/pr-comment, which calls the GitHub API itself: @@ -51,6 +52,14 @@ on: required: false type: string default: '' + pup-version: + description: >- + The pup release that builds the plugin, e.g. `2.0.0`, or `latest`. Use the release the + repository's own release build uses, so the check sees the same files. Not used with + `build-command`. + required: false + type: string + default: '2.0.0' runs-on: description: >- Runner label for the job that builds and checks, of runner version 2.336.0 or newer: the @@ -212,6 +221,16 @@ jobs: php-version: ${{ inputs.php-version }} coverage: none + # pup is a PHP archive, so PHP is set up first. One pup release builds, packages and names the + # zip below, so the name it looks for is always the name the package step made. The action's + # `read` input is not used: it reads from the workspace root, and the repository is in source/. + - name: Install pup + id: pup + if: ${{ inputs.build-command == '' }} + uses: $/.github/actions/pup + with: + pup-version: ${{ inputs.pup-version }} + - name: Find the Composer cache id: composer-cache working-directory: source @@ -280,6 +299,7 @@ jobs: working-directory: source env: BUILD_COMMAND: ${{ inputs.build-command }} + PUP_PHAR: ${{ steps.pup.outputs.phar }} COMPOSER_TOKEN: ${{ secrets.composer-token }} PACKAGE_TOKEN: ${{ secrets.package-token }} run: | @@ -299,11 +319,13 @@ jobs: exit 0 fi - composer run -q pup > /dev/null - composer -- pup build - version=$(php -d display_errors=0 bin/pup.phar get-version --dev) - composer -- pup package "$version" - name=$(php -d display_errors=0 bin/pup.phar zip-name "$version") + # The last line pup prints is the value, as the pup action reads it; PHP's own errors go to + # stderr so they never reach it. + pup() { php -d display_errors=stderr "$PUP_PHAR" "$@"; } + pup build + version=$(pup get-version --dev | tail -n 1) + pup package "$version" + name=$(pup zip-name "$version" | tail -n 1) unzip -q "$name.zip" -d "$PLUGIN_CHECK_BUILD_DIR" # The one contract every build has to meet, pup or not: a folder named after the slug, with diff --git a/templates/workflows/plugin-check.yml b/templates/workflows/plugin-check.yml index b19e7c3..e18bdac 100644 --- a/templates/workflows/plugin-check.yml +++ b/templates/workflows/plugin-check.yml @@ -6,7 +6,7 @@ name: Plugin Check # A file to copy into .github/workflows/ of a plugin repository. Keep the file name # plugin-check.yml: earlier runs of this file hold the baselines, and a new name starts with none. # The work is in stellarwp/plugin-toolbox/.github/workflows/plugin-check.yml. The repository needs -# a .puprc whose zip_name equals plugin-slug, and a `pup` Composer script; see the +# a .puprc whose zip_name equals plugin-slug; pup itself is installed by the workflow. See the # plugin-check-report action's README. # # Checks that do not fit the plugin go in a .plugin-check.json at the repository root, as @@ -41,3 +41,5 @@ jobs: uses: stellarwp/plugin-toolbox/.github/workflows/plugin-check.yml@main with: plugin-slug: my-plugin + # The pup release your release build uses. It builds the plugin for the check. + pup-version: '2.0.0' From 1a2770277fed2aec929542882888ff5cb0bc8c48 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Wed, 4 Nov 2026 14:30:23 -0300 Subject: [PATCH 23/31] Read the Plugin Check baseline from a committed file --- .../actions/plugin-check-report/action.yml | 74 ++++--------------- .github/workflows/plugin-check.yml | 21 ++++-- templates/workflows/plugin-check.yml | 14 ++-- 3 files changed, 37 insertions(+), 72 deletions(-) diff --git a/.github/actions/plugin-check-report/action.yml b/.github/actions/plugin-check-report/action.yml index 6112793..37fe69b 100644 --- a/.github/actions/plugin-check-report/action.yml +++ b/.github/actions/plugin-check-report/action.yml @@ -1,7 +1,8 @@ -# Compares a WordPress.org Plugin Check results file with a baseline: the counts of an earlier run -# of the same workflow, never a committed file. Only a rule with more errors than the baseline -# fails (baseline mode), or any error at all (strict mode). It also saves this run's counts as the -# next baseline and writes the short body for the pull request comment. +# Compares a WordPress.org Plugin Check results file with a baseline: the counts per rule in a file +# the repository commits, `.plugin-check-baseline.json`. Only a rule with more errors than the +# baseline fails (baseline mode), or any error at all (strict mode). It also saves this run's counts +# as a new baseline, to copy into the repository, and writes the short body for the pull request +# comment. # # It does not run Plugin Check and it does not post the comment. The reusable workflow # .github/workflows/plugin-check.yml does both, so this action stays usable with a results file @@ -12,7 +13,7 @@ # an actions/github-script step that requires it by path, so there is nothing to build: the file in # the repository is the file that runs. The same file runs from a command line; see the README. name: Plugin Check report -description: Compares Plugin Check results with an earlier run's baseline, and saves a new baseline +description: Compares Plugin Check results with the repository's baseline file, and saves a new baseline inputs: results: @@ -38,34 +39,23 @@ inputs: comment. Empty uses ``. required: false default: '' - baseline-wait: + baseline-file: description: >- - Seconds a pull request waits for the run of its base commit to finish, when that run is - still going. The reusable workflow lets its report job run for 30 minutes, so the default - leaves room for the work around the wait. + Path of the baseline file the repository commits: `{"errors": {code: n}, "warnings": {code: n}}`. + A relative path is read from the working directory. required: false - default: '1200' - token: - description: >- - Token that lists earlier runs and downloads their artifacts. It needs `actions: read`. - required: false - default: ${{ github.token }} + default: '.plugin-check-baseline.json' outputs: artifact-url: description: 'Link to the plugin-check-report artifact this run uploaded. Empty when there was nothing to upload.' value: ${{ steps.artifact.outputs.artifact-url }} - baseline-url: - description: 'Link to the run the baseline came from. Empty when there was no baseline.' - value: ${{ steps.baseline.outputs.url }} runs: using: 'composite' steps: # One folder per run under RUNNER_TEMP, cleared first, so a second use of the action in the same - # job never reads the first one's files. Also finds out which workflow's runs hold the baseline: - # in a called workflow GITHUB_WORKFLOW_REF is the caller's, and the runs belong to that file, not - # to the toolbox's workflow. + # job never reads the first one's files. - name: Set file paths id: paths uses: actions/github-script@v9 @@ -79,8 +69,8 @@ runs: const { runPaths } = require(`${process.env.ACTION_PATH}/plugin-check-report.js`) process.exitCode = runPaths({ core, env: process.env }) - # A baseline made from this run, uploaded even when the compare below fails. Later runs - # download it as their baseline: pull requests into this branch, and the next push. + # A baseline made from this run, uploaded even when the compare below fails. It is the easy way + # to update the committed file: download the artifact and copy its baseline.json. - name: Write a new baseline uses: actions/github-script@v9 env: @@ -92,9 +82,7 @@ runs: const { runUpdate } = require(`${process.env.ACTION_PATH}/plugin-check-report.js`) process.exitCode = runUpdate({ core, env: process.env }) - # The name and the baseline.json inside it are what the lookup in plugin-check-report.js looks - # for. Keep both: repositories that ran an earlier copy of this check upload the same names, so - # their old baselines keep working. + # The comment links to this artifact: the results, and the new baseline.json. - name: Upload the report id: artifact uses: actions/upload-artifact@v7 @@ -103,35 +91,6 @@ runs: path: ${{ steps.paths.outputs.report }} if-no-files-found: ignore - # The baseline: what the branch the code goes into looks like now, from the artifact of an - # earlier run on it. A pull request takes the run for its base commit; for a stacked pull - # request that is the run of the pull request below it. When that run is still going, it is - # waited for. When it has no baseline (it was cancelled by a newer push, say), the latest run on - # the base branch is used instead, and more errors than it are only reported, not failed: that - # run is older than the target. A push takes the run before it on the same branch, so a merge - # that adds errors fails there, where the whole team sees it. - # With no run to take it from, the compare checks nothing and passes with a warning. A failed - # lookup (GitHub API or download error) fails the step instead, so an outage never passes as - # "no baseline". - - name: Get the baseline - id: baseline - uses: actions/github-script@v9 - env: - ACTION_PATH: ${{ github.action_path }} - # The GitHub CLI downloads the artifact, and reads its token from here. - GH_TOKEN: ${{ inputs.token }} - EVENT: ${{ github.event_name }} - BASE_REF: ${{ github.event.pull_request.base.ref }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} - WAIT: ${{ inputs.baseline-wait }} - WORKFLOW: ${{ steps.paths.outputs.workflow }} - BASELINE: ${{ steps.paths.outputs.baseline }} - with: - github-token: ${{ inputs.token }} - script: | - const { runFetch } = require(`${process.env.ACTION_PATH}/plugin-check-report.js`) - process.exitCode = await runFetch({ github, core, exec, env: process.env }) - # Fails the step when the mode says so. Also writes the comment body and the same text to the # job summary. The step ends with an exit status, not an extra error annotation: the messages # above it already say what is wrong. @@ -142,11 +101,8 @@ runs: RESULTS: ${{ inputs.results }} COMMENT_FILE: ${{ inputs.comment-file }} PLUGIN_CHECK_MODE: ${{ inputs.mode }} - PLUGIN_CHECK_BASELINE: ${{ steps.paths.outputs.baseline }} - PLUGIN_CHECK_ADVISORY: ${{ steps.baseline.outputs.advisory }} + PLUGIN_CHECK_BASELINE: ${{ inputs.baseline-file }} COMMENT_MARKER: ${{ steps.paths.outputs.marker }} - BASELINE_LABEL: ${{ steps.baseline.outputs.label }} - BASELINE_URL: ${{ steps.baseline.outputs.url }} ARTIFACT_URL: ${{ steps.artifact.outputs.artifact-url }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} with: diff --git a/.github/workflows/plugin-check.yml b/.github/workflows/plugin-check.yml index 94c07f6..bd26543 100644 --- a/.github/workflows/plugin-check.yml +++ b/.github/workflows/plugin-check.yml @@ -390,17 +390,27 @@ jobs: # "start" is skipped for forks and non-pull-request runs, which would skip this job too. if: ${{ !cancelled() }} runs-on: ubuntu-latest - # Up to 20 minutes of it can be waiting for the target branch's run. - timeout-minutes: 30 + timeout-minutes: 10 - # Writes one sticky comment on the pull request, and reads an earlier run's artifact as the - # baseline. No checkout: the scripts come with the action. + # Writes one sticky comment on the pull request, and reads the repository's baseline file. The + # scripts come with the action. permissions: - actions: read + contents: read issues: write pull-requests: write steps: + # Only the baseline file, nothing else of the repository. It is JSON that the compare reads + # and never runs. The checkout is the pull request's merge commit, so a pull request that + # changes the file is compared with its own change, which a reviewer sees in the diff. + - name: Check out the baseline file + uses: actions/checkout@v7 + with: + path: source + sparse-checkout: .plugin-check-baseline.json + sparse-checkout-cone-mode: false + persist-credentials: false + - name: Set file paths run: | { @@ -432,6 +442,7 @@ jobs: comment-file: ${{ env.COMMENT_BODY }} comment-marker: ${{ env.COMMENT_MARKER }} mode: ${{ inputs.mode }} + baseline-file: ${{ github.workspace }}/source/.plugin-check-baseline.json - name: Post the result if: ${{ !cancelled() && env.COMMENT == 'true' }} diff --git a/templates/workflows/plugin-check.yml b/templates/workflows/plugin-check.yml index e18bdac..49d4194 100644 --- a/templates/workflows/plugin-check.yml +++ b/templates/workflows/plugin-check.yml @@ -1,13 +1,12 @@ name: Plugin Check # Runs WordPress.org's Plugin Check on the plugin as it ships, and compares the error counts with -# an earlier run: a pull request with the branch it targets, a push with the run before it. +# the baseline the repository commits, `.plugin-check-baseline.json`. # -# A file to copy into .github/workflows/ of a plugin repository. Keep the file name -# plugin-check.yml: earlier runs of this file hold the baselines, and a new name starts with none. -# The work is in stellarwp/plugin-toolbox/.github/workflows/plugin-check.yml. The repository needs -# a .puprc whose zip_name equals plugin-slug; pup itself is installed by the workflow. See the -# plugin-check-report action's README. +# A file to copy into .github/workflows/ of a plugin repository, under any name. The work is in +# stellarwp/plugin-toolbox/.github/workflows/plugin-check.yml. The repository needs a .puprc whose +# zip_name equals plugin-slug; pup itself is installed by the workflow. See the plugin-check-report +# action's README. # # Checks that do not fit the plugin go in a .plugin-check.json at the repository root, as # {"exclude-checks": ["check_name"]}; or pass the `exclude-checks` input, which replaces the file. @@ -17,7 +16,7 @@ name: Plugin Check on: pull_request: - # The branches pull requests target. A push there makes the baseline those pull requests read. + # The branches pull requests target. A push there keeps them honest after a merge. push: branches: [main] workflow_dispatch: @@ -34,7 +33,6 @@ jobs: plugin-check: # The most any job of the shared workflow asks for. A called workflow cannot go above this. permissions: - actions: read # read earlier runs and download their baseline contents: read # check out the repository issues: write # the sticky pull request comment pull-requests: write # the sticky pull request comment From 5bfea786460843b99eadb86cd97d18d6f2bc51f4 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Wed, 4 Nov 2026 14:31:38 -0300 Subject: [PATCH 24/31] Remove the Plugin Check run lookup --- .../plugin-check-report.js | 649 +----------------- .../plugin-check-report/action.test.js | 162 +---- .../plugin-check-report/exclude.test.js | 2 +- .../actions/plugin-check-report/fetch.test.js | 614 ----------------- 4 files changed, 22 insertions(+), 1405 deletions(-) delete mode 100644 tests/actions/plugin-check-report/fetch.test.js diff --git a/.github/actions/plugin-check-report/plugin-check-report.js b/.github/actions/plugin-check-report/plugin-check-report.js index 7d04405..b4534ab 100644 --- a/.github/actions/plugin-check-report/plugin-check-report.js +++ b/.github/actions/plugin-check-report/plugin-check-report.js @@ -1,30 +1,27 @@ 'use strict' /** - * Compares WordPress.org Plugin Check results with a baseline, and finds the baseline. + * Compares WordPress.org Plugin Check results with a baseline file. * - * The baseline is the count of errors and warnings per rule code of an earlier run of the same - * workflow, kept as the `plugin-check-report` artifact of that run. Nothing is committed. + * The baseline is the count of errors and warnings per rule code, kept in a file the repository + * commits, `.plugin-check-baseline.json`. A pull request is compared with the file in its own + * checkout, so a change to the baseline shows in the diff. * - * Two jobs: - * compare Counts the results, compares them with a baseline file, writes the pull request - * comment and the job summary, and says whether the check passes. With `update` it only + * One job: + * compare Counts the results, compares them with the baseline file, writes the pull request + * comment and the job summary, and says whether the check passes. With `--update` it only * writes the baseline file for this run. - * fetch Finds the earlier run that has a baseline and downloads its baseline.json. * - * The small functions are pure, so a test can call them with plain values. The functions that talk - * to GitHub take their client as an argument, so a test can pass a real Octokit with the network - * replaced, as the other actions do. The same code runs in three places: the action's steps - * (`runPaths`, `runUpdate`, `runFetch`, `runCompare`, through actions/github-script), a command line - * (`node plugin-check-report.js compare|fetch ...`), and the tests. + * The small functions are pure, so a test can call them with plain values. The same code runs in + * three places: the action's steps (`runPaths`, `runUpdate`, `runCompare`, through + * actions/github-script), a command line (`node plugin-check-report.js compare ...`), and the tests. * * Every function that decides an exit status returns it: 0 passes, 1 is a failed check or a missing - * baseline, 2 is a lookup that could not be trusted. A step sets `process.exitCode` from it, so the - * step fails without github-script adding a second error. + * file, another number is a file that could not be read as it should be. A step sets + * `process.exitCode` from it, so the step fails without github-script adding a second error. */ const fs = require('node:fs') -const os = require('node:os') const path = require('node:path') /** Title of every annotation. */ @@ -862,497 +859,15 @@ function actionLog(core) { } } -/** The artifact every run of the workflow uploads, with the baseline.json inside it. */ -const ARTIFACT = 'plugin-check-report' - -/** How long a pull request waits for the run of its base commit to show up at all, in seconds. */ -const GRACE_SECONDS = 120 -const POLL_SECONDS = 30 - -const FETCH_USAGE = - 'Usage: plugin-check-report.js fetch [--workflow ] [--sha ] [--wait ] [--before-run ] []' - -/** - * A lookup that cannot be trusted. It exits with 2, never 1, so "no baseline" is never mistaken for - * a typo or an outage. - * - * @param {string} reason What failed. - * - * @returns {CheckFailure} The failure to throw. - */ -function lookupFailed(reason) { - return new CheckFailure(`Could not look up Plugin Check runs: ${reason}`, 2) -} - -/** - * Checks a whole number from the command line. - * - * At most 18 digits, which keeps it inside a 64-bit integer, and a leading zero is not octal. - * - * @param {string} flag The option it belongs to, for the message. - * @param {string} value The text. - * - * @throws {CheckFailure} When it is not a whole number, or is too large. - * - * @returns {bigint} The number. - */ -function readWholeNumber(flag, value) { - if (!/^[0-9]+$/.test(value)) { - throw lookupFailed(`${flag} must be a whole number, not '${value}'`) - } - if (value.length > 18) { - throw lookupFailed(`${flag} is too large: '${value}'`) - } - - return BigInt(value) -} - -/** - * Checks the name of the workflow file. It goes into an API path, and a path or a ref there would - * ask for some other resource. - * - * @param {string} workflow The file name, such as `plugin-check.yml`. - * - * @throws {CheckFailure} When it is empty, or holds a `/` or an `@`. - * - * @returns {string} The name. - */ -function checkWorkflowFile(workflow) { - if (workflow === '' || /[/@]/.test(workflow)) { - throw lookupFailed(`--workflow must be a file name such as plugin-check.yml, not '${workflow}'`) - } - - return workflow -} - /** - * The file name of the calling workflow. In a called workflow GITHUB_WORKFLOW_REF is the caller's, like - * `example-org/my-plugin/.github/workflows/plugin-check.yml@refs/pull/12/merge`. The runs that - * hold the baseline belong to that file, not to the toolbox's workflow. - * - * @param {string} ref The value of GITHUB_WORKFLOW_REF. - * - * @returns {string} The file name, or '' when there is no ref. - */ -function workflowFromRef(ref) { - const withoutRef = String(ref ?? '').split('@')[0] - - return withoutRef.slice(withoutRef.lastIndexOf('/') + 1) -} - -/** - * Reads the command line of the lookup. - * - * Options come first, in any order, then the branch and an optional output file. An option with no - * value, or an empty one, fails here, so a typo never reads as "no baseline". - * - * @param {string[]} args The arguments. - * - * @throws {CheckFailure} For anything the lookup cannot use. - * - * @returns {{workflow: string, sha: string, wait: number, beforeRun: bigint|null, branch: string, - * output: string}} What to look up. - */ -function parseFetchArgs(args) { - const options = { workflow: 'plugin-check.yml', sha: '', wait: 0, beforeRun: null } - let rest = [...args] - - const needValue = () => { - if (rest.length < 2 || rest[1] === '') { - throw lookupFailed(`${rest[0]} needs a value. ${FETCH_USAGE}`) - } - } - - while (rest.length > 0) { - switch (rest[0]) { - case '--workflow': - needValue() - options.workflow = rest[1] - break - case '--sha': - needValue() - options.sha = rest[1] - break - case '--wait': - needValue() - options.wait = Number(readWholeNumber('--wait', rest[1])) - break - case '--before-run': - needValue() - options.beforeRun = readWholeNumber('--before-run', rest[1]) - break - default: - return finishFetchArgs(options, rest) - } - rest = rest.slice(2) - } - - return finishFetchArgs(options, rest) -} - -/** - * The branch and the output file, after the options. - * - * @param {object} options What the options said. - * @param {string[]} rest The arguments left. - * - * @throws {CheckFailure} When there is no branch. - * - * @returns {object} The options, the branch and the output file. - */ -function finishFetchArgs(options, rest) { - if (rest.length < 1 || rest[0] === '') { - throw lookupFailed(`a branch is needed. ${FETCH_USAGE}`) - } - - return { ...options, branch: rest[0], output: rest[1] || path.join(__dirname, 'baseline.json') } -} - -/** - * Lower-cases the letters A to Z only, as `ascii_downcase` does. - * - * @param {string} text The text. - * - * @returns {string} The text. - */ -function asciiLower(text) { - return text.replace(/[A-Z]/g, (letter) => letter.toLowerCase()) -} - -/** - * Whether a run belongs to this repository's own branches. - * - * The runs API matches `branch` and `head_sha` against the head of a pull request too, so a run for - * a pull request from a fork, from a branch named like the target, would otherwise be taken for a - * run of the target, and its artifact is whatever that fork's workflow chose to upload. - * - * @param {object} run A workflow run from the API. - * @param {string} repo `owner/name`. - * - * @returns {boolean} True when the run's head commit is in this repository. - */ -function isOwnRun(run, repo) { - return asciiLower(run.head_repository?.full_name || '') === asciiLower(repo) -} - -/** - * How many of this repository's own runs there are, and how many of them are not finished. - * - * @param {object[]} runs The runs from the API. - * @param {string} repo `owner/name`. - * - * @returns {{total: number, going: number}} The counts. - */ -function countRuns(runs, repo) { - const own = runs.filter((run) => isOwnRun(run, repo)) - - return { total: own.length, going: own.filter((run) => run.status !== 'completed').length } -} - -/** - * The runs to try, best first. - * - * Runs for the base commit come first, in the order the API gave them. Then the latest runs on the - * branch, newest first by run id. A run is tried once, and a run that is not older than - * `beforeRun` is skipped, so a run never compares with itself, and a re-run of an older push never - * compares with a newer one. - * - * @param {object} input - * @param {object[]} input.shaRuns Own runs for the base commit. - * @param {object[]} input.branchRuns Own runs on the branch. - * @param {bigint|null} input.beforeRun Skip runs with this id or a higher one. - * - * @returns {object[]} The runs. - */ -function orderCandidates({ shaRuns, branchRuns, beforeRun }) { - const newestFirst = [...branchRuns].sort((a, b) => (a.id < b.id ? 1 : a.id > b.id ? -1 : 0)) - const tried = new Set() - const candidates = [] - - for (const run of [...shaRuns, ...newestFirst]) { - if (beforeRun !== null && BigInt(run.id) >= beforeRun) { - continue - } - if (tried.has(String(run.id))) { - continue - } - tried.add(String(run.id)) - candidates.push(run) - } - - return candidates -} - -/** - * The runs of a workflow, from the API. - * - * @param {object} github A GitHub client. - * @param {object} query `owner`, `repo`, `workflow`, then the filters in the order they are sent. - * @param {object} log The log, for the reason a request failed. - * @param {string} reason What the caller was looking for, for the message. - * - * @throws {CheckFailure} When the request fails or the answer is not a list of runs. - * - * @returns {Promise} The runs. - */ -async function listRuns(github, { owner, repo, workflow, ...filters }, log, reason) { - let data - try { - ;({ data } = await github.request('GET /repos/{owner}/{repo}/actions/workflows/{workflow_id}/runs', { - owner, - repo, - workflow_id: workflow, - ...filters, - })) - } catch (error) { - log.err(`${error.message}${error.status ? ` (HTTP ${error.status})` : ''}`) - throw lookupFailed(reason) - } - - if (!Array.isArray(data?.workflow_runs)) { - throw lookupFailed(reason) - } - - return data.workflow_runs -} - -/** - * Waits for the run of the base commit to finish, when it is still going. - * - * A pull request opened right after a merge starts at the same time as the push run on the target - * branch. Waiting for it beats comparing with an older run. A run that does not show up at all is - * waited for only a short time, since the commit may never have had one. - * - * @param {object} input - * @param {object} input.github A GitHub client. - * @param {object} input.query `owner`, `repo`, `workflow`. - * @param {string} input.sha The base commit. - * @param {number} input.wait The most seconds to wait. - * @param {Function} input.now The time in whole seconds. - * @param {Function} input.sleep Waits for a number of seconds. - * @param {object} input.log The log. - */ -async function waitForBaseRun({ github, query, sha, wait, now, sleep, log }) { - const repo = `${query.owner}/${query.repo}` - const started = now() - const deadline = started + wait - - while (true) { - const runs = await listRuns(github, { ...query, head_sha: sha, per_page: 20 }, log, `runs for ${sha}`) - const { total, going } = countRuns(runs, repo) - - if (total === 0 && now() >= started + GRACE_SECONDS) { - return - } - if (total !== 0 && going === 0) { - return - } - if (now() >= deadline) { - log.err(`The run for ${sha} is still going after ${wait} seconds; trying older runs.`) - - return - } - - await sleep(POLL_SECONDS) - } -} - -/** - * Finds the earlier run that has a baseline, and downloads its baseline.json. - * - * Runs are tried in this order, and the first one with the artifact wins: runs for the base commit - * when `--sha` is given, then the latest runs on the branch. The artifact is downloaded with the - * GitHub CLI. A run that broke before the report job, whose artifact expired, or whose artifact has - * no baseline.json has nothing to give, and the next run is tried. - * - * @param {object} input - * @param {string[]} input.args The command line, see parseFetchArgs. - * @param {object} input.env The environment. GITHUB_REPOSITORY names the repository. - * @param {object} input.github A GitHub client. - * @param {object} input.exec `getExecOutput(command, args, options)`, as in @actions/exec. - * @param {object} input.log The log. - * @param {Function} input.now The time in whole seconds. Default: the clock. - * @param {Function} input.sleep Waits for a number of seconds. Default: a real wait. - * - * @returns {Promise<{status: number, id?: number, url?: string, match?: string}>} Status 0 with the - * run found and `exact` or `older` (the run is not for the base commit); 1 when no run has - * a baseline; 2 when the lookup itself failed, so "no baseline" is not known. - */ -async function fetchBaseline({ - args, - env, - github, - exec, - log, - now = () => Math.floor(Date.now() / 1000), - sleep = (seconds) => new Promise((resolve) => setTimeout(resolve, seconds * 1000)), -}) { - let temp = null - - try { - const { workflow, sha, wait, beforeRun, branch, output } = parseFetchArgs(args) - - const repo = env.GITHUB_REPOSITORY || (await repositoryFromGh(exec)) - checkWorkflowFile(workflow) - const [owner, name] = repo.split('/') - const query = { owner, repo: name, workflow } - - if (sha && wait > 0) { - await waitForBaseRun({ github, query, sha, wait, now, sleep, log }) - } - - const shaRuns = sha - ? await listRuns(github, { ...query, status: 'completed', head_sha: sha, per_page: 10 }, log, `runs for ${sha}`) - : [] - const branchRuns = await listRuns( - github, - { ...query, status: 'completed', branch, per_page: 100 }, - log, - `runs on ${branch}` - ) - - const candidates = orderCandidates({ - shaRuns: shaRuns.filter((run) => isOwnRun(run, repo)), - branchRuns: branchRuns.filter((run) => isOwnRun(run, repo)), - beforeRun, - }) - - temp = fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-fetch-')) - - for (const run of candidates) { - const file = await downloadBaseline({ github, exec, log, owner, repo: name, id: run.id, dir: path.join(temp, 'report') }) - if (file === null) { - continue - } - - fs.mkdirSync(path.dirname(output), { recursive: true }) - fs.copyFileSync(file, output) - - return { status: 0, id: run.id, url: run.html_url, match: sha && run.head_sha !== sha ? 'older' : 'exact' } - } - - log.err(`No Plugin Check run on ${branch}${sha ? ` or ${sha}` : ''} has a baseline.`) - - return { status: 1 } - } catch (error) { - const failure = error instanceof CheckFailure ? error : lookupFailed(error.message) - log.err(failure.message) - - return { status: failure.exitCode } - } finally { - if (temp) { - fs.rmSync(temp, { recursive: true, force: true }) - } - } -} - -/** - * Downloads the baseline.json of one run's artifact. - * - * @param {object} input - * @param {object} input.github A GitHub client. - * @param {object} input.exec `getExecOutput`. - * @param {object} input.log The log. - * @param {string} input.owner Repository owner. - * @param {string} input.repo Repository name. - * @param {number} input.id The run. - * @param {string} input.dir A folder to download into. - * - * @throws {CheckFailure} When the artifacts cannot be listed or the download fails. - * - * @returns {Promise} The path of the baseline.json, or null when the run has none. - */ -async function downloadBaseline({ github, exec, log, owner, repo, id, dir }) { - let data - try { - ;({ data } = await github.request('GET /repos/{owner}/{repo}/actions/runs/{run_id}/artifacts', { - owner, - repo, - run_id: id, - name: ARTIFACT, - })) - } catch (error) { - log.err(`${error.message}${error.status ? ` (HTTP ${error.status})` : ''}`) - throw lookupFailed(`artifacts of run ${id}`) - } - - if (!Array.isArray(data?.artifacts)) { - throw lookupFailed(`artifacts of run ${id}`) - } - if (!data.artifacts.some((artifact) => !artifact.expired)) { - return null - } - - fs.rmSync(dir, { recursive: true, force: true }) - const download = await exec.getExecOutput( - 'gh', - ['run', 'download', String(id), '--repo', `${owner}/${repo}`, '--name', ARTIFACT, '--dir', dir], - { silent: true, ignoreReturnCode: true } - ) - if (download.exitCode !== 0) { - if (download.stderr) { - log.err(download.stderr.trimEnd()) - } - throw lookupFailed(`download of the artifact of run ${id}`) - } - - const file = path.join(dir, 'baseline.json') - - return isFile(file) && fs.statSync(file).size > 0 ? file : null -} - -/** - * The repository of the current folder, from the GitHub CLI, for a run on a developer's machine. - * - * @param {object} exec `getExecOutput`. - * - * @throws {CheckFailure} When there is no repository. - * - * @returns {Promise} `owner/name`. - */ -async function repositoryFromGh(exec) { - const result = await exec.getExecOutput('gh', ['repo', 'view', '--json', 'nameWithOwner', '--jq', '.nameWithOwner'], { - silent: true, - ignoreReturnCode: true, - }) - const repo = result.stdout.trim() - - if (result.exitCode !== 0 || !repo) { - throw lookupFailed('no repository') - } - - return repo -} - -/** - * The label and the advisory flag for a baseline that was found. - * - * A baseline from an older run than the base commit's cannot fail the check, because the target - * may have changed since. - * - * @param {string} event `pull_request`, or any other event name. - * @param {string} match `exact` or `older`. - * - * @returns {{label: string, advisory: boolean}} How the comment names the baseline. - */ -function describeBaseline(event, match) { - if (match === 'older') { - return { label: 'Target branch, older run', advisory: true } - } - - return { label: event === 'pull_request' ? 'Target branch' : 'Previous run', advisory: false } -} - -/** - * The first step: where this run keeps its files, and which workflow's runs hold the baseline. + * The first step: where this run keeps its files. * * One folder per run under RUNNER_TEMP, cleared first, so a second use of the action in the same * job never reads the first one's files. * * @param {object} input * @param {object} input.core The toolkit github-script passes in. - * @param {object} input.env The environment: RUNNER_TEMP, GITHUB_WORKFLOW_REF, GITHUB_REPOSITORY and - * INPUT_COMMENT_MARKER. + * @param {object} input.env The environment: RUNNER_TEMP, GITHUB_REPOSITORY and INPUT_COMMENT_MARKER. * * @returns {number} 0. */ @@ -1360,15 +875,9 @@ function runPaths({ core, env }) { const dir = path.join(env.RUNNER_TEMP, 'plugin-check-report') fs.rmSync(dir, { recursive: true, force: true }) fs.mkdirSync(path.join(dir, 'report'), { recursive: true }) - fs.mkdirSync(path.join(dir, 'baseline'), { recursive: true }) - - const workflow = workflowFromRef(env.GITHUB_WORKFLOW_REF) core.setOutput('report', path.join(dir, 'report')) - core.setOutput('baseline', path.join(dir, 'baseline', 'baseline.json')) - core.setOutput('workflow', workflow) core.setOutput('marker', env.INPUT_COMMENT_MARKER || ``) - core.info(`Baselines are read from the runs of ${workflow}.`) return 0 } @@ -1398,52 +907,6 @@ function runUpdate({ core, env }) { }) } -/** - * The step that finds the baseline. A pull request takes the run for its base commit; for a stacked - * pull request that is the run of the pull request below it. When that run is still going, it is - * waited for. When it has no baseline, the latest run on the base branch is used instead, and - * more errors than it are only reported, not failed. Any other event takes the run before it on - * the same branch, so a merge that adds errors fails there, where the whole team sees it. - * - * Sets the outputs `label`, `url` and `advisory` when it finds one. - * - * @param {object} input - * @param {object} input.github A GitHub client. - * @param {object} input.core The toolkit. - * @param {object} input.exec `exec` from github-script. - * @param {object} input.env EVENT, BASE_REF, BASE_SHA, WAIT, WORKFLOW, BASELINE, GITHUB_REPOSITORY, - * GITHUB_RUN_ID and GITHUB_REF_NAME. - * @param {Function} input.now Optional clock. - * @param {Function} input.sleep Optional wait. - * - * @returns {Promise} 0 when a baseline was found or none exists, 2 when the lookup failed. - */ -async function runFetch({ github, core, exec, env, now, sleep }) { - const pullRequest = env.EVENT === 'pull_request' - const args = pullRequest - ? ['--workflow', env.WORKFLOW ?? '', '--sha', env.BASE_SHA ?? '', '--wait', env.WAIT ?? '', env.BASE_REF ?? '', env.BASELINE ?? ''] - : ['--workflow', env.WORKFLOW ?? '', '--before-run', env.GITHUB_RUN_ID ?? '', env.GITHUB_REF_NAME ?? '', env.BASELINE ?? ''] - - const found = await fetchBaseline({ args, env, github, exec, log: actionLog(core), now, sleep }) - - if (found.status === 1) { - core.info('No earlier run has a baseline.') - - return 0 - } - if (found.status !== 0) { - return found.status - } - - const { label, advisory } = describeBaseline(env.EVENT, found.match) - core.setOutput('label', label) - core.setOutput('url', found.url) - core.setOutput('advisory', String(advisory)) - core.info(`Baseline from ${found.url} (${found.match})`) - - return 0 -} - /** * The last step: fails the step when the mode says so. Also writes the comment body and the same * text to the job summary. @@ -1726,57 +1189,11 @@ function runExcludeChecks({ core, env }) { } /** - * A GitHub client for a run on a developer's machine. The token comes from GITHUB_TOKEN, else - * GH_TOKEN, else the GitHub CLI. - * - * @param {object} env The environment. - * @param {object} exec `getExecOutput`. - * - * @returns {Promise} A client. - */ -async function clientFromEnv(env, exec) { - let token = env.GITHUB_TOKEN || env.GH_TOKEN - - if (!token) { - const result = await exec.getExecOutput('gh', ['auth', 'token'], { silent: true, ignoreReturnCode: true }) - token = result.stdout.trim() - } - if (!token) { - throw lookupFailed('no token. Set GITHUB_TOKEN, or sign in with `gh auth login`') - } - - const { getOctokit } = await import('@actions/github') - - return getOctokit(token) -} - -/** - * `getExecOutput` for a command line, built on child_process. - * - * @param {object} env The environment the commands run with. - * - * @returns {{getExecOutput: Function}} The same shape as @actions/exec. - */ -function commandExec(env) { - const { spawnSync } = require('node:child_process') - - return { - getExecOutput: async (command, args = []) => { - const result = spawnSync(command, args, { encoding: 'utf8', env }) - - return { exitCode: result.status ?? 127, stdout: result.stdout ?? '', stderr: result.stderr ?? '' } - }, - } -} - -/** - * The command line: `compare` and `fetch`, with the arguments and the environment variables of the - * two bash scripts they replace. + * The command line: `compare` and `exclude-checks`. * * @param {string[]} argv The arguments after the script name. * @param {object} env The environment, default process.env. * @param {object} io `out` and `err`, each writing one line. Default: the process streams. - * @param {object} deps Stand-ins for a test: `github`, `exec`, `now`, `sleep`. * * @returns {Promise} The exit status. */ @@ -1786,8 +1203,7 @@ async function main( io = { out: (line) => process.stdout.write(`${line}\n`), err: (line) => process.stderr.write(`${line}\n`), - }, - deps = {} + } ) { const [command, ...args] = argv const log = commandLog(io) @@ -1796,25 +1212,6 @@ async function main( return compare({ args, env, log }) } - if (command === 'fetch') { - const exec = deps.exec ?? commandExec(env) - let github = deps.github - try { - github ??= await clientFromEnv(env, exec) - } catch (error) { - log.err(error.message) - - return 2 - } - - const found = await fetchBaseline({ args, env, github, exec, log, now: deps.now, sleep: deps.sleep }) - if (found.status === 0) { - log.out(`${found.id} ${found.url} ${found.match}`) - } - - return found.status - } - // Prints the flag `wp plugin check` takes, so a local run skips the same checks as CI: // `wp plugin check my-plugin $(node plugin-check-report.js exclude-checks)`. Prints nothing when // no check is excluded. EXCLUDE_CHECKS, when set, replaces the file, as the workflow input does. @@ -1844,7 +1241,7 @@ async function main( } } - io.err('Usage: plugin-check-report.js compare|fetch|exclude-checks ...') + io.err('Usage: plugin-check-report.js compare|exclude-checks ...') return 2 } @@ -1856,51 +1253,39 @@ if (require.main === module) { } module.exports = { - ARTIFACT, CheckFailure, MODES, actionLog, BADGE_TIME_ZONE, EXCLUDE_FILE, - asciiLower, badgeStamp, buildComment, buildRows, change, checkResultsText, - checkWorkflowFile, commandLog, compare, compareCodePoints, countPerCode, - countRuns, couldNotRunComment, cutBytes, - describeBaseline, displayCode, evaluate, excludedChecks, - fetchBaseline, finishBadge, formatBaseline, - isOwnRun, linksLine, main, - orderCandidates, parseExcludeFile, parseExcludeInput, - parseFetchArgs, parseResults, readBaseline, readIfFile, readJsonValues, - readWholeNumber, runCompare, runCouldNotRun, runExcludeChecks, - runFetch, runPaths, runUpdate, statusLine, - workflowFromRef, } diff --git a/tests/actions/plugin-check-report/action.test.js b/tests/actions/plugin-check-report/action.test.js index 712c1a6..d2c03da 100644 --- a/tests/actions/plugin-check-report/action.test.js +++ b/tests/actions/plugin-check-report/action.test.js @@ -7,51 +7,17 @@ const os = require('node:os') const path = require('node:path') const report = require('../../../.github/actions/plugin-check-report/plugin-check-report.js') -const { actionsFor, recordingExec } = require('../../support/actions.js') +const { actionsFor } = require('../../support/actions.js') const FIXTURES = path.join(__dirname, 'fixtures') const REPO = 'example-org/my-plugin' -const SHA = 'a'.repeat(40) const temp = (prefix) => fs.mkdtempSync(path.join(os.tmpdir(), `plugin-check-${prefix}-`)) -const RUNS = `/repos/${REPO}/actions/workflows/plugin-check.yml/runs` -const run = (id, sha, headRepo = REPO) => ({ - id, - html_url: `https://github.com/${REPO}/actions/runs/${id}`, - head_sha: sha, - status: 'completed', - head_repository: { full_name: headRepo }, -}) - -/** - * Exec rules for a baseline that downloads. - * - * @param {number} id The run. - * @param {string} baseline What its baseline.json holds. - * - * @returns {object[]} The rule. - */ -function downloads(id, baseline = '{"errors":{"Some.Rule":2},"warnings":{}}') { - return [ - { - command: 'gh', - args: new RegExp(`^run download ${id} `), - effect: (args) => { - const target = args[args.indexOf('--dir') + 1] - fs.mkdirSync(target, { recursive: true }) - fs.writeFileSync(path.join(target, 'baseline.json'), baseline) - }, - }, - ] -} - -const ARTIFACT = { body: { artifacts: [{ name: 'plugin-check-report', expired: false }] } } - describe('plugin-check-report action steps', () => { describe('runPaths', () => { - it('makes a clean folder for the run and names the workflow of the caller', async (t) => { - const { core, outputs, logged } = await actionsFor(t) + it('makes a clean folder for the run', async (t) => { + const { core, outputs } = await actionsFor(t) const tempDir = temp('paths') fs.mkdirSync(path.join(tempDir, 'plugin-check-report', 'report'), { recursive: true }) fs.writeFileSync(path.join(tempDir, 'plugin-check-report', 'report', 'old.txt'), 'from an earlier use') @@ -61,19 +27,15 @@ describe('plugin-check-report action steps', () => { env: { RUNNER_TEMP: tempDir, GITHUB_REPOSITORY: REPO, - GITHUB_WORKFLOW_REF: `${REPO}/.github/workflows/plugin-check.yml@refs/pull/12/merge`, }, }) assert.equal(status, 0) assert.deepEqual(outputs(), { report: path.join(tempDir, 'plugin-check-report', 'report'), - baseline: path.join(tempDir, 'plugin-check-report', 'baseline', 'baseline.json'), - workflow: 'plugin-check.yml', marker: ``, }) assert.deepEqual(fs.readdirSync(path.join(tempDir, 'plugin-check-report', 'report')), []) - assert.match(logged(), /Baselines are read from the runs of plugin-check\.yml\./) }) it('uses the marker the caller gives', async (t) => { @@ -81,7 +43,7 @@ describe('plugin-check-report action steps', () => { report.runPaths({ core, - env: { RUNNER_TEMP: temp('marker'), GITHUB_REPOSITORY: REPO, GITHUB_WORKFLOW_REF: 'a/b/.github/workflows/x.yml@r', INPUT_COMMENT_MARKER: '' }, + env: { RUNNER_TEMP: temp('marker'), GITHUB_REPOSITORY: REPO, INPUT_COMMENT_MARKER: '' }, }) assert.equal(outputs().marker, '') @@ -127,122 +89,6 @@ describe('plugin-check-report action steps', () => { }) }) - describe('runFetch', () => { - it('takes the run for the base commit of a pull request and labels it', async (t) => { - const baseline = path.join(temp('fetch'), 'baseline.json') - const { github, core, outputs, logged } = await actionsFor(t, [ - { path: `${RUNS}?head_sha=${SHA}&per_page=20`, body: { workflow_runs: [run(200, SHA)] } }, - { path: `${RUNS}?status=completed&head_sha=${SHA}&per_page=10`, body: { workflow_runs: [run(200, SHA)] } }, - { path: `${RUNS}?status=completed&branch=develop&per_page=100`, body: { workflow_runs: [] } }, - { path: `/repos/${REPO}/actions/runs/200/artifacts?name=plugin-check-report`, ...ARTIFACT }, - ]) - const { exec } = recordingExec(downloads(200)) - - const status = await report.runFetch({ - github, - core, - exec, - env: { GITHUB_REPOSITORY: REPO, EVENT: 'pull_request', BASE_REF: 'develop', BASE_SHA: SHA, WAIT: '1200', WORKFLOW: 'plugin-check.yml', BASELINE: baseline }, - now: () => 1000, - sleep: async () => {}, - }) - - assert.equal(status, 0) - assert.deepEqual(outputs(), { - label: 'Target branch', - url: `https://github.com/${REPO}/actions/runs/200`, - advisory: 'false', - }) - assert.match(logged(), /Baseline from https:\/\/github\.com\/example-org\/my-plugin\/actions\/runs\/200 \(exact\)/) - assert.ok(fs.existsSync(baseline)) - }) - - it('marks a baseline from an older run as advisory', async (t) => { - const { github, core, outputs } = await actionsFor(t, [ - { path: `${RUNS}?head_sha=${SHA}&per_page=20`, body: { workflow_runs: [] } }, - { path: `${RUNS}?status=completed&head_sha=${SHA}&per_page=10`, body: { workflow_runs: [] } }, - { path: `${RUNS}?status=completed&branch=develop&per_page=100`, body: { workflow_runs: [run(150, 'b'.repeat(40))] } }, - { path: `/repos/${REPO}/actions/runs/150/artifacts?name=plugin-check-report`, ...ARTIFACT }, - ]) - const { exec } = recordingExec(downloads(150)) - - await report.runFetch({ - github, - core, - exec, - env: { GITHUB_REPOSITORY: REPO, EVENT: 'pull_request', BASE_REF: 'develop', BASE_SHA: SHA, WAIT: '0', WORKFLOW: 'plugin-check.yml', BASELINE: path.join(temp('fetch'), 'baseline.json') }, - }) - - assert.deepEqual({ label: outputs().label, advisory: outputs().advisory }, { label: 'Target branch, older run', advisory: 'true' }) - }) - - it('takes the run before this one on the same branch for a push', async (t) => { - const { github, core, outputs } = await actionsFor(t, [ - { path: `${RUNS}?status=completed&branch=develop&per_page=100`, body: { workflow_runs: [run(300, SHA), run(150, SHA)] } }, - { path: `/repos/${REPO}/actions/runs/150/artifacts?name=plugin-check-report`, ...ARTIFACT }, - ]) - const { exec } = recordingExec(downloads(150)) - - const status = await report.runFetch({ - github, - core, - exec, - env: { GITHUB_REPOSITORY: REPO, EVENT: 'push', GITHUB_RUN_ID: '200', GITHUB_REF_NAME: 'develop', WORKFLOW: 'plugin-check.yml', BASELINE: path.join(temp('fetch'), 'baseline.json') }, - }) - - assert.equal(status, 0) - assert.equal(outputs().label, 'Previous run') - assert.match(outputs().url, /runs\/150$/) - }) - - it('passes with a message, and no outputs, when no run has a baseline', async (t) => { - const { github, core, outputs, logged } = await actionsFor(t, [ - { path: `${RUNS}?status=completed&branch=develop&per_page=100`, body: { workflow_runs: [] } }, - ]) - const { exec } = recordingExec() - - const status = await report.runFetch({ - github, - core, - exec, - env: { GITHUB_REPOSITORY: REPO, EVENT: 'push', GITHUB_RUN_ID: '200', GITHUB_REF_NAME: 'develop', WORKFLOW: 'plugin-check.yml', BASELINE: path.join(temp('fetch'), 'baseline.json') }, - }) - - assert.equal(status, 0) - assert.deepEqual(outputs(), {}) - assert.match(logged(), /No earlier run has a baseline\./) - }) - - it('fails with exit status 2 when the lookup fails, so an outage never passes as no baseline', async (t) => { - const { github, core, logged } = await actionsFor(t, []) - const { exec } = recordingExec() - - const status = await report.runFetch({ - github, - core, - exec, - env: { GITHUB_REPOSITORY: REPO, EVENT: 'push', GITHUB_RUN_ID: '200', GITHUB_REF_NAME: 'develop', WORKFLOW: 'plugin-check.yml', BASELINE: path.join(temp('fetch'), 'baseline.json') }, - }) - - assert.equal(status, 2) - assert.match(logged(), /Could not look up Plugin Check runs: runs on develop/) - }) - - it('fails with exit status 2 when the workflow file is unknown', async (t) => { - const { github, core } = await actionsFor(t, []) - const { exec } = recordingExec() - - const status = await report.runFetch({ - github, - core, - exec, - env: { GITHUB_REPOSITORY: REPO, EVENT: 'push', GITHUB_RUN_ID: '200', GITHUB_REF_NAME: 'develop', WORKFLOW: '', BASELINE: 'x' }, - }) - - assert.equal(status, 2) - }) - }) - describe('runCompare', () => { const base = () => ({ RESULTS: path.join(FIXTURES, 'errors-up.txt'), diff --git a/tests/actions/plugin-check-report/exclude.test.js b/tests/actions/plugin-check-report/exclude.test.js index ae2f3dd..1d0e410 100644 --- a/tests/actions/plugin-check-report/exclude.test.js +++ b/tests/actions/plugin-check-report/exclude.test.js @@ -236,6 +236,6 @@ describe('exclude-checks on the command line', () => { const status = await report.main(['nope'], {}, { out: () => {}, err: (line) => err.push(line) }) assert.equal(status, 2) - assert.match(err[0], /compare\|fetch\|exclude-checks/) + assert.match(err[0], /compare\|exclude-checks/) }) }) diff --git a/tests/actions/plugin-check-report/fetch.test.js b/tests/actions/plugin-check-report/fetch.test.js deleted file mode 100644 index dbd6aef..0000000 --- a/tests/actions/plugin-check-report/fetch.test.js +++ /dev/null @@ -1,614 +0,0 @@ -'use strict' - -const { describe, it } = require('node:test') -const assert = require('node:assert') -const fs = require('node:fs') -const os = require('node:os') -const path = require('node:path') - -const report = require('../../../.github/actions/plugin-check-report/plugin-check-report.js') -const { octokitFor, recordingExec } = require('../../support/actions.js') - -const REPO = 'example-org/my-plugin' -const SHA = 'a'.repeat(40) -const OLD_SHA = 'b'.repeat(40) - -/** - * The API paths the lookup asks for, as they go over the wire. - * - * @param {string} workflow The workflow file name. - */ -const API = (workflow = 'plugin-check.yml') => { - const runs = `/repos/${REPO}/actions/workflows/${workflow}/runs` - - return { - forSha: (sha) => `${runs}?status=completed&head_sha=${sha}&per_page=10`, - goingForSha: (sha) => `${runs}?head_sha=${sha}&per_page=20`, - onBranch: (branch) => `${runs}?status=completed&branch=${encodeURIComponent(branch)}&per_page=100`, - artifacts: (id) => `/repos/${REPO}/actions/runs/${id}/artifacts?name=plugin-check-report`, - } -} - -/** - * A run as the workflow runs API lists it. - * - * @param {number} id Run id. - * @param {string} sha Head commit. - * @param {string} headRepo Repository the run's head commit belongs to. A pull request from a - * fork has the fork's name here. - * - * @returns {object} The run. - */ -function run(id, sha = OLD_SHA, headRepo = REPO) { - return { - id, - html_url: `https://github.com/${REPO}/actions/runs/${id}`, - head_sha: sha, - status: 'completed', - head_repository: { full_name: headRepo }, - } -} - -/** - * A clock that only moves when the code sleeps, so the wait loop runs in no time. - * - * @returns {{now: Function, sleep: Function, slept: number[]}} The clock and the sleeps it saw. - */ -function fakeClock() { - const clock = { time: 1000, slept: [] } - - return { - now: () => clock.time, - sleep: async (seconds) => { - clock.slept.push(seconds) - clock.time += seconds - }, - slept: clock.slept, - } -} - -/** - * Runs the lookup against answers written down, the way the old tests ran the script against a - * stand-in `gh`. - * - * @param {string[]} args Arguments of the lookup. An output file is added unless `bare` is set. - * @param {object} answers Each API path mapped to the JSON body it answers with. A path with no - * answer is a 404, like a failed request. - * @param {object} baselines Each run id mapped to the baseline.json its artifact holds. A run - * with no entry has an artifact that cannot be downloaded. - * @param {object} options `bare` (no output file), `env`, `clock`, `files` (run id mapped to the - * exact text of the downloaded baseline.json). - * - * @returns {Promise<{status: number, stdout: string, stderr: string, paths: string[], calls: object[], - * output: string}>} The exit status, both streams, the API paths asked for in order, the - * commands run and the baseline file written. - */ -async function fetchBaseline(args, answers = {}, baselines = {}, options = {}) { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-fetch-')) - const output = path.join(dir, 'out', 'baseline.json') - - const rules = Object.entries(answers).map(([apiPath, body]) => ({ path: apiPath, body })) - const { github, requests } = await octokitFor(rules) - const { exec, calls } = recordingExec( - Object.entries(baselines).map(([id, baseline]) => ({ - command: 'gh', - args: new RegExp(`^run download ${id} `), - effect: (callArgs) => { - const target = callArgs[callArgs.indexOf('--dir') + 1] - fs.mkdirSync(target, { recursive: true }) - fs.writeFileSync(path.join(target, 'baseline.json'), options.files?.[id] ?? JSON.stringify(baseline)) - }, - })) - ) - - const stdout = [] - const stderr = [] - const clock = options.clock ?? fakeClock() - const status = await report.main( - ['fetch', ...args, ...(options.bare ? [] : [output])], - { GITHUB_REPOSITORY: REPO, ...options.env }, - { out: (line) => stdout.push(line), err: (line) => stderr.push(line) }, - { github, exec, now: clock.now, sleep: clock.sleep } - ) - - return { - status, - stdout: stdout.join('\n'), - stderr: stderr.join('\n'), - paths: requests.map((request) => request.path), - calls, - output: fs.existsSync(output) ? fs.readFileSync(output, 'utf8') : '', - } -} - -const BASELINE = { errors: { 'Some.Rule': 2 }, warnings: {} } -const ONE_ARTIFACT = { artifacts: [{ name: 'plugin-check-report', expired: false }] } -const EXPIRED = { artifacts: [{ name: 'plugin-check-report', expired: true }] } - -describe('plugin-check-report fetch', () => { - it('takes the run for the base commit and calls it exact', async () => { - const api = API() - const result = await fetchBaseline(['--sha', SHA, 'develop'], { - [api.forSha(SHA)]: { workflow_runs: [run(200, SHA)] }, - [api.onBranch('develop')]: { workflow_runs: [run(150)] }, - [api.artifacts(200)]: ONE_ARTIFACT, - }, { 200: BASELINE }) - - assert.equal(result.status, 0, result.stderr) - assert.equal(result.stdout, `200 https://github.com/${REPO}/actions/runs/200 exact`) - assert.deepEqual(JSON.parse(result.output), BASELINE) - }) - - it('falls back to an older run on the branch, skips an expired artifact, and calls it older', async () => { - const api = API() - const result = await fetchBaseline(['--sha', SHA, 'develop'], { - [api.forSha(SHA)]: { workflow_runs: [] }, - [api.onBranch('develop')]: { workflow_runs: [run(120), run(150)] }, - [api.artifacts(150)]: EXPIRED, - [api.artifacts(120)]: ONE_ARTIFACT, - }, { 120: BASELINE }) - - assert.equal(result.status, 0, result.stderr) - assert.equal(result.stdout, `120 https://github.com/${REPO}/actions/runs/120 older`) - // Newest first by run id, whatever order the API answered in. - assert.ok( - result.paths.findIndex((request) => request.includes('runs/150/')) < - result.paths.findIndex((request) => request.includes('runs/120/')) - ) - }) - - it('never compares a run with itself or with a newer run', async () => { - const api = API() - const result = await fetchBaseline(['--before-run', '150', 'develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(160), run(150), run(140)] }, - [api.artifacts(140)]: ONE_ARTIFACT, - }, { 140: BASELINE }) - - assert.equal(result.status, 0, result.stderr) - assert.match(result.stdout, /^140 /) - assert.ok(!result.paths.some((request) => /runs\/1[56]0\//.test(request))) - }) - - it('does not wait for a run for the base commit that has already finished', async () => { - const api = API() - const clock = fakeClock() - const result = await fetchBaseline(['--sha', SHA, '--wait', '1200', 'develop'], { - [api.goingForSha(SHA)]: { workflow_runs: [run(200, SHA)] }, - [api.forSha(SHA)]: { workflow_runs: [run(200, SHA)] }, - [api.onBranch('develop')]: { workflow_runs: [] }, - [api.artifacts(200)]: ONE_ARTIFACT, - }, { 200: BASELINE }, { clock }) - - assert.equal(result.status, 0, result.stderr) - assert.match(result.stdout, /^200 .* exact$/) - assert.deepEqual(clock.slept, []) - }) - - it('exits 1 when no run has a baseline', async () => { - const api = API() - const result = await fetchBaseline(['develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(150)] }, - [api.artifacts(150)]: { artifacts: [] }, - }) - - assert.equal(result.status, 1) - assert.match(result.stderr, /No Plugin Check run on develop has a baseline/) - assert.equal(result.output, '') - }) - - it('exits 2, not 1, when the lookup itself fails', async () => { - // No answer for the branch's runs: the request is a 404, like an API error. - const result = await fetchBaseline(['develop'], {}) - - assert.equal(result.status, 2) - assert.match(result.stderr, /Could not look up Plugin Check runs: runs on develop/) - }) - - it('exits 2 when the download fails', async () => { - const api = API() - const result = await fetchBaseline(['develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(150)] }, - [api.artifacts(150)]: ONE_ARTIFACT, - }) - - assert.equal(result.status, 2) - assert.match(result.stderr, /download of the artifact of run 150/) - }) - - it('exits 2 when the list of artifacts cannot be read', async () => { - const api = API() - const result = await fetchBaseline(['develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(150)] }, - }) - - assert.equal(result.status, 2) - assert.match(result.stderr, /artifacts of run 150/) - }) - - it('stops at a failed lookup even when an older run could have worked', async () => { - const api = API() - const result = await fetchBaseline(['develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(150), run(120)] }, - [api.artifacts(150)]: ONE_ARTIFACT, - [api.artifacts(120)]: ONE_ARTIFACT, - }, { 120: BASELINE }) - - assert.equal(result.status, 2) - assert.ok(!result.paths.some((request) => request.includes('runs/120/'))) - }) - - it('reads the runs of plugin-check.yml by default', async () => { - const result = await fetchBaseline(['develop'], {}) - - assert.ok(result.paths[0].includes('/workflows/plugin-check.yml/runs'), result.paths[0]) - }) - - it('reads the runs of the workflow file --workflow names', async () => { - const api = API('wporg-check.yml') - const result = await fetchBaseline(['--workflow', 'wporg-check.yml', 'develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(150)] }, - [api.artifacts(150)]: ONE_ARTIFACT, - }, { 150: BASELINE }) - - assert.equal(result.status, 0, result.stderr) - assert.ok(result.paths[0].includes('/workflows/wporg-check.yml/runs'), result.paths[0]) - }) - - it('ignores a run whose head commit is in a fork, even on a branch named like the target', async () => { - const api = API() - const result = await fetchBaseline(['--sha', SHA, 'develop'], { - // A fork's pull request run for the base commit, and a newer one on a fork branch "develop". - [api.forSha(SHA)]: { workflow_runs: [run(300, SHA, 'someone/my-plugin')] }, - [api.onBranch('develop')]: { workflow_runs: [run(250, OLD_SHA, 'someone/my-plugin'), run(150)] }, - [api.artifacts(150)]: ONE_ARTIFACT, - [api.artifacts(250)]: ONE_ARTIFACT, - [api.artifacts(300)]: ONE_ARTIFACT, - }, { 150: BASELINE, 250: { errors: { 'Some.Rule': 9999 }, warnings: {} }, 300: { errors: { 'Some.Rule': 9999 }, warnings: {} } }) - - assert.equal(result.status, 0, result.stderr) - assert.match(result.stdout, /^150 .* older$/) - assert.deepEqual(JSON.parse(result.output), BASELINE) - assert.ok( - !result.paths.some((request) => request.includes('/runs/250/') || request.includes('/runs/300/')), - 'no fork run was looked at' - ) - }) - - it('exits 1 when only fork runs exist', async () => { - const api = API() - const result = await fetchBaseline(['develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(250, OLD_SHA, 'someone/my-plugin')] }, - }) - - assert.equal(result.status, 1) - assert.match(result.stderr, /has a baseline/) - }) - - it('matches the repository name without regard to case', async () => { - const api = API() - const result = await fetchBaseline(['develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(150, OLD_SHA, 'Example-Org/My-Plugin')] }, - [api.artifacts(150)]: ONE_ARTIFACT, - }, { 150: BASELINE }) - - assert.equal(result.status, 0, result.stderr) - }) - - it('treats a run with no head repository as not ours', async () => { - const api = API() - const result = await fetchBaseline(['develop'], { - [api.onBranch('develop')]: { workflow_runs: [{ ...run(150), head_repository: null }] }, - }) - - assert.equal(result.status, 1) - }) - - it('does not wait for a fork run that is still going', async () => { - const api = API() - const clock = fakeClock() - const result = await fetchBaseline(['--sha', SHA, '--wait', '1', 'develop'], { - // Our run for the commit is done, and a fork's run for the same commit is not. Counted, the - // fork's run would make the lookup sleep 30 seconds and then give up waiting. - [api.goingForSha(SHA)]: { - workflow_runs: [run(200, SHA), { ...run(300, SHA, 'someone/my-plugin'), status: 'in_progress' }], - }, - [api.forSha(SHA)]: { workflow_runs: [run(200, SHA)] }, - [api.onBranch('develop')]: { workflow_runs: [] }, - [api.artifacts(200)]: ONE_ARTIFACT, - }, { 200: BASELINE }, { clock }) - - assert.equal(result.status, 0, result.stderr) - assert.match(result.stdout, /^200 .* exact$/) - assert.doesNotMatch(result.stderr, /still going/) - assert.deepEqual(clock.slept, []) - }) - - it('exits 2, not 1, for an option value that is empty or not a whole number', async () => { - const cases = [ - [['--wait', '20m', 'develop'], /--wait must be a whole number, not '20m'/], - [['--wait', '', 'develop'], /--wait needs a value/], - [['--before-run', 'abc', 'develop'], /--before-run must be a whole number, not 'abc'/], - [['--before-run', '-5', 'develop'], /--before-run must be a whole number/], - [['--before-run', '9'.repeat(19), 'develop'], /--before-run is too large/], - [['--sha', '', 'develop'], /--sha needs a value/], - [['', 'baseline.json'], /a branch is needed/], - ] - - for (const [args, message] of cases) { - const result = await fetchBaseline(args, {}, {}, { bare: true }) - - assert.equal(result.status, 2, args.join(' ')) - assert.match(result.stderr, message, args.join(' ')) - assert.deepEqual(result.paths, [], `${args.join(' ')} asked the API nothing`) - } - }) - - it('exits 2 when an option or the branch is missing altogether', async () => { - const cases = [ - [['--wait'], /--wait needs a value/], - [['--workflow'], /--workflow needs a value/], - [[], /a branch is needed/], - ] - - for (const [args, message] of cases) { - const result = await fetchBaseline(args, {}, {}, { bare: true }) - - assert.equal(result.status, 2, args.join(' ')) - assert.match(result.stderr, message, args.join(' ')) - } - }) - - it('reads a run id with a leading zero as a decimal number', async () => { - const api = API() - const result = await fetchBaseline(['--before-run', '0150', 'develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(150), run(120)] }, - [api.artifacts(120)]: ONE_ARTIFACT, - }, { 120: BASELINE }) - - assert.equal(result.status, 0, result.stderr) - assert.match(result.stdout, /^120 /) - }) - - it('takes a run id of 18 digits for --before-run', async () => { - const api = API() - const result = await fetchBaseline(['--before-run', '999999999999999999', 'develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(150)] }, - [api.artifacts(150)]: ONE_ARTIFACT, - }, { 150: BASELINE }) - - assert.equal(result.status, 0, result.stderr) - assert.match(result.stdout, /^150 /) - }) - - it('refuses a --workflow that is a path or a ref, before calling the API', async () => { - for (const workflow of ['.github/workflows/plugin-check.yml', 'plugin-check.yml@refs/heads/develop']) { - const result = await fetchBaseline(['--workflow', workflow, 'develop'], {}) - - assert.equal(result.status, 2, workflow) - assert.match(result.stderr, /--workflow must be a file name/) - assert.deepEqual(result.paths, []) - } - }) - - it('asks the GitHub CLI for the repository when GITHUB_REPOSITORY is not set', async () => { - const { github, requests } = await octokitFor([]) - const { exec, calls } = recordingExec([{ command: 'gh', args: /^repo view/, stdout: `${REPO}\n` }]) - const found = await report.fetchBaseline({ - args: ['develop'], - env: {}, - github, - exec, - log: report.commandLog({ out: () => {}, err: () => {} }), - }) - - assert.equal(calls[0].command, 'gh') - assert.match(calls[0].args.join(' '), /^repo view/) - assert.equal(requests[0].path.startsWith(`/repos/${REPO}/`), true) - assert.equal(found.status, 2, 'the runs are not answered, so the lookup fails after the repository') - }) - - it('exits 2 when there is no repository at all', async () => { - const result = await fetchBaseline(['develop'], {}, {}, { env: { GITHUB_REPOSITORY: '' } }) - - assert.equal(result.status, 2) - assert.match(result.stderr, /no repository/) - }) - - it('takes the next run when the artifact has no baseline.json, or an empty one', async () => { - const api = API() - const result = await fetchBaseline(['develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(150), run(140), run(130)] }, - [api.artifacts(150)]: ONE_ARTIFACT, - [api.artifacts(140)]: ONE_ARTIFACT, - [api.artifacts(130)]: ONE_ARTIFACT, - }, { 150: {}, 140: {}, 130: BASELINE }, { files: { 150: '' } }) - - assert.equal(result.status, 0, result.stderr) - assert.match(result.stdout, /^140 /, 'an empty file is skipped, and a non-empty one is taken') - }) - - it('creates the folder of the output file', async () => { - const api = API() - const result = await fetchBaseline(['develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(150)] }, - [api.artifacts(150)]: ONE_ARTIFACT, - }, { 150: BASELINE }) - - assert.equal(result.status, 0) - assert.deepEqual(JSON.parse(result.output), BASELINE) - }) - - it('asks for the pages of runs in the order the old script did, with the filters in the query', async () => { - const api = API() - const result = await fetchBaseline(['--sha', SHA, '--wait', '5', 'develop'], { - [api.goingForSha(SHA)]: { workflow_runs: [run(200, SHA)] }, - [api.forSha(SHA)]: { workflow_runs: [run(200, SHA)] }, - [api.onBranch('develop')]: { workflow_runs: [] }, - [api.artifacts(200)]: ONE_ARTIFACT, - }, { 200: BASELINE }) - - assert.deepEqual(result.paths, [ - api.goingForSha(SHA), - api.forSha(SHA), - api.onBranch('develop'), - api.artifacts(200), - ]) - }) - - it('encodes a branch name with a slash in the query', async () => { - const api = API() - const result = await fetchBaseline(['feature/x y'], {}) - - assert.deepEqual(result.paths, [api.onBranch('feature/x y')]) - assert.match(result.paths[0], /branch=feature%2Fx%20y&/) - }) - - it('downloads with the GitHub CLI, for the repository and the artifact name', async () => { - const api = API() - const result = await fetchBaseline(['develop'], { - [api.onBranch('develop')]: { workflow_runs: [run(150)] }, - [api.artifacts(150)]: ONE_ARTIFACT, - }, { 150: BASELINE }) - - assert.equal(result.calls.length, 1) - assert.equal(result.calls[0].command, 'gh') - assert.deepEqual(result.calls[0].args.slice(0, 7), ['run', 'download', '150', '--repo', REPO, '--name', 'plugin-check-report']) - }) - - describe('the wait for the run of the base commit', () => { - it('polls every 30 seconds until the run is done', async () => { - const api = API() - const clock = fakeClock() - const { github, requests } = await octokitFor([ - { - path: api.goingForSha(SHA), - responses: [ - { body: { workflow_runs: [{ ...run(200, SHA), status: 'in_progress' }] } }, - { body: { workflow_runs: [{ ...run(200, SHA), status: 'in_progress' }] } }, - { body: { workflow_runs: [run(200, SHA)] } }, - ], - }, - { path: api.forSha(SHA), body: { workflow_runs: [run(200, SHA)] } }, - { path: api.onBranch('develop'), body: { workflow_runs: [] } }, - { path: api.artifacts(200), body: ONE_ARTIFACT }, - ]) - const { exec } = recordingExec([ - { - command: 'gh', - args: /^run download 200 /, - effect: (args) => { - const target = args[args.indexOf('--dir') + 1] - fs.mkdirSync(target, { recursive: true }) - fs.writeFileSync(path.join(target, 'baseline.json'), JSON.stringify(BASELINE)) - }, - }, - ]) - const output = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-wait-')), 'baseline.json') - - const status = await report.main( - ['fetch', '--sha', SHA, '--wait', '1200', 'develop', output], - { GITHUB_REPOSITORY: REPO }, - { out: () => {}, err: () => {} }, - { github, exec, now: clock.now, sleep: clock.sleep } - ) - - assert.equal(status, 0) - assert.deepEqual(clock.slept, [30, 30]) - assert.equal(requests.filter((request) => request.path === api.goingForSha(SHA)).length, 3) - }) - - it('gives up after the wait, says so, and goes on with older runs', async () => { - const api = API() - const clock = fakeClock() - const stillGoing = { workflow_runs: [{ ...run(200, SHA), status: 'in_progress' }] } - const result = await fetchBaseline(['--sha', SHA, '--wait', '60', 'develop'], { - [api.goingForSha(SHA)]: stillGoing, - [api.forSha(SHA)]: { workflow_runs: [] }, - [api.onBranch('develop')]: { workflow_runs: [run(150)] }, - [api.artifacts(150)]: ONE_ARTIFACT, - }, { 150: BASELINE }, { clock }) - - assert.equal(result.status, 0, result.stderr) - assert.match(result.stdout, /^150 .* older$/) - assert.match(result.stderr, new RegExp(`The run for ${SHA} is still going after 60 seconds; trying older runs\\.`)) - assert.deepEqual(clock.slept, [30, 30]) - }) - - it('stops waiting for a run that never shows up after 120 seconds', async () => { - const api = API() - const clock = fakeClock() - const result = await fetchBaseline(['--sha', SHA, '--wait', '1200', 'develop'], { - [api.goingForSha(SHA)]: { workflow_runs: [] }, - [api.forSha(SHA)]: { workflow_runs: [] }, - [api.onBranch('develop')]: { workflow_runs: [run(150)] }, - [api.artifacts(150)]: ONE_ARTIFACT, - }, { 150: BASELINE }, { clock }) - - assert.equal(result.status, 0, result.stderr) - assert.deepEqual(clock.slept, [30, 30, 30, 30]) - assert.doesNotMatch(result.stderr, /still going/) - }) - - it('says it is still going, not that the run never showed up, when the wait is short', async () => { - const api = API() - const clock = fakeClock() - const result = await fetchBaseline(['--sha', SHA, '--wait', '20', 'develop'], { - [api.goingForSha(SHA)]: { workflow_runs: [] }, - [api.forSha(SHA)]: { workflow_runs: [] }, - [api.onBranch('develop')]: { workflow_runs: [] }, - }, {}, { clock }) - - assert.equal(result.status, 1) - assert.match(result.stderr, /still going after 20 seconds/) - }) - - it('does not wait at all with --wait 0, or without --sha', async () => { - const api = API() - const clock = fakeClock() - const result = await fetchBaseline(['--wait', '300', 'develop'], { - [api.onBranch('develop')]: { workflow_runs: [] }, - }, {}, { clock }) - - assert.equal(result.status, 1) - assert.deepEqual(result.paths, [api.onBranch('develop')]) - }) - - it('exits 2 when polling fails', async () => { - const result = await fetchBaseline(['--sha', SHA, '--wait', '60', 'develop'], {}) - - assert.equal(result.status, 2) - assert.match(result.stderr, new RegExp(`runs for ${SHA}`)) - }) - }) -}) - -describe('plugin-check-report fetch helpers', () => { - it('reads the workflow file from the workflow ref', () => { - assert.equal( - report.workflowFromRef('example-org/my-plugin/.github/workflows/plugin-check.yml@refs/pull/12/merge'), - 'plugin-check.yml' - ) - assert.equal(report.workflowFromRef(''), '') - assert.equal(report.workflowFromRef(undefined), '') - }) - - it('lower-cases only A to Z', () => { - assert.equal(report.asciiLower('Example-Org/ÄB'), 'example-org/Äb') - }) - - it('names the baseline for the event and the match', () => { - assert.deepEqual(report.describeBaseline('pull_request', 'exact'), { label: 'Target branch', advisory: false }) - assert.deepEqual(report.describeBaseline('pull_request', 'older'), { label: 'Target branch, older run', advisory: true }) - assert.deepEqual(report.describeBaseline('push', 'exact'), { label: 'Previous run', advisory: false }) - }) - - it('orders runs: the base commit first, then the branch, newest first, each once', () => { - const ordered = report.orderCandidates({ - shaRuns: [run(5), run(9)], - branchRuns: [run(7), run(9), run(11)], - beforeRun: null, - }) - - assert.deepEqual(ordered.map((candidate) => candidate.id), [5, 9, 11, 7]) - }) -}) From 5f52ba88158c0d01b31ae1bec4febc79e4ac6915 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Wed, 4 Nov 2026 14:32:38 -0300 Subject: [PATCH 25/31] Remove the advisory baseline and the baseline label --- .../plugin-check-report.js | 46 +++++-------------- .../plugin-check-report/compare.test.js | 27 ----------- .../expected/errors-up-advisory-uppercase.md | 11 ----- .../expected/errors-up-advisory-uppercase.out | 2 - .../errors-up-advisory-uppercase.summary | 11 ----- .../fixtures/expected/errors-up-advisory.md | 11 ----- .../fixtures/expected/errors-up-advisory.out | 2 - .../expected/errors-up-advisory.summary | 11 ----- .../fixtures/expected/label-and-url.md | 9 ---- .../fixtures/expected/label-and-url.out | 1 - .../fixtures/expected/label-and-url.summary | 9 ---- .../fixtures/expected/status.json | 3 -- tests/actions/plugin-check-report/golden.js | 18 -------- 13 files changed, 12 insertions(+), 149 deletions(-) delete mode 100644 tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.md delete mode 100644 tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.out delete mode 100644 tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.summary delete mode 100644 tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.md delete mode 100644 tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.out delete mode 100644 tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.summary delete mode 100644 tests/actions/plugin-check-report/fixtures/expected/label-and-url.md delete mode 100644 tests/actions/plugin-check-report/fixtures/expected/label-and-url.out delete mode 100644 tests/actions/plugin-check-report/fixtures/expected/label-and-url.summary diff --git a/.github/actions/plugin-check-report/plugin-check-report.js b/.github/actions/plugin-check-report/plugin-check-report.js index b4534ab..1d153d0 100644 --- a/.github/actions/plugin-check-report/plugin-check-report.js +++ b/.github/actions/plugin-check-report/plugin-check-report.js @@ -402,14 +402,12 @@ function buildRows(now, base) { * @param {object} input * @param {object[]} input.rows From buildRows. * @param {string} input.mode `baseline` or `strict`. - * @param {boolean} input.advisory The baseline is from an older commit than the target, so more - * errors are only reported. * @param {boolean} input.noBaseline There was nothing to compare with: only the totals count. * * @returns {object} `messages` (`{level, text}` in print order), `totals`, `wentUp`, `wentDown`, * `errorsUp`, `upList` and `failed`. */ -function evaluate({ rows, mode, advisory, noBaseline }) { +function evaluate({ rows, mode, noBaseline }) { const totals = { errorsBase: 0, errorsNow: 0, warningsBase: 0, warningsNow: 0 } const messages = [] let failed = false @@ -452,18 +450,11 @@ function evaluate({ rows, mode, advisory, noBaseline }) { } errorsUp++ - if (advisory) { - messages.push({ - level: 'warning', - text: `${shown} has ${found} errors, an older run of the target branch has ${allowed}. Not enforced, because the target may have changed since.`, - }) - } else { - failed = true - messages.push({ - level: 'error', - text: `${shown} has ${found} errors, the baseline has ${allowed}. The full list is in the plugin-check-results artifact.`, - }) - } + failed = true + messages.push({ + level: 'error', + text: `${shown} has ${found} errors, the baseline has ${allowed}. The full list is in the plugin-check-results artifact.`, + }) } else if (found < allowed) { wentDown++ messages.push({ level: 'notice', text: `${shown} (${kind}) went down from ${allowed} to ${found}.` }) @@ -509,13 +500,12 @@ function change(now, base) { * @param {string} input.mode `baseline` or `strict`. * @param {boolean} input.failed The check fails. * @param {boolean} input.noBaseline Nothing was compared. - * @param {boolean} input.advisory The baseline is from an older run of the target branch. * @param {number} input.errorsUp How many rules have more errors. * @param {number} input.errorsNow This run's error total. * * @returns {string} The line. */ -function statusLine({ mode, failed, noBaseline, advisory, errorsUp, errorsNow }) { +function statusLine({ mode, failed, noBaseline, errorsUp, errorsNow }) { if (mode === 'strict' && failed) { return `❌ Plugin Check failed: ${errorsNow} errors, and strict mode allows none` } @@ -525,9 +515,6 @@ function statusLine({ mode, failed, noBaseline, advisory, errorsUp, errorsNow }) if (noBaseline) { return '⚠️ Plugin Check ran, but there was no baseline to compare with' } - if (errorsUp > 0 && advisory) { - return `⚠️ Plugin Check passed, not enforced: ${errorsUp} rule(s) have more errors than an older run of the target branch` - } if (failed) { return errorsUp === 1 ? '❌ Plugin Check failed: 1 rule has more errors than the baseline allows' @@ -615,9 +602,8 @@ function buildComment({ marker, status, badge, column, noBaseline, totals, wentU * the results file. * @param {object} input.env Environment. PLUGIN_CHECK_MODE (`baseline`, the default, or `strict`), * COMMENT_MARKER, PLUGIN_CHECK_BASELINE (default: baseline.json next - * to this file), BASELINE_LABEL, BASELINE_URL, PLUGIN_CHECK_ADVISORY - * (`true` when the baseline is from an older commit than the target), - * COMMENT_BADGE, RUN_URL, ARTIFACT_URL, GITHUB_STEP_SUMMARY. + * to this file), COMMENT_BADGE, RUN_URL, ARTIFACT_URL, + * GITHUB_STEP_SUMMARY. * @param {object} input.log Where messages go: `out`, `err`, `error`, `warning`, `notice`. * * @returns {number} The exit status: 0 passes, 1 fails the check, another number is a file that @@ -720,8 +706,7 @@ function compareOrThrow({ args, env, log }) { base = readBaseline(fs.readFileSync(baselinePath, 'utf8')) } - const advisory = env.PLUGIN_CHECK_ADVISORY === 'true' - const verdict = evaluate({ rows: buildRows(now, base), mode, advisory, noBaseline }) + const verdict = evaluate({ rows: buildRows(now, base), mode, noBaseline }) for (const { level, text: message } of verdict.messages) { log[level](message) } @@ -738,18 +723,11 @@ function compareOrThrow({ args, env, log }) { runUrl = `${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}` } - // The column is named after where the baseline came from ("Target branch", "Previous run"), and - // links to that run when known. - let column = env.BASELINE_LABEL || 'Baseline' - if (env.BASELINE_URL) { - column = `[${column}](${env.BASELINE_URL})` - } - const body = buildComment({ marker: env.COMMENT_MARKER || '', - status: statusLine({ mode, advisory, noBaseline, ...verdict, errorsNow: verdict.totals.errorsNow }), + status: statusLine({ mode, noBaseline, ...verdict, errorsNow: verdict.totals.errorsNow }), badge: env.COMMENT_BADGE || '', - column, + column: 'Baseline', noBaseline, totals: verdict.totals, wentUp: verdict.wentUp, diff --git a/tests/actions/plugin-check-report/compare.test.js b/tests/actions/plugin-check-report/compare.test.js index 2006b2d..e2a8186 100644 --- a/tests/actions/plugin-check-report/compare.test.js +++ b/tests/actions/plugin-check-report/compare.test.js @@ -358,23 +358,6 @@ describe('plugin-check-report compare', () => { assert.match(run.comment, new RegExp(`Went up: \`${DIRECT_QUERY}\` warning \\(\\+2\\)`)) }) - it('only warns when the baseline is from an older run of the target branch', () => { - const run = compareWith([fixture('errors-up.txt')], { - PLUGIN_CHECK_BASELINE: BASELINE, - PLUGIN_CHECK_ADVISORY: 'true', - }) - - assert.equal(run.status, 0) - assert.match(run.stdout, /::warning title=Plugin Check::.*Not enforced/) - assert.match(run.comment, /⚠️ Plugin Check passed, not enforced/) - }) - - it('takes only the exact word true as advisory', () => { - const run = compareWith([fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: BASELINE, PLUGIN_CHECK_ADVISORY: 'TRUE' }) - - assert.equal(run.status, 1) - }) - it('treats an empty baseline file, and null, as no counts at all', () => { for (const content of ['', 'null', ' \n']) { const run = compareWith([fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: baselineFile(content) }) @@ -534,16 +517,6 @@ describe('plugin-check-report compare', () => { assert.equal(run.comment.split('\n')[0], marker) }) - it('names the baseline column after where it came from, with a link', () => { - const run = compareWith([fixture('same-as-baseline.txt')], { - PLUGIN_CHECK_BASELINE: BASELINE, - BASELINE_LABEL: 'Target branch', - BASELINE_URL: 'https://github.com/o/r/actions/runs/1', - }) - - assert.match(run.comment, /\| Now \| \[Target branch\]\(https:\/\/github\.com\/o\/r\/actions\/runs\/1\) \| Change \|/) - }) - it('lists at most five rules and never grows past fifteen lines', () => { const run = compareWith([resultsWithNewCodes(7)], { PLUGIN_CHECK_BASELINE: BASELINE }) diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.md b/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.md deleted file mode 100644 index c5e77f1..0000000 --- a/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.md +++ /dev/null @@ -1,11 +0,0 @@ - -❌ Plugin Check failed: 1 rule has more errors than the baseline allows - -| | Now | Baseline | Change | -|:--|--:|--:|:--| -| Errors | 4 | 3 | +1 | -| Warnings | 1 | 1 | ±0 | - -Went up: `WordPress.Security.EscapeOutput.OutputNotEscaped` (+1) - -📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.out b/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.out deleted file mode 100644 index 90bf791..0000000 --- a/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.out +++ /dev/null @@ -1,2 +0,0 @@ -::error title=Plugin Check::WordPress.Security.EscapeOutput.OutputNotEscaped has 3 errors, the baseline has 2. The full list is in the plugin-check-results artifact. -Plugin Check found more errors than the baseline allows. diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.summary b/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.summary deleted file mode 100644 index c5e77f1..0000000 --- a/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory-uppercase.summary +++ /dev/null @@ -1,11 +0,0 @@ - -❌ Plugin Check failed: 1 rule has more errors than the baseline allows - -| | Now | Baseline | Change | -|:--|--:|--:|:--| -| Errors | 4 | 3 | +1 | -| Warnings | 1 | 1 | ±0 | - -Went up: `WordPress.Security.EscapeOutput.OutputNotEscaped` (+1) - -📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.md b/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.md deleted file mode 100644 index be36055..0000000 --- a/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.md +++ /dev/null @@ -1,11 +0,0 @@ - -⚠️ Plugin Check passed, not enforced: 1 rule(s) have more errors than an older run of the target branch - -| | Now | Baseline | Change | -|:--|--:|--:|:--| -| Errors | 4 | 3 | +1 | -| Warnings | 1 | 1 | ±0 | - -Went up: `WordPress.Security.EscapeOutput.OutputNotEscaped` (+1) - -📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.out b/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.out deleted file mode 100644 index 2418fe5..0000000 --- a/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.out +++ /dev/null @@ -1,2 +0,0 @@ -::warning title=Plugin Check::WordPress.Security.EscapeOutput.OutputNotEscaped has 3 errors, an older run of the target branch has 2. Not enforced, because the target may have changed since. -Plugin Check error counts are within the baseline. diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.summary b/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.summary deleted file mode 100644 index be36055..0000000 --- a/tests/actions/plugin-check-report/fixtures/expected/errors-up-advisory.summary +++ /dev/null @@ -1,11 +0,0 @@ - -⚠️ Plugin Check passed, not enforced: 1 rule(s) have more errors than an older run of the target branch - -| | Now | Baseline | Change | -|:--|--:|--:|:--| -| Errors | 4 | 3 | +1 | -| Warnings | 1 | 1 | ±0 | - -Went up: `WordPress.Security.EscapeOutput.OutputNotEscaped` (+1) - -📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/label-and-url.md b/tests/actions/plugin-check-report/fixtures/expected/label-and-url.md deleted file mode 100644 index 1a85ac8..0000000 --- a/tests/actions/plugin-check-report/fixtures/expected/label-and-url.md +++ /dev/null @@ -1,9 +0,0 @@ - -✅ Plugin Check passed - -| | Now | [Target branch](https://github.com/o/r/actions/runs/1) | Change | -|:--|--:|--:|:--| -| Errors | 3 | 3 | ±0 | -| Warnings | 1 | 1 | ±0 | - -📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/label-and-url.out b/tests/actions/plugin-check-report/fixtures/expected/label-and-url.out deleted file mode 100644 index 0d8c877..0000000 --- a/tests/actions/plugin-check-report/fixtures/expected/label-and-url.out +++ /dev/null @@ -1 +0,0 @@ -Plugin Check error counts are within the baseline. diff --git a/tests/actions/plugin-check-report/fixtures/expected/label-and-url.summary b/tests/actions/plugin-check-report/fixtures/expected/label-and-url.summary deleted file mode 100644 index 1a85ac8..0000000 --- a/tests/actions/plugin-check-report/fixtures/expected/label-and-url.summary +++ /dev/null @@ -1,9 +0,0 @@ - -✅ Plugin Check passed - -| | Now | [Target branch](https://github.com/o/r/actions/runs/1) | Change | -|:--|--:|--:|:--| -| Errors | 3 | 3 | ±0 | -| Warnings | 1 | 1 | ±0 | - -📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/status.json b/tests/actions/plugin-check-report/fixtures/expected/status.json index ad302bd..45e1c09 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/status.json +++ b/tests/actions/plugin-check-report/fixtures/expected/status.json @@ -5,13 +5,10 @@ "rule-up-total-same": 1, "errors-down": 0, "warnings-up": 0, - "errors-up-advisory": 0, - "errors-up-advisory-uppercase": 1, "strict-no-baseline": 1, "strict-errors-down": 1, "strict-clean": 0, "strict-warnings-only": 0, - "label-and-url": 0, "seven-new-codes": 1, "six-new-codes": 1, "badge-and-artifact": 1, diff --git a/tests/actions/plugin-check-report/golden.js b/tests/actions/plugin-check-report/golden.js index 11bdda3..1823152 100644 --- a/tests/actions/plugin-check-report/golden.js +++ b/tests/actions/plugin-check-report/golden.js @@ -73,18 +73,6 @@ const CASES = [ { name: 'rule-up-total-same', results: 'rule-up-total-same.txt', baseline: BASELINE, env: RUN }, { name: 'errors-down', results: 'errors-down.txt', baseline: BASELINE, env: RUN }, { name: 'warnings-up', results: 'warnings-up.txt', baseline: BASELINE, env: RUN }, - { - name: 'errors-up-advisory', - results: 'errors-up.txt', - baseline: BASELINE, - env: { ...RUN, PLUGIN_CHECK_ADVISORY: 'true' }, - }, - { - name: 'errors-up-advisory-uppercase', - results: 'errors-up.txt', - baseline: BASELINE, - env: { ...RUN, PLUGIN_CHECK_ADVISORY: 'TRUE' }, - }, { name: 'strict-no-baseline', results: 'errors-down.txt', env: { ...RUN, PLUGIN_CHECK_MODE: 'strict' } }, { name: 'strict-errors-down', @@ -94,12 +82,6 @@ const CASES = [ }, { name: 'strict-clean', results: 'clean.txt', env: { ...RUN, PLUGIN_CHECK_MODE: 'strict' } }, { name: 'strict-warnings-only', text: warningsOnly, env: { ...RUN, PLUGIN_CHECK_MODE: 'strict' } }, - { - name: 'label-and-url', - results: 'same-as-baseline.txt', - baseline: BASELINE, - env: { ...RUN, BASELINE_LABEL: 'Target branch', BASELINE_URL: 'https://github.com/o/r/actions/runs/1' }, - }, { name: 'seven-new-codes', text: () => newCodes(7), baseline: BASELINE, env: RUN }, { name: 'six-new-codes', text: () => newCodes(6), baseline: BASELINE, env: RUN }, { From 1d9431dbdce18d7c502ec5a9c6f3e516e41d6f71 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Wed, 4 Nov 2026 14:35:09 -0300 Subject: [PATCH 26/31] Tell the author how to make and lower the Plugin Check baseline --- .../plugin-check-report.js | 160 +++++++++++++++--- .../plugin-check-report/action.test.js | 4 +- .../plugin-check-report/compare.test.js | 145 +++++++++++++++- .../plugin-check-report/exclude.test.js | 2 +- .../fixtures/expected/artifact-only.md | 2 +- .../fixtures/expected/artifact-only.out | 2 +- .../fixtures/expected/artifact-only.summary | 9 +- .../expected/badge-and-artifact.summary | 7 + .../fixtures/expected/clean-no-baseline.md | 2 +- .../fixtures/expected/clean-no-baseline.out | 2 +- .../expected/clean-no-baseline.summary | 9 +- .../fixtures/expected/errors-down.md | 2 +- .../fixtures/expected/errors-down.out | 1 + .../fixtures/expected/errors-down.summary | 9 +- .../fixtures/expected/errors-up.summary | 7 + .../expected/marker-with-line-breaks.summary | 7 + .../fixtures/expected/marker.md | 2 +- .../fixtures/expected/marker.out | 2 +- .../fixtures/expected/marker.summary | 9 +- .../fixtures/expected/rule-up-total-same.md | 2 +- .../fixtures/expected/rule-up-total-same.out | 1 + .../expected/rule-up-total-same.summary | 9 +- .../fixtures/expected/seven-new-codes.md | 2 +- .../fixtures/expected/seven-new-codes.out | 1 + .../fixtures/expected/seven-new-codes.summary | 9 +- .../fixtures/expected/six-new-codes.md | 2 +- .../fixtures/expected/six-new-codes.out | 1 + .../fixtures/expected/six-new-codes.summary | 9 +- .../fixtures/expected/strict-clean.out | 2 +- .../fixtures/expected/strict-no-baseline.out | 2 +- .../expected/strict-warnings-only.out | 2 +- 31 files changed, 379 insertions(+), 46 deletions(-) diff --git a/.github/actions/plugin-check-report/plugin-check-report.js b/.github/actions/plugin-check-report/plugin-check-report.js index 1d153d0..58815e2 100644 --- a/.github/actions/plugin-check-report/plugin-check-report.js +++ b/.github/actions/plugin-check-report/plugin-check-report.js @@ -27,6 +27,9 @@ const path = require('node:path') /** Title of every annotation. */ const TITLE = 'Plugin Check' +/** The baseline file a repository commits, at its root. */ +const BASELINE_FILE = '.plugin-check-baseline.json' + /** The pull request comment must stay short. Everything else is behind the links. */ const MAX_COMMENT_LINES = 15 const MAX_RULES_LISTED = 5 @@ -54,11 +57,13 @@ class CheckFailure extends Error { /** * @param {string} message What went wrong, in plain words. * @param {number} exitCode The exit status for the step. + * @param {boolean} annotate Show the message as an error annotation, not only in the log. */ - constructor(message, exitCode = 1) { + constructor(message, exitCode = 1, annotate = false) { super(message) this.name = 'CheckFailure' this.exitCode = exitCode + this.annotate = annotate } } @@ -403,15 +408,17 @@ function buildRows(now, base) { * @param {object[]} input.rows From buildRows. * @param {string} input.mode `baseline` or `strict`. * @param {boolean} input.noBaseline There was nothing to compare with: only the totals count. + * @param {string} input.baselineName The baseline file's name, for the hint to update it. * * @returns {object} `messages` (`{level, text}` in print order), `totals`, `wentUp`, `wentDown`, - * `errorsUp`, `upList` and `failed`. + * `errorsDown` (how many rules have fewer errors), `errorsUp`, `upList` and `failed`. */ -function evaluate({ rows, mode, noBaseline }) { +function evaluate({ rows, mode, noBaseline, baselineName = BASELINE_FILE }) { const totals = { errorsBase: 0, errorsNow: 0, warningsBase: 0, warningsNow: 0 } const messages = [] let failed = false let wentDown = 0 + let errorsDown = 0 let wentUp = 0 let errorsUp = 0 let upList = '' @@ -457,10 +464,22 @@ function evaluate({ rows, mode, noBaseline }) { }) } else if (found < allowed) { wentDown++ + if (kind === 'errors') { + errorsDown++ + } messages.push({ level: 'notice', text: `${shown} (${kind}) went down from ${allowed} to ${found}.` }) } } + // Fewer errors than the file says leaves room for them to come back unnoticed. Strict mode does + // not use the file, so it asks for nothing. + if (mode !== 'strict' && errorsDown > 0) { + messages.push({ + level: 'notice', + text: `Error counts went down. Lower the baseline so they cannot come back: commit the baseline.json from this run's plugin-check-report artifact as ${baselineName}.`, + }) + } + // Strict mode ignores the baseline for pass or fail: any error fails. The counts and the table // still compare with the baseline, so the comment shows what changed. if (mode === 'strict') { @@ -470,7 +489,7 @@ function evaluate({ rows, mode, noBaseline }) { } } - return { messages, totals, wentUp, wentDown, errorsUp, upList, failed } + return { messages, totals, wentUp, wentDown, errorsDown, errorsUp, upList, failed } } /** @@ -513,7 +532,7 @@ function statusLine({ mode, failed, noBaseline, errorsUp, errorsNow }) { return '✅ Plugin Check passed: no errors' } if (noBaseline) { - return '⚠️ Plugin Check ran, but there was no baseline to compare with' + return '⚠️ Plugin Check ran, but there is no baseline file yet' } if (failed) { return errorsUp === 1 @@ -554,12 +573,13 @@ function linksLine({ artifactUrl, runUrl, resultsPath }) { * @param {object} input.totals From evaluate. * @param {number} input.wentUp How many rules went up. * @param {number} input.wentDown How many rules went down. + * @param {boolean} input.updateHint An error count went down: ask for the baseline file to be updated. * @param {string} input.upList The first rules that went up, ready to print. * @param {string} input.links From linksLine. * * @returns {string} The body, cut to MAX_COMMENT_LINES lines, with no final line break. */ -function buildComment({ marker, status, badge, column, noBaseline, totals, wentUp, wentDown, upList, links }) { +function buildComment({ marker, status, badge, column, noBaseline, totals, wentUp, wentDown, updateHint = false, upList, links }) { let body = `${marker}\n${status}\n\n` if (badge) { @@ -582,7 +602,7 @@ function buildComment({ marker, status, badge, column, noBaseline, totals, wentU body += `Went up: ${upList}\n` } if (wentDown > 0) { - body += `Rules that went down: ${wentDown}.\n` + body += `Rules that went down: ${wentDown}.${updateHint ? ' Update the baseline file, see the job summary.' : ''}\n` } // A blank line separates the notes above from the links, when there are notes. if (wentUp > 0 || wentDown > 0) { @@ -594,6 +614,85 @@ function buildComment({ marker, status, badge, column, noBaseline, totals, wentU return body.split('\n').slice(0, MAX_COMMENT_LINES).join('\n').replace(/\n+$/, '') } +/** + * Reads the baseline file, and names it when it cannot be used. + * + * Plugin Check reports it as an error annotation: a committed file can be broken by a merge conflict + * or by hand, and the author has to fix it. + * + * @param {string} file The path. + * @param {string} name The file's name, as the messages show it. + * + * @throws {CheckFailure} When it is not a baseline. + * + * @returns {{errors: Map, warnings: Map}} Counts. + */ +function readBaselineFile(file, name) { + try { + return readBaseline(fs.readFileSync(file, 'utf8')) + } catch (error) { + if (!(error instanceof CheckFailure)) { + throw error + } + + const reason = error.message + .replace(/^The baseline is not valid: /, '') + .replace(/^jq: parse error: /, 'it is not JSON: ') + + throw new CheckFailure( + `The baseline file ${name} is not valid: ${reason} Fix it, or replace it with the baseline.json from this run's plugin-check-report artifact.`, + error.exitCode, + true + ) + } +} + +/** + * The part of the job summary that says how to change the baseline file, or '' when there is nothing + * to do. The comment stays short; this is where the commands go. + * + * @param {object} input + * @param {string} input.mode `baseline` or `strict`. + * @param {boolean} input.noBaseline There is no baseline file. + * @param {object} input.verdict From evaluate. + * @param {string} input.baselineName The baseline file's name. + * @param {object} input.env GITHUB_RUN_ID and GITHUB_REPOSITORY, for the real command. + * + * @returns {string} Markdown with a final line break, or ''. + */ +function updateSection({ mode, noBaseline, verdict, baselineName, env }) { + if (mode === 'strict') { + return '' + } + + const lines = [] + if (noBaseline) { + lines.push(`There is no ${baselineName} yet. To start one, add the file from this run:`) + } else if (verdict.errorsDown > 0) { + lines.push(`Error counts went down. To lower the baseline, replace ${baselineName} with the file from this run:`) + } else if (verdict.failed) { + lines.push( + `If the new errors are on purpose, raise their counts in ${baselineName} in this pull request, so reviewers see it. The file from this run has the new counts:` + ) + } else { + return '' + } + + const run = env.GITHUB_RUN_ID || '' + const repo = env.GITHUB_REPOSITORY || '/' + + return [ + '', + ...lines, + '', + '```bash', + `gh run download ${run} --repo ${repo} --name plugin-check-report --dir /tmp/plugin-check`, + `cp /tmp/plugin-check/baseline.json ${baselineName}`, + '```', + '', + ].join('\n') +} + /** * Compares a results file with a baseline, or writes the baseline for the results. * @@ -614,7 +713,12 @@ function compare({ args, env, log }) { return compareOrThrow({ args, env, log }) } catch (error) { if (error instanceof CheckFailure) { - log.err(error.message) + // A baseline file that cannot be read is the author's to fix, so it shows in the checks panel. + if (error.annotate) { + log.error(error.message) + } else { + log.err(error.message) + } return error.exitCode } @@ -669,7 +773,10 @@ function compareOrThrow({ args, env, log }) { return 1 } - const baselinePath = env.PLUGIN_CHECK_BASELINE || path.join(__dirname, 'baseline.json') + // The default is the committed file in the current folder, so `compare --update` from the plugin + // root writes it in place. + const baselinePath = env.PLUGIN_CHECK_BASELINE || path.resolve(BASELINE_FILE) + const baselineName = path.basename(baselinePath) if (!isFile(resultsPath)) { log.error(`Results file not found: ${resultsPath}`) @@ -694,19 +801,24 @@ function compareOrThrow({ args, env, log }) { return 0 } - // No earlier run to compare with: report the counts and pass. Failing here would block every pull - // request whenever the target branch has no finished run (its first run, or expired artifacts). + // No baseline file yet: report the counts and pass. Failing here would block the very pull request + // that adds the workflow, before anyone has a result to make the file from. Strict mode does not + // use the file, so it has nothing to say. const noBaseline = !isFile(baselinePath) let base = { errors: new Map(), warnings: new Map() } if (noBaseline) { - log.warning( - `No baseline to compare with (${baselinePath}), so nothing was compared. Run the workflow on the target branch to make one.` - ) + if (mode === 'strict') { + log.out('No baseline file. Strict mode does not need one.') + } else { + log.warning( + `No baseline file at ${baselineName}, so nothing was compared. To start one, commit the baseline.json from this run's plugin-check-report artifact as ${baselineName}.` + ) + } } else { - base = readBaseline(fs.readFileSync(baselinePath, 'utf8')) + base = readBaselineFile(baselinePath, baselineName) } - const verdict = evaluate({ rows: buildRows(now, base), mode, noBaseline }) + const verdict = evaluate({ rows: buildRows(now, base), mode, noBaseline, baselineName }) for (const { level, text: message } of verdict.messages) { log[level](message) } @@ -732,6 +844,7 @@ function compareOrThrow({ args, env, log }) { totals: verdict.totals, wentUp: verdict.wentUp, wentDown: verdict.wentDown, + updateHint: mode !== 'strict' && verdict.errorsDown > 0, upList: verdict.upList, links: linksLine({ artifactUrl: env.ARTIFACT_URL || '', runUrl, resultsPath }), }) @@ -740,7 +853,8 @@ function compareOrThrow({ args, env, log }) { writeText(commentFile, `${body}\n`, 'write') } if (env.GITHUB_STEP_SUMMARY) { - writeText(env.GITHUB_STEP_SUMMARY, `${body}\n`, 'append') + const how = updateSection({ mode, noBaseline, verdict, baselineName, env }) + writeText(env.GITHUB_STEP_SUMMARY, `${body}\n${how}`, 'append') } if (verdict.failed) { @@ -1167,7 +1281,7 @@ function runExcludeChecks({ core, env }) { } /** - * The command line: `compare` and `exclude-checks`. + * The command line: `compare`, `update` and `exclude-checks`. * * @param {string[]} argv The arguments after the script name. * @param {object} env The environment, default process.env. @@ -1190,6 +1304,11 @@ async function main( return compare({ args, env, log }) } + // `update ` is `compare --update `: writes the baseline file for the results. + if (command === 'update') { + return compare({ args: ['--update', ...args], env, log }) + } + // Prints the flag `wp plugin check` takes, so a local run skips the same checks as CI: // `wp plugin check my-plugin $(node plugin-check-report.js exclude-checks)`. Prints nothing when // no check is excluded. EXCLUDE_CHECKS, when set, replaces the file, as the workflow input does. @@ -1219,7 +1338,7 @@ async function main( } } - io.err('Usage: plugin-check-report.js compare|exclude-checks ...') + io.err('Usage: plugin-check-report.js compare|update|exclude-checks ...') return 2 } @@ -1235,6 +1354,7 @@ module.exports = { MODES, actionLog, BADGE_TIME_ZONE, + BASELINE_FILE, EXCLUDE_FILE, badgeStamp, buildComment, @@ -1258,6 +1378,7 @@ module.exports = { parseExcludeInput, parseResults, readBaseline, + readBaselineFile, readIfFile, readJsonValues, runCompare, @@ -1266,4 +1387,5 @@ module.exports = { runPaths, runUpdate, statusLine, + updateSection, } diff --git a/tests/actions/plugin-check-report/action.test.js b/tests/actions/plugin-check-report/action.test.js index d2c03da..acdda74 100644 --- a/tests/actions/plugin-check-report/action.test.js +++ b/tests/actions/plugin-check-report/action.test.js @@ -107,7 +107,7 @@ describe('plugin-check-report action steps', () => { assert.match(logged(), /Plugin Check found more errors than the baseline allows\./) const comment = fs.readFileSync(commentFile, 'utf8') assert.match(comment, /❌ Plugin Check failed: 1 rule has more errors than the baseline allows/) - assert.equal(summary(), comment) + assert.ok(summary().startsWith(comment), 'the summary is the comment, and then how to update the baseline') }) it('passes, and writes no comment file when none is asked for', async (t) => { @@ -146,7 +146,7 @@ describe('plugin-check-report action steps', () => { report.runCompare({ core, env: { ...base(), RESULTS: results, PLUGIN_CHECK_BASELINE: '/nonexistent/baseline.json' } }) - assert.match(logged(), /::warning title=Plugin Check::No baseline to compare with/) + assert.match(logged(), /::warning title=Plugin Check::No baseline file at baseline\.json, so nothing was compared\./) const failing = report.runCompare({ core, env: { ...base(), RESULTS: results, PLUGIN_CHECK_BASELINE: path.join(FIXTURES, 'baseline.json') } }) assert.equal(failing, 1) assert.match(logged(), /::error title=Plugin Check::100%25done has 1 errors/) diff --git a/tests/actions/plugin-check-report/compare.test.js b/tests/actions/plugin-check-report/compare.test.js index e2a8186..8a51999 100644 --- a/tests/actions/plugin-check-report/compare.test.js +++ b/tests/actions/plugin-check-report/compare.test.js @@ -311,12 +311,91 @@ describe('plugin-check-report compare', () => { const run = compareWith([fixture('clean.txt')]) assert.equal(run.status, 0) - assert.match(run.stdout, /::warning title=Plugin Check::No baseline to compare with/) - assert.match(run.comment, /⚠️ Plugin Check ran, but there was no baseline to compare with/) + assert.match( + run.stdout, + /::warning title=Plugin Check::No baseline file at no-baseline\.json, so nothing was compared\. To start one, commit the baseline\.json from this run's plugin-check-report artifact as no-baseline\.json\./ + ) + assert.match(run.comment, /⚠️ Plugin Check ran, but there is no baseline file yet/) assert.match(run.comment, /\| Errors {3}\| 0 \|/) assert.doesNotMatch(run.comment, /Change/) }) + it('says nothing about a missing baseline file in strict mode, which does not use one', () => { + const run = compareWith([fixture('clean.txt')], { PLUGIN_CHECK_MODE: 'strict' }) + + assert.equal(run.status, 0) + assert.doesNotMatch(run.stdout, /::warning/) + assert.match(run.stdout, /No baseline file\. Strict mode does not need one\./) + assert.match(run.comment, /✅ Plugin Check passed: no errors/) + assert.equal(run.summary, run.comment, 'nothing to update, so the summary is the comment') + }) + + it('asks for the baseline file to be lowered when an error count went down', () => { + const run = compareWith([fixture('errors-down.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + + assert.equal(run.status, 0) + assert.match( + run.stdout, + /::notice title=Plugin Check::Error counts went down\. Lower the baseline so they cannot come back: commit the baseline\.json from this run's plugin-check-report artifact as baseline\.json\./ + ) + assert.match(run.comment, /Rules that went down: \d+\. Update the baseline file, see the job summary\./) + }) + + it('does not ask for an update when only warnings went down, or in strict mode', () => { + const warningsDown = baselineFile({ ...JSON.parse(fs.readFileSync(BASELINE, 'utf8')), warnings: { [DIRECT_QUERY]: 9 } }) + const quiet = compareWith([fixture('same-as-baseline.txt')], { PLUGIN_CHECK_BASELINE: warningsDown }) + + assert.match(quiet.comment, /Rules that went down: 1\.\n/) + assert.doesNotMatch(quiet.stdout, /Error counts went down/) + assert.equal(quiet.summary, quiet.comment) + + const strict = compareWith([fixture('errors-down.txt')], { PLUGIN_CHECK_BASELINE: BASELINE, PLUGIN_CHECK_MODE: 'strict' }) + + assert.doesNotMatch(strict.stdout, /Error counts went down/) + assert.doesNotMatch(strict.comment, /Update the baseline file/) + }) + + it('puts the command to update the baseline in the job summary, not in the comment', () => { + const run = compareWith([fixture('errors-down.txt')], { + PLUGIN_CHECK_BASELINE: BASELINE, + GITHUB_RUN_ID: '42', + GITHUB_REPOSITORY: 'example-org/my-plugin', + GITHUB_SERVER_URL: 'https://github.com', + }) + + assert.ok(run.summary.startsWith(run.comment)) + assert.match(run.summary, /Error counts went down\. To lower the baseline, replace baseline\.json with the file from this run:/) + assert.match(run.summary, /gh run download 42 --repo example-org\/my-plugin --name plugin-check-report --dir \/tmp\/plugin-check/) + assert.match(run.summary, /cp \/tmp\/plugin-check\/baseline\.json baseline\.json/) + assert.doesNotMatch(run.comment, /gh run download/) + }) + + it('tells the author how to raise a count on purpose when the check fails', () => { + const run = compareWith([fixture('errors-up.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + + assert.equal(run.status, 1) + assert.match(run.summary, /raise their counts in baseline\.json in this pull request, so reviewers see it/) + assert.match(run.summary, /gh run download --repo \/ /, 'placeholders outside a run') + }) + + it('fails with the file named, as an error annotation, when the baseline file is not valid', () => { + const cases = { + 'not JSON': ['{"errors": ', /it is not JSON: /], + 'a merge conflict': ['<<<<<<< HEAD\n{"errors":{}}\n=======\n{"errors":{"a":1}}\n>>>>>>> branch\n', /it is not JSON: /], + 'the wrong shape': ['[1]', /it must be an object with "errors" and "warnings"\./], + } + + for (const [what, [content, reason]] of Object.entries(cases)) { + const run = compareWith([fixture('clean.txt')], { PLUGIN_CHECK_BASELINE: baselineFile(content) }) + + assert.notEqual(run.status, 0, what) + assert.match(run.stdout, /::error title=Plugin Check::The baseline file baseline\.json is not valid: /, what) + assert.match(run.stdout, reason, what) + assert.match(run.stdout, /Fix it, or replace it with the baseline\.json from this run's plugin-check-report artifact\./, what) + assert.equal(run.comment, '', what) + } + }) + it('passes when every count matches the baseline', () => { const run = compareWith([fixture('same-as-baseline.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) @@ -409,7 +488,7 @@ describe('plugin-check-report compare', () => { const run = compareWith([resultsOf([{ type: 'ERROR', code: 'a' }])], { PLUGIN_CHECK_BASELINE: baselineFile(content) }) assert.equal(run.status, 5, content) - assert.match(run.stderr, /more than 9007199254740991, which cannot be compared exactly/) + assert.match(run.stdout, /::error title=Plugin Check::The baseline file baseline\.json is not valid: .*more than 9007199254740991, which cannot be compared exactly/) assert.equal(run.comment, '', content) } @@ -535,7 +614,7 @@ describe('plugin-check-report compare', () => { }) it('writes the same text to the job summary', () => { - const run = compareWith([fixture('errors-down.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) + const run = compareWith([fixture('same-as-baseline.txt')], { PLUGIN_CHECK_BASELINE: BASELINE }) assert.equal(run.summary, run.comment) }) @@ -579,3 +658,61 @@ describe('the baseline file text', () => { assert.equal(report.formatBaseline(counts), '{\n "errors": {},\n "warnings": {\n "__proto__": 1\n }\n}\n') }) }) + +describe('the compare command line', () => { + /** + * Runs the command line in a new folder, so the default baseline path lands there. + * + * @param {string[]} argv The arguments after the script name. + * + * @returns {Promise<{status: number, lines: string[], dir: string}>} The status, what it printed and the folder. + */ + async function inNewFolder(argv) { + const dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-cli-'))) + const before = process.cwd() + const lines = [] + + process.chdir(dir) + try { + const status = await report.main(argv, {}, { out: (line) => lines.push(line), err: (line) => lines.push(line) }) + + return { status, lines, dir } + } finally { + process.chdir(before) + } + } + + const pinned = () => fs.readFileSync(path.join(FIXTURES, 'expected', 'update-errors-up.json'), 'utf8') + + it('writes .plugin-check-baseline.json in the current folder by default', async () => { + const { status, lines, dir } = await inNewFolder(['compare', '--update', fixture('errors-up.txt')]) + + assert.equal(status, 0) + assert.equal(fs.readFileSync(path.join(dir, '.plugin-check-baseline.json'), 'utf8'), pinned()) + assert.match(lines.join('\n'), /Wrote .*\.plugin-check-baseline\.json: 2 error codes, 1 warning codes/) + }) + + it('takes `update` as a short name for `compare --update`', async () => { + const { status, dir } = await inNewFolder(['update', fixture('errors-up.txt')]) + + assert.equal(status, 0) + assert.equal(fs.readFileSync(path.join(dir, '.plugin-check-baseline.json'), 'utf8'), pinned()) + }) + + it('compares with the file it wrote', async () => { + const dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-check-cli-'))) + fs.writeFileSync(path.join(dir, '.plugin-check-baseline.json'), pinned()) + const before = process.cwd() + const lines = [] + + process.chdir(dir) + try { + const status = await report.main(['compare', fixture('errors-up.txt')], {}, { out: (line) => lines.push(line), err: (line) => lines.push(line) }) + + assert.equal(status, 0) + assert.match(lines.join('\n'), /within the baseline/) + } finally { + process.chdir(before) + } + }) +}) diff --git a/tests/actions/plugin-check-report/exclude.test.js b/tests/actions/plugin-check-report/exclude.test.js index 1d0e410..70601e6 100644 --- a/tests/actions/plugin-check-report/exclude.test.js +++ b/tests/actions/plugin-check-report/exclude.test.js @@ -236,6 +236,6 @@ describe('exclude-checks on the command line', () => { const status = await report.main(['nope'], {}, { out: () => {}, err: (line) => err.push(line) }) assert.equal(status, 2) - assert.match(err[0], /compare\|exclude-checks/) + assert.match(err[0], /compare\|update\|exclude-checks/) }) }) diff --git a/tests/actions/plugin-check-report/fixtures/expected/artifact-only.md b/tests/actions/plugin-check-report/fixtures/expected/artifact-only.md index f6e1a8c..324adff 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/artifact-only.md +++ b/tests/actions/plugin-check-report/fixtures/expected/artifact-only.md @@ -1,5 +1,5 @@ -⚠️ Plugin Check ran, but there was no baseline to compare with +⚠️ Plugin Check ran, but there is no baseline file yet | | Now | |:--|--:| diff --git a/tests/actions/plugin-check-report/fixtures/expected/artifact-only.out b/tests/actions/plugin-check-report/fixtures/expected/artifact-only.out index ffed42c..fae3456 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/artifact-only.out +++ b/tests/actions/plugin-check-report/fixtures/expected/artifact-only.out @@ -1,2 +1,2 @@ -::warning title=Plugin Check::No baseline to compare with (/nonexistent/baseline.json), so nothing was compared. Run the workflow on the target branch to make one. +::warning title=Plugin Check::No baseline file at baseline.json, so nothing was compared. To start one, commit the baseline.json from this run's plugin-check-report artifact as baseline.json. Plugin Check ran with no baseline, so nothing was compared. diff --git a/tests/actions/plugin-check-report/fixtures/expected/artifact-only.summary b/tests/actions/plugin-check-report/fixtures/expected/artifact-only.summary index f6e1a8c..092a5b2 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/artifact-only.summary +++ b/tests/actions/plugin-check-report/fixtures/expected/artifact-only.summary @@ -1,5 +1,5 @@ -⚠️ Plugin Check ran, but there was no baseline to compare with +⚠️ Plugin Check ran, but there is no baseline file yet | | Now | |:--|--:| @@ -7,3 +7,10 @@ | Warnings | 0 | 📎 [Full results and new baseline](https://github.com/o/r/actions/runs/1/artifacts/2) + +There is no baseline.json yet. To start one, add the file from this run: + +```bash +gh run download --repo / --name plugin-check-report --dir /tmp/plugin-check +cp /tmp/plugin-check/baseline.json baseline.json +``` diff --git a/tests/actions/plugin-check-report/fixtures/expected/badge-and-artifact.summary b/tests/actions/plugin-check-report/fixtures/expected/badge-and-artifact.summary index c9247b3..4375d48 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/badge-and-artifact.summary +++ b/tests/actions/plugin-check-report/fixtures/expected/badge-and-artifact.summary @@ -11,3 +11,10 @@ Went up: `WordPress.Security.EscapeOutput.OutputNotEscaped` (+1) 📎 [Full results and new baseline](https://github.com/o/r/actions/runs/1/artifacts/2) · [Run](https://example.test/actions/runs/1) + +If the new errors are on purpose, raise their counts in baseline.json in this pull request, so reviewers see it. The file from this run has the new counts: + +```bash +gh run download --repo / --name plugin-check-report --dir /tmp/plugin-check +cp /tmp/plugin-check/baseline.json baseline.json +``` diff --git a/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.md b/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.md index 40ace2c..a465c6b 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.md +++ b/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.md @@ -1,5 +1,5 @@ -⚠️ Plugin Check ran, but there was no baseline to compare with +⚠️ Plugin Check ran, but there is no baseline file yet | | Now | |:--|--:| diff --git a/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.out b/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.out index ffed42c..fae3456 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.out +++ b/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.out @@ -1,2 +1,2 @@ -::warning title=Plugin Check::No baseline to compare with (/nonexistent/baseline.json), so nothing was compared. Run the workflow on the target branch to make one. +::warning title=Plugin Check::No baseline file at baseline.json, so nothing was compared. To start one, commit the baseline.json from this run's plugin-check-report artifact as baseline.json. Plugin Check ran with no baseline, so nothing was compared. diff --git a/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.summary b/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.summary index 40ace2c..8d892fd 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.summary +++ b/tests/actions/plugin-check-report/fixtures/expected/clean-no-baseline.summary @@ -1,5 +1,5 @@ -⚠️ Plugin Check ran, but there was no baseline to compare with +⚠️ Plugin Check ran, but there is no baseline file yet | | Now | |:--|--:| @@ -7,3 +7,10 @@ | Warnings | 0 | 📎 [Run](https://example.test/actions/runs/1) + +There is no baseline.json yet. To start one, add the file from this run: + +```bash +gh run download --repo / --name plugin-check-report --dir /tmp/plugin-check +cp /tmp/plugin-check/baseline.json baseline.json +``` diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-down.md b/tests/actions/plugin-check-report/fixtures/expected/errors-down.md index 00b7a4c..d8cc887 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/errors-down.md +++ b/tests/actions/plugin-check-report/fixtures/expected/errors-down.md @@ -6,6 +6,6 @@ | Errors | 2 | 3 | -1 | | Warnings | 1 | 1 | ±0 | -Rules that went down: 1. +Rules that went down: 1. Update the baseline file, see the job summary. 📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-down.out b/tests/actions/plugin-check-report/fixtures/expected/errors-down.out index df39f8a..ebb31c4 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/errors-down.out +++ b/tests/actions/plugin-check-report/fixtures/expected/errors-down.out @@ -1,2 +1,3 @@ ::notice title=Plugin Check::WordPress.Security.EscapeOutput.OutputNotEscaped (errors) went down from 2 to 1. +::notice title=Plugin Check::Error counts went down. Lower the baseline so they cannot come back: commit the baseline.json from this run's plugin-check-report artifact as baseline.json. Plugin Check error counts are within the baseline. diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-down.summary b/tests/actions/plugin-check-report/fixtures/expected/errors-down.summary index 00b7a4c..ec490bc 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/errors-down.summary +++ b/tests/actions/plugin-check-report/fixtures/expected/errors-down.summary @@ -6,6 +6,13 @@ | Errors | 2 | 3 | -1 | | Warnings | 1 | 1 | ±0 | -Rules that went down: 1. +Rules that went down: 1. Update the baseline file, see the job summary. 📎 [Run](https://example.test/actions/runs/1) + +Error counts went down. To lower the baseline, replace baseline.json with the file from this run: + +```bash +gh run download --repo / --name plugin-check-report --dir /tmp/plugin-check +cp /tmp/plugin-check/baseline.json baseline.json +``` diff --git a/tests/actions/plugin-check-report/fixtures/expected/errors-up.summary b/tests/actions/plugin-check-report/fixtures/expected/errors-up.summary index c5e77f1..8c8c95f 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/errors-up.summary +++ b/tests/actions/plugin-check-report/fixtures/expected/errors-up.summary @@ -9,3 +9,10 @@ Went up: `WordPress.Security.EscapeOutput.OutputNotEscaped` (+1) 📎 [Run](https://example.test/actions/runs/1) + +If the new errors are on purpose, raise their counts in baseline.json in this pull request, so reviewers see it. The file from this run has the new counts: + +```bash +gh run download --repo / --name plugin-check-report --dir /tmp/plugin-check +cp /tmp/plugin-check/baseline.json baseline.json +``` diff --git a/tests/actions/plugin-check-report/fixtures/expected/marker-with-line-breaks.summary b/tests/actions/plugin-check-report/fixtures/expected/marker-with-line-breaks.summary index fa5abf2..8a706aa 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/marker-with-line-breaks.summary +++ b/tests/actions/plugin-check-report/fixtures/expected/marker-with-line-breaks.summary @@ -13,3 +13,10 @@ line 12 line 13 line 14 line 15 + +If the new errors are on purpose, raise their counts in baseline.json in this pull request, so reviewers see it. The file from this run has the new counts: + +```bash +gh run download --repo / --name plugin-check-report --dir /tmp/plugin-check +cp /tmp/plugin-check/baseline.json baseline.json +``` diff --git a/tests/actions/plugin-check-report/fixtures/expected/marker.md b/tests/actions/plugin-check-report/fixtures/expected/marker.md index df1c76e..2be0d1f 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/marker.md +++ b/tests/actions/plugin-check-report/fixtures/expected/marker.md @@ -1,5 +1,5 @@ -⚠️ Plugin Check ran, but there was no baseline to compare with +⚠️ Plugin Check ran, but there is no baseline file yet | | Now | |:--|--:| diff --git a/tests/actions/plugin-check-report/fixtures/expected/marker.out b/tests/actions/plugin-check-report/fixtures/expected/marker.out index ffed42c..fae3456 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/marker.out +++ b/tests/actions/plugin-check-report/fixtures/expected/marker.out @@ -1,2 +1,2 @@ -::warning title=Plugin Check::No baseline to compare with (/nonexistent/baseline.json), so nothing was compared. Run the workflow on the target branch to make one. +::warning title=Plugin Check::No baseline file at baseline.json, so nothing was compared. To start one, commit the baseline.json from this run's plugin-check-report artifact as baseline.json. Plugin Check ran with no baseline, so nothing was compared. diff --git a/tests/actions/plugin-check-report/fixtures/expected/marker.summary b/tests/actions/plugin-check-report/fixtures/expected/marker.summary index df1c76e..8137bfd 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/marker.summary +++ b/tests/actions/plugin-check-report/fixtures/expected/marker.summary @@ -1,5 +1,5 @@ -⚠️ Plugin Check ran, but there was no baseline to compare with +⚠️ Plugin Check ran, but there is no baseline file yet | | Now | |:--|--:| @@ -7,3 +7,10 @@ | Warnings | 0 | 📎 [Run](https://example.test/actions/runs/1) + +There is no baseline.json yet. To start one, add the file from this run: + +```bash +gh run download --repo / --name plugin-check-report --dir /tmp/plugin-check +cp /tmp/plugin-check/baseline.json baseline.json +``` diff --git a/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.md b/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.md index 04294cc..99a03fa 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.md +++ b/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.md @@ -7,6 +7,6 @@ | Warnings | 1 | 1 | ±0 | Went up: `WordPress.Security.EscapeOutput.OutputNotEscaped` (+1) -Rules that went down: 1. +Rules that went down: 1. Update the baseline file, see the job summary. 📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.out b/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.out index 68830be..caaeea3 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.out +++ b/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.out @@ -1,3 +1,4 @@ ::error title=Plugin Check::WordPress.Security.EscapeOutput.OutputNotEscaped has 3 errors, the baseline has 2. The full list is in the plugin-check-results artifact. ::notice title=Plugin Check::WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents (errors) went down from 1 to 0. +::notice title=Plugin Check::Error counts went down. Lower the baseline so they cannot come back: commit the baseline.json from this run's plugin-check-report artifact as baseline.json. Plugin Check found more errors than the baseline allows. diff --git a/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.summary b/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.summary index 04294cc..2386d65 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.summary +++ b/tests/actions/plugin-check-report/fixtures/expected/rule-up-total-same.summary @@ -7,6 +7,13 @@ | Warnings | 1 | 1 | ±0 | Went up: `WordPress.Security.EscapeOutput.OutputNotEscaped` (+1) -Rules that went down: 1. +Rules that went down: 1. Update the baseline file, see the job summary. 📎 [Run](https://example.test/actions/runs/1) + +Error counts went down. To lower the baseline, replace baseline.json with the file from this run: + +```bash +gh run download --repo / --name plugin-check-report --dir /tmp/plugin-check +cp /tmp/plugin-check/baseline.json baseline.json +``` diff --git a/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.md b/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.md index 0a97eea..c407f25 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.md +++ b/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.md @@ -7,6 +7,6 @@ | Warnings | 0 | 1 | -1 | Went up (top 5 of 7): `Test.Rule.Number1` (+1), `Test.Rule.Number2` (+1), `Test.Rule.Number3` (+1), `Test.Rule.Number4` (+1), `Test.Rule.Number5` (+1) -Rules that went down: 3. +Rules that went down: 3. Update the baseline file, see the job summary. 📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.out b/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.out index c8c976c..fd225d1 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.out +++ b/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.out @@ -8,4 +8,5 @@ ::notice title=Plugin Check::WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents (errors) went down from 1 to 0. ::notice title=Plugin Check::WordPress.Security.EscapeOutput.OutputNotEscaped (errors) went down from 2 to 0. ::notice title=Plugin Check::WordPress.DB.DirectDatabaseQuery.DirectQuery (warnings) went down from 1 to 0. +::notice title=Plugin Check::Error counts went down. Lower the baseline so they cannot come back: commit the baseline.json from this run's plugin-check-report artifact as baseline.json. Plugin Check found more errors than the baseline allows. diff --git a/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.summary b/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.summary index 0a97eea..3f2176c 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.summary +++ b/tests/actions/plugin-check-report/fixtures/expected/seven-new-codes.summary @@ -7,6 +7,13 @@ | Warnings | 0 | 1 | -1 | Went up (top 5 of 7): `Test.Rule.Number1` (+1), `Test.Rule.Number2` (+1), `Test.Rule.Number3` (+1), `Test.Rule.Number4` (+1), `Test.Rule.Number5` (+1) -Rules that went down: 3. +Rules that went down: 3. Update the baseline file, see the job summary. 📎 [Run](https://example.test/actions/runs/1) + +Error counts went down. To lower the baseline, replace baseline.json with the file from this run: + +```bash +gh run download --repo / --name plugin-check-report --dir /tmp/plugin-check +cp /tmp/plugin-check/baseline.json baseline.json +``` diff --git a/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.md b/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.md index 9960632..131efef 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.md +++ b/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.md @@ -7,6 +7,6 @@ | Warnings | 0 | 1 | -1 | Went up (top 5 of 6): `Test.Rule.Number1` (+1), `Test.Rule.Number2` (+1), `Test.Rule.Number3` (+1), `Test.Rule.Number4` (+1), `Test.Rule.Number5` (+1) -Rules that went down: 3. +Rules that went down: 3. Update the baseline file, see the job summary. 📎 [Run](https://example.test/actions/runs/1) diff --git a/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.out b/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.out index 733d898..ecd7d0f 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.out +++ b/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.out @@ -7,4 +7,5 @@ ::notice title=Plugin Check::WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents (errors) went down from 1 to 0. ::notice title=Plugin Check::WordPress.Security.EscapeOutput.OutputNotEscaped (errors) went down from 2 to 0. ::notice title=Plugin Check::WordPress.DB.DirectDatabaseQuery.DirectQuery (warnings) went down from 1 to 0. +::notice title=Plugin Check::Error counts went down. Lower the baseline so they cannot come back: commit the baseline.json from this run's plugin-check-report artifact as baseline.json. Plugin Check found more errors than the baseline allows. diff --git a/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.summary b/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.summary index 9960632..11d5606 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.summary +++ b/tests/actions/plugin-check-report/fixtures/expected/six-new-codes.summary @@ -7,6 +7,13 @@ | Warnings | 0 | 1 | -1 | Went up (top 5 of 6): `Test.Rule.Number1` (+1), `Test.Rule.Number2` (+1), `Test.Rule.Number3` (+1), `Test.Rule.Number4` (+1), `Test.Rule.Number5` (+1) -Rules that went down: 3. +Rules that went down: 3. Update the baseline file, see the job summary. 📎 [Run](https://example.test/actions/runs/1) + +Error counts went down. To lower the baseline, replace baseline.json with the file from this run: + +```bash +gh run download --repo / --name plugin-check-report --dir /tmp/plugin-check +cp /tmp/plugin-check/baseline.json baseline.json +``` diff --git a/tests/actions/plugin-check-report/fixtures/expected/strict-clean.out b/tests/actions/plugin-check-report/fixtures/expected/strict-clean.out index ffed42c..afbd6dd 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/strict-clean.out +++ b/tests/actions/plugin-check-report/fixtures/expected/strict-clean.out @@ -1,2 +1,2 @@ -::warning title=Plugin Check::No baseline to compare with (/nonexistent/baseline.json), so nothing was compared. Run the workflow on the target branch to make one. +No baseline file. Strict mode does not need one. Plugin Check ran with no baseline, so nothing was compared. diff --git a/tests/actions/plugin-check-report/fixtures/expected/strict-no-baseline.out b/tests/actions/plugin-check-report/fixtures/expected/strict-no-baseline.out index bc6ddf6..ee0d0f0 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/strict-no-baseline.out +++ b/tests/actions/plugin-check-report/fixtures/expected/strict-no-baseline.out @@ -1,3 +1,3 @@ -::warning title=Plugin Check::No baseline to compare with (/nonexistent/baseline.json), so nothing was compared. Run the workflow on the target branch to make one. +No baseline file. Strict mode does not need one. ::error title=Plugin Check::2 errors found. Strict mode fails on any error. Plugin Check found errors, and strict mode allows none. diff --git a/tests/actions/plugin-check-report/fixtures/expected/strict-warnings-only.out b/tests/actions/plugin-check-report/fixtures/expected/strict-warnings-only.out index ffed42c..afbd6dd 100644 --- a/tests/actions/plugin-check-report/fixtures/expected/strict-warnings-only.out +++ b/tests/actions/plugin-check-report/fixtures/expected/strict-warnings-only.out @@ -1,2 +1,2 @@ -::warning title=Plugin Check::No baseline to compare with (/nonexistent/baseline.json), so nothing was compared. Run the workflow on the target branch to make one. +No baseline file. Strict mode does not need one. Plugin Check ran with no baseline, so nothing was compared. From 189ffa61e67e6c1443956997863f11d3199c24df Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Wed, 4 Nov 2026 14:37:31 -0300 Subject: [PATCH 27/31] Check the target branch too when the repository has no Plugin Check baseline file --- .../actions/plugin-check-report/action.yml | 16 +++ .../plugin-check-report.js | 108 ++++++++++++----- .github/workflows/plugin-check.yml | 110 +++++++++++++++--- .../plugin-check-report/compare.test.js | 86 ++++++++++++++ 4 files changed, 274 insertions(+), 46 deletions(-) diff --git a/.github/actions/plugin-check-report/action.yml b/.github/actions/plugin-check-report/action.yml index 37fe69b..db5e674 100644 --- a/.github/actions/plugin-check-report/action.yml +++ b/.github/actions/plugin-check-report/action.yml @@ -45,6 +45,20 @@ inputs: A relative path is read from the working directory. required: false default: '.plugin-check-baseline.json' + base-results: + description: >- + Path of the Plugin Check results of the pull request's target branch. Used as the baseline when + there is no baseline file, so a pull request is still compared with something. Empty when + nothing was checked on the target branch. Not used in `strict` mode. + required: false + default: '' + base-required: + description: >- + `true` when the caller checked the target branch because there is no baseline file. A missing + `base-results` file then fails the compare, so a target branch that could not be checked never + passes as "no baseline". + required: false + default: 'false' outputs: artifact-url: @@ -102,6 +116,8 @@ runs: COMMENT_FILE: ${{ inputs.comment-file }} PLUGIN_CHECK_MODE: ${{ inputs.mode }} PLUGIN_CHECK_BASELINE: ${{ inputs.baseline-file }} + PLUGIN_CHECK_BASE_RESULTS: ${{ inputs.base-results }} + PLUGIN_CHECK_BASE_REQUIRED: ${{ inputs.base-required }} COMMENT_MARKER: ${{ steps.paths.outputs.marker }} ARTIFACT_URL: ${{ steps.artifact.outputs.artifact-url }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/actions/plugin-check-report/plugin-check-report.js b/.github/actions/plugin-check-report/plugin-check-report.js index 58815e2..c856b95 100644 --- a/.github/actions/plugin-check-report/plugin-check-report.js +++ b/.github/actions/plugin-check-report/plugin-check-report.js @@ -409,11 +409,12 @@ function buildRows(now, base) { * @param {string} input.mode `baseline` or `strict`. * @param {boolean} input.noBaseline There was nothing to compare with: only the totals count. * @param {string} input.baselineName The baseline file's name, for the hint to update it. + * @param {boolean} input.fromFile The baseline is the committed file, which can be lowered. * * @returns {object} `messages` (`{level, text}` in print order), `totals`, `wentUp`, `wentDown`, * `errorsDown` (how many rules have fewer errors), `errorsUp`, `upList` and `failed`. */ -function evaluate({ rows, mode, noBaseline, baselineName = BASELINE_FILE }) { +function evaluate({ rows, mode, noBaseline, baselineName = BASELINE_FILE, fromFile = true }) { const totals = { errorsBase: 0, errorsNow: 0, warningsBase: 0, warningsNow: 0 } const messages = [] let failed = false @@ -473,7 +474,7 @@ function evaluate({ rows, mode, noBaseline, baselineName = BASELINE_FILE }) { // Fewer errors than the file says leaves room for them to come back unnoticed. Strict mode does // not use the file, so it asks for nothing. - if (mode !== 'strict' && errorsDown > 0) { + if (mode !== 'strict' && fromFile && errorsDown > 0) { messages.push({ level: 'notice', text: `Error counts went down. Lower the baseline so they cannot come back: commit the baseline.json from this run's plugin-check-report artifact as ${baselineName}.`, @@ -574,12 +575,13 @@ function linksLine({ artifactUrl, runUrl, resultsPath }) { * @param {number} input.wentUp How many rules went up. * @param {number} input.wentDown How many rules went down. * @param {boolean} input.updateHint An error count went down: ask for the baseline file to be updated. + * @param {string} input.note One line under the table, such as where the baseline came from, or ''. * @param {string} input.upList The first rules that went up, ready to print. * @param {string} input.links From linksLine. * * @returns {string} The body, cut to MAX_COMMENT_LINES lines, with no final line break. */ -function buildComment({ marker, status, badge, column, noBaseline, totals, wentUp, wentDown, updateHint = false, upList, links }) { +function buildComment({ marker, status, badge, column, noBaseline, totals, wentUp, wentDown, updateHint = false, note = '', upList, links }) { let body = `${marker}\n${status}\n\n` if (badge) { @@ -596,6 +598,9 @@ function buildComment({ marker, status, badge, column, noBaseline, totals, wentU body += `| Warnings | ${totals.warningsNow} | ${totals.warningsBase} | ${change(totals.warningsNow, totals.warningsBase)} |\n\n` } + if (note) { + body += `${note}\n` + } if (wentUp > MAX_RULES_LISTED) { body += `Went up (top ${MAX_RULES_LISTED} of ${wentUp}): ${upList}\n` } else if (wentUp > 0) { @@ -605,7 +610,7 @@ function buildComment({ marker, status, badge, column, noBaseline, totals, wentU body += `Rules that went down: ${wentDown}.${updateHint ? ' Update the baseline file, see the job summary.' : ''}\n` } // A blank line separates the notes above from the links, when there are notes. - if (wentUp > 0 || wentDown > 0) { + if (wentUp > 0 || wentDown > 0 || note) { body += '\n' } body += links @@ -647,26 +652,59 @@ function readBaselineFile(file, name) { } } +/** + * The target branch's results as a baseline, for a pull request in a repository with no baseline + * file. The workflow checked the target branch's commit in the same run, so its counts are what the + * pull request is compared with. + * + * @param {object} env PLUGIN_CHECK_BASE_RESULTS (the results file) and PLUGIN_CHECK_BASE_REQUIRED. + * @param {string} name The baseline file's name, for the message. + * + * @throws {CheckFailure} When the results are missing or are not Plugin Check output. Never passes + * without a baseline, because the workflow asked for one. + * + * @returns {{errors: Map, warnings: Map}} Counts. + */ +function readTargetResults(env, name) { + const file = env.PLUGIN_CHECK_BASE_RESULTS || '' + + if (!file || !isFile(file)) { + throw new CheckFailure( + `The target branch could not be checked, so there is nothing to compare with. Look at the job that checks the target branch in this run. To skip that check, commit the baseline.json from this run's plugin-check-report artifact as ${name}.`, + 1, + true + ) + } + + const text = fs.readFileSync(file, 'utf8') + const refusal = checkResultsText(text) + if (refusal) { + throw new CheckFailure(`The results of the target branch cannot be used. ${refusal}`, 1, true) + } + + return countPerCode(parseResults(text)) +} + /** * The part of the job summary that says how to change the baseline file, or '' when there is nothing * to do. The comment stays short; this is where the commands go. * * @param {object} input * @param {string} input.mode `baseline` or `strict`. - * @param {boolean} input.noBaseline There is no baseline file. + * @param {boolean} input.noFile There is no baseline file. * @param {object} input.verdict From evaluate. * @param {string} input.baselineName The baseline file's name. * @param {object} input.env GITHUB_RUN_ID and GITHUB_REPOSITORY, for the real command. * * @returns {string} Markdown with a final line break, or ''. */ -function updateSection({ mode, noBaseline, verdict, baselineName, env }) { +function updateSection({ mode, noFile, verdict, baselineName, env }) { if (mode === 'strict') { return '' } const lines = [] - if (noBaseline) { + if (noFile) { lines.push(`There is no ${baselineName} yet. To start one, add the file from this run:`) } else if (verdict.errorsDown > 0) { lines.push(`Error counts went down. To lower the baseline, replace ${baselineName} with the file from this run:`) @@ -700,9 +738,11 @@ function updateSection({ mode, noBaseline, verdict, baselineName, env }) { * @param {string[]} input.args Arguments: `--update` and `--comment-file ` in any order, then * the results file. * @param {object} input.env Environment. PLUGIN_CHECK_MODE (`baseline`, the default, or `strict`), - * COMMENT_MARKER, PLUGIN_CHECK_BASELINE (default: baseline.json next - * to this file), COMMENT_BADGE, RUN_URL, ARTIFACT_URL, - * GITHUB_STEP_SUMMARY. + * COMMENT_MARKER, PLUGIN_CHECK_BASELINE (default: .plugin-check-baseline.json + * in the current folder), PLUGIN_CHECK_BASE_RESULTS (the target branch's results, + * used as the baseline when there is no file) and + * PLUGIN_CHECK_BASE_REQUIRED (`true`: they must exist), COMMENT_BADGE, + * RUN_URL, ARTIFACT_URL, GITHUB_STEP_SUMMARY. * @param {object} input.log Where messages go: `out`, `err`, `error`, `warning`, `notice`. * * @returns {number} The exit status: 0 passes, 1 fails the check, another number is a file that @@ -801,24 +841,33 @@ function compareOrThrow({ args, env, log }) { return 0 } - // No baseline file yet: report the counts and pass. Failing here would block the very pull request - // that adds the workflow, before anyone has a result to make the file from. Strict mode does not - // use the file, so it has nothing to say. - const noBaseline = !isFile(baselinePath) + // Where the baseline comes from: the committed file, else (baseline mode, pull request) the target + // branch's results, else nothing. With no baseline at all, the counts are reported and the check + // passes: failing would block a push before anyone has a result to make the file from. + const hasFile = isFile(baselinePath) + let source = hasFile ? 'file' : 'none' let base = { errors: new Map(), warnings: new Map() } - if (noBaseline) { - if (mode === 'strict') { - log.out('No baseline file. Strict mode does not need one.') - } else { - log.warning( - `No baseline file at ${baselineName}, so nothing was compared. To start one, commit the baseline.json from this run's plugin-check-report artifact as ${baselineName}.` - ) - } - } else { + + if (hasFile) { base = readBaselineFile(baselinePath, baselineName) + } else if (mode === 'strict') { + // Strict mode does not use a baseline, so it has nothing to say and builds nothing. + log.out('No baseline file. Strict mode does not need one.') + } else if (env.PLUGIN_CHECK_BASE_RESULTS || env.PLUGIN_CHECK_BASE_REQUIRED === 'true') { + base = readTargetResults(env, baselineName) + source = 'target' + log.warning( + `No baseline file at ${baselineName}, so the target branch was checked too, and its results are the baseline for this run. That makes the run slower. To stop it, commit the baseline.json from this run's plugin-check-report artifact as ${baselineName}.` + ) + } else { + log.warning( + `No baseline file at ${baselineName}, so nothing was compared. To start one, commit the baseline.json from this run's plugin-check-report artifact as ${baselineName}.` + ) } - const verdict = evaluate({ rows: buildRows(now, base), mode, noBaseline, baselineName }) + const noBaseline = source === 'none' + const verdict = evaluate({ rows: buildRows(now, base), mode, noBaseline, baselineName, fromFile: hasFile }) + for (const { level, text: message } of verdict.messages) { log[level](message) } @@ -839,12 +888,16 @@ function compareOrThrow({ args, env, log }) { marker: env.COMMENT_MARKER || '', status: statusLine({ mode, noBaseline, ...verdict, errorsNow: verdict.totals.errorsNow }), badge: env.COMMENT_BADGE || '', - column: 'Baseline', + column: source === 'target' ? 'Target branch' : 'Baseline', noBaseline, totals: verdict.totals, wentUp: verdict.wentUp, wentDown: verdict.wentDown, - updateHint: mode !== 'strict' && verdict.errorsDown > 0, + updateHint: mode !== 'strict' && hasFile && verdict.errorsDown > 0, + note: + source === 'target' + ? `ℹ️ There is no baseline file, so the baseline is the target branch's results. See the job summary to add one.` + : '', upList: verdict.upList, links: linksLine({ artifactUrl: env.ARTIFACT_URL || '', runUrl, resultsPath }), }) @@ -853,7 +906,7 @@ function compareOrThrow({ args, env, log }) { writeText(commentFile, `${body}\n`, 'write') } if (env.GITHUB_STEP_SUMMARY) { - const how = updateSection({ mode, noBaseline, verdict, baselineName, env }) + const how = updateSection({ mode, noFile: !hasFile, verdict, baselineName, env }) writeText(env.GITHUB_STEP_SUMMARY, `${body}\n${how}`, 'append') } @@ -1379,6 +1432,7 @@ module.exports = { parseResults, readBaseline, readBaselineFile, + readTargetResults, readIfFile, readJsonValues, runCompare, diff --git a/.github/workflows/plugin-check.yml b/.github/workflows/plugin-check.yml index bd26543..c2e2132 100644 --- a/.github/workflows/plugin-check.yml +++ b/.github/workflows/plugin-check.yml @@ -11,12 +11,19 @@ name: Plugin Check # .github/actions/pup action, so the repository does not need a copy of it or a Composer script for # it. A repository without pup passes `build-command` instead; see the plugin-check-report README. # -# Three jobs, so the third-party action never holds a token that can write. The comments are written +# Four jobs, so the third-party action never holds a token that can write. The comments are written # by .github/actions/pr-comment, which calls the GitHub API itself: # start posts "in progress" on the pull request. -# check builds the plugin and runs wordpress/plugin-check-action. Read-only token. +# plan lists the excluded checks, and decides whether the target branch is checked too. +# check builds the plugin and runs wordpress/plugin-check-action. Read-only token. It runs for +# the pull request ("head") and, when the repository has no baseline file, for the commit +# the pull request targets ("base"), in parallel. # report compares the results with the baseline and replaces the comment with the result. # +# The baseline is the repository's `.plugin-check-baseline.json`, read from the pull request's own +# checkout. Without that file, a pull request is compared with the target branch's results instead, +# which costs a second build and check; commit the file to stop that. +# # The action also tries to post its own comment. With a read-only token that call fails (it logs # "Failed to post PR comment" and carries on), so the only comment is ours. # @@ -154,10 +161,66 @@ jobs: ![Plugin Check started](${{ steps.badge.outputs.url }}) + # Works out, from the repository's files, what the check job needs to know. Cheap: a sparse + # checkout of two files. A bad .plugin-check.json fails here in seconds, before any build. + plan: + name: Plugin Check plan + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + matrix: ${{ steps.target.outputs.matrix }} + base: ${{ steps.target.outputs.base }} + checks: ${{ steps.exclude.outputs.checks }} + steps: + - uses: actions/checkout@v7 + with: + path: source + sparse-checkout: | + .plugin-check.json + .plugin-check-baseline.json + sparse-checkout-cone-mode: false + persist-credentials: false + + # The checks to skip: the `exclude-checks` input when it names any, else the list in the + # repository's .plugin-check.json. Like the report job, this needs a runner of version 2.336.0 + # or newer for `$/`; the two small jobs run on ubuntu-latest, which has it. + - name: List the excluded checks + id: exclude + uses: $/.github/actions/plugin-check-report/exclude-checks + with: + exclude-checks: ${{ inputs.exclude-checks }} + config-file: ${{ github.workspace }}/source/.plugin-check.json + + # A pull request in baseline mode, in a repository with no baseline file, is compared with the + # target branch, so that commit is built and checked too. Strict mode needs no baseline, and a + # push has no target to compare with. + - name: Decide whether to check the target branch + id: target + env: + EVENT: ${{ github.event_name }} + MODE: ${{ inputs.mode }} + run: | + if [ "$EVENT" = pull_request ] && [ "$MODE" != strict ] && [ ! -f source/.plugin-check-baseline.json ]; then + echo 'matrix={"target":["head","base"]}' >> "$GITHUB_OUTPUT" + echo 'base=true' >> "$GITHUB_OUTPUT" + echo "::notice title=Plugin Check::There is no .plugin-check-baseline.json, so the target branch is built and checked too. Commit the baseline.json from the plugin-check-report artifact of this run to stop that." + else + echo 'matrix={"target":["head"]}' >> "$GITHUB_OUTPUT" + echo 'base=false' >> "$GITHUB_OUTPUT" + fi + check: - name: Plugin Check run + name: Plugin Check run (${{ matrix.target }}) + needs: plan runs-on: ${{ inputs.runs-on }} + strategy: + # The pull request and its target branch are separate checks: one failing must not stop the other. + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + # Build, WordPress start and the check together take about 15 minutes for a large plugin. The cap is # for a container that stops answering. timeout-minutes: 30 @@ -178,21 +241,12 @@ jobs: - uses: actions/checkout@v7 with: path: source + # "base" is the commit the pull request targets, as it is without the pull request. + ref: ${{ matrix.target == 'base' && github.event.pull_request.base.sha || '' }} submodules: ${{ inputs.submodules }} # Nothing here pushes over git, and the job goes on to run third-party code. persist-credentials: false - # The checks to skip: the `exclude-checks` input when it names any, else the list in the - # repository's .plugin-check.json. First, so a bad file fails in seconds, before the build, with - # a message that says what to fix. Like the report job, this needs a runner of version 2.336.0 - # or newer for `$/`; the `runs-on` input says so. - - name: List the excluded checks - id: exclude - uses: $/.github/actions/plugin-check-report/exclude-checks - with: - exclude-checks: ${{ inputs.exclude-checks }} - config-file: ${{ github.workspace }}/source/.plugin-check.json - # The action's wp-env pulls these images when it starts, and pulling them is part of the # time that step takes. Started now, in the background, the download overlaps with the # build. The images are the ones wp-env's generated files name (FROM wordpress..., FROM @@ -352,9 +406,10 @@ jobs: echo "Checking $dir ($(find "$dir" -type f | wc -l | xargs) files)." # Saved now, before the third-party action below runs, so nothing it does can end up in a - # cache that later runs restore. Skipped when this exact commit already saved one. + # cache that later runs restore. Skipped when this exact commit already saved one, and for the + # target branch, which only reads the cache. - name: Save the build cache - if: ${{ inputs.cache-paths != '' && steps.build-cache.outputs.cache-hit != 'true' }} + if: ${{ matrix.target == 'head' && inputs.cache-paths != '' && steps.build-cache.outputs.cache-hit != 'true' }} uses: actions/cache/save@v6 with: path: ${{ steps.cache-paths.outputs.paths }} @@ -376,17 +431,21 @@ jobs: # Fails on any error, which is expected while a plugin still has known problems: the report # job decides pass or fail from the counts. The action writes its results to # $RUNNER_TEMP/plugin-check-results.txt and uploads them as the plugin-check-results - # artifact. Its `slug` input is not passed: it would rename that artifact. + # artifact. Its `slug` input is passed for the target branch only: it renames that artifact to + # plugin-check-results-, so the two do not clash. - name: Run Plugin Check continue-on-error: true uses: wordpress/plugin-check-action@v1 with: build-dir: ${{ github.workspace }}/build/${{ inputs.plugin-slug }} - exclude-checks: ${{ steps.exclude.outputs.checks }} + exclude-checks: ${{ needs.plan.outputs.checks }} + # The target branch's results go into an artifact of their own. The slug is the folder the + # plugin is in anyway, so the checks see the same thing; only the artifact's name changes. + slug: ${{ matrix.target == 'base' && inputs.plugin-slug || '' }} report: name: Plugin Check - needs: [start, check] + needs: [start, plan, check] # "start" is skipped for forks and non-pull-request runs, which would skip this job too. if: ${{ !cancelled() }} runs-on: ubuntu-latest @@ -415,6 +474,7 @@ jobs: run: | { echo "RESULTS=$RUNNER_TEMP/results/plugin-check-results.txt" + echo "BASE_RESULTS=$RUNNER_TEMP/base-results/plugin-check-results.txt" echo "COMMENT_BODY=$RUNNER_TEMP/plugin-check-comment.md" } >> "$GITHUB_ENV" @@ -427,6 +487,16 @@ jobs: name: plugin-check-results path: ${{ runner.temp }}/results + # The target branch's results, when the plan said to check it. A missing artifact is not an + # error here: the compare fails with a message that says the target could not be checked. + - name: Download the target branch's results + if: ${{ needs.plan.outputs.base == 'true' }} + continue-on-error: true + uses: actions/download-artifact@v8 + with: + name: plugin-check-results-${{ inputs.plugin-slug }} + path: ${{ runner.temp }}/base-results + # `$/` is this repository at the commit that runs this workflow, so the action is always the # same revision as the workflow, whichever ref the caller used, and it needs no checkout. It # needs a runner of version 2.336.0 or newer; this job runs on ubuntu-latest. Not ./, which @@ -443,6 +513,8 @@ jobs: comment-marker: ${{ env.COMMENT_MARKER }} mode: ${{ inputs.mode }} baseline-file: ${{ github.workspace }}/source/.plugin-check-baseline.json + base-results: ${{ needs.plan.outputs.base == 'true' && env.BASE_RESULTS || '' }} + base-required: ${{ needs.plan.outputs.base == 'true' }} - name: Post the result if: ${{ !cancelled() && env.COMMENT == 'true' }} diff --git a/tests/actions/plugin-check-report/compare.test.js b/tests/actions/plugin-check-report/compare.test.js index 8a51999..d5c61c2 100644 --- a/tests/actions/plugin-check-report/compare.test.js +++ b/tests/actions/plugin-check-report/compare.test.js @@ -634,6 +634,92 @@ describe('plugin-check-report compare', () => { }) }) +describe('compare with no baseline file, in a pull request', () => { + // The workflow checked the target branch's commit in the same run, and passes its results. + const baseResults = () => fixture('same-as-baseline.txt') + + it('compares with the target branch and says so', () => { + const run = compareWith([fixture('errors-up.txt')], { PLUGIN_CHECK_BASE_RESULTS: baseResults(), PLUGIN_CHECK_BASE_REQUIRED: 'true' }) + + assert.equal(run.status, 1) + assert.match( + run.stdout, + /::warning title=Plugin Check::No baseline file at no-baseline\.json, so the target branch was checked too, and its results are the baseline for this run\. That makes the run slower\. To stop it, commit the baseline\.json from this run's plugin-check-report artifact as no-baseline\.json\./ + ) + assert.match(run.stdout, new RegExp(`::error title=Plugin Check::${ESCAPE} has 3 errors, the baseline has 2`)) + assert.match(run.comment, /\| Now \| Target branch \| Change \|/) + assert.match(run.comment, /ℹ️ There is no baseline file, so the baseline is the target branch's results\. See the job summary to add one\./) + assert.match(run.summary, /There is no no-baseline\.json yet\. To start one, add the file from this run:/) + }) + + it('passes when the pull request has no more errors than the target branch', () => { + const run = compareWith([fixture('same-as-baseline.txt')], { PLUGIN_CHECK_BASE_RESULTS: baseResults(), PLUGIN_CHECK_BASE_REQUIRED: 'true' }) + + assert.equal(run.status, 0) + assert.match(run.comment, /✅ Plugin Check passed/) + assert.match(run.comment, /\| Errors {3}\| 3 \| 3 \| ±0 \|/) + }) + + it('does not ask to lower a baseline file that does not exist', () => { + const run = compareWith([fixture('errors-down.txt')], { PLUGIN_CHECK_BASE_RESULTS: baseResults(), PLUGIN_CHECK_BASE_REQUIRED: 'true' }) + + assert.equal(run.status, 0) + assert.match(run.stdout, /went down from/, 'the per-rule notices stay') + assert.doesNotMatch(run.stdout, /Error counts went down/) + assert.doesNotMatch(run.comment, /Update the baseline file/) + }) + + it('keeps the comment within fifteen lines, links last, with the note', () => { + const run = compareWith([resultsWithNewCodes(7)], { PLUGIN_CHECK_BASE_RESULTS: baseResults(), PLUGIN_CHECK_BASE_REQUIRED: 'true' }) + const lines = run.comment.replace(/\n$/, '').split('\n') + + assert.ok(lines.length <= 15, `${lines.length} lines`) + assert.match(lines.at(-1), /^📎 /) + assert.ok(lines.some((line) => line.startsWith('ℹ️ There is no baseline file'))) + }) + + it('fails, never passes, when the target branch could not be checked', () => { + for (const env of [{}, { PLUGIN_CHECK_BASE_RESULTS: '/nonexistent/base-results.txt' }]) { + const run = compareWith([fixture('clean.txt')], { PLUGIN_CHECK_BASE_REQUIRED: 'true', ...env }) + + assert.equal(run.status, 1) + assert.match(run.stdout, /::error title=Plugin Check::The target branch could not be checked, so there is nothing to compare with\./) + assert.equal(run.comment, '', 'no comment body, so the workflow writes "could not run"') + } + }) + + it('fails when the results of the target branch are not Plugin Check output', () => { + const run = compareWith([fixture('clean.txt')], { PLUGIN_CHECK_BASE_RESULTS: fixture('garbage.txt'), PLUGIN_CHECK_BASE_REQUIRED: 'true' }) + + assert.equal(run.status, 1) + assert.match(run.stdout, /::error title=Plugin Check::The results of the target branch cannot be used\. The results file is not Plugin Check output/) + }) + + it('uses the baseline file when there is one, and ignores the target branch', () => { + const run = compareWith([fixture('errors-up.txt')], { + PLUGIN_CHECK_BASELINE: BASELINE, + PLUGIN_CHECK_BASE_RESULTS: fixture('garbage.txt'), + PLUGIN_CHECK_BASE_REQUIRED: 'true', + }) + + assert.equal(run.status, 1) + assert.match(run.comment, /\| Now \| Baseline \| Change \|/) + assert.doesNotMatch(run.stdout, /target branch was checked/) + }) + + it('leaves strict mode alone, which needs no baseline', () => { + const run = compareWith([fixture('clean.txt')], { + PLUGIN_CHECK_MODE: 'strict', + PLUGIN_CHECK_BASE_RESULTS: fixture('garbage.txt'), + PLUGIN_CHECK_BASE_REQUIRED: 'true', + }) + + assert.equal(run.status, 0) + assert.match(run.stdout, /No baseline file\. Strict mode does not need one\./) + assert.doesNotMatch(run.stdout, /target branch/) + }) +}) + describe('the baseline file text', () => { it('sorts codes by Unicode code point, not by UTF-16 unit', () => { const counts = report.countPerCode([ From f7ea8f04817d3a4dad63aa7b24a29ee82d984757 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Wed, 4 Nov 2026 14:37:51 -0300 Subject: [PATCH 28/31] Say when a pull request changes the Plugin Check baseline file --- .../actions/plugin-check-report/action.yml | 7 +++++ .../plugin-check-report.js | 28 +++++++++++++++---- .github/workflows/plugin-check.yml | 16 +++++++++++ .../plugin-check-report/compare.test.js | 19 +++++++++++++ 4 files changed, 65 insertions(+), 5 deletions(-) diff --git a/.github/actions/plugin-check-report/action.yml b/.github/actions/plugin-check-report/action.yml index db5e674..64ac08f 100644 --- a/.github/actions/plugin-check-report/action.yml +++ b/.github/actions/plugin-check-report/action.yml @@ -59,6 +59,12 @@ inputs: passes as "no baseline". required: false default: 'false' + baseline-changed: + description: >- + `true` when the pull request changes the baseline file. The comment then says so, because the + pull request is compared with its own change and a reviewer should look at the file's diff. + required: false + default: 'false' outputs: artifact-url: @@ -118,6 +124,7 @@ runs: PLUGIN_CHECK_BASELINE: ${{ inputs.baseline-file }} PLUGIN_CHECK_BASE_RESULTS: ${{ inputs.base-results }} PLUGIN_CHECK_BASE_REQUIRED: ${{ inputs.base-required }} + PLUGIN_CHECK_BASELINE_CHANGED: ${{ inputs.baseline-changed }} COMMENT_MARKER: ${{ steps.paths.outputs.marker }} ARTIFACT_URL: ${{ steps.artifact.outputs.artifact-url }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/actions/plugin-check-report/plugin-check-report.js b/.github/actions/plugin-check-report/plugin-check-report.js index c856b95..078c9fe 100644 --- a/.github/actions/plugin-check-report/plugin-check-report.js +++ b/.github/actions/plugin-check-report/plugin-check-report.js @@ -619,6 +619,24 @@ function buildComment({ marker, status, badge, column, noBaseline, totals, wentU return body.split('\n').slice(0, MAX_COMMENT_LINES).join('\n').replace(/\n+$/, '') } +/** + * The one line under the table that says where the baseline came from, when that is worth saying. + * + * @param {object} input + * @param {string} input.source `file`, `target` or `none`. + * @param {boolean} input.changed The pull request changes the baseline file, so the comparison is + * with its own change. A reviewer should look at the diff of the file. + * + * @returns {string} The line, or ''. + */ +function baselineNote({ source, changed }) { + if (source === 'target') { + return "ℹ️ There is no baseline file, so the baseline is the target branch's results. See the job summary to add one." + } + + return source === 'file' && changed ? 'ℹ️ This pull request changes the baseline file.' : '' +} + /** * Reads the baseline file, and names it when it cannot be used. * @@ -741,7 +759,9 @@ function updateSection({ mode, noFile, verdict, baselineName, env }) { * COMMENT_MARKER, PLUGIN_CHECK_BASELINE (default: .plugin-check-baseline.json * in the current folder), PLUGIN_CHECK_BASE_RESULTS (the target branch's results, * used as the baseline when there is no file) and - * PLUGIN_CHECK_BASE_REQUIRED (`true`: they must exist), COMMENT_BADGE, + * PLUGIN_CHECK_BASE_REQUIRED (`true`: they must exist), + * PLUGIN_CHECK_BASELINE_CHANGED (`true`: this pull request changes the + * baseline file), COMMENT_BADGE, * RUN_URL, ARTIFACT_URL, GITHUB_STEP_SUMMARY. * @param {object} input.log Where messages go: `out`, `err`, `error`, `warning`, `notice`. * @@ -894,10 +914,7 @@ function compareOrThrow({ args, env, log }) { wentUp: verdict.wentUp, wentDown: verdict.wentDown, updateHint: mode !== 'strict' && hasFile && verdict.errorsDown > 0, - note: - source === 'target' - ? `ℹ️ There is no baseline file, so the baseline is the target branch's results. See the job summary to add one.` - : '', + note: baselineNote({ source, changed: env.PLUGIN_CHECK_BASELINE_CHANGED === 'true' }), upList: verdict.upList, links: linksLine({ artifactUrl: env.ARTIFACT_URL || '', runUrl, resultsPath }), }) @@ -1431,6 +1448,7 @@ module.exports = { parseExcludeInput, parseResults, readBaseline, + baselineNote, readBaselineFile, readTargetResults, readIfFile, diff --git a/.github/workflows/plugin-check.yml b/.github/workflows/plugin-check.yml index c2e2132..37f5ea7 100644 --- a/.github/workflows/plugin-check.yml +++ b/.github/workflows/plugin-check.yml @@ -468,8 +468,23 @@ jobs: path: source sparse-checkout: .plugin-check-baseline.json sparse-checkout-cone-mode: false + # The merge commit and its two parents, to see whether the pull request changes the file. + fetch-depth: 2 persist-credentials: false + # The comment says so when a pull request changes the baseline: it is compared with its own + # change, which a reviewer should look at. + - name: Check whether the pull request changes the baseline file + id: changed + if: ${{ github.event_name == 'pull_request' }} + working-directory: source + run: | + if git diff --quiet HEAD^1 HEAD -- .plugin-check-baseline.json; then + echo 'changed=false' >> "$GITHUB_OUTPUT" + else + echo 'changed=true' >> "$GITHUB_OUTPUT" + fi + - name: Set file paths run: | { @@ -515,6 +530,7 @@ jobs: baseline-file: ${{ github.workspace }}/source/.plugin-check-baseline.json base-results: ${{ needs.plan.outputs.base == 'true' && env.BASE_RESULTS || '' }} base-required: ${{ needs.plan.outputs.base == 'true' }} + baseline-changed: ${{ steps.changed.outputs.changed == 'true' }} - name: Post the result if: ${{ !cancelled() && env.COMMENT == 'true' }} diff --git a/tests/actions/plugin-check-report/compare.test.js b/tests/actions/plugin-check-report/compare.test.js index d5c61c2..7fa3079 100644 --- a/tests/actions/plugin-check-report/compare.test.js +++ b/tests/actions/plugin-check-report/compare.test.js @@ -720,6 +720,25 @@ describe('compare with no baseline file, in a pull request', () => { }) }) +describe('the note about a pull request that changes the baseline file', () => { + it('says so under the table, and keeps the comment within fifteen lines', () => { + const run = compareWith([resultsWithNewCodes(7)], { PLUGIN_CHECK_BASELINE: BASELINE, PLUGIN_CHECK_BASELINE_CHANGED: 'true' }) + const lines = run.comment.replace(/\n$/, '').split('\n') + + assert.ok(lines.includes('ℹ️ This pull request changes the baseline file.')) + assert.ok(lines.length <= 15, `${lines.length} lines`) + assert.match(lines.at(-1), /^📎 /) + }) + + it('says nothing when the file is not changed, or does not exist', () => { + const same = compareWith([fixture('same-as-baseline.txt')], { PLUGIN_CHECK_BASELINE: BASELINE, PLUGIN_CHECK_BASELINE_CHANGED: 'false' }) + const none = compareWith([fixture('clean.txt')], { PLUGIN_CHECK_BASELINE_CHANGED: 'true' }) + + assert.doesNotMatch(same.comment, /changes the baseline file/) + assert.doesNotMatch(none.comment, /changes the baseline file/) + }) +}) + describe('the baseline file text', () => { it('sorts codes by Unicode code point, not by UTF-16 unit', () => { const counts = report.countPerCode([ From abd240d7957b2726a65c39d47504f8f0a3b57643 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Wed, 4 Nov 2026 15:08:31 -0300 Subject: [PATCH 29/31] End the reason of an invalid Plugin Check baseline message with a full stop --- .github/actions/plugin-check-report/plugin-check-report.js | 5 ++++- tests/actions/plugin-check-report/compare.test.js | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/actions/plugin-check-report/plugin-check-report.js b/.github/actions/plugin-check-report/plugin-check-report.js index 078c9fe..3460a5b 100644 --- a/.github/actions/plugin-check-report/plugin-check-report.js +++ b/.github/actions/plugin-check-report/plugin-check-report.js @@ -662,8 +662,11 @@ function readBaselineFile(file, name) { .replace(/^The baseline is not valid: /, '') .replace(/^jq: parse error: /, 'it is not JSON: ') + // A parser's message has no full stop of its own, and the sentence after it needs one. + const stop = /[.!?]$/.test(reason) ? '' : '.' + throw new CheckFailure( - `The baseline file ${name} is not valid: ${reason} Fix it, or replace it with the baseline.json from this run's plugin-check-report artifact.`, + `The baseline file ${name} is not valid: ${reason}${stop} Fix it, or replace it with the baseline.json from this run's plugin-check-report artifact.`, error.exitCode, true ) diff --git a/tests/actions/plugin-check-report/compare.test.js b/tests/actions/plugin-check-report/compare.test.js index 7fa3079..5598cbf 100644 --- a/tests/actions/plugin-check-report/compare.test.js +++ b/tests/actions/plugin-check-report/compare.test.js @@ -391,7 +391,7 @@ describe('plugin-check-report compare', () => { assert.notEqual(run.status, 0, what) assert.match(run.stdout, /::error title=Plugin Check::The baseline file baseline\.json is not valid: /, what) assert.match(run.stdout, reason, what) - assert.match(run.stdout, /Fix it, or replace it with the baseline\.json from this run's plugin-check-report artifact\./, what) + assert.match(run.stdout, /[.!?] Fix it, or replace it with the baseline\.json from this run's plugin-check-report artifact\./, what) assert.equal(run.comment, '', what) } }) From a700fc2fd8fb62b4cd9986e26b62419d770f4c37 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Wed, 4 Nov 2026 15:08:31 -0300 Subject: [PATCH 30/31] Document the committed Plugin Check baseline --- .github/actions/plugin-check-report/README.md | 165 ++++++++++++------ 1 file changed, 114 insertions(+), 51 deletions(-) diff --git a/.github/actions/plugin-check-report/README.md b/.github/actions/plugin-check-report/README.md index 0826804..5c203cf 100644 --- a/.github/actions/plugin-check-report/README.md +++ b/.github/actions/plugin-check-report/README.md @@ -1,8 +1,9 @@ # plugin-check-report Compares a [WordPress.org Plugin Check](https://github.com/WordPress/plugin-check) results file with -a baseline: the counts of an earlier run of the same workflow. It saves this run's counts as the -next baseline and writes a short body for the pull request comment. +a baseline: the number of errors and warnings per rule, kept in a file the repository commits, +`.plugin-check-baseline.json`. It writes a short body for the pull request comment, and a new +baseline to copy into the repository. Most repositories do not use this action directly. They call the reusable workflow [`.github/workflows/plugin-check.yml`](../../workflows/plugin-check.yml), which builds the plugin, @@ -14,7 +15,6 @@ action and posts the comment. A caller to copy is in jobs: plugin-check: permissions: - actions: read contents: read issues: write pull-requests: write @@ -33,7 +33,9 @@ jobs: to the release the repository's own release build uses, so the check sees the same files. - Ignore rules that keep dev-only files out of the zip: a `.distignore`, `export-ignore` lines in `.gitattributes` (pup reads both), or pup's default ignore list. -- The caller file named `plugin-check.yml`. See [The baseline](#the-baseline). +- A baseline file, `.plugin-check-baseline.json` at the root. Without it the check still works, but + slower: see [The baseline](#the-baseline). Keep it out of the zip: Plugin Check flags hidden files, + and pup's default ignore list already drops them. The JavaScript toolchain comes from files the repository already has: `.nvmrc` sets up Node, `package-lock.json` turns on the npm cache, and `bun.lock` or `bun.lockb` sets up bun. @@ -87,28 +89,80 @@ guess at the top folder. ## The baseline -The baseline is never a committed file. It is the `baseline.json` in the `plugin-check-report` -artifact of an earlier run of the same workflow file: +The baseline is `.plugin-check-baseline.json`, a file the repository commits: -- A pull request compares with the run for its base commit. For a stacked pull request that is - the run of the pull request below it. A run still going is waited for, up to 20 minutes. -- If that run has no baseline (it was cancelled, say), the newest run on the target branch is used. - More errors than that older run only warn, because the target may have changed since. -- A push compares with the run before it on the same branch. -- With no earlier run at all, nothing is compared and the check passes with a warning. A failed - lookup (an API or download error) fails instead, so an outage never passes as "no baseline". +```json +{ + "errors": { + "WordPress.Security.EscapeOutput.OutputNotEscaped": 12 + }, + "warnings": { + "WordPress.DB.DirectDatabaseQuery.DirectQuery": 3 + } +} +``` + +It holds a count for each rule code, and no file names or line numbers, so it only changes when a +rule gains or loses an error somewhere, not when code moves. Only a rule with more errors than the +file allows fails the check. + +A pull request is compared with the file in its own checkout, the pull request merged into its +target. So a pull request that fixes errors lowers the file in the same diff, and one that adds an +error on purpose raises it in the same diff, where a reviewer sees it. The comment says "This pull +request changes the baseline file" when it does. The tradeoff: a pull request can raise a count to +hide a new error. The change is in the diff, in a file with an obvious name. To require a reviewer +for it, add a line such as `/.plugin-check-baseline.json @my-org/leads` to CODEOWNERS. + +A push is compared with the file at that commit, so a merge that adds errors fails on the target +branch, where the whole team sees it. + +### Making and updating the file + +Every run uploads the `plugin-check-report` artifact, with a `baseline.json` made from that run's +results. To start the file, or to lower it after errors were fixed, download that artifact and copy +the file. The job summary shows the command with the run number filled in: + +```bash +gh run download --repo my-org/my-plugin --name plugin-check-report --dir /tmp/plugin-check +cp /tmp/plugin-check/baseline.json .plugin-check-baseline.json +``` -Only runs of the repository's own branches count. A pull request from a fork has a run of the same -workflow, and the runs list matches it by branch name and commit, so a fork's run on a branch called -`develop` would look like a run of `develop`. Its artifact is whatever the fork's workflow uploaded, -so runs whose head commit is in another repository are skipped. +Or locally, from the plugin's root, with the results of a local Plugin Check run: -The earlier runs are found by the caller's workflow file name, read from `GITHUB_WORKFLOW_REF`. -Renaming the caller file starts again with no baseline. Artifacts expire after the repository's -retention period (90 days by default), so a branch with no run for that long starts with none too. +```bash +node .github/actions/plugin-check-report/plugin-check-report.js update results.txt +``` + +The workflow never commits the file itself. That would need a write token on every pull request, +could not push to a fork, and would change the baseline without the person who owns the change. + +When error counts go down and the file is not updated, the check passes with a notice and the +comment asks for the update. Slack in the file lets a later pull request add errors back, so it is +worth lowering the file when errors are fixed. If two pull requests change the same count, git +reports a conflict in the file: merge the target branch, then make the file again from the new run +instead of merging the numbers by hand. + +### With no baseline file + +The check is slower, not weaker: -For a pull request to have a baseline, the branch it targets needs runs: list it under `push` in -the caller. +- **A pull request** (in `baseline` mode) builds and checks the target branch's commit too, in a + second job that runs in parallel with the first, and compares with those results. The comment's + column is then "Target branch", and a line says there is no baseline file. If the target branch + cannot be built or checked, the check fails with a message that says so: it never passes without a + baseline. The cost is a second build and Plugin Check run, so the job minutes double, though the + wait does not. +- **A push** has no target to compare with. The counts are reported and the check passes with a + warning that says how to start the file. +- **`strict` mode** needs no baseline, so it builds nothing extra and says nothing. + +Commit the file to stop the second build. + +### A file that is not valid + +A file that is not JSON (a merge conflict left markers in it, say) or not the shape above fails the +check with an error annotation that names the file and says what is wrong. The comment then says +Plugin Check could not run. Replace it with the `baseline.json` of the run's artifact. ## Modes @@ -161,6 +215,11 @@ grows past 15 lines. Pull requests from forks get no comment: their token cannot The workflow writes it with the [pr-comment](../pr-comment/) action. The comment is found by its first line, not by its author. +One line under the table says where the baseline came from when that matters: that the baseline is +the target branch's results because there is no baseline file, or that the pull request changes the +baseline file. When an error count went down, the "Rules that went down" line asks for the file to be +updated. The commands are in the job summary, not in the comment. + The action builds the body. Under the status line it puts a "Check Finished" badge with the time in Eastern Time (`06:08 PM EDT` in summer, `EST` in winter), made by `finishBadge()` in `plugin-check-report.js`, so no caller passes one. When the build or the check broke and there was @@ -200,14 +259,15 @@ in the same job does not see them there. ```yaml permissions: - actions: read # read earlier runs and download their baseline - contents: read # check out the repository + contents: read # check out the repository, and read the baseline file issues: write # the sticky comment pull-requests: write # the sticky comment ``` A called workflow can only lower these. If the caller grants less than a job asks for, GitHub does not start the run. The job that runs the Plugin Check action lowers itself to `contents: read`. +A caller from an earlier version that still grants `actions: read` works: a caller may give more +than a job asks for. ## Inputs of the action @@ -217,51 +277,49 @@ not start the run. The job that runs the Plugin Check action lowers itself to `c | `comment-file` | no | none | Where to write the comment body | | `mode` | no | `baseline` | `baseline` or `strict` | | `comment-marker` | no | `` | First line of the comment body | -| `baseline-wait` | no | `1200` | Seconds to wait for the base commit's run. A whole number, or the check fails | -| `token` | no | `github.token` | Needs `actions: read` | +| `baseline-file` | no | `.plugin-check-baseline.json` | The committed baseline. A relative path is read from the working directory | +| `base-results` | no | none | Results of the target branch, used as the baseline when there is no baseline file. Not used in `strict` mode | +| `base-required` | no | `false` | `true` when the caller checked the target branch: a missing `base-results` file then fails the compare | +| `baseline-changed` | no | `false` | `true` when the pull request changes the baseline file; the comment says so | | Output | What it is | |---|---| | `artifact-url` | The `plugin-check-report` artifact of this run | -| `baseline-url` | The run the baseline came from. Empty with no baseline | ## Running it locally -You need Node 20 or newer, and for the baseline the GitHub CLI signed in (`gh auth login`). Run -`npm install` once in the toolbox checkout, which installs the GitHub client the lookup uses. The -`compare` and `fetch` commands take the same arguments and environment variables as the two bash -scripts did. +You need Node 20 or newer. For a pull request's baseline, check out its branch: the file is in it. -1. Build the plugin and run Plugin Check in a local WordPress, for example with wp-env: +1. Build the plugin and run Plugin Check in a local WordPress, for example with wp-env. The excluded + checks come from the same `.plugin-check.json` as in CI: ```bash npx wp-env start - npx wp-env run cli wp plugin check my-plugin --format=json > results.txt + npx wp-env run cli wp plugin check my-plugin --format=json \ + $(node .github/actions/plugin-check-report/plugin-check-report.js exclude-checks) > results.txt ``` Keep only the command's output in the file. A notice from wp-env at the top fails the compare with "not Plugin Check output", on purpose. -2. Get a baseline from the branch you compare with: +2. Compare, from the plugin's root, where `.plugin-check-baseline.json` is: ```bash - GITHUB_REPOSITORY=my-org/my-plugin \ - node .github/actions/plugin-check-report/plugin-check-report.js fetch develop baseline.json + node .github/actions/plugin-check-report/plugin-check-report.js compare --comment-file comment.md results.txt ``` - The token comes from `GITHUB_TOKEN`, else `GH_TOKEN`, else `gh auth token`. The exit status is 0 - when a baseline was written, 1 when no run has one, and 2 when the lookup itself failed. + Exit 0 means the check passes. `comment.md` holds what the pull request would show. Add + `PLUGIN_CHECK_MODE=strict` to see the strict result, or `PLUGIN_CHECK_BASELINE=path` to use + another file. With no baseline file, the counts are shown and nothing is compared. -3. Compare: +3. To make or update the file from these results: ```bash - PLUGIN_CHECK_BASELINE=baseline.json \ - node .github/actions/plugin-check-report/plugin-check-report.js compare --comment-file comment.md results.txt + node .github/actions/plugin-check-report/plugin-check-report.js update results.txt ``` - Exit 0 means the check passes. `comment.md` holds what the pull request would show. Add - `PLUGIN_CHECK_MODE=strict` to see the strict result. Skip step 2 to see the counts with no - baseline. `compare --update results.txt` writes the baseline for a results file instead. + `update` is short for `compare --update`. It writes `.plugin-check-baseline.json` in the current + folder, with the same text as the `baseline.json` of the CI artifact. ## Differences from the earlier bash scripts @@ -271,8 +329,13 @@ show with a hand-edited baseline or an odd results file, except the first two: - The mode `ratchet` is now called `baseline`. The old word is refused, and the message says what to use. -- The `workflow-file` input is gone. The runs that hold the baseline are the calling workflow's, - which the run itself knows. +- The `workflow-file`, `baseline-wait`, `token` and `baseline-url` pieces are gone, and so is the + "older run" status and the "Target branch" / "Previous run" column names. The baseline is a + committed file now, not the artifact of an earlier run, so nothing is looked up or waited for. + The `fetch` command is gone with them. +- The texts for a missing baseline, for error counts that went down and for an invalid baseline + file are new, and the job summary has the commands to update the file. These are the only + expected files under `fixtures/expected/` that no longer come from the bash script. - The `badge-url` input is gone. The action makes the "Check Finished" badge itself, and also writes the "could not run" body that a shell step in the workflow wrote before. Both look as they did. - A rule with an empty code is counted like any other. Bash misread its row and showed wrong totals. @@ -287,14 +350,14 @@ show with a hand-edited baseline or an odd results file, except the first two: ## How it is tested The work is in `plugin-check-report.js`: small functions that take plain values, so a test calls -them directly. `tests/actions/plugin-check-report/` has five files, run with `node --test`: +them directly. `tests/actions/plugin-check-report/` has four files, run with `node --test`: - `compare.test.js` runs the compare in-process on the sample results files in `fixtures/`, and checks the comment, the job summary and the annotations against the exact files in - `fixtures/expected/`. Those files were written by the original bash script, so they pin that the - JavaScript prints what it printed. -- `fetch.test.js` runs the lookup against a real GitHub client with the network replaced by written - answers, a fake clock for the wait, and a stand-in for the `gh` download. + `fixtures/expected/`. Most of those files were written by the original bash script, so they pin + that the JavaScript prints what it printed; the ones listed under "Differences" were rewritten for + the committed baseline. It also covers the missing file, the target branch as a baseline, the + invalid file and the command line. - `action.test.js` calls the step functions the way `actions/github-script` does. - `badge.test.js` checks the badge URL against what the old shell step printed, from the real `date` command, for both sides of both daylight saving changes, midnight, noon and the end of a From f0af2c9403f26527cb84489ab92e8136f9225234 Mon Sep 17 00:00:00 2001 From: Paulo Iankoski Date: Wed, 4 Nov 2026 19:11:30 -0300 Subject: [PATCH 31/31] Build the Plugin Check zip with the zip-name and release-zip actions --- .github/actions/plugin-check-report/README.md | 21 ++--- .github/workflows/plugin-check.yml | 80 ++++++++++++------- 2 files changed, 57 insertions(+), 44 deletions(-) diff --git a/.github/actions/plugin-check-report/README.md b/.github/actions/plugin-check-report/README.md index 5c203cf..afd715d 100644 --- a/.github/actions/plugin-check-report/README.md +++ b/.github/actions/plugin-check-report/README.md @@ -28,7 +28,7 @@ jobs: - A `.puprc` whose `build` makes the plugin as it ships, and whose `zip_name` equals `plugin-slug`. The zip's top folder is named after `zip_name`, and Plugin Check takes the slug from the folder name. -- Nothing for pup itself. The workflow installs it with the shared [pup](../pup/) action, so the +- Nothing for pup itself. The workflow downloads it in the shared [zip-name](../zip-name/) and [release-zip](../release-zip/) actions, so the repository needs no copy of `pup.phar` and no `pup` Composer script. Set the `pup-version` input to the release the repository's own release build uses, so the check sees the same files. - Ignore rules that keep dev-only files out of the zip: a `.distignore`, `export-ignore` lines in @@ -42,19 +42,14 @@ The JavaScript toolchain comes from files the repository already has: `.nvmrc` s ## How the plugin is built -By default, with pup, the same steps the release zip uses, from the root of the checkout. `pup` -here is `php pup.phar`, with the phar the shared pup action installed: +By default, with pup, the same QA build the release zip makes, from the root of the checkout (the +`source/` folder of the job). Two shared actions do it: [zip-name](../zip-name/) reads the version +and names the zip (`production: no`, `prerelease: dev`), and [release-zip](../release-zip/) runs +`pup build` and `pup package ` with `check: no` and `i18n: no`, since this workflow needs +neither. The workflow then unpacks the zip into `$PLUGIN_CHECK_BUILD_DIR`. -```bash -pup build -version=$(pup get-version --dev | tail -n 1) -pup package "$version" -name=$(pup zip-name "$version" | tail -n 1) -unzip -q "$name.zip" -d "$PLUGIN_CHECK_BUILD_DIR" -``` - -One pup release builds, packages and names the zip, so the name looked for is always the name the -package step made. The last line pup prints is the value, as the pup action reads it. +One pup release (the `pup-version` input) reads the version, builds, packages and names the zip, so +the name looked for is always the name the package step made. So `.puprc` and the ignore files decide what is checked. If the release needs a step the check should see too, such as making the POT file, put it in `.puprc`'s `build`, not in the workflow. diff --git a/.github/workflows/plugin-check.yml b/.github/workflows/plugin-check.yml index 37f5ea7..508ba87 100644 --- a/.github/workflows/plugin-check.yml +++ b/.github/workflows/plugin-check.yml @@ -8,8 +8,8 @@ name: Plugin Check # # The plugin is built with pup by default (`pup build`, then `pup package`), so the repository's # .puprc and ignore files decide what is checked, the same as the release. pup comes from the shared -# .github/actions/pup action, so the repository does not need a copy of it or a Composer script for -# it. A repository without pup passes `build-command` instead; see the plugin-check-report README. +# zip-name and release-zip actions, so the repository does not need a copy of it or a Composer +# script for it. A repository without pup passes `build-command` instead; see the plugin-check-report README. # # Four jobs, so the third-party action never holds a token that can write. The comments are written # by .github/actions/pr-comment, which calls the GitHub API itself: @@ -275,16 +275,6 @@ jobs: php-version: ${{ inputs.php-version }} coverage: none - # pup is a PHP archive, so PHP is set up first. One pup release builds, packages and names the - # zip below, so the name it looks for is always the name the package step made. The action's - # `read` input is not used: it reads from the workspace root, and the repository is in source/. - - name: Install pup - id: pup - if: ${{ inputs.build-command == '' }} - uses: $/.github/actions/pup - with: - pup-version: ${{ inputs.pup-version }} - - name: Find the Composer cache id: composer-cache working-directory: source @@ -346,16 +336,49 @@ jobs: restore-keys: | plugin-check-${{ runner.os }}-${{ hashFiles('source/package-lock.json', 'source/bun.lock', 'source/bun.lockb') }}- - # The same pup steps the release zip uses, so .puprc and the ignore files decide what is checked. + # pup builds and packages the plugin, so .puprc and the ignore files decide what is checked, + # the same as the release. The version and the zip name come from the zip-name action, the zip + # from release-zip: the same QA build the zip workflow makes, without `pup check` and `pup i18n`, + # which this workflow does not need. pup runs on the PHP set up above. The repository is in + # source/, so both actions run there. + - name: Name the zip + id: zip-name + if: ${{ inputs.build-command == '' }} + uses: $/.github/actions/zip-name + with: + production: 'no' + prerelease: dev + pup-version: ${{ inputs.pup-version }} + working-directory: source + # The secrets reach this step only, never the job, so the third-party action later in this - # job does not see them in its environment. - - name: Build the plugin + # job does not see them in its environment. The build runs with them in its environment, as + # pup build needs them to install private packages. + - name: Build the plugin with pup + id: zip + if: ${{ inputs.build-command == '' }} + uses: $/.github/actions/release-zip + with: + version: ${{ steps.zip-name.outputs.version }} + production: 'no' + check: 'no' + i18n: 'no' + pup-version: ${{ inputs.pup-version }} + working-directory: source + env: + COMPOSER_AUTH: >- + ${{ secrets.composer-token != '' && format('{{"github-oauth": {{"github.com": "{0}"}}}}', secrets.composer-token) || '' }} + NODE_AUTH_TOKEN: ${{ secrets.package-token }} + BUN_AUTH_TOKEN: ${{ secrets.package-token }} + GITHUB_TOKEN: ${{ secrets.package-token != '' && secrets.package-token || github.token }} + + - name: Build the plugin with build-command + if: ${{ inputs.build-command != '' }} working-directory: source env: BUILD_COMMAND: ${{ inputs.build-command }} - PUP_PHAR: ${{ steps.pup.outputs.phar }} - COMPOSER_TOKEN: ${{ secrets.composer-token }} PACKAGE_TOKEN: ${{ secrets.package-token }} + COMPOSER_TOKEN: ${{ secrets.composer-token }} run: | if [ -n "$COMPOSER_TOKEN" ]; then COMPOSER_AUTH=$(jq -cn --arg token "$COMPOSER_TOKEN" '{"github-oauth": {"github.com": $token}}') @@ -366,21 +389,16 @@ jobs: fi unset COMPOSER_TOKEN PACKAGE_TOKEN mkdir -p "$PLUGIN_CHECK_BUILD_DIR" + echo "Building with build-command." + bash -eo pipefail -c "$BUILD_COMMAND" - if [ -n "$BUILD_COMMAND" ]; then - echo "Building with build-command." - bash -eo pipefail -c "$BUILD_COMMAND" - exit 0 - fi - - # The last line pup prints is the value, as the pup action reads it; PHP's own errors go to - # stderr so they never reach it. - pup() { php -d display_errors=stderr "$PUP_PHAR" "$@"; } - pup build - version=$(pup get-version --dev | tail -n 1) - pup package "$version" - name=$(pup zip-name "$version" | tail -n 1) - unzip -q "$name.zip" -d "$PLUGIN_CHECK_BUILD_DIR" + - name: Unpack the zip + if: ${{ inputs.build-command == '' }} + env: + ZIP_PATH: ${{ steps.zip.outputs.zip-path }} + run: | + mkdir -p "$PLUGIN_CHECK_BUILD_DIR" + unzip -q "$ZIP_PATH" -d "$PLUGIN_CHECK_BUILD_DIR" # The one contract every build has to meet, pup or not: a folder named after the slug, with # the main plugin file in it. Plugin Check takes the slug from the folder name, so a wrong