diff --git a/.github/workflows/sdk-api-drift.yml b/.github/workflows/sdk-api-drift.yml index 90f7c37c4..90ddaa1dd 100644 --- a/.github/workflows/sdk-api-drift.yml +++ b/.github/workflows/sdk-api-drift.yml @@ -76,12 +76,24 @@ jobs: node build/tasks/check-sdk-api-drift.mts > report.md || code=$? cat report.md + # An answer is 0, or 16 with a bit for each thing to report, so it + # never shares a number with node's own codes: 1 for an uncaught + # exception, 13 for a top-level await that never settled, above 128 + # for a signal. Any other code is a run that crashed before it could + # answer, and is reported as one that could not check rather than + # read as one that found something. + code=${code:-0} + if [ "$code" -ne 0 ] && { [ "$code" -lt 17 ] || [ "$code" -gt 19 ]; }; then + echo "::error::the check exited ${code} without answering" + code=18 + fi + # Both bits are read: being behind says nothing about whether the # tags could be read, so neither answer is allowed to hide the # other. Reported after the issue is filed, so an API nobody could # reach cannot hold back a report about a reference that is missing. - echo "behind=$(( (${code:-0} & 1) != 0 ))" >> "$GITHUB_OUTPUT" - echo "unchecked=$(( (${code:-0} & 2) != 0 ))" >> "$GITHUB_OUTPUT" + echo "behind=$(( (code & 1) != 0 ))" >> "$GITHUB_OUTPUT" + echo "unchecked=$(( (code & 2) != 0 ))" >> "$GITHUB_OUTPUT" - name: Say so, once if: steps.compare.outputs.behind == '1' diff --git a/.github/workflows/vendored-sync.yml b/.github/workflows/vendored-sync.yml index da5a6fb75..8a1f84618 100644 --- a/.github/workflows/vendored-sync.yml +++ b/.github/workflows/vendored-sync.yml @@ -53,12 +53,24 @@ jobs: node build/tasks/check-vendored.mts > report.md || code=$? cat report.md + # An answer is 0, or 16 with a bit for each thing to report, so it + # never shares a number with node's own codes: 1 for an uncaught + # exception, 13 for a top-level await that never settled, above 128 + # for a signal. Any other code is a run that crashed before it could + # answer, and is reported as one that could not check rather than + # read as one that found something. + code=${code:-0} + if [ "$code" -ne 0 ] && { [ "$code" -lt 17 ] || [ "$code" -gt 19 ]; }; then + echo "::error::the check exited ${code} without answering" + code=18 + fi + # Both bits are read: one file drifting says nothing about whether # another was reachable, so neither answer is allowed to hide the # other. Reported after the issue is filed, so a file nobody could # reach cannot hold back a report about one that drifted. - echo "drifted=$(( (${code:-0} & 1) != 0 ))" >> "$GITHUB_OUTPUT" - echo "unchecked=$(( (${code:-0} & 2) != 0 ))" >> "$GITHUB_OUTPUT" + echo "drifted=$(( (code & 1) != 0 ))" >> "$GITHUB_OUTPUT" + echo "unchecked=$(( (code & 2) != 0 ))" >> "$GITHUB_OUTPUT" - name: Say so, once if: steps.compare.outputs.drifted == '1' env: diff --git a/build/shared/check-answers.test.mts b/build/shared/check-answers.test.mts new file mode 100644 index 000000000..a1df8041c --- /dev/null +++ b/build/shared/check-answers.test.mts @@ -0,0 +1,201 @@ +/** + * @file Regression tests for how the scheduled checks' answers are read. + * @author The OpenINF Authors & Friends + * @license MIT OR Apache-2.0 OR BlueOak-1.0.0 + * @module {type ES6Module} build/shared/check-answers.test + * + * `check-vendored.mts` and `check-sdk-api-drift.mts` answer through their + * exit code, and the workflows that run them turn that code into whether to + * file an issue and whether to fail. Node picks exit codes of its own when a + * script crashes, so what is checked here is that none of those is read as an + * answer: a crash must fail the job rather than file an empty report. + * + * The workflows' own step scripts are run, with `node` replaced by a stand-in + * that exits with whatever code a case needs, so the shell under test is the + * shell GitHub runs. + */ + +import { deepStrictEqual, strictEqual } from 'node:assert/strict'; +import { execFile, spawnSync } from 'node:child_process'; +import { + chmod, + cp, + mkdir, + mkdtemp, + readFile, + rm, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; +import { parse } from 'yaml'; + +const execute = promisify(execFile); +const root = fileURLToPath(new URL('../../', import.meta.url)); + +/** Each workflow, and the output that says it found something to report. */ +const WORKFLOWS = [ + { file: '.github/workflows/vendored-sync.yml', found: 'drifted' }, + { file: '.github/workflows/sdk-api-drift.yml', found: 'behind' }, +]; + +/** + * The script of the step that runs the check and reads its answer. + * @param {string} file The workflow, relative to the repository root. + * @returns {Promise} The step's `run:` script. + */ +async function compareStep(file: string) { + const workflow = parse(await readFile(join(root, file), 'utf8')) as { + jobs: { check: { steps: { id?: string; run?: string }[] } }; + }; + const step = workflow.jobs.check.steps.find(({ id }) => id === 'compare'); + + if (step?.run === undefined) throw new Error(`${file} has no compare step`); + return step.run; +} + +/** + * Runs a step script the way the runner does when a step names no shell, and + * returns what it wrote to `GITHUB_OUTPUT`. + * @param {string} script The step's `run:` script. + * @param {string} cwd Where to run it. + * @param {{ exit?: string; path?: string }} fake The stand-in `node`, if any. + * @returns {Promise>} The step's outputs. + */ +async function runStep( + script: string, + cwd: string, + fake: { exit?: string; path?: string } = {} +) { + const output = join(cwd, 'github-output'); + const file = join(cwd, 'step.sh'); + const env = { ...process.env }; + + env.GITHUB_OUTPUT = output; + if (fake.exit !== undefined) env.FAKE_EXIT = fake.exit; + if (fake.path !== undefined) env.PATH = fake.path; + + await writeFile(output, ''); + await writeFile(file, script); + await execute('bash', ['-e', file], { cwd, env }); + + return Object.fromEntries( + (await readFile(output, 'utf8')) + .split('\n') + .filter(Boolean) + .map((line) => line.split('=') as [string, string]) + ); +} + +/** + * A workspace whose `node` exits with `FAKE_EXIT`, standing in for a check + * that answered with that code or crashed with it. + * @returns Its path, the PATH that finds the stand-in, and cleanup. + */ +async function fakeNode() { + const cwd = await mkdtemp(join(tmpdir(), 'check-answers-test-')); + const bin = join(cwd, 'bin'); + + await mkdir(bin); + await writeFile( + join(bin, 'node'), + '#!/bin/sh\necho report\nexit "$FAKE_EXIT"\n' + ); + await chmod(join(bin, 'node'), 0o755); + + return { + cwd, + path: `${bin}:${process.env.PATH ?? ''}`, + cleanup: () => rm(cwd, { force: true, recursive: true }), + }; +} + +/** + * The code node itself exits with when a script goes wrong in a given way. + * @param {string[]} args What to run. + * @returns {number} Its exit code. + */ +const crashCode = (args: string[]) => + spawnSync(process.execPath, args, { stdio: 'ignore' }).status ?? -1; + +const CRASHES = { + 'an uncaught exception': crashCode(['-e', 'throw new Error("boom")']), + 'a top-level await that never settles': crashCode([ + '--input-type=module', + '-e', + 'await new Promise(() => {})', + ]), + // What a shell reports for a process killed by SIGKILL. + 'a killed process': 137, +}; + +for (const { file, found } of WORKFLOWS) { + test(`${file} reads each answer the task gives`, async () => { + const script = await compareStep(file); + const { cwd, path, cleanup } = await fakeNode(); + + try { + for (const [code, expected] of [ + ['0', { [found]: '0', unchecked: '0' }], + ['17', { [found]: '1', unchecked: '0' }], + ['18', { [found]: '0', unchecked: '1' }], + ['19', { [found]: '1', unchecked: '1' }], + ] as const) { + deepStrictEqual( + await runStep(script, cwd, { exit: code, path }), + expected, + `exit ${code}` + ); + } + } finally { + await cleanup(); + } + }); + + for (const [crash, code] of Object.entries(CRASHES)) { + test(`${file} fails on ${crash} rather than filing a report`, async () => { + const script = await compareStep(file); + const { cwd, path, cleanup } = await fakeNode(); + + try { + deepStrictEqual( + await runStep(script, cwd, { exit: String(code), path }), + { [found]: '0', unchecked: '1' }, + `exit ${code}` + ); + } finally { + await cleanup(); + } + }); + } +} + +test('check-vendored answers "could not compare" with a code node never picks', async () => { + // A workspace holding the task and none of the files it compares, so it + // answers without reaching the network. + const cwd = await mkdtemp(join(tmpdir(), 'check-answers-test-')); + const task = join(cwd, 'build/tasks/check-vendored.mts'); + + try { + await cp(join(root, 'build/tasks/check-vendored.mts'), task); + + const status = spawnSync(process.execPath, [task], { + cwd, + stdio: 'ignore', + }).status; + + strictEqual(status, 18); + deepStrictEqual( + await runStep( + await compareStep('.github/workflows/vendored-sync.yml'), + cwd + ), + { drifted: '0', unchecked: '1' } + ); + } finally { + await rm(cwd, { force: true, recursive: true }); + } +}); diff --git a/build/tasks/check-sdk-api-drift.mts b/build/tasks/check-sdk-api-drift.mts index e7ea8fcb0..94951b264 100644 --- a/build/tasks/check-sdk-api-drift.mts +++ b/build/tasks/check-sdk-api-drift.mts @@ -28,9 +28,9 @@ * the directories in this checkout. * * The exit code carries both answers at once, the way `check-vendored.mts` - * does: bit 1 is set when the portal is behind, bit 2 when the tags could not - * be read. Being behind is a thing to act on and an unreachable host is not, - * so neither hides the other. + * does: 0 when the portal is current, otherwise 16 with bit 1 set when it is + * behind and bit 2 when the tags could not be read. Being behind is a thing + * to act on and an unreachable host is not, so neither hides the other. */ import { readdir } from 'node:fs/promises'; @@ -59,8 +59,14 @@ const TIMEOUT = 30_000; const PER_PAGE = 100; const PAGES = 20; -/** Bits of the exit code. Both can be set; neither masks the other. */ +/** + * Bits of the exit code. Both can be set; neither masks the other. An answer + * also sets `ANSWERED`, so that none is a code node exits with of its own + * accord -- 1 for an uncaught exception, 13 for a top-level await that never + * settled -- and a run that crashed cannot be read as one that answered. + */ const MATCHED = 0; +const ANSWERED = 16; const BEHIND = 1; const UNCHECKED = 2; @@ -161,6 +167,8 @@ if (answer.problem !== undefined) { } process.exitCode = - MATCHED | - (behind ? BEHIND : MATCHED) | - (answer.problem === undefined ? MATCHED : UNCHECKED); + !behind && answer.problem === undefined + ? MATCHED + : ANSWERED | + (behind ? BEHIND : MATCHED) | + (answer.problem === undefined ? MATCHED : UNCHECKED); diff --git a/build/tasks/check-vendored.mts b/build/tasks/check-vendored.mts index 8a79a2164..26a233131 100644 --- a/build/tasks/check-vendored.mts +++ b/build/tasks/check-vendored.mts @@ -10,9 +10,10 @@ * that have nothing to do with it. * * The exit code carries both answers at once, since one file drifting says - * nothing about whether another was reachable: bit 1 is set when a copy has - * drifted, bit 2 when one could not be compared. Drift is a thing to act on - * and an upstream nobody can reach is not, so neither hides the other. + * nothing about whether another was reachable: 0 when every copy matched, + * otherwise 16 with bit 1 set when a copy has drifted and bit 2 when one + * could not be compared. Drift is a thing to act on and an upstream nobody + * can reach is not, so neither hides the other. */ import { readFile } from 'node:fs/promises'; @@ -43,8 +44,14 @@ const VENDORED: Vendored[] = [ /** How long to wait on an upstream before giving up, in milliseconds. */ const TIMEOUT = 30_000; -/** Bits of the exit code. Both can be set; neither masks the other. */ +/** + * Bits of the exit code. Both can be set; neither masks the other. An answer + * also sets `ANSWERED`, so that none is a code node exits with of its own + * accord -- 1 for an uncaught exception, 13 for a top-level await that never + * settled -- and a run that crashed cannot be read as one that answered. + */ const MATCHED = 0; +const ANSWERED = 16; const DRIFTED = 1; const UNCHECKED = 2; @@ -148,6 +155,8 @@ if (drifted.length > 0) { } process.exitCode = - MATCHED | - (drifted.length > 0 ? DRIFTED : MATCHED) | - (unchecked.length > 0 ? UNCHECKED : MATCHED); + drifted.length === 0 && unchecked.length === 0 + ? MATCHED + : ANSWERED | + (drifted.length > 0 ? DRIFTED : MATCHED) | + (unchecked.length > 0 ? UNCHECKED : MATCHED);