Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions .github/workflows/sdk-api-drift.yml
Original file line number Diff line number Diff line change
Expand Up @@ -76,12 +76,24 @@ jobs:
node build/tasks/check-sdk-api-drift.mts > report.md || code=$?
cat report.md

# An answer is 0, or 16 with a bit for each thing to report, so it
# never shares a number with node's own codes: 1 for an uncaught
# exception, 13 for a top-level await that never settled, above 128
# for a signal. Any other code is a run that crashed before it could
# answer, and is reported as one that could not check rather than
# read as one that found something.
code=${code:-0}
if [ "$code" -ne 0 ] && { [ "$code" -lt 17 ] || [ "$code" -gt 19 ]; }; then
echo "::error::the check exited ${code} without answering"
code=18
fi

# Both bits are read: being behind says nothing about whether the
# tags could be read, so neither answer is allowed to hide the
# other. Reported after the issue is filed, so an API nobody could
# reach cannot hold back a report about a reference that is missing.
echo "behind=$(( (${code:-0} & 1) != 0 ))" >> "$GITHUB_OUTPUT"
echo "unchecked=$(( (${code:-0} & 2) != 0 ))" >> "$GITHUB_OUTPUT"
echo "behind=$(( (code & 1) != 0 ))" >> "$GITHUB_OUTPUT"
echo "unchecked=$(( (code & 2) != 0 ))" >> "$GITHUB_OUTPUT"

- name: Say so, once
if: steps.compare.outputs.behind == '1'
Expand Down
16 changes: 14 additions & 2 deletions .github/workflows/vendored-sync.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,24 @@ jobs:
node build/tasks/check-vendored.mts > report.md || code=$?
cat report.md

# An answer is 0, or 16 with a bit for each thing to report, so it
# never shares a number with node's own codes: 1 for an uncaught
# exception, 13 for a top-level await that never settled, above 128
# for a signal. Any other code is a run that crashed before it could
# answer, and is reported as one that could not check rather than
# read as one that found something.
code=${code:-0}
if [ "$code" -ne 0 ] && { [ "$code" -lt 17 ] || [ "$code" -gt 19 ]; }; then
echo "::error::the check exited ${code} without answering"
code=18
fi

# Both bits are read: one file drifting says nothing about whether
# another was reachable, so neither answer is allowed to hide the
# other. Reported after the issue is filed, so a file nobody could
# reach cannot hold back a report about one that drifted.
echo "drifted=$(( (${code:-0} & 1) != 0 ))" >> "$GITHUB_OUTPUT"
echo "unchecked=$(( (${code:-0} & 2) != 0 ))" >> "$GITHUB_OUTPUT"
echo "drifted=$(( (code & 1) != 0 ))" >> "$GITHUB_OUTPUT"
echo "unchecked=$(( (code & 2) != 0 ))" >> "$GITHUB_OUTPUT"
- name: Say so, once
if: steps.compare.outputs.drifted == '1'
env:
Expand Down
201 changes: 201 additions & 0 deletions build/shared/check-answers.test.mts
Original file line number Diff line number Diff line change
@@ -0,0 +1,201 @@
/**
* @file Regression tests for how the scheduled checks' answers are read.
* @author The OpenINF Authors & Friends
* @license MIT OR Apache-2.0 OR BlueOak-1.0.0
* @module {type ES6Module} build/shared/check-answers.test
*
* `check-vendored.mts` and `check-sdk-api-drift.mts` answer through their
* exit code, and the workflows that run them turn that code into whether to
* file an issue and whether to fail. Node picks exit codes of its own when a
* script crashes, so what is checked here is that none of those is read as an
* answer: a crash must fail the job rather than file an empty report.
*
* The workflows' own step scripts are run, with `node` replaced by a stand-in
* that exits with whatever code a case needs, so the shell under test is the
* shell GitHub runs.
*/

import { deepStrictEqual, strictEqual } from 'node:assert/strict';
import { execFile, spawnSync } from 'node:child_process';
import {
chmod,
cp,
mkdir,
mkdtemp,
readFile,
rm,
writeFile,
} from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { test } from 'node:test';
import { fileURLToPath } from 'node:url';
import { promisify } from 'node:util';
import { parse } from 'yaml';

const execute = promisify(execFile);
const root = fileURLToPath(new URL('../../', import.meta.url));

/** Each workflow, and the output that says it found something to report. */
const WORKFLOWS = [
{ file: '.github/workflows/vendored-sync.yml', found: 'drifted' },
{ file: '.github/workflows/sdk-api-drift.yml', found: 'behind' },
];

/**
* The script of the step that runs the check and reads its answer.
* @param {string} file The workflow, relative to the repository root.
* @returns {Promise<string>} The step's `run:` script.
*/
async function compareStep(file: string) {
const workflow = parse(await readFile(join(root, file), 'utf8')) as {
jobs: { check: { steps: { id?: string; run?: string }[] } };
};
const step = workflow.jobs.check.steps.find(({ id }) => id === 'compare');

if (step?.run === undefined) throw new Error(`${file} has no compare step`);
return step.run;
}

/**
* Runs a step script the way the runner does when a step names no shell, and
* returns what it wrote to `GITHUB_OUTPUT`.
* @param {string} script The step's `run:` script.
* @param {string} cwd Where to run it.
* @param {{ exit?: string; path?: string }} fake The stand-in `node`, if any.
* @returns {Promise<Record<string, string>>} The step's outputs.
*/
async function runStep(
script: string,
cwd: string,
fake: { exit?: string; path?: string } = {}
) {
const output = join(cwd, 'github-output');
const file = join(cwd, 'step.sh');
const env = { ...process.env };

env.GITHUB_OUTPUT = output;
if (fake.exit !== undefined) env.FAKE_EXIT = fake.exit;
if (fake.path !== undefined) env.PATH = fake.path;

await writeFile(output, '');
await writeFile(file, script);
await execute('bash', ['-e', file], { cwd, env });

return Object.fromEntries(
(await readFile(output, 'utf8'))
.split('\n')
.filter(Boolean)
.map((line) => line.split('=') as [string, string])
);
}

/**
* A workspace whose `node` exits with `FAKE_EXIT`, standing in for a check
* that answered with that code or crashed with it.
* @returns Its path, the PATH that finds the stand-in, and cleanup.
*/
async function fakeNode() {
const cwd = await mkdtemp(join(tmpdir(), 'check-answers-test-'));
const bin = join(cwd, 'bin');

await mkdir(bin);
await writeFile(
join(bin, 'node'),
'#!/bin/sh\necho report\nexit "$FAKE_EXIT"\n'
);
await chmod(join(bin, 'node'), 0o755);

return {
cwd,
path: `${bin}:${process.env.PATH ?? ''}`,
cleanup: () => rm(cwd, { force: true, recursive: true }),
};
}

/**
* The code node itself exits with when a script goes wrong in a given way.
* @param {string[]} args What to run.
* @returns {number} Its exit code.
*/
const crashCode = (args: string[]) =>
spawnSync(process.execPath, args, { stdio: 'ignore' }).status ?? -1;

const CRASHES = {
'an uncaught exception': crashCode(['-e', 'throw new Error("boom")']),
'a top-level await that never settles': crashCode([
'--input-type=module',
'-e',
'await new Promise(() => {})',
]),
// What a shell reports for a process killed by SIGKILL.
'a killed process': 137,
};

for (const { file, found } of WORKFLOWS) {
test(`${file} reads each answer the task gives`, async () => {
const script = await compareStep(file);
const { cwd, path, cleanup } = await fakeNode();

try {
for (const [code, expected] of [
['0', { [found]: '0', unchecked: '0' }],
['17', { [found]: '1', unchecked: '0' }],
['18', { [found]: '0', unchecked: '1' }],
['19', { [found]: '1', unchecked: '1' }],
] as const) {
deepStrictEqual(
await runStep(script, cwd, { exit: code, path }),
expected,
`exit ${code}`
);
}
} finally {
await cleanup();
}
});

for (const [crash, code] of Object.entries(CRASHES)) {
test(`${file} fails on ${crash} rather than filing a report`, async () => {
const script = await compareStep(file);
const { cwd, path, cleanup } = await fakeNode();

try {
deepStrictEqual(
await runStep(script, cwd, { exit: String(code), path }),
{ [found]: '0', unchecked: '1' },
`exit ${code}`
);
} finally {
await cleanup();
}
});
}
}

test('check-vendored answers "could not compare" with a code node never picks', async () => {
// A workspace holding the task and none of the files it compares, so it
// answers without reaching the network.
const cwd = await mkdtemp(join(tmpdir(), 'check-answers-test-'));
const task = join(cwd, 'build/tasks/check-vendored.mts');

try {
await cp(join(root, 'build/tasks/check-vendored.mts'), task);

const status = spawnSync(process.execPath, [task], {
cwd,
stdio: 'ignore',
}).status;

strictEqual(status, 18);
deepStrictEqual(
await runStep(
await compareStep('.github/workflows/vendored-sync.yml'),
cwd
),
{ drifted: '0', unchecked: '1' }
);
} finally {
await rm(cwd, { force: true, recursive: true });
}
});
22 changes: 15 additions & 7 deletions build/tasks/check-sdk-api-drift.mts
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@
* the directories in this checkout.
*
* The exit code carries both answers at once, the way `check-vendored.mts`
* does: bit 1 is set when the portal is behind, bit 2 when the tags could not
* be read. Being behind is a thing to act on and an unreachable host is not,
* so neither hides the other.
* does: 0 when the portal is current, otherwise 16 with bit 1 set when it is
* behind and bit 2 when the tags could not be read. Being behind is a thing
* to act on and an unreachable host is not, so neither hides the other.
*/

import { readdir } from 'node:fs/promises';
Expand Down Expand Up @@ -59,8 +59,14 @@ const TIMEOUT = 30_000;
const PER_PAGE = 100;
const PAGES = 20;

/** Bits of the exit code. Both can be set; neither masks the other. */
/**
* Bits of the exit code. Both can be set; neither masks the other. An answer
* also sets `ANSWERED`, so that none is a code node exits with of its own
* accord -- 1 for an uncaught exception, 13 for a top-level await that never
* settled -- and a run that crashed cannot be read as one that answered.
*/
const MATCHED = 0;
const ANSWERED = 16;
const BEHIND = 1;
const UNCHECKED = 2;

Expand Down Expand Up @@ -161,6 +167,8 @@ if (answer.problem !== undefined) {
}

process.exitCode =
MATCHED |
(behind ? BEHIND : MATCHED) |
(answer.problem === undefined ? MATCHED : UNCHECKED);
!behind && answer.problem === undefined
? MATCHED
: ANSWERED |
(behind ? BEHIND : MATCHED) |
(answer.problem === undefined ? MATCHED : UNCHECKED);
23 changes: 16 additions & 7 deletions build/tasks/check-vendored.mts
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@
* that have nothing to do with it.
*
* The exit code carries both answers at once, since one file drifting says
* nothing about whether another was reachable: bit 1 is set when a copy has
* drifted, bit 2 when one could not be compared. Drift is a thing to act on
* and an upstream nobody can reach is not, so neither hides the other.
* nothing about whether another was reachable: 0 when every copy matched,
* otherwise 16 with bit 1 set when a copy has drifted and bit 2 when one
* could not be compared. Drift is a thing to act on and an upstream nobody
* can reach is not, so neither hides the other.
*/

import { readFile } from 'node:fs/promises';
Expand Down Expand Up @@ -43,8 +44,14 @@ const VENDORED: Vendored[] = [
/** How long to wait on an upstream before giving up, in milliseconds. */
const TIMEOUT = 30_000;

/** Bits of the exit code. Both can be set; neither masks the other. */
/**
* Bits of the exit code. Both can be set; neither masks the other. An answer
* also sets `ANSWERED`, so that none is a code node exits with of its own
* accord -- 1 for an uncaught exception, 13 for a top-level await that never
* settled -- and a run that crashed cannot be read as one that answered.
*/
const MATCHED = 0;
const ANSWERED = 16;
const DRIFTED = 1;
const UNCHECKED = 2;

Expand Down Expand Up @@ -148,6 +155,8 @@ if (drifted.length > 0) {
}

process.exitCode =
MATCHED |
(drifted.length > 0 ? DRIFTED : MATCHED) |
(unchecked.length > 0 ? UNCHECKED : MATCHED);
drifted.length === 0 && unchecked.length === 0
? MATCHED
: ANSWERED |
(drifted.length > 0 ? DRIFTED : MATCHED) |
(unchecked.length > 0 ? UNCHECKED : MATCHED);
Loading