Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions build.zig
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,16 @@ pub fn build(b: *std.Build) void {
json_schema_test.setCwd(b.path("."));
json_schema_test.expectExitCode(0);
json_schema_test.has_side_effects = true;
const test_bundle_test = b.addSystemCommand(&.{ "bash", "-c", "TOOL_BIN=\"$0\" exec bash tests/test_test_bundle.sh" });
test_bundle_test.addArtifactArg(exe);
test_bundle_test.setCwd(b.path("."));
test_bundle_test.expectExitCode(0);
test_bundle_test.has_side_effects = true;
const stable_output_test = b.addSystemCommand(&.{ "bash", "-c", "TOOL_BIN=\"$0\" exec bash tests/test_output_stability.sh" });
stable_output_test.addArtifactArg(exe);
stable_output_test.setCwd(b.path("."));
stable_output_test.expectExitCode(0);
stable_output_test.has_side_effects = true;

const test_step = b.step("test", "Run the unit tests, the command-line suites and the differential run");
test_step.dependOn(&b.addRunArtifact(unit).step);
Expand All @@ -110,4 +120,6 @@ pub fn build(b: *std.Build) void {
test_step.dependOn(&usage_test.step);
test_step.dependOn(&pending_test.step);
test_step.dependOn(&json_schema_test.step);
test_step.dependOn(&test_bundle_test.step);
test_step.dependOn(&stable_output_test.step);
}
1 change: 1 addition & 0 deletions changelog.d/+test-bundle.feature.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add `test-bundle` and `test-compare` to capture and compare normalized flaky-test evidence bundles across sites, with schema-versioned JSON outputs and documentation.
60 changes: 60 additions & 0 deletions docs/orgmode/howto/compare-flaky-test-bundles.org
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#+TITLE: Compare flaky-test evidence bundles across sites
#+OPTIONS: toc:nil todo:nil num:nil author:nil title:nil
#+STARTUP: content

* Compare flaky-test evidence bundles across sites

This guide records two test runs as comparable evidence bundles and compares their differing fields.
It does not decide pass or fail.

** 1. Capture one bundle per test run

Save each run's stdout and stderr, then run =test-bundle= with the same stable test identity for that test:

#+begin_example
eb-keep-logs test-bundle \
--test-id qe.pw.scf-si \
--suite-revision 5f6a9cc \
--eessi-target x86_64/intel/sapphirerapids \
--stdout run-a.stdout \
--stderr run-a.stderr \
--input qe.in \
--seed 42 \
--module QuantumESPRESSO/7.4=/cvmfs/eessi/software/QuantumESPRESSO/7.4.lua \
--allocation "nodes=1 ntasks=32" \
--binding "rank0->core0, rank1->core1" \
--cpu "AMD EPYC 7B13 (32)" \
--gpu "none" \
--out run-a.bundle.json
#+end_example

Run it again for another site or run, changing only what actually differs, and write e.g. =run-b.bundle.json=.

** 2. Compare bundles

For a text summary:

#+begin_example
eb-keep-logs test-compare run-a.bundle.json run-b.bundle.json
#+end_example

For machine-readable output:

#+begin_example
eb-keep-logs test-compare --json run-a.bundle.json run-b.bundle.json > compare.json
#+end_example

The summary lists only fields that differ across bundles:

- stable test identity and suite revision;
- normalized stdout/stderr hashes;
- input and seed hashes;
- module list and module paths;
- EESSI target;
- scheduler allocation and binding;
- CPU and GPU facts.

** See also

- [[file:../reference/cli.org][The eb-keep-logs command]]
- [[file:../reference/json-schemas.org][JSON output schemas]]
1 change: 1 addition & 0 deletions docs/orgmode/howto/index.org
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ The title names the task.
who-ran-an-executable
check-prefix-use
review-unused-software
compare-flaky-test-bundles
what-a-dry-run-would-build
rotate-a-log-tree
mark-licensed-logs
Expand Down
82 changes: 79 additions & 3 deletions docs/orgmode/reference/cli.org
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

* The eb-keep-logs command

=eb-keep-logs= keeps the logs of a failed EasyBuild build, compresses and removes the old runs of a log tree, says from the kept logs why a run's builds failed, lists how a package's builds went run by run, reads who loaded a module from Lmod's usage log, reads who ran an executable from an XALT run record, lists paths under an install prefix from those records, classifies installed software by its recorded use, and reads what an EasyBuild dry-run or missing-modules report still has to build.
=eb-keep-logs= keeps the logs of a failed EasyBuild build, compresses and removes the old runs of a log tree, says from the kept logs why a run's builds failed, lists how a package's builds went run by run, reads who loaded a module from Lmod's usage log, reads who ran an executable from an XALT run record, lists paths under an install prefix from those records, classifies installed software by its recorded use, reads what an EasyBuild dry-run or missing-modules report still has to build, writes a flaky-test evidence bundle, and compares such bundles across sites.
This page describes the Zig build; the shell script =eb-keep-logs= in the repository behaves the same, except as [[file:compatibility.org][Compatibility]] lists: it has no =why=, =history=, =loads=, =runs=, =used=, =usage= or =pending=.

** Synopsis
Expand All @@ -19,6 +19,8 @@ eb-keep-logs runs --exec PATH [--json] FILE
eb-keep-logs used --prefix PREFIX [--json] [--require-none] FILE...
eb-keep-logs usage --software DIR --modules DIR [--since EPOCH] [--unused | --active] [--json] FILE...
eb-keep-logs pending [--json] FILE
eb-keep-logs test-bundle --test-id ID --suite-revision REV --eessi-target TARGET --stdout FILE --stderr FILE [--input FILE ...] [--module NAME=PATH ...] --seed SEED --allocation TEXT --binding TEXT --cpu TEXT --gpu TEXT [--out FILE]
eb-keep-logs test-compare [--json] BUNDLE...
eb-keep-logs -h | --help
eb-keep-logs --version
#+end_example
Expand Down Expand Up @@ -53,6 +55,20 @@ eb-keep-logs --version
| =--unused= | =usage= | Print only installed =Name/Version= entries with no hit. |
| =--active= | =usage= | Print only installed =Name/Version= entries with a hit, in the listing format =eb-stack stack gap --site= reads. |
| =--json= | =usage= | Print one line of JSON instead of text. |
| =--test-id ID=, =--test-id=ID= | =test-bundle= | Required. Stable test identity shared by the same test across sites and runs. |
| =--suite-revision REV=, =--suite-revision=REV= | =test-bundle= | Required. Test-suite revision string. |
| =--eessi-target TARGET=, =--eessi-target=TARGET= | =test-bundle= | Required. EESSI target identifier. |
| =--stdout FILE=, =--stdout=FILE= | =test-bundle= | Required. Captured stdout to normalize and hash. |
| =--stderr FILE=, =--stderr=FILE= | =test-bundle= | Required. Captured stderr to normalize and hash. |
| =--input FILE=, =--input=FILE= | =test-bundle= | Optional, repeatable. One input file to hash. |
| =--module NAME=PATH=, =--module=NAME=PATH= | =test-bundle= | Optional, repeatable. One loaded module with its modulefile path. |
| =--seed SEED=, =--seed=SEED= | =test-bundle= | Required. Seed value to hash and record. |
| =--allocation TEXT=, =--allocation=TEXT= | =test-bundle= | Required. Scheduler allocation summary. |
| =--binding TEXT=, =--binding=TEXT= | =test-bundle= | Required. Scheduler binding summary. |
| =--cpu TEXT=, =--cpu=TEXT= | =test-bundle= | Required. CPU facts. |
| =--gpu TEXT=, =--gpu=TEXT= | =test-bundle= | Required. GPU facts (or =none=). |
| =--out FILE=, =--out=FILE= | =test-bundle= | Optional. Write the bundle JSON to =FILE= instead of stdout. |
| =--json= | =test-compare= | Print one JSON summary of differing fields instead of text. |
| =-h=, =--help= | | Print the usage on stdout and exit 0. |
| =--version= | | Print the version line on stdout and exit 0. |

Expand Down Expand Up @@ -86,7 +102,7 @@ eb-keep-logs --version

| =MESSAGE= | Cause |
|-----------------------------------------------------------------+-----------------------------------------------------------------------------------------------|
| =unknown command 'WORD' (failed, rotate, why, history, --help)= | The first argument is none of the commands. |
| =unknown command 'WORD' (failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle, test-compare, --help)= | The first argument is none of the commands. |
| =failed needs STATUS NAME LOGDIR [TMP_LOGDIR]= | =failed= got fewer than three arguments. |
| =STATUS must be a number, not 'STATUS'= | =STATUS= is empty or holds something other than digits. |
| =LOGDIR LOGDIR is not a directory= | =LOGDIR= does not name a directory that can be opened. |
Expand Down Expand Up @@ -424,6 +440,21 @@ With =--json= the document has =schema_version= 1, =kind= ="pending"=, and an =e
| =--json= | Print one line of JSON instead of the text lines. |
| =FILE= | The report, as =eb --dry-run= or =eb --missing-modules= printed it. A file that cannot be read, or that is neither report, is refused. |

** test-bundle

=test-bundle= records one test run as a stable flaky-test evidence bundle.
It normalizes stdout and stderr to LF line endings, trims trailing spaces and tabs on each line, and ensures a trailing newline when there is output.
It hashes those normalized streams, hashes each =--input= file, hashes =--seed=, and records modules with modulefile paths, EESSI target, scheduler allocation and binding, CPU and GPU facts, and suite revision.
With =--out FILE= it writes JSON there; without it, it prints the JSON.
The JSON shape is schema_version 1 in [[file:json-schemas.org][JSON schemas]].

** test-compare

=test-compare= reads two or more =test-bundle= JSON documents and reports only which fields differ.
It does not decide pass or fail.
Compared fields are: test identity, suite revision, EESSI target, normalized stdout/stderr hashes, input and seed hashes, module list and paths, scheduler allocation and binding, and CPU/GPU facts.
With =--json= it prints schema_version 1 JSON; without it, a text summary.

*** --help

The usage =-h=, =--help= and no arguments print on stdout, the same from the shell script:
Expand Down Expand Up @@ -594,6 +625,51 @@ otherwise lose, and keep the pipeline's log tree bounded.
FILE
The report, as eb --dry-run or eb --missing-modules printed it.
Required.
eb-keep-logs test-bundle --test-id ID --suite-revision REV --eessi-target TARGET
... --stdout FILE --stderr FILE [--input FILE ...]
... [--module NAME=PATH ...] --seed SEED
... --allocation TEXT --binding TEXT --cpu TEXT --gpu TEXT
[--out FILE]
Write one flaky-test evidence bundle as JSON schema_version 1, with a
stable test identity (ID), normalized stdout and stderr, input and
seed hashes, full module list and module paths, EESSI target,
scheduler allocation and binding, CPU and GPU facts, and test-suite
revision. Without --out, print it on stdout. Zig build only.
--test-id ID, --test-id=ID
Stable test identity. Required.
--suite-revision REV, --suite-revision=REV
Test-suite revision. Required.
--eessi-target TARGET, --eessi-target=TARGET
EESSI target of the run. Required.
--stdout FILE, --stdout=FILE
Captured stdout, normalized and hashed. Required.
--stderr FILE, --stderr=FILE
Captured stderr, normalized and hashed. Required.
--input FILE, --input=FILE
Input file to hash. Optional; may be repeated. Default: none.
--module NAME=PATH, --module=NAME=PATH
Loaded module and its modulefile path. Optional; may be
repeated. Default: none.
--seed SEED, --seed=SEED
Seed value to hash. Required.
--allocation TEXT, --allocation=TEXT
Scheduler allocation summary. Required.
--binding TEXT, --binding=TEXT
Scheduler binding summary. Required.
--cpu TEXT, --cpu=TEXT
CPU facts. Required.
--gpu TEXT, --gpu=TEXT
GPU facts, or none. Required.
--out FILE, --out=FILE
Write JSON to FILE. Default: stdout.
eb-keep-logs test-compare [--json] BUNDLE...
Summarize which evidence fields differ across two or more
test-bundle JSON files, without deciding pass or fail. --json prints
schema_version 1. Zig build only.
--json
Print one line of JSON instead of text. Default: text.
BUNDLE
Two or more test-bundle JSON files. Required.
eb-keep-logs -h | --help
Print this help on stdout and exit 0.
eb-keep-logs --version
Expand All @@ -615,7 +691,7 @@ Environment:
directory of zone files.

Exit status:
0 -h, --help or --version, or a failed, rotate, why, history, loads, runs, used, usage or pending that
0 -h, --help or --version, or a failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle or test-compare that
accepted its arguments, whatever it could keep, compress, remove or
read: what it could not do is reported on stderr. used with
--require-none also exits 2 when any path was listed.
Expand Down
7 changes: 5 additions & 2 deletions docs/orgmode/reference/json-schemas.org
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@

* JSON output schemas

The Zig build's =--json= on =why=, =history=, =loads=, =runs=, =used=, =usage= and =pending= prints one line of JSON with =schema_version= 1.
The Zig build's =--json= on =why=, =history=, =loads=, =runs=, =used=, =usage=, =pending= and =test-compare= prints one line of JSON with =schema_version= 1.
The =test-bundle= command always writes one JSON document with =schema_version= 1.
The shell script refuses those commands; see [[file:compatibility.org][Compatibility]].
Each document is validated in CI against a JSON Schema file in the repository tree under =docs/orgmode/reference/json/=.
The schemas use draft 2020-12; a new =schema_version= may drop or rename a key, or change what one means.
Expand All @@ -18,9 +19,11 @@ The schemas use draft 2020-12; a new =schema_version= may drop or rename a key,
| =used= | =docs/orgmode/reference/json/used.schema.json= |
| =usage= | =docs/orgmode/reference/json/usage.schema.json= |
| =pending= | =docs/orgmode/reference/json/pending.schema.json= |
| =test-bundle= | =docs/orgmode/reference/json/test-bundle.schema.json= |
| =test-compare= | =docs/orgmode/reference/json/test-compare.schema.json= |

Field names and semantics for =why=, =history=, =used= and =usage= are under [[file:cli.org][The eb-keep-logs command]].
=loads=, =runs= and =pending= document their JSON shape in the same page.
=loads=, =runs=, =pending=, =test-bundle= and =test-compare= document their JSON shape in the same page.

** See also

Expand Down
103 changes: 103 additions & 0 deletions docs/orgmode/reference/json/test-bundle.schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://stackhands.github.io/eb-keep-logs/reference/json/test-bundle.schema.json",
"title": "eb-keep-logs test-bundle",
"type": "object",
"additionalProperties": false,
"required": [
"schema_version",
"kind",
"test_id",
"suite_revision",
"eessi_target",
"stdout",
"stderr",
"inputs",
"input_hash_sha256",
"seed",
"modules",
"scheduler",
"hardware"
],
"properties": {
"schema_version": { "type": "integer", "const": 1 },
"kind": { "type": "string", "const": "test-bundle" },
"test_id": { "type": "string", "minLength": 1 },
"suite_revision": { "type": "string", "minLength": 1 },
"eessi_target": { "type": "string", "minLength": 1 },
"stdout": { "$ref": "#/$defs/stream" },
"stderr": { "$ref": "#/$defs/stream" },
"inputs": {
"type": "array",
"items": { "$ref": "#/$defs/input" }
},
"input_hash_sha256": { "$ref": "#/$defs/sha256" },
"seed": { "$ref": "#/$defs/seed" },
"modules": {
"type": "array",
"items": { "$ref": "#/$defs/module" }
},
"scheduler": { "$ref": "#/$defs/scheduler" },
"hardware": { "$ref": "#/$defs/hardware" }
},
"$defs": {
"sha256": {
"type": "string",
"pattern": "^[0-9a-f]{64}$"
},
"stream": {
"type": "object",
"additionalProperties": false,
"required": ["normalized", "sha256"],
"properties": {
"normalized": { "type": "string" },
"sha256": { "$ref": "#/$defs/sha256" }
}
},
"input": {
"type": "object",
"additionalProperties": false,
"required": ["path", "sha256"],
"properties": {
"path": { "type": "string", "minLength": 1 },
"sha256": { "$ref": "#/$defs/sha256" }
}
},
"seed": {
"type": "object",
"additionalProperties": false,
"required": ["value", "sha256"],
"properties": {
"value": { "type": "string" },
"sha256": { "$ref": "#/$defs/sha256" }
}
},
"module": {
"type": "object",
"additionalProperties": false,
"required": ["name", "path"],
"properties": {
"name": { "type": "string", "minLength": 1 },
"path": { "type": "string", "minLength": 1 }
}
},
"scheduler": {
"type": "object",
"additionalProperties": false,
"required": ["allocation", "binding"],
"properties": {
"allocation": { "type": "string", "minLength": 1 },
"binding": { "type": "string", "minLength": 1 }
}
},
"hardware": {
"type": "object",
"additionalProperties": false,
"required": ["cpu", "gpu"],
"properties": {
"cpu": { "type": "string", "minLength": 1 },
"gpu": { "type": "string", "minLength": 1 }
}
}
}
}
Loading
Loading