From ab83d862cfe36789be59cd9e8c56cc5dfc189d73 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Tue, 17 Nov 2026 22:59:21 +0000 Subject: [PATCH] feat: add test-bundle and test-compare evidence workflows --- build.zig | 12 + changelog.d/+test-bundle.feature.md | 1 + .../howto/compare-flaky-test-bundles.org | 60 ++ docs/orgmode/howto/index.org | 1 + docs/orgmode/reference/cli.org | 82 ++- docs/orgmode/reference/json-schemas.org | 7 +- .../reference/json/test-bundle.schema.json | 103 +++ .../reference/json/test-compare.schema.json | 45 ++ eb-keep-logs | 51 +- src/main.zig | 53 +- src/test_bundle.zig | 617 ++++++++++++++++++ tests/test_json_schemas.sh | 61 ++ tests/test_output_stability.sh | 44 ++ tests/test_test_bundle.sh | 88 +++ 14 files changed, 1215 insertions(+), 10 deletions(-) create mode 100644 changelog.d/+test-bundle.feature.md create mode 100644 docs/orgmode/howto/compare-flaky-test-bundles.org create mode 100644 docs/orgmode/reference/json/test-bundle.schema.json create mode 100644 docs/orgmode/reference/json/test-compare.schema.json create mode 100644 src/test_bundle.zig create mode 100755 tests/test_output_stability.sh create mode 100755 tests/test_test_bundle.sh diff --git a/build.zig b/build.zig index ad7027d..7f36e40 100644 --- a/build.zig +++ b/build.zig @@ -97,6 +97,16 @@ pub fn build(b: *std.Build) void { json_schema_test.setCwd(b.path(".")); json_schema_test.expectExitCode(0); json_schema_test.has_side_effects = true; + const test_bundle_test = b.addSystemCommand(&.{ "bash", "-c", "TOOL_BIN=\"$0\" exec bash tests/test_test_bundle.sh" }); + test_bundle_test.addArtifactArg(exe); + test_bundle_test.setCwd(b.path(".")); + test_bundle_test.expectExitCode(0); + test_bundle_test.has_side_effects = true; + const stable_output_test = b.addSystemCommand(&.{ "bash", "-c", "TOOL_BIN=\"$0\" exec bash tests/test_output_stability.sh" }); + stable_output_test.addArtifactArg(exe); + stable_output_test.setCwd(b.path(".")); + stable_output_test.expectExitCode(0); + stable_output_test.has_side_effects = true; const test_step = b.step("test", "Run the unit tests, the command-line suites and the differential run"); test_step.dependOn(&b.addRunArtifact(unit).step); @@ -110,4 +120,6 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&usage_test.step); test_step.dependOn(&pending_test.step); test_step.dependOn(&json_schema_test.step); + test_step.dependOn(&test_bundle_test.step); + test_step.dependOn(&stable_output_test.step); } diff --git a/changelog.d/+test-bundle.feature.md b/changelog.d/+test-bundle.feature.md new file mode 100644 index 0000000..8a03e39 --- /dev/null +++ b/changelog.d/+test-bundle.feature.md @@ -0,0 +1 @@ +Add `test-bundle` and `test-compare` to capture and compare normalized flaky-test evidence bundles across sites, with schema-versioned JSON outputs and documentation. diff --git a/docs/orgmode/howto/compare-flaky-test-bundles.org b/docs/orgmode/howto/compare-flaky-test-bundles.org new file mode 100644 index 0000000..3604748 --- /dev/null +++ b/docs/orgmode/howto/compare-flaky-test-bundles.org @@ -0,0 +1,60 @@ +#+TITLE: Compare flaky-test evidence bundles across sites +#+OPTIONS: toc:nil todo:nil num:nil author:nil title:nil +#+STARTUP: content + +* Compare flaky-test evidence bundles across sites + +This guide records two test runs as comparable evidence bundles and compares their differing fields. +It does not decide pass or fail. + +** 1. Capture one bundle per test run + +Save each run's stdout and stderr, then run =test-bundle= with the same stable test identity for that test: + +#+begin_example +eb-keep-logs test-bundle \ + --test-id qe.pw.scf-si \ + --suite-revision 5f6a9cc \ + --eessi-target x86_64/intel/sapphirerapids \ + --stdout run-a.stdout \ + --stderr run-a.stderr \ + --input qe.in \ + --seed 42 \ + --module QuantumESPRESSO/7.4=/cvmfs/eessi/software/QuantumESPRESSO/7.4.lua \ + --allocation "nodes=1 ntasks=32" \ + --binding "rank0->core0, rank1->core1" \ + --cpu "AMD EPYC 7B13 (32)" \ + --gpu "none" \ + --out run-a.bundle.json +#+end_example + +Run it again for another site or run, changing only what actually differs, and write e.g. =run-b.bundle.json=. + +** 2. Compare bundles + +For a text summary: + +#+begin_example +eb-keep-logs test-compare run-a.bundle.json run-b.bundle.json +#+end_example + +For machine-readable output: + +#+begin_example +eb-keep-logs test-compare --json run-a.bundle.json run-b.bundle.json > compare.json +#+end_example + +The summary lists only fields that differ across bundles: + +- stable test identity and suite revision; +- normalized stdout/stderr hashes; +- input and seed hashes; +- module list and module paths; +- EESSI target; +- scheduler allocation and binding; +- CPU and GPU facts. + +** See also + +- [[file:../reference/cli.org][The eb-keep-logs command]] +- [[file:../reference/json-schemas.org][JSON output schemas]] diff --git a/docs/orgmode/howto/index.org b/docs/orgmode/howto/index.org index 8cd5d8f..e988485 100644 --- a/docs/orgmode/howto/index.org +++ b/docs/orgmode/howto/index.org @@ -18,6 +18,7 @@ The title names the task. who-ran-an-executable check-prefix-use review-unused-software + compare-flaky-test-bundles what-a-dry-run-would-build rotate-a-log-tree mark-licensed-logs diff --git a/docs/orgmode/reference/cli.org b/docs/orgmode/reference/cli.org index 59bb383..d8c9db8 100644 --- a/docs/orgmode/reference/cli.org +++ b/docs/orgmode/reference/cli.org @@ -4,7 +4,7 @@ * The eb-keep-logs command -=eb-keep-logs= keeps the logs of a failed EasyBuild build, compresses and removes the old runs of a log tree, says from the kept logs why a run's builds failed, lists how a package's builds went run by run, reads who loaded a module from Lmod's usage log, reads who ran an executable from an XALT run record, lists paths under an install prefix from those records, classifies installed software by its recorded use, and reads what an EasyBuild dry-run or missing-modules report still has to build. +=eb-keep-logs= keeps the logs of a failed EasyBuild build, compresses and removes the old runs of a log tree, says from the kept logs why a run's builds failed, lists how a package's builds went run by run, reads who loaded a module from Lmod's usage log, reads who ran an executable from an XALT run record, lists paths under an install prefix from those records, classifies installed software by its recorded use, reads what an EasyBuild dry-run or missing-modules report still has to build, writes a flaky-test evidence bundle, and compares such bundles across sites. This page describes the Zig build; the shell script =eb-keep-logs= in the repository behaves the same, except as [[file:compatibility.org][Compatibility]] lists: it has no =why=, =history=, =loads=, =runs=, =used=, =usage= or =pending=. ** Synopsis @@ -19,6 +19,8 @@ eb-keep-logs runs --exec PATH [--json] FILE eb-keep-logs used --prefix PREFIX [--json] [--require-none] FILE... eb-keep-logs usage --software DIR --modules DIR [--since EPOCH] [--unused | --active] [--json] FILE... eb-keep-logs pending [--json] FILE +eb-keep-logs test-bundle --test-id ID --suite-revision REV --eessi-target TARGET --stdout FILE --stderr FILE [--input FILE ...] [--module NAME=PATH ...] --seed SEED --allocation TEXT --binding TEXT --cpu TEXT --gpu TEXT [--out FILE] +eb-keep-logs test-compare [--json] BUNDLE... eb-keep-logs -h | --help eb-keep-logs --version #+end_example @@ -53,6 +55,20 @@ eb-keep-logs --version | =--unused= | =usage= | Print only installed =Name/Version= entries with no hit. | | =--active= | =usage= | Print only installed =Name/Version= entries with a hit, in the listing format =eb-stack stack gap --site= reads. | | =--json= | =usage= | Print one line of JSON instead of text. | +| =--test-id ID=, =--test-id=ID= | =test-bundle= | Required. Stable test identity shared by the same test across sites and runs. | +| =--suite-revision REV=, =--suite-revision=REV= | =test-bundle= | Required. Test-suite revision string. | +| =--eessi-target TARGET=, =--eessi-target=TARGET= | =test-bundle= | Required. EESSI target identifier. | +| =--stdout FILE=, =--stdout=FILE= | =test-bundle= | Required. Captured stdout to normalize and hash. | +| =--stderr FILE=, =--stderr=FILE= | =test-bundle= | Required. Captured stderr to normalize and hash. | +| =--input FILE=, =--input=FILE= | =test-bundle= | Optional, repeatable. One input file to hash. | +| =--module NAME=PATH=, =--module=NAME=PATH= | =test-bundle= | Optional, repeatable. One loaded module with its modulefile path. | +| =--seed SEED=, =--seed=SEED= | =test-bundle= | Required. Seed value to hash and record. | +| =--allocation TEXT=, =--allocation=TEXT= | =test-bundle= | Required. Scheduler allocation summary. | +| =--binding TEXT=, =--binding=TEXT= | =test-bundle= | Required. Scheduler binding summary. | +| =--cpu TEXT=, =--cpu=TEXT= | =test-bundle= | Required. CPU facts. | +| =--gpu TEXT=, =--gpu=TEXT= | =test-bundle= | Required. GPU facts (or =none=). | +| =--out FILE=, =--out=FILE= | =test-bundle= | Optional. Write the bundle JSON to =FILE= instead of stdout. | +| =--json= | =test-compare= | Print one JSON summary of differing fields instead of text. | | =-h=, =--help= | | Print the usage on stdout and exit 0. | | =--version= | | Print the version line on stdout and exit 0. | @@ -86,7 +102,7 @@ eb-keep-logs --version | =MESSAGE= | Cause | |-----------------------------------------------------------------+-----------------------------------------------------------------------------------------------| -| =unknown command 'WORD' (failed, rotate, why, history, --help)= | The first argument is none of the commands. | +| =unknown command 'WORD' (failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle, test-compare, --help)= | The first argument is none of the commands. | | =failed needs STATUS NAME LOGDIR [TMP_LOGDIR]= | =failed= got fewer than three arguments. | | =STATUS must be a number, not 'STATUS'= | =STATUS= is empty or holds something other than digits. | | =LOGDIR LOGDIR is not a directory= | =LOGDIR= does not name a directory that can be opened. | @@ -424,6 +440,21 @@ With =--json= the document has =schema_version= 1, =kind= ="pending"=, and an =e | =--json= | Print one line of JSON instead of the text lines. | | =FILE= | The report, as =eb --dry-run= or =eb --missing-modules= printed it. A file that cannot be read, or that is neither report, is refused. | +** test-bundle + +=test-bundle= records one test run as a stable flaky-test evidence bundle. +It normalizes stdout and stderr to LF line endings, trims trailing spaces and tabs on each line, and ensures a trailing newline when there is output. +It hashes those normalized streams, hashes each =--input= file, hashes =--seed=, and records modules with modulefile paths, EESSI target, scheduler allocation and binding, CPU and GPU facts, and suite revision. +With =--out FILE= it writes JSON there; without it, it prints the JSON. +The JSON shape is schema_version 1 in [[file:json-schemas.org][JSON schemas]]. + +** test-compare + +=test-compare= reads two or more =test-bundle= JSON documents and reports only which fields differ. +It does not decide pass or fail. +Compared fields are: test identity, suite revision, EESSI target, normalized stdout/stderr hashes, input and seed hashes, module list and paths, scheduler allocation and binding, and CPU/GPU facts. +With =--json= it prints schema_version 1 JSON; without it, a text summary. + *** --help The usage =-h=, =--help= and no arguments print on stdout, the same from the shell script: @@ -594,6 +625,51 @@ otherwise lose, and keep the pipeline's log tree bounded. FILE The report, as eb --dry-run or eb --missing-modules printed it. Required. + eb-keep-logs test-bundle --test-id ID --suite-revision REV --eessi-target TARGET + ... --stdout FILE --stderr FILE [--input FILE ...] + ... [--module NAME=PATH ...] --seed SEED + ... --allocation TEXT --binding TEXT --cpu TEXT --gpu TEXT + [--out FILE] + Write one flaky-test evidence bundle as JSON schema_version 1, with a + stable test identity (ID), normalized stdout and stderr, input and + seed hashes, full module list and module paths, EESSI target, + scheduler allocation and binding, CPU and GPU facts, and test-suite + revision. Without --out, print it on stdout. Zig build only. + --test-id ID, --test-id=ID + Stable test identity. Required. + --suite-revision REV, --suite-revision=REV + Test-suite revision. Required. + --eessi-target TARGET, --eessi-target=TARGET + EESSI target of the run. Required. + --stdout FILE, --stdout=FILE + Captured stdout, normalized and hashed. Required. + --stderr FILE, --stderr=FILE + Captured stderr, normalized and hashed. Required. + --input FILE, --input=FILE + Input file to hash. Optional; may be repeated. Default: none. + --module NAME=PATH, --module=NAME=PATH + Loaded module and its modulefile path. Optional; may be + repeated. Default: none. + --seed SEED, --seed=SEED + Seed value to hash. Required. + --allocation TEXT, --allocation=TEXT + Scheduler allocation summary. Required. + --binding TEXT, --binding=TEXT + Scheduler binding summary. Required. + --cpu TEXT, --cpu=TEXT + CPU facts. Required. + --gpu TEXT, --gpu=TEXT + GPU facts, or none. Required. + --out FILE, --out=FILE + Write JSON to FILE. Default: stdout. + eb-keep-logs test-compare [--json] BUNDLE... + Summarize which evidence fields differ across two or more + test-bundle JSON files, without deciding pass or fail. --json prints + schema_version 1. Zig build only. + --json + Print one line of JSON instead of text. Default: text. + BUNDLE + Two or more test-bundle JSON files. Required. eb-keep-logs -h | --help Print this help on stdout and exit 0. eb-keep-logs --version @@ -615,7 +691,7 @@ Environment: directory of zone files. Exit status: - 0 -h, --help or --version, or a failed, rotate, why, history, loads, runs, used, usage or pending that + 0 -h, --help or --version, or a failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle or test-compare that accepted its arguments, whatever it could keep, compress, remove or read: what it could not do is reported on stderr. used with --require-none also exits 2 when any path was listed. diff --git a/docs/orgmode/reference/json-schemas.org b/docs/orgmode/reference/json-schemas.org index 86ea16d..8f4aa9b 100644 --- a/docs/orgmode/reference/json-schemas.org +++ b/docs/orgmode/reference/json-schemas.org @@ -4,7 +4,8 @@ * JSON output schemas -The Zig build's =--json= on =why=, =history=, =loads=, =runs=, =used=, =usage= and =pending= prints one line of JSON with =schema_version= 1. +The Zig build's =--json= on =why=, =history=, =loads=, =runs=, =used=, =usage=, =pending= and =test-compare= prints one line of JSON with =schema_version= 1. +The =test-bundle= command always writes one JSON document with =schema_version= 1. The shell script refuses those commands; see [[file:compatibility.org][Compatibility]]. Each document is validated in CI against a JSON Schema file in the repository tree under =docs/orgmode/reference/json/=. The schemas use draft 2020-12; a new =schema_version= may drop or rename a key, or change what one means. @@ -18,9 +19,11 @@ The schemas use draft 2020-12; a new =schema_version= may drop or rename a key, | =used= | =docs/orgmode/reference/json/used.schema.json= | | =usage= | =docs/orgmode/reference/json/usage.schema.json= | | =pending= | =docs/orgmode/reference/json/pending.schema.json= | +| =test-bundle= | =docs/orgmode/reference/json/test-bundle.schema.json= | +| =test-compare= | =docs/orgmode/reference/json/test-compare.schema.json= | Field names and semantics for =why=, =history=, =used= and =usage= are under [[file:cli.org][The eb-keep-logs command]]. -=loads=, =runs= and =pending= document their JSON shape in the same page. +=loads=, =runs=, =pending=, =test-bundle= and =test-compare= document their JSON shape in the same page. ** See also diff --git a/docs/orgmode/reference/json/test-bundle.schema.json b/docs/orgmode/reference/json/test-bundle.schema.json new file mode 100644 index 0000000..2eb69db --- /dev/null +++ b/docs/orgmode/reference/json/test-bundle.schema.json @@ -0,0 +1,103 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://stackhands.github.io/eb-keep-logs/reference/json/test-bundle.schema.json", + "title": "eb-keep-logs test-bundle", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "kind", + "test_id", + "suite_revision", + "eessi_target", + "stdout", + "stderr", + "inputs", + "input_hash_sha256", + "seed", + "modules", + "scheduler", + "hardware" + ], + "properties": { + "schema_version": { "type": "integer", "const": 1 }, + "kind": { "type": "string", "const": "test-bundle" }, + "test_id": { "type": "string", "minLength": 1 }, + "suite_revision": { "type": "string", "minLength": 1 }, + "eessi_target": { "type": "string", "minLength": 1 }, + "stdout": { "$ref": "#/$defs/stream" }, + "stderr": { "$ref": "#/$defs/stream" }, + "inputs": { + "type": "array", + "items": { "$ref": "#/$defs/input" } + }, + "input_hash_sha256": { "$ref": "#/$defs/sha256" }, + "seed": { "$ref": "#/$defs/seed" }, + "modules": { + "type": "array", + "items": { "$ref": "#/$defs/module" } + }, + "scheduler": { "$ref": "#/$defs/scheduler" }, + "hardware": { "$ref": "#/$defs/hardware" } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "stream": { + "type": "object", + "additionalProperties": false, + "required": ["normalized", "sha256"], + "properties": { + "normalized": { "type": "string" }, + "sha256": { "$ref": "#/$defs/sha256" } + } + }, + "input": { + "type": "object", + "additionalProperties": false, + "required": ["path", "sha256"], + "properties": { + "path": { "type": "string", "minLength": 1 }, + "sha256": { "$ref": "#/$defs/sha256" } + } + }, + "seed": { + "type": "object", + "additionalProperties": false, + "required": ["value", "sha256"], + "properties": { + "value": { "type": "string" }, + "sha256": { "$ref": "#/$defs/sha256" } + } + }, + "module": { + "type": "object", + "additionalProperties": false, + "required": ["name", "path"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "path": { "type": "string", "minLength": 1 } + } + }, + "scheduler": { + "type": "object", + "additionalProperties": false, + "required": ["allocation", "binding"], + "properties": { + "allocation": { "type": "string", "minLength": 1 }, + "binding": { "type": "string", "minLength": 1 } + } + }, + "hardware": { + "type": "object", + "additionalProperties": false, + "required": ["cpu", "gpu"], + "properties": { + "cpu": { "type": "string", "minLength": 1 }, + "gpu": { "type": "string", "minLength": 1 } + } + } + } +} diff --git a/docs/orgmode/reference/json/test-compare.schema.json b/docs/orgmode/reference/json/test-compare.schema.json new file mode 100644 index 0000000..cab85c5 --- /dev/null +++ b/docs/orgmode/reference/json/test-compare.schema.json @@ -0,0 +1,45 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://stackhands.github.io/eb-keep-logs/reference/json/test-compare.schema.json", + "title": "eb-keep-logs test-compare --json", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "kind", "bundle_count", "differing_fields"], + "properties": { + "schema_version": { "type": "integer", "const": 1 }, + "kind": { "type": "string", "const": "test-compare" }, + "bundle_count": { "type": "integer", "minimum": 2 }, + "differing_fields": { + "type": "array", + "items": { "$ref": "#/$defs/field" } + } + }, + "$defs": { + "field": { + "type": "object", + "additionalProperties": false, + "required": ["field", "values"], + "properties": { + "field": { "type": "string", "minLength": 1 }, + "values": { + "type": "array", + "minItems": 2, + "items": { "$ref": "#/$defs/value_group" } + } + } + }, + "value_group": { + "type": "object", + "additionalProperties": false, + "required": ["value", "bundles"], + "properties": { + "value": { "type": "string" }, + "bundles": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + } + } + } + } +} diff --git a/eb-keep-logs b/eb-keep-logs index a0d28ee..ed75f5b 100755 --- a/eb-keep-logs +++ b/eb-keep-logs @@ -179,6 +179,51 @@ otherwise lose, and keep the pipeline's log tree bounded. FILE The report, as eb --dry-run or eb --missing-modules printed it. Required. + eb-keep-logs test-bundle --test-id ID --suite-revision REV --eessi-target TARGET + ... --stdout FILE --stderr FILE [--input FILE ...] + ... [--module NAME=PATH ...] --seed SEED + ... --allocation TEXT --binding TEXT --cpu TEXT --gpu TEXT + [--out FILE] + Write one flaky-test evidence bundle as JSON schema_version 1, with a + stable test identity (ID), normalized stdout and stderr, input and + seed hashes, full module list and module paths, EESSI target, + scheduler allocation and binding, CPU and GPU facts, and test-suite + revision. Without --out, print it on stdout. Zig build only. + --test-id ID, --test-id=ID + Stable test identity. Required. + --suite-revision REV, --suite-revision=REV + Test-suite revision. Required. + --eessi-target TARGET, --eessi-target=TARGET + EESSI target of the run. Required. + --stdout FILE, --stdout=FILE + Captured stdout, normalized and hashed. Required. + --stderr FILE, --stderr=FILE + Captured stderr, normalized and hashed. Required. + --input FILE, --input=FILE + Input file to hash. Optional; may be repeated. Default: none. + --module NAME=PATH, --module=NAME=PATH + Loaded module and its modulefile path. Optional; may be + repeated. Default: none. + --seed SEED, --seed=SEED + Seed value to hash. Required. + --allocation TEXT, --allocation=TEXT + Scheduler allocation summary. Required. + --binding TEXT, --binding=TEXT + Scheduler binding summary. Required. + --cpu TEXT, --cpu=TEXT + CPU facts. Required. + --gpu TEXT, --gpu=TEXT + GPU facts, or none. Required. + --out FILE, --out=FILE + Write JSON to FILE. Default: stdout. + eb-keep-logs test-compare [--json] BUNDLE... + Summarize which evidence fields differ across two or more + test-bundle JSON files, without deciding pass or fail. --json prints + schema_version 1. Zig build only. + --json + Print one line of JSON instead of text. Default: text. + BUNDLE + Two or more test-bundle JSON files. Required. eb-keep-logs -h | --help Print this help on stdout and exit 0. eb-keep-logs --version @@ -200,7 +245,7 @@ Environment: directory of zone files. Exit status: - 0 -h, --help or --version, or a failed, rotate, why, history, loads, runs, used, usage or pending that + 0 -h, --help or --version, or a failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle or test-compare that accepted its arguments, whatever it could keep, compress, remove or read: what it could not do is reported on stderr. used with --require-none also exits 2 when any path was listed. @@ -768,7 +813,7 @@ case "${1:-}" in --version) echo "eb-keep-logs ${VERSION}" ;; failed) shift; cmd_failed "$@" ;; rotate) shift; cmd_rotate "$@" ;; - why | history | loads | runs | used | usage | pending) die "$1 is in the Zig build only" ;; + why | history | loads | runs | used | usage | pending | test-bundle | test-compare) die "$1 is in the Zig build only" ;; "") usage; exit 2 ;; - *) die "unknown command '$1' (failed, rotate, why, history, loads, runs, used, usage, pending, --help)" ;; + *) die "unknown command '$1' (failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle, test-compare, --help)" ;; esac diff --git a/src/main.zig b/src/main.zig index ef22ba8..e5e0a42 100644 --- a/src/main.zig +++ b/src/main.zig @@ -17,6 +17,7 @@ const xalt_runs = @import("runs.zig"); const pending = @import("pending.zig"); const used_cmd = @import("used.zig"); const usage_cmd = @import("usage.zig"); +const test_bundle = @import("test_bundle.zig"); const syslog = @import("syslog.zig"); const Allocator = std.mem.Allocator; @@ -193,6 +194,51 @@ const usage_text = \\ FILE \\ The report, as eb --dry-run or eb --missing-modules printed it. \\ Required. + \\ eb-keep-logs test-bundle --test-id ID --suite-revision REV --eessi-target TARGET + \\ ... --stdout FILE --stderr FILE [--input FILE ...] + \\ ... [--module NAME=PATH ...] --seed SEED + \\ ... --allocation TEXT --binding TEXT --cpu TEXT --gpu TEXT + \\ [--out FILE] + \\ Write one flaky-test evidence bundle as JSON schema_version 1, with a + \\ stable test identity (ID), normalized stdout and stderr, input and + \\ seed hashes, full module list and module paths, EESSI target, + \\ scheduler allocation and binding, CPU and GPU facts, and test-suite + \\ revision. Without --out, print it on stdout. Zig build only. + \\ --test-id ID, --test-id=ID + \\ Stable test identity. Required. + \\ --suite-revision REV, --suite-revision=REV + \\ Test-suite revision. Required. + \\ --eessi-target TARGET, --eessi-target=TARGET + \\ EESSI target of the run. Required. + \\ --stdout FILE, --stdout=FILE + \\ Captured stdout, normalized and hashed. Required. + \\ --stderr FILE, --stderr=FILE + \\ Captured stderr, normalized and hashed. Required. + \\ --input FILE, --input=FILE + \\ Input file to hash. Optional; may be repeated. Default: none. + \\ --module NAME=PATH, --module=NAME=PATH + \\ Loaded module and its modulefile path. Optional; may be + \\ repeated. Default: none. + \\ --seed SEED, --seed=SEED + \\ Seed value to hash. Required. + \\ --allocation TEXT, --allocation=TEXT + \\ Scheduler allocation summary. Required. + \\ --binding TEXT, --binding=TEXT + \\ Scheduler binding summary. Required. + \\ --cpu TEXT, --cpu=TEXT + \\ CPU facts. Required. + \\ --gpu TEXT, --gpu=TEXT + \\ GPU facts, or none. Required. + \\ --out FILE, --out=FILE + \\ Write JSON to FILE. Default: stdout. + \\ eb-keep-logs test-compare [--json] BUNDLE... + \\ Summarize which evidence fields differ across two or more + \\ test-bundle JSON files, without deciding pass or fail. --json prints + \\ schema_version 1. Zig build only. + \\ --json + \\ Print one line of JSON instead of text. Default: text. + \\ BUNDLE + \\ Two or more test-bundle JSON files. Required. \\ eb-keep-logs -h | --help \\ Print this help on stdout and exit 0. \\ eb-keep-logs --version @@ -214,7 +260,7 @@ const usage_text = \\ directory of zone files. \\ \\Exit status: - \\ 0 -h, --help or --version, or a failed, rotate, why, history, loads, runs, used, usage or pending that + \\ 0 -h, --help or --version, or a failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle or test-compare that \\ accepted its arguments, whatever it could keep, compress, remove or \\ read: what it could not do is reported on stderr. used with \\ --require-none also exits 2 when any path was listed. @@ -1141,11 +1187,13 @@ pub fn main(init: std.process.Init) u8 { if (std.mem.eql(u8, cmd, "used")) return used_cmd.cmdUsed(gpa, args[2..]); if (std.mem.eql(u8, cmd, "usage")) return usage_cmd.cmdUsage(gpa, args[2..]); if (std.mem.eql(u8, cmd, "pending")) return pending.cmdPending(gpa, args[2..]); + if (std.mem.eql(u8, cmd, "test-bundle")) return test_bundle.cmdTestBundle(gpa, args[2..]); + if (std.mem.eql(u8, cmd, "test-compare")) return test_bundle.cmdTestCompare(gpa, args[2..]); if (cmd.len == 0) { cli.put(1, usage_text); return cli.usage_exit; } - die("unknown command '{s}' (failed, rotate, why, history, loads, runs, used, usage, pending, --help)", .{cmd}); + die("unknown command '{s}' (failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle, test-compare, --help)", .{cmd}); } test { @@ -1157,6 +1205,7 @@ test { _ = pending; _ = used_cmd; _ = usage_cmd; + _ = test_bundle; _ = syslog; } diff --git a/src/test_bundle.zig b/src/test_bundle.zig new file mode 100644 index 0000000..68ad4c7 --- /dev/null +++ b/src/test_bundle.zig @@ -0,0 +1,617 @@ +//! `test-bundle`: record a comparable flaky-test evidence bundle. +//! `test-compare`: summarize differing fields across such bundles. +const std = @import("std"); +const common = @import("zig_common"); +const cli = common.cli; +const fsx = common.fsx; +const sys = common.sys; + +const Allocator = std.mem.Allocator; +const tool = "eb-keep-logs"; +const read_limit: usize = 64 * 1024 * 1024; + +pub const bundle_schema_version: u32 = 1; +pub const compare_schema_version: u32 = 1; + +fn die(comptime fmt: []const u8, args: anytype) noreturn { + cli.die(tool, fmt, args); +} + +fn warn(comptime fmt: []const u8, args: anytype) void { + cli.err(tool ++ ": " ++ fmt ++ "\n", args); +} + +fn readFile(gpa: Allocator, path: []const u8) ?[]u8 { + const opened = fsx.openRegular(fsx.cwd, path, false) catch |err| { + warn("cannot read {s}: {s}", .{ path, @errorName(err) }); + return null; + }; + defer sys.close(opened.fd); + return fsx.readAll(gpa, opened.fd, read_limit) catch |err| { + warn("cannot read {s}: {s}", .{ path, @errorName(err) }); + return null; + }; +} + +fn normalizeText(gpa: Allocator, text: []const u8) ![]u8 { + var out: std.ArrayList(u8) = .empty; + errdefer out.deinit(gpa); + var i: usize = 0; + var wrote_any = false; + while (i < text.len) { + const line_start = i; + while (i < text.len and text[i] != '\n' and text[i] != '\r') : (i += 1) {} + var line = text[line_start..i]; + line = std.mem.trimEnd(u8, line, " \t"); + try out.appendSlice(gpa, line); + wrote_any = wrote_any or line.len > 0; + if (i < text.len) { + if (text[i] == '\r' and i + 1 < text.len and text[i + 1] == '\n') i += 1; + i += 1; + try out.append(gpa, '\n'); + wrote_any = true; + } + } + if (!wrote_any) return try out.toOwnedSlice(gpa); + if (out.items.len == 0 or out.items[out.items.len - 1] != '\n') try out.append(gpa, '\n'); + return try out.toOwnedSlice(gpa); +} + +fn sha256Hex(gpa: Allocator, bytes: []const u8) ![]u8 { + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(bytes, &digest, .{}); + var out: [64]u8 = undefined; + _ = std.fmt.bufPrint(&out, "{x}", .{digest}) catch unreachable; + return try gpa.dupe(u8, &out); +} + +const InputRecord = struct { + path: []const u8, + sha256: []const u8, +}; + +const ModuleRecord = struct { + name: []const u8, + path: []const u8, +}; + +const StreamRecord = struct { + normalized: []const u8, + sha256: []const u8, +}; + +const SchedulerRecord = struct { + allocation: []const u8, + binding: []const u8, +}; + +const HardwareRecord = struct { + cpu: []const u8, + gpu: []const u8, +}; + +const SeedRecord = struct { + value: []const u8, + sha256: []const u8, +}; + +const BundleRecord = struct { + schema_version: u32 = bundle_schema_version, + kind: []const u8 = "test-bundle", + test_id: []const u8, + suite_revision: []const u8, + eessi_target: []const u8, + stdout: StreamRecord, + stderr: StreamRecord, + inputs: []const InputRecord, + input_hash_sha256: []const u8, + seed: SeedRecord, + modules: []const ModuleRecord, + scheduler: SchedulerRecord, + hardware: HardwareRecord, +}; + +fn lessModule(_: void, a: ModuleRecord, b: ModuleRecord) bool { + if (std.mem.lessThan(u8, a.name, b.name)) return true; + if (std.mem.lessThan(u8, b.name, a.name)) return false; + return std.mem.lessThan(u8, a.path, b.path); +} + +fn lessInput(_: void, a: InputRecord, b: InputRecord) bool { + return std.mem.lessThan(u8, a.path, b.path); +} + +fn parseModule(gpa: Allocator, raw: []const u8) !ModuleRecord { + const sep = std.mem.indexOfScalar(u8, raw, '=') orelse + die("test-bundle needs --module NAME=PATH", .{}); + const name = std.mem.trim(u8, raw[0..sep], " \t"); + const path = std.mem.trim(u8, raw[sep + 1 ..], " \t"); + if (name.len == 0 or path.len == 0) die("test-bundle needs --module NAME=PATH", .{}); + return .{ + .name = try gpa.dupe(u8, name), + .path = try gpa.dupe(u8, path), + }; +} + +fn optionValue( + args: []const []const u8, + i: *usize, + arg: []const u8, + comptime option: []const u8, + comptime message: []const u8, +) []const u8 { + const value = if (std.mem.startsWith(u8, arg, option ++ "=")) + arg[option.len + 1 ..] + else blk: { + i.* += 1; + if (i.* >= args.len) die(message, .{}); + break :blk args[i.*]; + }; + if (value.len == 0 or value[0] == '-') die(message, .{}); + return value; +} + +fn combinedInputHash(gpa: Allocator, inputs: []const InputRecord) ![]u8 { + var hash = std.crypto.hash.sha2.Sha256.init(.{}); + for (inputs) |input| { + hash.update(input.path); + hash.update("\x00"); + hash.update(input.sha256); + hash.update("\n"); + } + var digest: [32]u8 = undefined; + hash.final(&digest); + var out: [64]u8 = undefined; + _ = std.fmt.bufPrint(&out, "{x}", .{digest}) catch unreachable; + return try gpa.dupe(u8, &out); +} + +fn writeOutput(path: []const u8, body: []const u8) !void { + const fd = try sys.openat(fsx.cwd, path, .{ + .ACCMODE = .WRONLY, + .CREAT = true, + .TRUNC = true, + .NOFOLLOW = true, + }, 0o644); + defer sys.close(fd); + try sys.writeAll(fd, body); +} + +pub fn cmdTestBundle(gpa: Allocator, args: []const []const u8) u8 { + var arena_state = std.heap.ArenaAllocator.init(gpa); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var test_id: []const u8 = ""; + var suite_revision: []const u8 = ""; + var eessi_target: []const u8 = ""; + var seed: []const u8 = ""; + var allocation: []const u8 = ""; + var binding: []const u8 = ""; + var cpu: []const u8 = ""; + var gpu: []const u8 = ""; + var stdout_path: []const u8 = ""; + var stderr_path: []const u8 = ""; + var out_path: ?[]const u8 = null; + var inputs: std.ArrayList([]const u8) = .empty; + defer inputs.deinit(arena); + var modules: std.ArrayList(ModuleRecord) = .empty; + defer modules.deinit(arena); + + var i: usize = 0; + while (i < args.len) : (i += 1) { + const arg = args[i]; + if (std.mem.eql(u8, arg, "--test-id") or std.mem.startsWith(u8, arg, "--test-id=")) { + test_id = optionValue(args, &i, arg, "--test-id", "test-bundle needs --test-id"); + } else if (std.mem.eql(u8, arg, "--suite-revision") or std.mem.startsWith(u8, arg, "--suite-revision=")) { + suite_revision = optionValue(args, &i, arg, "--suite-revision", "test-bundle needs --suite-revision"); + } else if (std.mem.eql(u8, arg, "--eessi-target") or std.mem.startsWith(u8, arg, "--eessi-target=")) { + eessi_target = optionValue(args, &i, arg, "--eessi-target", "test-bundle needs --eessi-target"); + } else if (std.mem.eql(u8, arg, "--seed") or std.mem.startsWith(u8, arg, "--seed=")) { + seed = optionValue(args, &i, arg, "--seed", "test-bundle needs --seed"); + } else if (std.mem.eql(u8, arg, "--allocation") or std.mem.startsWith(u8, arg, "--allocation=")) { + allocation = optionValue(args, &i, arg, "--allocation", "test-bundle needs --allocation"); + } else if (std.mem.eql(u8, arg, "--binding") or std.mem.startsWith(u8, arg, "--binding=")) { + binding = optionValue(args, &i, arg, "--binding", "test-bundle needs --binding"); + } else if (std.mem.eql(u8, arg, "--cpu") or std.mem.startsWith(u8, arg, "--cpu=")) { + cpu = optionValue(args, &i, arg, "--cpu", "test-bundle needs --cpu"); + } else if (std.mem.eql(u8, arg, "--gpu") or std.mem.startsWith(u8, arg, "--gpu=")) { + gpu = optionValue(args, &i, arg, "--gpu", "test-bundle needs --gpu"); + } else if (std.mem.eql(u8, arg, "--stdout") or std.mem.startsWith(u8, arg, "--stdout=")) { + stdout_path = optionValue(args, &i, arg, "--stdout", "test-bundle needs --stdout"); + } else if (std.mem.eql(u8, arg, "--stderr") or std.mem.startsWith(u8, arg, "--stderr=")) { + stderr_path = optionValue(args, &i, arg, "--stderr", "test-bundle needs --stderr"); + } else if (std.mem.eql(u8, arg, "--out") or std.mem.startsWith(u8, arg, "--out=")) { + out_path = optionValue(args, &i, arg, "--out", "test-bundle needs --out"); + } else if (std.mem.eql(u8, arg, "--input") or std.mem.startsWith(u8, arg, "--input=")) { + const value = optionValue(args, &i, arg, "--input", "test-bundle needs --input PATH"); + inputs.append(arena, value) catch die("out of memory", .{}); + } else if (std.mem.eql(u8, arg, "--module") or std.mem.startsWith(u8, arg, "--module=")) { + const value = optionValue(args, &i, arg, "--module", "test-bundle needs --module NAME=PATH"); + modules.append(arena, parseModule(arena, value) catch die("out of memory", .{})) catch die("out of memory", .{}); + } else if (arg.len > 0 and arg[0] == '-') { + die("test-bundle does not know '{s}'", .{arg}); + } else { + die("test-bundle does not know '{s}'", .{arg}); + } + } + + if (test_id.len == 0) die("test-bundle needs --test-id", .{}); + if (suite_revision.len == 0) die("test-bundle needs --suite-revision", .{}); + if (eessi_target.len == 0) die("test-bundle needs --eessi-target", .{}); + if (stdout_path.len == 0) die("test-bundle needs --stdout", .{}); + if (stderr_path.len == 0) die("test-bundle needs --stderr", .{}); + if (allocation.len == 0) die("test-bundle needs --allocation", .{}); + if (binding.len == 0) die("test-bundle needs --binding", .{}); + if (cpu.len == 0) die("test-bundle needs --cpu", .{}); + if (gpu.len == 0) die("test-bundle needs --gpu", .{}); + + const stdout_raw = readFile(gpa, stdout_path) orelse return 2; + defer gpa.free(stdout_raw); + const stderr_raw = readFile(gpa, stderr_path) orelse return 2; + defer gpa.free(stderr_raw); + + const stdout_norm = normalizeText(arena, stdout_raw) catch die("out of memory", .{}); + const stderr_norm = normalizeText(arena, stderr_raw) catch die("out of memory", .{}); + const stdout_hash = sha256Hex(arena, stdout_norm) catch die("out of memory", .{}); + const stderr_hash = sha256Hex(arena, stderr_norm) catch die("out of memory", .{}); + const seed_hash = sha256Hex(arena, seed) catch die("out of memory", .{}); + + var input_records: std.ArrayList(InputRecord) = .empty; + defer input_records.deinit(arena); + for (inputs.items) |path| { + const content = readFile(gpa, path) orelse return 2; + defer gpa.free(content); + input_records.append(arena, .{ + .path = arena.dupe(u8, path) catch die("out of memory", .{}), + .sha256 = sha256Hex(arena, content) catch die("out of memory", .{}), + }) catch die("out of memory", .{}); + } + std.mem.sort(InputRecord, input_records.items, {}, lessInput); + std.mem.sort(ModuleRecord, modules.items, {}, lessModule); + const input_hash = combinedInputHash(arena, input_records.items) catch die("out of memory", .{}); + + const bundle: BundleRecord = .{ + .test_id = test_id, + .suite_revision = suite_revision, + .eessi_target = eessi_target, + .stdout = .{ .normalized = stdout_norm, .sha256 = stdout_hash }, + .stderr = .{ .normalized = stderr_norm, .sha256 = stderr_hash }, + .inputs = input_records.items, + .input_hash_sha256 = input_hash, + .seed = .{ .value = seed, .sha256 = seed_hash }, + .modules = modules.items, + .scheduler = .{ .allocation = allocation, .binding = binding }, + .hardware = .{ .cpu = cpu, .gpu = gpu }, + }; + var out: std.Io.Writer.Allocating = .init(arena); + defer out.deinit(); + std.json.Stringify.value(bundle, .{}, &out.writer) catch die("out of memory", .{}); + out.writer.writeAll("\n") catch die("out of memory", .{}); + + const body = out.toOwnedSlice() catch die("out of memory", .{}); + if (out_path) |p| { + writeOutput(p, body) catch |err| + die("cannot write {s}: {t}", .{ p, err }); + } else { + cli.out("{s}", .{body}); + } + return 0; +} + +const BundleCore = struct { + label: []const u8, + test_id: []const u8, + suite_revision: []const u8, + eessi_target: []const u8, + stdout_hash: []const u8, + stderr_hash: []const u8, + input_hash: []const u8, + seed_hash: []const u8, + modules_key: []const u8, + allocation: []const u8, + binding: []const u8, + cpu: []const u8, + gpu: []const u8, +}; + +const FieldDiff = struct { + field: []const u8, + values: []const ValueGroup, +}; + +const ValueGroup = struct { + value: []const u8, + bundles: []const []const u8, +}; + +const CompareJson = struct { + schema_version: u32 = compare_schema_version, + kind: []const u8 = "test-compare", + bundle_count: usize, + differing_fields: []const FieldDiff, +}; + +fn getString(obj: std.json.ObjectMap, key: []const u8) error{BadBundle}![]const u8 { + const v = obj.get(key) orelse return error.BadBundle; + return switch (v) { + .string => |s| s, + else => error.BadBundle, + }; +} + +fn modulesKey(gpa: Allocator, modules: std.json.Value) ![]u8 { + const arr = switch (modules) { + .array => |a| a, + else => return error.BadBundle, + }; + var rows: std.ArrayList([]const u8) = .empty; + defer rows.deinit(gpa); + for (arr.items) |item| { + const obj = switch (item) { + .object => |o| o, + else => return error.BadBundle, + }; + const name = try getString(obj, "name"); + const path = try getString(obj, "path"); + rows.append(gpa, try std.fmt.allocPrint(gpa, "{s}\x00{s}", .{ name, path })) catch return error.OutOfMemory; + } + std.mem.sort([]const u8, rows.items, {}, struct { + fn lt(_: void, a: []const u8, b: []const u8) bool { + return std.mem.lessThan(u8, a, b); + } + }.lt); + var out: std.ArrayList(u8) = .empty; + errdefer out.deinit(gpa); + for (rows.items) |row| { + const sep = std.mem.indexOfScalar(u8, row, 0) orelse continue; + try out.print(gpa, "{s}={s}\n", .{ row[0..sep], row[sep + 1 ..] }); + } + return try out.toOwnedSlice(gpa); +} + +fn parseBundle(gpa: Allocator, label: []const u8, text: []const u8) !BundleCore { + const parsed = std.json.parseFromSlice(std.json.Value, gpa, text, .{}) catch return error.BadBundle; + defer parsed.deinit(); + const root = switch (parsed.value) { + .object => |o| o, + else => return error.BadBundle, + }; + const version = switch (root.get("schema_version") orelse return error.BadBundle) { + .integer => |n| n, + else => return error.BadBundle, + }; + if (version != bundle_schema_version) return error.BadBundle; + const kind = try getString(root, "kind"); + if (!std.mem.eql(u8, kind, "test-bundle")) return error.BadBundle; + const test_id = try getString(root, "test_id"); + const suite_revision = try getString(root, "suite_revision"); + const eessi_target = try getString(root, "eessi_target"); + const input_hash = try getString(root, "input_hash_sha256"); + + const stdout_obj = switch (root.get("stdout") orelse return error.BadBundle) { + .object => |o| o, + else => return error.BadBundle, + }; + const stderr_obj = switch (root.get("stderr") orelse return error.BadBundle) { + .object => |o| o, + else => return error.BadBundle, + }; + const seed_obj = switch (root.get("seed") orelse return error.BadBundle) { + .object => |o| o, + else => return error.BadBundle, + }; + const scheduler_obj = switch (root.get("scheduler") orelse return error.BadBundle) { + .object => |o| o, + else => return error.BadBundle, + }; + const hardware_obj = switch (root.get("hardware") orelse return error.BadBundle) { + .object => |o| o, + else => return error.BadBundle, + }; + const modules_obj = root.get("modules") orelse return error.BadBundle; + const key = try modulesKey(gpa, modules_obj); + + return .{ + .label = try gpa.dupe(u8, label), + .test_id = try gpa.dupe(u8, test_id), + .suite_revision = try gpa.dupe(u8, suite_revision), + .eessi_target = try gpa.dupe(u8, eessi_target), + .stdout_hash = try gpa.dupe(u8, try getString(stdout_obj, "sha256")), + .stderr_hash = try gpa.dupe(u8, try getString(stderr_obj, "sha256")), + .input_hash = try gpa.dupe(u8, input_hash), + .seed_hash = try gpa.dupe(u8, try getString(seed_obj, "sha256")), + .modules_key = key, + .allocation = try gpa.dupe(u8, try getString(scheduler_obj, "allocation")), + .binding = try gpa.dupe(u8, try getString(scheduler_obj, "binding")), + .cpu = try gpa.dupe(u8, try getString(hardware_obj, "cpu")), + .gpu = try gpa.dupe(u8, try getString(hardware_obj, "gpu")), + }; +} + +fn appendFieldDiff( + gpa: Allocator, + arena: Allocator, + diffs: *std.ArrayList(FieldDiff), + bundles: []const BundleCore, + comptime name: []const u8, + comptime valueOf: fn (BundleCore) []const u8, +) !void { + var seen: std.StringHashMapUnmanaged(usize) = .empty; + defer seen.deinit(gpa); + var values: std.ArrayList([]const u8) = .empty; + defer values.deinit(gpa); + var labels: std.ArrayList(std.ArrayList([]const u8)) = .empty; + defer { + for (labels.items) |*x| x.deinit(gpa); + labels.deinit(gpa); + } + for (bundles) |bundle| { + const value = valueOf(bundle); + const got = try seen.getOrPut(gpa, value); + if (!got.found_existing) { + got.value_ptr.* = values.items.len; + try values.append(gpa, value); + const group: std.ArrayList([]const u8) = .empty; + try labels.append(gpa, group); + } + try labels.items[got.value_ptr.*].append(gpa, bundle.label); + } + if (values.items.len <= 1) return; + var groups: std.ArrayList(ValueGroup) = .empty; + for (values.items, 0..) |value, idx| { + try groups.append(arena, .{ + .value = try arena.dupe(u8, value), + .bundles = try labels.items[idx].toOwnedSlice(arena), + }); + } + try diffs.append(arena, .{ + .field = name, + .values = try groups.toOwnedSlice(arena), + }); +} + +fn valueTestId(b: BundleCore) []const u8 { + return b.test_id; +} +fn valueSuiteRevision(b: BundleCore) []const u8 { + return b.suite_revision; +} +fn valueEessiTarget(b: BundleCore) []const u8 { + return b.eessi_target; +} +fn valueStdoutHash(b: BundleCore) []const u8 { + return b.stdout_hash; +} +fn valueStderrHash(b: BundleCore) []const u8 { + return b.stderr_hash; +} +fn valueInputHash(b: BundleCore) []const u8 { + return b.input_hash; +} +fn valueSeedHash(b: BundleCore) []const u8 { + return b.seed_hash; +} +fn valueModules(b: BundleCore) []const u8 { + return b.modules_key; +} +fn valueAllocation(b: BundleCore) []const u8 { + return b.allocation; +} +fn valueBinding(b: BundleCore) []const u8 { + return b.binding; +} +fn valueCpu(b: BundleCore) []const u8 { + return b.cpu; +} +fn valueGpu(b: BundleCore) []const u8 { + return b.gpu; +} + +pub fn cmdTestCompare(gpa: Allocator, args: []const []const u8) u8 { + var json_out = false; + var files: std.ArrayList([]const u8) = .empty; + defer files.deinit(gpa); + for (args) |arg| { + if (std.mem.eql(u8, arg, "--json")) { + json_out = true; + } else if (arg.len > 0 and arg[0] == '-') { + die("test-compare does not know '{s}'", .{arg}); + } else { + files.append(gpa, arg) catch die("out of memory", .{}); + } + } + if (files.items.len < 2) die("test-compare needs at least two bundle files", .{}); + + var arena_state = std.heap.ArenaAllocator.init(gpa); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var bundles: std.ArrayList(BundleCore) = .empty; + defer bundles.deinit(arena); + for (files.items) |path| { + const text = readFile(gpa, path) orelse return 2; + defer gpa.free(text); + const parsed = parseBundle(arena, path, text) catch { + warn("test-compare: {s} is not a test-bundle schema_version {d} document", .{ path, bundle_schema_version }); + return 2; + }; + bundles.append(arena, parsed) catch die("out of memory", .{}); + } + + var diffs: std.ArrayList(FieldDiff) = .empty; + appendFieldDiff(arena, arena, &diffs, bundles.items, "test_id", valueTestId) catch die("out of memory", .{}); + appendFieldDiff(arena, arena, &diffs, bundles.items, "suite_revision", valueSuiteRevision) catch die("out of memory", .{}); + appendFieldDiff(arena, arena, &diffs, bundles.items, "eessi_target", valueEessiTarget) catch die("out of memory", .{}); + appendFieldDiff(arena, arena, &diffs, bundles.items, "stdout_sha256", valueStdoutHash) catch die("out of memory", .{}); + appendFieldDiff(arena, arena, &diffs, bundles.items, "stderr_sha256", valueStderrHash) catch die("out of memory", .{}); + appendFieldDiff(arena, arena, &diffs, bundles.items, "input_hash_sha256", valueInputHash) catch die("out of memory", .{}); + appendFieldDiff(arena, arena, &diffs, bundles.items, "seed_sha256", valueSeedHash) catch die("out of memory", .{}); + appendFieldDiff(arena, arena, &diffs, bundles.items, "modules", valueModules) catch die("out of memory", .{}); + appendFieldDiff(arena, arena, &diffs, bundles.items, "scheduler.allocation", valueAllocation) catch die("out of memory", .{}); + appendFieldDiff(arena, arena, &diffs, bundles.items, "scheduler.binding", valueBinding) catch die("out of memory", .{}); + appendFieldDiff(arena, arena, &diffs, bundles.items, "hardware.cpu", valueCpu) catch die("out of memory", .{}); + appendFieldDiff(arena, arena, &diffs, bundles.items, "hardware.gpu", valueGpu) catch die("out of memory", .{}); + + if (json_out) { + const doc: CompareJson = .{ + .bundle_count = bundles.items.len, + .differing_fields = diffs.items, + }; + var writer: std.Io.Writer.Allocating = .init(gpa); + defer writer.deinit(); + std.json.Stringify.value(doc, .{}, &writer.writer) catch die("out of memory", .{}); + writer.writer.writeAll("\n") catch die("out of memory", .{}); + const body = writer.toOwnedSlice() catch die("out of memory", .{}); + defer gpa.free(body); + cli.out("{s}", .{body}); + return 0; + } + + var out: std.ArrayList(u8) = .empty; + defer out.deinit(gpa); + out.print(gpa, "Compared {d} bundles.\n", .{bundles.items.len}) catch die("out of memory", .{}); + if (diffs.items.len == 0) { + out.appendSlice(gpa, "No differing fields.\n") catch die("out of memory", .{}); + } else { + out.appendSlice(gpa, "Differing fields:\n") catch die("out of memory", .{}); + for (diffs.items) |diff| { + out.print(gpa, "- {s}\n", .{diff.field}) catch die("out of memory", .{}); + for (diff.values) |value_group| { + out.print(gpa, " value={s}\n", .{value_group.value}) catch die("out of memory", .{}); + for (value_group.bundles) |label| { + out.print(gpa, " {s}\n", .{label}) catch die("out of memory", .{}); + } + } + } + } + const body = out.toOwnedSlice(gpa) catch die("out of memory", .{}); + defer gpa.free(body); + cli.out("{s}", .{body}); + return 0; +} + +test "normalize text unifies line endings and trailing blanks" { + const got = try normalizeText(std.testing.allocator, "a \r\nb\t\rend"); + defer std.testing.allocator.free(got); + try std.testing.expectEqualStrings("a\nb\nend\n", got); +} + +test "test-compare sees differing fields" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const gpa = arena_state.allocator(); + const one = + \\{"schema_version":1,"kind":"test-bundle","test_id":"qe.suite.test1","suite_revision":"abc","eessi_target":"zen4","stdout":{"normalized":"ok\n","sha256":"1"},"stderr":{"normalized":"","sha256":"2"},"inputs":[],"input_hash_sha256":"3","seed":{"value":"7","sha256":"4"},"modules":[{"name":"A","path":"/a"}],"scheduler":{"allocation":"nodes=1","binding":"core"},"hardware":{"cpu":"cpu-a","gpu":"gpu-a"}} + ; + const two = + \\{"schema_version":1,"kind":"test-bundle","test_id":"qe.suite.test1","suite_revision":"abc","eessi_target":"zen4","stdout":{"normalized":"ok\n","sha256":"9"},"stderr":{"normalized":"","sha256":"2"},"inputs":[],"input_hash_sha256":"3","seed":{"value":"7","sha256":"4"},"modules":[{"name":"A","path":"/a"}],"scheduler":{"allocation":"nodes=2","binding":"core"},"hardware":{"cpu":"cpu-a","gpu":"gpu-a"}} + ; + const b1 = try parseBundle(gpa, "one.json", one); + const b2 = try parseBundle(gpa, "two.json", two); + const arena = gpa; + var diffs: std.ArrayList(FieldDiff) = .empty; + try appendFieldDiff(gpa, arena, &diffs, &.{ b1, b2 }, "stdout_sha256", valueStdoutHash); + try appendFieldDiff(gpa, arena, &diffs, &.{ b1, b2 }, "scheduler.allocation", valueAllocation); + try std.testing.expectEqual(@as(usize, 2), diffs.items.len); +} diff --git a/tests/test_json_schemas.sh b/tests/test_json_schemas.sh index f491e62..506bf22 100755 --- a/tests/test_json_schemas.sh +++ b/tests/test_json_schemas.sh @@ -64,6 +64,67 @@ run "runs" validate runs runs --exec gmx --json "${work}/runs.json" run "pending dry-run" validate pending pending --json "${work}/dry.txt" run "used" validate used used --prefix "${software}/Foo/1.0" --json "${work}/xalt.json" run "usage" validate usage usage --software "${software}" --modules "${modules}" --json "${work}/xalt.json" "${work}/lmod.log" +cat > "${work}/stdout-a.log" <<'EOF' +line one +line two +EOF +cat > "${work}/stderr-a.log" <<'EOF' +warn +EOF +cat > "${work}/stdout-b.log" <<'EOF' +line one +line three +EOF +cat > "${work}/stderr-b.log" <<'EOF' +warn +EOF +echo alpha > "${work}/input-a.txt" +echo beta > "${work}/input-b.txt" +run "test-bundle" validate test-bundle test-bundle \ + --test-id qe.pw.scf-si \ + --suite-revision abc1234 \ + --eessi-target x86_64/intel/sapphirerapids \ + --stdout "${work}/stdout-a.log" \ + --stderr "${work}/stderr-a.log" \ + --input "${work}/input-a.txt" \ + --input "${work}/input-b.txt" \ + --module QuantumESPRESSO/7.4=/cvmfs/eessi/software/QuantumESPRESSO/7.4.lua \ + --seed 42 \ + --allocation "nodes=1" \ + --binding "core" \ + --cpu "cpu-a" \ + --gpu "none" +"${tool}" test-bundle \ + --test-id qe.pw.scf-si \ + --suite-revision abc1234 \ + --eessi-target x86_64/intel/sapphirerapids \ + --stdout "${work}/stdout-a.log" \ + --stderr "${work}/stderr-a.log" \ + --input "${work}/input-a.txt" \ + --input "${work}/input-b.txt" \ + --module QuantumESPRESSO/7.4=/cvmfs/eessi/software/QuantumESPRESSO/7.4.lua \ + --seed 42 \ + --allocation "nodes=1" \ + --binding "core" \ + --cpu "cpu-a" \ + --gpu "none" \ + --out "${work}/bundle-a.json" +"${tool}" test-bundle \ + --test-id qe.pw.scf-si \ + --suite-revision abc1234 \ + --eessi-target x86_64/intel/sapphirerapids \ + --stdout "${work}/stdout-b.log" \ + --stderr "${work}/stderr-b.log" \ + --input "${work}/input-a.txt" \ + --input "${work}/input-b.txt" \ + --module QuantumESPRESSO/7.4=/cvmfs/eessi/software/QuantumESPRESSO/7.4.lua \ + --seed 42 \ + --allocation "nodes=2" \ + --binding "core" \ + --cpu "cpu-b" \ + --gpu "none" \ + --out "${work}/bundle-b.json" +run "test-compare" bash -c "\"${tool}\" test-compare --json \"${work}/bundle-a.json\" \"${work}/bundle-b.json\" | python3 \"${repo_root}/tests/json_schema_validate.py\" \"${schema_dir}/test-compare.schema.json\"" echo "${fails} failed" [ "${fails}" -eq 0 ] \ No newline at end of file diff --git a/tests/test_output_stability.sh b/tests/test_output_stability.sh new file mode 100755 index 0000000..747d554 --- /dev/null +++ b/tests/test_output_stability.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Assert selected existing command outputs stay byte-identical. +# TOOL_BIN=zig-out/bin/eb-keep-logs bash tests/test_output_stability.sh +set -u + +tool="${TOOL_BIN:?TOOL_BIN names the Zig build}" +case "${tool}" in /*) ;; *) tool="${PWD}/${tool}" ;; esac + +work=$(mktemp -d "${TMPDIR:-/tmp}/eb-keep-logs-stability.XXXXXX") +trap 'rm -rf -- "${work}"' EXIT +mkdir -p "${work}/logs" "${work}/empty" + +expect() { + local label=$1 expected_status=$2 expected_out=$3 expected_err=$4 + shift 4 + "$@" > "${work}/out" 2> "${work}/err" + status=$? + if [ "${status}" -ne "${expected_status}" ]; then + echo "FAIL ${label}: status ${status} != ${expected_status}" >&2 + return 1 + fi + printf '%s' "${expected_out}" > "${work}/exp.out" + printf '%s' "${expected_err}" > "${work}/exp.err" + if ! /usr/bin/cmp -s "${work}/exp.out" "${work}/out"; then + echo "FAIL ${label}: stdout differs" >&2 + /usr/bin/diff -u "${work}/exp.out" "${work}/out" >&2 || true + return 1 + fi + if ! /usr/bin/cmp -s "${work}/exp.err" "${work}/err"; then + echo "FAIL ${label}: stderr differs" >&2 + /usr/bin/diff -u "${work}/exp.err" "${work}/err" >&2 || true + return 1 + fi + echo "ok ${label}" +} + +( + cd "${work}" || exit 1 + expect "failed status 0 no-op" 0 "" "" "${tool}" failed 0 Foo logs + expect "rotate empty tree" 0 "eb-keep-logs rotate: 0 files compressed, 0 runs deleted under logs +" "" "${tool}" rotate logs --compress-after 5 --delete-after 9 + PATH="${work}/empty" expect "failed no eb on PATH" 0 "No EasyBuild log to keep for Foo: eb is not on PATH. +" "" "${tool}" failed 3 Foo logs +) diff --git a/tests/test_test_bundle.sh b/tests/test_test_bundle.sh new file mode 100755 index 0000000..1769491 --- /dev/null +++ b/tests/test_test_bundle.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# Validate test-bundle and test-compare outputs and schemas. +# TOOL_BIN=zig-out/bin/eb-keep-logs bash tests/test_test_bundle.sh +set -u + +repo_root=$(cd "$(dirname "$0")/.." && pwd) +schema_dir="${repo_root}/docs/orgmode/reference/json" +tool="${TOOL_BIN:?TOOL_BIN names the Zig build}" +case "${tool}" in /*) ;; *) tool="${PWD}/${tool}" ;; esac + +if ! python3 -c "import jsonschema" 2>/dev/null; then + python3 -m pip install -q --user jsonschema +fi + +work=$(mktemp -d "${TMPDIR:-/tmp}/eb-keep-logs-bundle.XXXXXX") +trap 'rm -rf -- "${work}"' EXIT + +cat > "${work}/stdout-a.log" <<'EOF' +line one +line two +EOF +cat > "${work}/stderr-a.log" <<'EOF' +warn +EOF +cat > "${work}/stdout-b.log" <<'EOF' +line one +line two changed +EOF +cat > "${work}/stderr-b.log" <<'EOF' +warn +EOF +cat > "${work}/in-a.txt" <<'EOF' +alpha +EOF +cat > "${work}/in-b.txt" <<'EOF' +beta +EOF + +"${tool}" test-bundle \ + --test-id qe.pw.scf-si \ + --suite-revision 5f6a9cc \ + --eessi-target x86_64/intel/sapphirerapids \ + --stdout "${work}/stdout-a.log" \ + --stderr "${work}/stderr-a.log" \ + --input "${work}/in-a.txt" \ + --input "${work}/in-b.txt" \ + --module QuantumESPRESSO/7.4=/cvmfs/eessi/software/QuantumESPRESSO/7.4.lua \ + --module FFTW/3.3=/cvmfs/eessi/software/FFTW/3.3.lua \ + --seed 42 \ + --allocation "nodes=1 ntasks=32" \ + --binding "rank0->core0, rank1->core1" \ + --cpu "AMD EPYC 7B13 (32)" \ + --gpu "none" \ + --out "${work}/bundle-a.json" + +"${tool}" test-bundle \ + --test-id qe.pw.scf-si \ + --suite-revision 5f6a9cc \ + --eessi-target x86_64/intel/sapphirerapids \ + --stdout "${work}/stdout-b.log" \ + --stderr "${work}/stderr-b.log" \ + --input "${work}/in-a.txt" \ + --input "${work}/in-b.txt" \ + --module QuantumESPRESSO/7.4=/cvmfs/eessi/software/QuantumESPRESSO/7.4.lua \ + --module FFTW/3.3=/cvmfs/eessi/software/FFTW/3.3.lua \ + --seed 42 \ + --allocation "nodes=2 ntasks=64" \ + --binding "rank0->core0, rank1->core2" \ + --cpu "AMD EPYC 7B13 (64)" \ + --gpu "none" \ + --out "${work}/bundle-b.json" + +python3 "${repo_root}/tests/json_schema_validate.py" "${schema_dir}/test-bundle.schema.json" < "${work}/bundle-a.json" +python3 "${repo_root}/tests/json_schema_validate.py" "${schema_dir}/test-bundle.schema.json" < "${work}/bundle-b.json" + +"${tool}" test-compare --json "${work}/bundle-a.json" "${work}/bundle-b.json" > "${work}/compare.json" +python3 "${repo_root}/tests/json_schema_validate.py" "${schema_dir}/test-compare.schema.json" < "${work}/compare.json" + +python3 - "${work}/compare.json" <<'PY' +import json, sys +doc = json.load(open(sys.argv[1], "r", encoding="utf-8")) +fields = {f["field"] for f in doc["differing_fields"]} +assert "stdout_sha256" in fields, fields +assert "scheduler.allocation" in fields, fields +assert "hardware.cpu" in fields, fields +PY + +echo "ok test-bundle/test-compare"