diff --git a/COMPAT.md b/COMPAT.md index 337e7ec..7edd3d3 100644 --- a/COMPAT.md +++ b/COMPAT.md @@ -29,6 +29,7 @@ on the same invocations. | `--version` | `eb-keep-logs 0.1.0` | `eb-keep-logs 0.2.1 (TARGET, MODE, zig VERSION, COMMIT)`; the first two words are unchanged | | `why` | refused with `eb-keep-logs: why is in the Zig build only`, exit 2 | says why the run's builds failed, from their kept logs, read through gzip when rotated | | `history` | refused with `eb-keep-logs: history is in the Zig build only`, exit 2 | lists how each kept build of a package went, run by run | +| `failures` | refused with `eb-keep-logs: failures is in the Zig build only`, exit 2 | lists failed builds across runs, filtered by recipe or date | | `loads` | refused with `eb-keep-logs: loads is in the Zig build only`, exit 2 | prints who loaded a module, from Lmod's ModuleUsageTracking lines | | `runs` | refused with `eb-keep-logs: runs is in the Zig build only`, exit 2 | prints who ran an executable, from XALT run records | | `used` | refused with `eb-keep-logs: used is in the Zig build only`, exit 2 | lists paths under a prefix in XALT and Lmod records | diff --git a/README.md b/README.md index 225c4c7..2beaedb 100644 --- a/README.md +++ b/README.md @@ -105,13 +105,15 @@ EB_KEEP_LOGS_NOW=$(date -d 2027-09-20 +%s) \ eb-keep-logs why --latest --root /srv/build-logs eb-keep-logs why 20261117_040615 --root /srv/build-logs --target zen4 --json eb-keep-logs history hello --root /srv/build-logs +eb-keep-logs failures --package hello --since 2026-11-17 --root /srv/build-logs ``` `why` reads the logs a run kept, as `failed` kept them or `rotate` compressed them, and prints for each failed easyconfig its package and easyconfig, the EasyBuild step it failed in, EasyBuild's message, and the line of its log that shows the error, the first compiler or linker error or else EasyBuild's own error line, with the path, the line number and three lines either side. For a failure class, feed the kept log to eb-stack. -`history` lists each kept build of a package, run by run, with its outcome and how long EasyBuild took. -`--json` prints either as one line of JSON with `schema_version` 1, for a pipeline's run summary. -Both are in the Zig build only. +`history` lists each kept build of a package, run by run, with its outcome and how long EasyBuild took, and accepts `--since`, `--until` and `--failed`. +`failures` lists failed builds across runs with their failed EasyBuild step and duration, filtered by `--package`, `--target`, `--since` and `--until`; `why` accepts `--package`, `--since` and `--until` as well. +`--json` prints any of these as one line of JSON with `schema_version` 1, for a pipeline's run summary. +All three are in the Zig build only. ## Emit syslog and journald events diff --git a/build.zig b/build.zig index b74dc73..bd9456a 100644 --- a/build.zig +++ b/build.zig @@ -112,6 +112,11 @@ pub fn build(b: *std.Build) void { stable_output_test.setCwd(b.path(".")); stable_output_test.expectExitCode(0); stable_output_test.has_side_effects = true; + const failures_test = b.addSystemCommand(&.{ "bash", "-c", "TOOL_BIN=\"$0\" exec bash tests/test_failures.sh" }); + failures_test.addArtifactArg(exe); + failures_test.setCwd(b.path(".")); + failures_test.expectExitCode(0); + failures_test.has_side_effects = true; const metrics_test = b.addSystemCommand(&.{ "bash", "-c", "TOOL_BIN=\"$0\" exec bash tests/test_metrics.sh" }); metrics_test.addArtifactArg(exe); metrics_test.setCwd(b.path(".")); @@ -133,5 +138,6 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&test_bundle_test.step); test_step.dependOn(&lifecycle_test.step); test_step.dependOn(&stable_output_test.step); + test_step.dependOn(&failures_test.step); test_step.dependOn(&metrics_test.step); } diff --git a/changelog.d/+filter-failed-runs.feature.md b/changelog.d/+filter-failed-runs.feature.md new file mode 100644 index 0000000..34e558c --- /dev/null +++ b/changelog.d/+filter-failed-runs.feature.md @@ -0,0 +1 @@ +`eb-keep-logs failures` and opt-in `--package`, `--since`, `--until` and `--failed` filters on `why` and `history` filter failed builds across `YYYYMMDD_HHMMSS` runs by recipe or date, with schema-validated `--json` output and unchanged default command outputs. diff --git a/docs/orgmode/howto/find-why-a-build-failed.org b/docs/orgmode/howto/find-why-a-build-failed.org index c7fa3ee..39d967d 100644 --- a/docs/orgmode/howto/find-why-a-build-failed.org +++ b/docs/orgmode/howto/find-why-a-build-failed.org @@ -85,7 +85,29 @@ Use this after a run in which builds failed, to see for each the EasyBuild step =hello= takes every easyconfig whose name starts with =hello-=; =hello-2.12.1-GCCcore-13.3.0= takes that one alone. A duration of =-= is one EasyBuild did not print, as when it stopped before building; ask =why= about that run for the reason. -5. To put the reasons in a script's own summary, read the JSON =--json= prints: + Pass =--since DATE=, =--until DATE= and =--failed= to limit =history= to failed builds in a date window. +5. List failed builds across runs, filtered by recipe or date: + + #+begin_src bash + eb-keep-logs failures --root ~/build-logs --package hello --since 2026-11-17 + #+end_src + + #+begin_example + /home/alice/build-logs: 8 failed builds in 7 runs + + RUN TARGET STEP DURATION NAME + 20261117_035825 sapphirerapids fetch 48s hello-2.12.1-GCCcore-14.2.0 + 20261117_040032 sapphirerapids patch 41s hello-2.12.1-GCCcore-14.2.0 + 20261117_040119 sapphirerapids - - hello-2.12.1-GCCcore-14.2.0 + 20261117_040311 sapphirerapids configure 2m24s hello-2.12.1-GCCcore-14.2.0 + 20261117_040615 sapphirerapids build 3m06s hello-2.12.1-GCCcore-14.2.0 + 20261117_040928 sapphirerapids build 2m30s hello-2.12.1-GCCcore-14.2.0 + 20261117_041222 sapphirerapids sanitycheck 2m54s hello-2.12.1-GCCcore-13.3.0 + 20261117_041222 sapphirerapids test 2m33s hello-2.12.1-GCCcore-14.2.0 + #+end_example + + Leave out =--package= to list failures across every recipe in the date range, or pass =--package PACKAGE= to =why= to inspect one recipe within a run. +6. To put the reasons in a script's own summary, read the JSON =--json= prints: #+begin_src bash eb-keep-logs why 20261117_041222 --root ~/build-logs --json \ diff --git a/docs/orgmode/reference/cli.org b/docs/orgmode/reference/cli.org index 69dd84c..7bac753 100644 --- a/docs/orgmode/reference/cli.org +++ b/docs/orgmode/reference/cli.org @@ -4,16 +4,17 @@ * The eb-keep-logs command -=eb-keep-logs= keeps the logs of a failed EasyBuild build, compresses and removes the old runs of a log tree, says from the kept logs why a run's builds failed, lists how a package's builds went run by run, reads who loaded a module from Lmod's usage log, reads who ran an executable from an XALT run record, lists paths under an install prefix from those records, classifies installed software by its recorded use, reads what an EasyBuild dry-run or missing-modules report still has to build, writes a flaky-test evidence bundle, and compares such bundles across sites. -This page describes the Zig build; the shell script =eb-keep-logs= in the repository behaves the same, except as [[file:compatibility.org][Compatibility]] lists: it has no =why=, =history=, =loads=, =runs=, =used=, =usage= or =pending=. +=eb-keep-logs= keeps the logs of a failed EasyBuild build, compresses and removes the old runs of a log tree, says from the kept logs why a run's builds failed, lists how a package's builds went run by run, lists failed builds across runs filtered by recipe or date, reads who loaded a module from Lmod's usage log, reads who ran an executable from an XALT run record, lists paths under an install prefix from those records, classifies installed software by its recorded use, reads what an EasyBuild dry-run or missing-modules report still has to build, writes a flaky-test evidence bundle, and compares such bundles across sites. +This page describes the Zig build; the shell script =eb-keep-logs= in the repository behaves the same, except as [[file:compatibility.org][Compatibility]] lists: it has no =why=, =history=, =failures=, =loads=, =runs=, =used=, =usage= or =pending=. ** Synopsis #+begin_example eb-keep-logs failed [--syslog TARGET] [--metrics-file PATH] STATUS NAME LOGDIR [TMP_LOGDIR] eb-keep-logs rotate ROOT --compress-after DAYS [--delete-after DAYS] [--dry-run] [--lock-timeout SECONDS] [--syslog TARGET] [--metrics-file PATH] -eb-keep-logs why [RUN | --latest] [--target TARGET] [--root ROOT] [--json] -eb-keep-logs history PACKAGE [--target TARGET] [--root ROOT] [--json] +eb-keep-logs why [RUN | --latest] [--target TARGET] [--package PACKAGE] [--since DATE] [--until DATE] [--root ROOT] [--json] +eb-keep-logs history PACKAGE [--target TARGET] [--since DATE] [--until DATE] [--failed] [--root ROOT] [--json] +eb-keep-logs failures [--package PACKAGE] [--target TARGET] [--since DATE] [--until DATE] [--root ROOT] [--json] eb-keep-logs loads --module NAME [--json] FILE eb-keep-logs runs --exec PATH [--json] FILE eb-keep-logs used --prefix PREFIX [--json] [--require-none] FILE... @@ -48,9 +49,13 @@ eb-keep-logs --version | =--syslog TARGET=, =--syslog=TARGET= | =failed=, =rotate= | Emit an RFC 5424 or native =systemd-journald= event to =TARGET= (=journald=, =syslog=, =unix:///path=, =journal+unix:///path=, =udp://host:port=, or =tcp://host:port=); overrides =EB_KEEP_LOGS_SYSLOG=. An empty value disables emission. | | =--metrics-file PATH=, =--metrics-file=PATH= | =failed=, =rotate= | Atomically update the Prometheus text exposition at =PATH=, except on =rotate --dry-run=, which writes no metrics. Without it, no metrics file is read or written. | | =--latest= | =why= | Read the newest run under =--root=, by its name; with =--target=, the newest that has that target. The default without =RUN=. | -| =--target TARGET=, =--target=TARGET= | =why=, =history= | Read only the builds in the run's directory =TARGET=; for =history=, in each run's, a run without one passed over. Without it, those in the run's own directory and then in each directory in it, by name. | -| =--root ROOT=, =--root=ROOT= | =why=, =history= | The directory that holds the runs, directories named =YYYYMMDD_HHMMSS=, not symbolic links. Without it, the current directory. | -| =--json= | =why=, =history=, =loads=, =runs=, =pending= | Print one line of JSON instead of the report; its keys are under [[file:json-schemas.org][JSON schemas]] and below. | +| =--target TARGET=, =--target=TARGET= | =why=, =history=, =failures= | Read only the builds in the run's directory =TARGET=; for =history= and =failures=, in each run's, a run without one passed over. Without it, those in the run's own directory and then in each directory in it, by name. | +| =--package PACKAGE=, =--package=PACKAGE= | =why=, =failures= | Read only builds whose easyconfig name matches =PACKAGE= (prefix up to a dash, or =NAME/VERSION=). Default: all builds. | +| =--since DATE=, =--since=DATE= | =why=, =history=, =failures= | Bound candidate runs to those at or after =DATE= (=YYYY-MM-DD=, =YYYYMMDD=, or =YYYYMMDD_HHMMSS=); for =why=, applies when picking =--latest=. | +| =--until DATE=, =--until=DATE= | =why=, =history=, =failures= | Bound candidate runs to those at or before =DATE= (=YYYY-MM-DD=, =YYYYMMDD=, or =YYYYMMDD_HHMMSS=); for =why=, applies when picking =--latest=. | +| =--failed= | =history= | Print only failed builds. | +| =--root ROOT=, =--root=ROOT= | =why=, =history=, =failures= | The directory that holds the runs, directories named =YYYYMMDD_HHMMSS=, not symbolic links. Without it, the current directory. | +| =--json= | =why=, =history=, =failures=, =loads=, =runs=, =pending= | Print one line of JSON instead of the report; its keys are under [[file:json-schemas.org][JSON schemas]] and below. | | =--software DIR=, =--software=DIR= | =usage= | Required. Read installed software from the direct =Name/Version= directories below =DIR=. | | =--modules DIR=, =--modules=DIR= | =usage= | Required. Count an Lmod record only when its module-file path is =DIR= or below it. | | =--since EPOCH=, =--since=EPOCH= | =usage= | Count records at or after the non-negative epoch time; leave out records with no numeric time. | @@ -105,7 +110,7 @@ eb-keep-logs --version | =MESSAGE= | Cause | |-----------------------------------------------------------------+-----------------------------------------------------------------------------------------------| -| =unknown command 'WORD' (failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle, test-compare, --help)= | The first argument is none of the commands. | +| =unknown command 'WORD' (failed, rotate, why, history, failures, loads, runs, used, usage, pending, test-bundle, test-compare, lifecycle, --help)= | The first argument is none of the commands. | | =failed needs STATUS NAME LOGDIR [TMP_LOGDIR]= | =failed= got fewer than three arguments. | | =STATUS must be a number, not 'STATUS'= | =STATUS= is empty or holds something other than digits. | | =LOGDIR LOGDIR is not a directory= | =LOGDIR= does not name a directory that can be opened. | @@ -347,6 +352,22 @@ With =--json=, =history= prints one line of JSON instead, its keys in this order What =history= cannot read it reports on stderr as =why= does, a run that cannot be opened with =cannot open PATH: ERROR= too, and goes on with the rest; a build whose output could not be read is still listed. +*** failures + +=failures= only reads, scanning the runs under =ROOT= oldest first and filtering by =--since=, =--until=, =--target= and =--package=. +For each failed build it reads the kept session log (or =NAME.stdout.log= when no session log exists) to report the failed EasyBuild step and duration: + +#+begin_example +tests/fixtures/build-logs: 8 failed builds in 7 runs + +RUN TARGET STEP DURATION NAME +20261117_035825 sapphirerapids fetch 48s hello-2.12.1-GCCcore-14.2.0 +20261117_040032 sapphirerapids patch 41s hello-2.12.1-GCCcore-14.2.0 +#+end_example + +With no matching failure, it prints =ROOT: 0 failed builds in 0 runs=. +With =--json=, =failures= prints one line of JSON with =schema_version= 1, =kind= ="failures"=, =root=, =package=, =target=, =since=, =until=, =counts= (={"runs": N, "failed": N}=), and =failures= (each item with =run=, =target=, =name=, =package=, =easyconfig=, =step=, =message=, =duration_s=, and =log=). + ** loads =loads --module NAME [--json] FILE= prints who loaded =NAME=, from Lmod's ModuleUsageTracking lines in =FILE=. @@ -575,7 +596,8 @@ otherwise lose, and keep the pipeline's log tree bounded. --metrics-file PATH, --metrics-file=PATH Atomically record when this rotation completed, unless --dry-run is given. Default: none. - eb-keep-logs why [RUN | --latest] [--target TARGET] [--root ROOT] [--json] + eb-keep-logs why [RUN | --latest] [--target TARGET] [--package PACKAGE] + [--since DATE] [--until DATE] [--root ROOT] [--json] Say why the builds of a run failed: for each failed easyconfig, its package and easyconfig, the EasyBuild step it failed in, EasyBuild's message, and the line of its kept log that shows the error: the first @@ -591,12 +613,24 @@ otherwise lose, and keep the pipeline's log tree bounded. --target TARGET, --target=TARGET Only the builds in the run's directory TARGET. Default: those in the run's own directory and in every directory in it. + --package PACKAGE, --package=PACKAGE + Only builds matching PACKAGE (an easyconfig name without .eb, + a prefix up to a dash, or NAME/VERSION). Default: all builds. + --since DATE, --since=DATE + Only consider runs at or after DATE when picking --latest. + DATE is YYYY-MM-DD, YYYYMMDD, or YYYYMMDD_HHMMSS. + Default: none. + --until DATE, --until=DATE + Only consider runs at or before DATE when picking --latest. + DATE is YYYY-MM-DD, YYYYMMDD, or YYYYMMDD_HHMMSS. + Default: none. --root ROOT, --root=ROOT The directory that holds the runs. Default: the current directory. --json Print one line of JSON instead, with schema_version 1. - eb-keep-logs history PACKAGE [--target TARGET] [--root ROOT] [--json] + eb-keep-logs history PACKAGE [--target TARGET] [--since DATE] + [--until DATE] [--failed] [--root ROOT] [--json] Say how each kept build of PACKAGE went, oldest run first: success, failed or unknown, and how long EasyBuild took, from its output. PACKAGE is an easyconfig's name or its start up to a dash, as hello @@ -606,6 +640,37 @@ otherwise lose, and keep the pipeline's log tree bounded. --target TARGET, --target=TARGET Only the builds in each run's directory TARGET. Default: those in each run's own directory and in every directory in it. + --since DATE, --since=DATE + Only read runs at or after DATE (YYYY-MM-DD, YYYYMMDD, or + YYYYMMDD_HHMMSS). Default: none. + --until DATE, --until=DATE + Only read runs at or before DATE (YYYY-MM-DD, YYYYMMDD, or + YYYYMMDD_HHMMSS). Default: none. + --failed + Only list failed builds. + --root ROOT, --root=ROOT + The directory that holds the runs. Default: the current + directory. + --json + Print one line of JSON instead, with schema_version 1. + eb-keep-logs failures [--package PACKAGE] [--target TARGET] + [--since DATE] [--until DATE] [--root ROOT] [--json] + List failed builds across runs under --root, oldest run first. + Each row gives the run, target, failed step, duration and build + name. Zig build only. Options come in any order. + --package PACKAGE, --package=PACKAGE + Only failed builds matching PACKAGE (an easyconfig name + without .eb, a prefix up to a dash, or NAME/VERSION). Default: + all packages. + --target TARGET, --target=TARGET + Only the builds in each run's directory TARGET. Default: those + in each run's own directory and in every directory in it. + --since DATE, --since=DATE + Only read runs at or after DATE (YYYY-MM-DD, YYYYMMDD, or + YYYYMMDD_HHMMSS). Default: none. + --until DATE, --until=DATE + Only read runs at or before DATE (YYYY-MM-DD, YYYYMMDD, or + YYYYMMDD_HHMMSS). Default: none. --root ROOT, --root=ROOT The directory that holds the runs. Default: the current directory. @@ -803,7 +868,7 @@ Environment: directory of zone files. Exit status: - 0 -h, --help or --version, or a failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle, test-compare or lifecycle that + 0 -h, --help or --version, or a failed, rotate, why, history, failures, loads, runs, used, usage, pending, test-bundle, test-compare or lifecycle that accepted its arguments, whatever it could keep, compress, remove or read: what it could not do is reported on stderr. used with --require-none also exits 2 when any path was listed. diff --git a/docs/orgmode/reference/json-schemas.org b/docs/orgmode/reference/json-schemas.org index 21f0ddb..e0c7b74 100644 --- a/docs/orgmode/reference/json-schemas.org +++ b/docs/orgmode/reference/json-schemas.org @@ -4,7 +4,7 @@ * JSON output schemas -The Zig build's =--json= on =why=, =history=, =loads=, =runs=, =used=, =usage=, =pending=, =test-compare= and =lifecycle= prints one line of JSON with =schema_version= 1. +The Zig build's =--json= on =why=, =history=, =failures=, =loads=, =runs=, =used=, =usage=, =pending=, =test-compare= and =lifecycle= prints one line of JSON with =schema_version= 1. The =test-bundle= command always writes one JSON document with =schema_version= 1. The shell script refuses those commands; see [[file:compatibility.org][Compatibility]]. Each document is validated in CI against a JSON Schema file in the repository tree under =docs/orgmode/reference/json/=. @@ -14,6 +14,7 @@ The schemas use draft 2020-12; a new =schema_version= may drop or rename a key, |----------+-------------------------------| | =why= | =docs/orgmode/reference/json/why.schema.json= | | =history= | =docs/orgmode/reference/json/history.schema.json= | +| =failures= | =docs/orgmode/reference/json/failures.schema.json= | | =loads= | =docs/orgmode/reference/json/loads.schema.json= | | =runs= | =docs/orgmode/reference/json/runs.schema.json= | | =used= | =docs/orgmode/reference/json/used.schema.json= | @@ -23,7 +24,7 @@ The schemas use draft 2020-12; a new =schema_version= may drop or rename a key, | =test-compare= | =docs/orgmode/reference/json/test-compare.schema.json= | | =lifecycle= | =docs/orgmode/reference/json/lifecycle.schema.json= | -Field names and semantics for =why=, =history=, =used=, =usage= and =lifecycle= are under [[file:cli.org][The eb-keep-logs command]]. +Field names and semantics for =why=, =history=, =failures=, =used=, =usage= and =lifecycle= are under [[file:cli.org][The eb-keep-logs command]]. =loads=, =runs=, =pending=, =test-bundle= and =test-compare= document their JSON shape in the same page. ** See also diff --git a/docs/orgmode/reference/json/failures.schema.json b/docs/orgmode/reference/json/failures.schema.json new file mode 100644 index 0000000..df793c3 --- /dev/null +++ b/docs/orgmode/reference/json/failures.schema.json @@ -0,0 +1,48 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://stackhands.github.io/eb-keep-logs/reference/json/failures.schema.json", + "title": "eb-keep-logs failures --json", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "kind", "root", "package", "target", "since", "until", "counts", "failures"], + "properties": { + "schema_version": { "type": "integer", "const": 1 }, + "kind": { "type": "string", "const": "failures" }, + "root": { "type": "string" }, + "package": { "type": ["string", "null"] }, + "target": { "type": ["string", "null"] }, + "since": { "type": ["string", "null"] }, + "until": { "type": ["string", "null"] }, + "counts": { + "type": "object", + "additionalProperties": false, + "required": ["runs", "failed"], + "properties": { + "runs": { "type": "integer", "minimum": 0 }, + "failed": { "type": "integer", "minimum": 0 } + } + }, + "failures": { + "type": "array", + "items": { "$ref": "#/$defs/failure" } + } + }, + "$defs": { + "failure": { + "type": "object", + "additionalProperties": false, + "required": ["run", "target", "name", "package", "easyconfig", "step", "message", "duration_s", "log"], + "properties": { + "run": { "type": "string" }, + "target": { "type": ["string", "null"] }, + "name": { "type": "string" }, + "package": { "type": ["string", "null"] }, + "easyconfig": { "type": ["string", "null"] }, + "step": { "type": ["string", "null"] }, + "message": { "type": ["string", "null"] }, + "duration_s": { "type": ["integer", "null"], "minimum": 0 }, + "log": { "type": "string" } + } + } + } +} diff --git a/eb-keep-logs b/eb-keep-logs index a2ab89c..4ec979f 100755 --- a/eb-keep-logs +++ b/eb-keep-logs @@ -62,7 +62,8 @@ otherwise lose, and keep the pipeline's log tree bounded. --metrics-file PATH, --metrics-file=PATH Atomically record when this rotation completed, unless --dry-run is given. Default: none. - eb-keep-logs why [RUN | --latest] [--target TARGET] [--root ROOT] [--json] + eb-keep-logs why [RUN | --latest] [--target TARGET] [--package PACKAGE] + [--since DATE] [--until DATE] [--root ROOT] [--json] Say why the builds of a run failed: for each failed easyconfig, its package and easyconfig, the EasyBuild step it failed in, EasyBuild's message, and the line of its kept log that shows the error: the first @@ -78,12 +79,24 @@ otherwise lose, and keep the pipeline's log tree bounded. --target TARGET, --target=TARGET Only the builds in the run's directory TARGET. Default: those in the run's own directory and in every directory in it. + --package PACKAGE, --package=PACKAGE + Only builds matching PACKAGE (an easyconfig name without .eb, + a prefix up to a dash, or NAME/VERSION). Default: all builds. + --since DATE, --since=DATE + Only consider runs at or after DATE when picking --latest. + DATE is YYYY-MM-DD, YYYYMMDD, or YYYYMMDD_HHMMSS. + Default: none. + --until DATE, --until=DATE + Only consider runs at or before DATE when picking --latest. + DATE is YYYY-MM-DD, YYYYMMDD, or YYYYMMDD_HHMMSS. + Default: none. --root ROOT, --root=ROOT The directory that holds the runs. Default: the current directory. --json Print one line of JSON instead, with schema_version 1. - eb-keep-logs history PACKAGE [--target TARGET] [--root ROOT] [--json] + eb-keep-logs history PACKAGE [--target TARGET] [--since DATE] + [--until DATE] [--failed] [--root ROOT] [--json] Say how each kept build of PACKAGE went, oldest run first: success, failed or unknown, and how long EasyBuild took, from its output. PACKAGE is an easyconfig's name or its start up to a dash, as hello @@ -93,6 +106,37 @@ otherwise lose, and keep the pipeline's log tree bounded. --target TARGET, --target=TARGET Only the builds in each run's directory TARGET. Default: those in each run's own directory and in every directory in it. + --since DATE, --since=DATE + Only read runs at or after DATE (YYYY-MM-DD, YYYYMMDD, or + YYYYMMDD_HHMMSS). Default: none. + --until DATE, --until=DATE + Only read runs at or before DATE (YYYY-MM-DD, YYYYMMDD, or + YYYYMMDD_HHMMSS). Default: none. + --failed + Only list failed builds. + --root ROOT, --root=ROOT + The directory that holds the runs. Default: the current + directory. + --json + Print one line of JSON instead, with schema_version 1. + eb-keep-logs failures [--package PACKAGE] [--target TARGET] + [--since DATE] [--until DATE] [--root ROOT] [--json] + List failed builds across runs under --root, oldest run first. + Each row gives the run, target, failed step, duration and build + name. Zig build only. Options come in any order. + --package PACKAGE, --package=PACKAGE + Only failed builds matching PACKAGE (an easyconfig name + without .eb, a prefix up to a dash, or NAME/VERSION). Default: + all packages. + --target TARGET, --target=TARGET + Only the builds in each run's directory TARGET. Default: those + in each run's own directory and in every directory in it. + --since DATE, --since=DATE + Only read runs at or after DATE (YYYY-MM-DD, YYYYMMDD, or + YYYYMMDD_HHMMSS). Default: none. + --until DATE, --until=DATE + Only read runs at or before DATE (YYYY-MM-DD, YYYYMMDD, or + YYYYMMDD_HHMMSS). Default: none. --root ROOT, --root=ROOT The directory that holds the runs. Default: the current directory. @@ -290,7 +334,7 @@ Environment: directory of zone files. Exit status: - 0 -h, --help or --version, or a failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle, test-compare or lifecycle that + 0 -h, --help or --version, or a failed, rotate, why, history, failures, loads, runs, used, usage, pending, test-bundle, test-compare or lifecycle that accepted its arguments, whatever it could keep, compress, remove or read: what it could not do is reported on stderr. used with --require-none also exits 2 when any path was listed. @@ -1029,7 +1073,7 @@ case "${1:-}" in --version) echo "eb-keep-logs ${VERSION}" ;; failed) shift; cmd_failed "$@" ;; rotate) shift; cmd_rotate "$@" ;; - why | history | loads | runs | used | usage | pending | test-bundle | test-compare | lifecycle) die "$1 is in the Zig build only" ;; + why | history | failures | loads | runs | used | usage | pending | test-bundle | test-compare | lifecycle) die "$1 is in the Zig build only" ;; "") usage; exit 2 ;; - *) die "unknown command '$1' (failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle, test-compare, lifecycle, --help)" ;; + *) die "unknown command '$1' (failed, rotate, why, history, failures, loads, runs, used, usage, pending, test-bundle, test-compare, lifecycle, --help)" ;; esac diff --git a/src/main.zig b/src/main.zig index 9a861ef..2e8fab0 100644 --- a/src/main.zig +++ b/src/main.zig @@ -79,7 +79,8 @@ const usage_text = \\ --metrics-file PATH, --metrics-file=PATH \\ Atomically record when this rotation completed, unless --dry-run \\ is given. Default: none. - \\ eb-keep-logs why [RUN | --latest] [--target TARGET] [--root ROOT] [--json] + \\ eb-keep-logs why [RUN | --latest] [--target TARGET] [--package PACKAGE] + \\ [--since DATE] [--until DATE] [--root ROOT] [--json] \\ Say why the builds of a run failed: for each failed easyconfig, its \\ package and easyconfig, the EasyBuild step it failed in, EasyBuild's \\ message, and the line of its kept log that shows the error: the first @@ -95,12 +96,24 @@ const usage_text = \\ --target TARGET, --target=TARGET \\ Only the builds in the run's directory TARGET. Default: those in \\ the run's own directory and in every directory in it. + \\ --package PACKAGE, --package=PACKAGE + \\ Only builds matching PACKAGE (an easyconfig name without .eb, + \\ a prefix up to a dash, or NAME/VERSION). Default: all builds. + \\ --since DATE, --since=DATE + \\ Only consider runs at or after DATE when picking --latest. + \\ DATE is YYYY-MM-DD, YYYYMMDD, or YYYYMMDD_HHMMSS. + \\ Default: none. + \\ --until DATE, --until=DATE + \\ Only consider runs at or before DATE when picking --latest. + \\ DATE is YYYY-MM-DD, YYYYMMDD, or YYYYMMDD_HHMMSS. + \\ Default: none. \\ --root ROOT, --root=ROOT \\ The directory that holds the runs. Default: the current \\ directory. \\ --json \\ Print one line of JSON instead, with schema_version 1. - \\ eb-keep-logs history PACKAGE [--target TARGET] [--root ROOT] [--json] + \\ eb-keep-logs history PACKAGE [--target TARGET] [--since DATE] + \\ [--until DATE] [--failed] [--root ROOT] [--json] \\ Say how each kept build of PACKAGE went, oldest run first: success, \\ failed or unknown, and how long EasyBuild took, from its output. \\ PACKAGE is an easyconfig's name or its start up to a dash, as hello @@ -110,6 +123,37 @@ const usage_text = \\ --target TARGET, --target=TARGET \\ Only the builds in each run's directory TARGET. Default: those in \\ each run's own directory and in every directory in it. + \\ --since DATE, --since=DATE + \\ Only read runs at or after DATE (YYYY-MM-DD, YYYYMMDD, or + \\ YYYYMMDD_HHMMSS). Default: none. + \\ --until DATE, --until=DATE + \\ Only read runs at or before DATE (YYYY-MM-DD, YYYYMMDD, or + \\ YYYYMMDD_HHMMSS). Default: none. + \\ --failed + \\ Only list failed builds. + \\ --root ROOT, --root=ROOT + \\ The directory that holds the runs. Default: the current + \\ directory. + \\ --json + \\ Print one line of JSON instead, with schema_version 1. + \\ eb-keep-logs failures [--package PACKAGE] [--target TARGET] + \\ [--since DATE] [--until DATE] [--root ROOT] [--json] + \\ List failed builds across runs under --root, oldest run first. + \\ Each row gives the run, target, failed step, duration and build + \\ name. Zig build only. Options come in any order. + \\ --package PACKAGE, --package=PACKAGE + \\ Only failed builds matching PACKAGE (an easyconfig name + \\ without .eb, a prefix up to a dash, or NAME/VERSION). Default: + \\ all packages. + \\ --target TARGET, --target=TARGET + \\ Only the builds in each run's directory TARGET. Default: those + \\ in each run's own directory and in every directory in it. + \\ --since DATE, --since=DATE + \\ Only read runs at or after DATE (YYYY-MM-DD, YYYYMMDD, or + \\ YYYYMMDD_HHMMSS). Default: none. + \\ --until DATE, --until=DATE + \\ Only read runs at or before DATE (YYYY-MM-DD, YYYYMMDD, or + \\ YYYYMMDD_HHMMSS). Default: none. \\ --root ROOT, --root=ROOT \\ The directory that holds the runs. Default: the current \\ directory. @@ -307,7 +351,7 @@ const usage_text = \\ directory of zone files. \\ \\Exit status: - \\ 0 -h, --help or --version, or a failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle, test-compare or lifecycle that + \\ 0 -h, --help or --version, or a failed, rotate, why, history, failures, loads, runs, used, usage, pending, test-bundle, test-compare or lifecycle that \\ accepted its arguments, whatever it could keep, compress, remove or \\ read: what it could not do is reported on stderr. used with \\ --require-none also exits 2 when any path was listed. @@ -1341,6 +1385,7 @@ pub fn main(init: std.process.Init) u8 { if (std.mem.eql(u8, cmd, "rotate")) return cmdRotate(c, args[2..]); if (std.mem.eql(u8, cmd, "why")) return why.cmdWhy(gpa, args[2..]); if (std.mem.eql(u8, cmd, "history")) return why.cmdHistory(gpa, args[2..]); + if (std.mem.eql(u8, cmd, "failures")) return why.cmdFailures(gpa, args[2..]); if (std.mem.eql(u8, cmd, "loads")) return loads.cmdLoads(gpa, args[2..]); if (std.mem.eql(u8, cmd, "runs")) return xalt_runs.cmdRuns(gpa, args[2..]); if (std.mem.eql(u8, cmd, "used")) return used_cmd.cmdUsed(gpa, args[2..]); @@ -1353,7 +1398,7 @@ pub fn main(init: std.process.Init) u8 { cli.put(1, usage_text); return cli.usage_exit; } - die("unknown command '{s}' (failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle, test-compare, lifecycle, --help)", .{cmd}); + die("unknown command '{s}' (failed, rotate, why, history, failures, loads, runs, used, usage, pending, test-bundle, test-compare, lifecycle, --help)", .{cmd}); } test { diff --git a/src/why.zig b/src/why.zig index cd0e246..041c195 100644 --- a/src/why.zig +++ b/src/why.zig @@ -2,7 +2,8 @@ //! and say why its builds failed: for each, the EasyBuild step it failed //! in, EasyBuild's message, and the line of its log that shows the error, //! with the lines around it. `history`: how each kept build of a package -//! went, run by run, from EasyBuild's output. +//! went, run by run, from EasyBuild's output. `failures`: list failed +//! builds across runs under `--root`, filtered by recipe or date. const std = @import("std"); const common = @import("zig_common"); const sys = common.sys; @@ -754,24 +755,148 @@ fn emit(gpa: Allocator, r: anytype, as_json: bool, comptime writeText: fn (*std. } const why_help = - \\eb-keep-logs why [RUN | --latest] [--target TARGET] [--root ROOT] [--json] + \\eb-keep-logs why [RUN | --latest] [--target TARGET] [--package PACKAGE] [--since DATE] [--until DATE] [--root ROOT] [--json] \\ \\Say why the builds of one run failed, from the logs it kept. \\With no RUN, the newest run under --root is read. --root defaults to \\the current directory. A run is a directory named YYYYMMDD_HHMMSS. - \\--target reads only that directory of the run. --json prints one document. + \\--target reads only that directory of the run. --package filters builds + \\by recipe. --since and --until bound --latest by run date. --json prints + \\one document. \\ ; const history_help = - \\eb-keep-logs history PACKAGE [--target TARGET] [--root ROOT] [--json] + \\eb-keep-logs history PACKAGE [--target TARGET] [--since DATE] [--until DATE] [--failed] [--root ROOT] [--json] \\ \\PACKAGE is the easyconfig's name without .eb, or NAME/VERSION. \\--root defaults to the current directory. A run is a directory named YYYYMMDD_HHMMSS. - \\--target reads only that directory of each run. --json prints one document. + \\--target reads only that directory of each run. --since and --until bound + \\runs by date. --failed lists only failed builds. --json prints one document. \\ ; +const failures_help = + \\eb-keep-logs failures [--package PACKAGE] [--target TARGET] [--since DATE] [--until DATE] [--root ROOT] [--json] + \\ + \\List failed builds across runs under --root, oldest run first. + \\--package takes an easyconfig name without .eb, a prefix up to a dash, + \\or NAME/VERSION. --since and --until take YYYY-MM-DD, YYYYMMDD, or + \\YYYYMMDD_HHMMSS. --root defaults to the current directory. --json prints + \\one document. + \\ +; + +fn isLeapYear(year: u32) bool { + return (year % 4 == 0 and year % 100 != 0) or (year % 400 == 0); +} + +fn daysInMonth(year: u32, month: u32) u32 { + return switch (month) { + 1, 3, 5, 7, 8, 10, 12 => 31, + 4, 6, 9, 11 => 30, + 2 => if (isLeapYear(year)) 29 else 28, + else => 0, + }; +} + +fn parseDigits(s: []const u8) ?u32 { + if (!isNumber(s)) return null; + return std.fmt.parseInt(u32, s, 10) catch null; +} + +/// Normalize a `--since` or `--until` date (`YYYY-MM-DD`, `YYYYMMDD`, or +/// `YYYYMMDD_HHMMSS`) into a 15-byte `YYYYMMDD_HHMMSS` bound for comparing +/// against run directory names. +fn parseDateBound(raw: []const u8, is_until: bool) error{InvalidDate}![15]u8 { + var out: [15]u8 = undefined; + var y_s: []const u8 = undefined; + var m_s: []const u8 = undefined; + var d_s: []const u8 = undefined; + var hh_s: []const u8 = if (is_until) "23" else "00"; + var mm_s: []const u8 = if (is_until) "59" else "00"; + var ss_s: []const u8 = if (is_until) "59" else "00"; + + if (raw.len == 10 and raw[4] == '-' and raw[7] == '-') { + y_s = raw[0..4]; + m_s = raw[5..7]; + d_s = raw[8..10]; + } else if (raw.len == 8) { + y_s = raw[0..4]; + m_s = raw[4..6]; + d_s = raw[6..8]; + } else if (raw.len == 15 and isRunName(raw)) { + y_s = raw[0..4]; + m_s = raw[4..6]; + d_s = raw[6..8]; + hh_s = raw[9..11]; + mm_s = raw[11..13]; + ss_s = raw[13..15]; + } else { + return error.InvalidDate; + } + + const year = parseDigits(y_s) orelse return error.InvalidDate; + const month = parseDigits(m_s) orelse return error.InvalidDate; + const day = parseDigits(d_s) orelse return error.InvalidDate; + if (month < 1 or month > 12) return error.InvalidDate; + if (day < 1 or day > daysInMonth(year, month)) return error.InvalidDate; + + const hh = parseDigits(hh_s) orelse return error.InvalidDate; + const mm = parseDigits(mm_s) orelse return error.InvalidDate; + const ss = parseDigits(ss_s) orelse return error.InvalidDate; + if (hh > 23 or mm > 59 or ss > 59) return error.InvalidDate; + + @memcpy(out[0..4], y_s); + @memcpy(out[4..6], m_s); + @memcpy(out[6..8], d_s); + out[8] = '_'; + @memcpy(out[9..11], hh_s); + @memcpy(out[11..13], mm_s); + @memcpy(out[13..15], ss_s); + return out; +} + +const DateFilter = struct { + since_bound: ?[15]u8 = null, + until_bound: ?[15]u8 = null, + + fn init(since_raw: ?[]const u8, until_raw: ?[]const u8) DateFilter { + var f: DateFilter = .{}; + if (since_raw) |s| { + f.since_bound = parseDateBound(s, false) catch + die("--since needs YYYY-MM-DD, YYYYMMDD, or YYYYMMDD_HHMMSS", .{}); + } + if (until_raw) |u| { + f.until_bound = parseDateBound(u, true) catch + die("--until needs YYYY-MM-DD, YYYYMMDD, or YYYYMMDD_HHMMSS", .{}); + } + if (f.since_bound != null and f.until_bound != null) { + if (std.mem.order(u8, &f.since_bound.?, &f.until_bound.?) == .gt) { + die("--until must not be earlier than --since", .{}); + } + } + return f; + } + + fn active(self: DateFilter) bool { + return self.since_bound != null or self.until_bound != null; + } + + fn matches(self: DateFilter, run: []const u8) bool { + if (self.since_bound) |sb| if (std.mem.order(u8, run, &sb) == .lt) return false; + if (self.until_bound) |ub| if (std.mem.order(u8, run, &ub) == .gt) return false; + return true; + } +}; + +fn normalizePackage(arena: Allocator, raw: []const u8) Allocator.Error![]const u8 { + // A module name, NAME/VERSION, is the easyconfig's NAME-VERSION. + const package = try arena.dupe(u8, raw); + std.mem.replaceScalar(u8, package, '/', '-'); + return package; +} + fn wantsHelp(args: []const [:0]const u8) bool { for (args) |a| { if (std.mem.eql(u8, a, "-h") or std.mem.eql(u8, a, "--help")) return true; @@ -779,8 +904,8 @@ fn wantsHelp(args: []const [:0]const u8) bool { return false; } -/// `why [RUN | --latest] [--target TARGET] [--root ROOT] [--json]`: print -/// why the builds of a run failed, from the logs it kept. Returns 0 once +/// `why [RUN | --latest] [--target TARGET] [--package PACKAGE] [--since DATE] [--until DATE] [--root ROOT] [--json]`: +/// print why the builds of a run failed, from the logs it kept. Returns 0 once /// its arguments are accepted, whatever it could read; a refused /// invocation exits 2. `-h` and `--help` print what to pass and return 0. pub fn cmdWhy(gpa: Allocator, args: []const [:0]const u8) u8 { @@ -791,6 +916,9 @@ pub fn cmdWhy(gpa: Allocator, args: []const [:0]const u8) u8 { var run_arg: ?[]const u8 = null; var latest = false; var target: ?[]const u8 = null; + var package_arg: ?[]const u8 = null; + var since_raw: ?[]const u8 = null; + var until_raw: ?[]const u8 = null; var root_arg: ?[]const u8 = null; var as_json = false; var i: usize = 0; @@ -802,6 +930,12 @@ pub fn cmdWhy(gpa: Allocator, args: []const [:0]const u8) u8 { as_json = true; } else if (optionValue(args, &i, "--target", "a target")) |v| { target = v; + } else if (optionValue(args, &i, "--package", "a package")) |v| { + package_arg = v; + } else if (optionValue(args, &i, "--since", "a date")) |v| { + since_raw = v; + } else if (optionValue(args, &i, "--until", "a date")) |v| { + until_raw = v; } else if (optionValue(args, &i, "--root", "a directory")) |v| { root_arg = v; } else if (a.len > 0 and a[0] == '-') { @@ -812,12 +946,15 @@ pub fn cmdWhy(gpa: Allocator, args: []const [:0]const u8) u8 { } } if (run_arg != null and latest) die("why takes RUN or --latest, not both", .{}); + if (run_arg != null and (since_raw != null or until_raw != null)) die("why takes RUN or --since/--until, not both", .{}); if (run_arg) |r| if (r.len == 0) die("RUN must not be empty", .{}); + const date_filter = DateFilter.init(since_raw, until_raw); var arena_state = std.heap.ArenaAllocator.init(gpa); defer arena_state.deinit(); const arena = arena_state.allocator(); const oom = "out of memory"; + const package_filter: ?[]const u8 = if (package_arg) |p| normalizePackage(arena, p) catch die(oom, .{}) else null; const root_s = root_arg orelse "."; const root = fsx.openDir(fsx.cwd, root_s, true) catch die("ROOT {s} is not a directory", .{root_s}); @@ -827,10 +964,15 @@ pub fn cmdWhy(gpa: Allocator, args: []const [:0]const u8) u8 { var k = runs.len; while (k > 0) { k -= 1; + if (!date_filter.matches(runs[k])) continue; const t = target orelse break :blk runs[k]; const p = std.fs.path.join(arena, &.{ runs[k], t }) catch die(oom, .{}); if (isDirAt(root, p)) break :blk runs[k]; } + if (date_filter.active()) { + if (target) |t| die("no run under {s} in the date range has a target {s}", .{ root_s, t }); + die("no run under {s} matches the date range", .{root_s}); + } if (target) |t| die("no run under {s} has a target {s}. Give --target a directory of that run", .{ root_s, t }); die("no run under {s}; it holds no directory named YYYYMMDD_HHMMSS. Give --root the directory of the runs, or name one run", .{root_s}); }; @@ -852,6 +994,7 @@ pub fn cmdWhy(gpa: Allocator, args: []const [:0]const u8) u8 { defer sys.close(dir); const builds = buildsIn(gpa, arena, dir, dir_s) catch die(oom, .{}); for (builds) |b| { + if (package_filter) |pf| if (!isBuildOf(b.name, pf)) continue; const v = outcomeOf(gpa, arena, dir, dir_s, b) catch die(oom, .{}); counts.add(v.outcome); if (v.outcome != .failed) continue; @@ -930,8 +1073,8 @@ fn writeHistory(w: *std.Io.Writer, r: HistoryReport) std.Io.Writer.Error!void { for (r.builds) |b| try writeRow(w, .{ b.run, b.target orelse "-", @tagName(b.outcome), durationText(&buf, b.duration_s), b.name }, widths); } -/// `history PACKAGE [--target TARGET] [--root ROOT] [--json]`: print how -/// each build of PACKAGE kept under ROOT went, oldest run first. Returns 0 +/// `history PACKAGE [--target TARGET] [--since DATE] [--until DATE] [--failed] [--root ROOT] [--json]`: +/// print how each build of PACKAGE kept under ROOT went, oldest run first. Returns 0 /// once its arguments are accepted, whatever it could read; a refused /// invocation exits 2. pub fn cmdHistory(gpa: Allocator, args: []const [:0]const u8) u8 { @@ -941,6 +1084,9 @@ pub fn cmdHistory(gpa: Allocator, args: []const [:0]const u8) u8 { } var package_arg: ?[]const u8 = null; var target: ?[]const u8 = null; + var since_raw: ?[]const u8 = null; + var until_raw: ?[]const u8 = null; + var failed_only = false; var root_arg: ?[]const u8 = null; var as_json = false; var i: usize = 0; @@ -948,8 +1094,14 @@ pub fn cmdHistory(gpa: Allocator, args: []const [:0]const u8) u8 { const a = args[i]; if (std.mem.eql(u8, a, "--json")) { as_json = true; + } else if (std.mem.eql(u8, a, "--failed")) { + failed_only = true; } else if (optionValue(args, &i, "--target", "a target")) |v| { target = v; + } else if (optionValue(args, &i, "--since", "a date")) |v| { + since_raw = v; + } else if (optionValue(args, &i, "--until", "a date")) |v| { + until_raw = v; } else if (optionValue(args, &i, "--root", "a directory")) |v| { root_arg = v; } else if (a.len > 0 and a[0] == '-') { @@ -961,14 +1113,13 @@ pub fn cmdHistory(gpa: Allocator, args: []const [:0]const u8) u8 { } const package_s = package_arg orelse die("history needs PACKAGE: the easyconfig name without .eb, or NAME/VERSION", .{}); if (package_s.len == 0) die("PACKAGE must not be empty", .{}); + const date_filter = DateFilter.init(since_raw, until_raw); var arena_state = std.heap.ArenaAllocator.init(gpa); defer arena_state.deinit(); const arena = arena_state.allocator(); const oom = "out of memory"; - // A module name, NAME/VERSION, is the easyconfig's NAME-VERSION. - const package = arena.dupe(u8, package_s) catch die(oom, .{}); - std.mem.replaceScalar(u8, package, '/', '-'); + const package = normalizePackage(arena, package_s) catch die(oom, .{}); const root_s = root_arg orelse "."; const root = fsx.openDir(fsx.cwd, root_s, true) catch die("ROOT {s} is not a directory", .{root_s}); @@ -977,6 +1128,7 @@ pub fn cmdHistory(gpa: Allocator, args: []const [:0]const u8) u8 { var kept: std.ArrayList(Kept) = .empty; const runs = runsIn(gpa, arena, root, root_s) catch die(oom, .{}); for (runs) |r| { + if (!date_filter.matches(r)) continue; const run_s = std.fs.path.join(arena, &.{ root_s, r }) catch die(oom, .{}); const run = fsx.openDir(root, r, false) catch |err| { warn("cannot open {s}: {t}", .{ run_s, err }); @@ -994,6 +1146,7 @@ pub fn cmdHistory(gpa: Allocator, args: []const [:0]const u8) u8 { for (builds) |b| { if (!isBuildOf(b.name, package)) continue; const v = outcomeOf(gpa, arena, dir, dir_s, b) catch die(oom, .{}); + if (failed_only and v.outcome != .failed) continue; counts.add(v.outcome); const log = std.fs.path.join(arena, &.{ dir_s, b.stdout orelse b.session.? }) catch die(oom, .{}); kept.append(arena, .{ @@ -1017,6 +1170,164 @@ pub fn cmdHistory(gpa: Allocator, args: []const [:0]const u8) u8 { return 0; } +const FailedRunItem = struct { + run: []const u8, + target: ?[]const u8, + name: []const u8, + package: ?[]const u8, + easyconfig: ?[]const u8, + step: ?[]const u8, + message: ?[]const u8, + duration_s: ?u64, + log: []const u8, +}; + +const FailuresCounts = struct { + runs: usize, + failed: usize, +}; + +const FailuresReport = struct { + schema_version: u32 = schema_version, + kind: []const u8 = "failures", + root: []const u8, + package: ?[]const u8, + target: ?[]const u8, + since: ?[]const u8, + until: ?[]const u8, + counts: FailuresCounts, + failures: []const FailedRunItem, +}; + +fn writeFailures(w: *std.Io.Writer, r: FailuresReport) std.Io.Writer.Error!void { + const total = r.failures.len; + if (total == 0) { + try w.print("{s}: 0 failed builds in 0 runs\n", .{r.root}); + return; + } + try w.print("{s}: {d} failed build{s} in {d} run{s}\n\n", .{ r.root, total, plural(total), r.counts.runs, plural(r.counts.runs) }); + const header = [5][]const u8{ "RUN", "TARGET", "STEP", "DURATION", "NAME" }; + var widths: [4]usize = undefined; + for (header[0..4], &widths) |h, *n| n.* = h.len; + var buf: [24]u8 = undefined; + for (r.failures) |f| { + const target_s: []const u8 = f.target orelse "-"; + const step_s: []const u8 = f.step orelse "-"; + widths[0] = @max(widths[0], f.run.len); + widths[1] = @max(widths[1], target_s.len); + widths[2] = @max(widths[2], step_s.len); + widths[3] = @max(widths[3], durationText(&buf, f.duration_s).len); + } + try writeRow(w, header, widths); + for (r.failures) |f| { + try writeRow(w, .{ f.run, f.target orelse "-", f.step orelse "-", durationText(&buf, f.duration_s), f.name }, widths); + } +} + +/// `failures [--package PACKAGE] [--target TARGET] [--since DATE] [--until DATE] [--root ROOT] [--json]`: +/// list failed builds across runs under ROOT, oldest run first. Returns 0 once +/// its arguments are accepted, whatever it could read; a refused invocation +/// exits 2. `-h` and `--help` print what to pass and return 0. +pub fn cmdFailures(gpa: Allocator, args: []const [:0]const u8) u8 { + if (wantsHelp(args)) { + cli.put(1, failures_help); + return 0; + } + var package_arg: ?[]const u8 = null; + var target: ?[]const u8 = null; + var since_raw: ?[]const u8 = null; + var until_raw: ?[]const u8 = null; + var root_arg: ?[]const u8 = null; + var as_json = false; + var i: usize = 0; + while (i < args.len) : (i += 1) { + const a = args[i]; + if (std.mem.eql(u8, a, "--json")) { + as_json = true; + } else if (optionValue(args, &i, "--package", "a package")) |v| { + package_arg = v; + } else if (optionValue(args, &i, "--target", "a target")) |v| { + target = v; + } else if (optionValue(args, &i, "--since", "a date")) |v| { + since_raw = v; + } else if (optionValue(args, &i, "--until", "a date")) |v| { + until_raw = v; + } else if (optionValue(args, &i, "--root", "a directory")) |v| { + root_arg = v; + } else if (a.len > 0 and a[0] == '-') { + die("unknown option {s}", .{a}); + } else { + die("failures takes no positional argument, got '{s}'", .{a}); + } + } + const date_filter = DateFilter.init(since_raw, until_raw); + + var arena_state = std.heap.ArenaAllocator.init(gpa); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const oom = "out of memory"; + const package_filter: ?[]const u8 = if (package_arg) |p| normalizePackage(arena, p) catch die(oom, .{}) else null; + + const root_s = root_arg orelse "."; + const root = fsx.openDir(fsx.cwd, root_s, true) catch die("ROOT {s} is not a directory", .{root_s}); + defer sys.close(root); + var failed_runs: usize = 0; + var items: std.ArrayList(FailedRunItem) = .empty; + const runs = runsIn(gpa, arena, root, root_s) catch die(oom, .{}); + for (runs) |r| { + if (!date_filter.matches(r)) continue; + const run_s = std.fs.path.join(arena, &.{ root_s, r }) catch die(oom, .{}); + const run = fsx.openDir(root, r, false) catch |err| { + warn("cannot open {s}: {t}", .{ run_s, err }); + continue; + }; + defer sys.close(run); + const targets: []const []const u8 = if (target) |t| + (if (isDirAt(run, t)) arena.dupe([]const u8, &.{t}) catch die(oom, .{}) else &.{}) + else + targetsIn(gpa, arena, run, run_s) catch die(oom, .{}); + var run_had_failure = false; + for (targets) |t| { + const dir, const dir_s = (openBuilds(arena, run, run_s, t) catch die(oom, .{})) orelse continue; + defer sys.close(dir); + const builds = buildsIn(gpa, arena, dir, dir_s) catch die(oom, .{}); + for (builds) |b| { + if (package_filter) |pf| if (!isBuildOf(b.name, pf)) continue; + const v = outcomeOf(gpa, arena, dir, dir_s, b) catch die(oom, .{}); + if (v.outcome != .failed) continue; + const f = failureOf(gpa, arena, dir, dir_s, t, b, v) catch die(oom, .{}); + run_had_failure = true; + items.append(arena, .{ + .run = r, + .target = f.target, + .name = f.name, + .package = f.package, + .easyconfig = f.easyconfig, + .step = f.step, + .message = f.message, + .duration_s = f.duration_s, + .log = f.log.?, + }) catch die(oom, .{}); + } + } + if (run_had_failure) failed_runs += 1; + } + const report: FailuresReport = .{ + .root = utf8(arena, root_s) catch die(oom, .{}), + .package = if (package_arg) |p| utf8(arena, p) catch die(oom, .{}) else null, + .target = if (target) |t| utf8(arena, t) catch die(oom, .{}) else null, + .since = if (since_raw) |s| utf8(arena, s) catch die(oom, .{}) else null, + .until = if (until_raw) |u| utf8(arena, u) catch die(oom, .{}) else null, + .counts = .{ + .runs = failed_runs, + .failed = items.items.len, + }, + .failures = items.items, + }; + emit(gpa, report, as_json, writeFailures); + return 0; +} + const testing = std.testing; test "isCompilerOrLinkerError: a compiler's error at a source line, a linker's error" { @@ -1284,3 +1595,34 @@ test "isBuildOf takes a name or its start up to a dash" { try testing.expect(!isBuildOf(name, "hello-2.12")); try testing.expect(isBuildOf("hello-world-1.0", "hello")); } + +test "parseDateBound normalizes dates and rejects invalid ones" { + try testing.expectEqualStrings("20261117_000000", &(try parseDateBound("2026-11-17", false))); + try testing.expectEqualStrings("20261117_235959", &(try parseDateBound("2026-11-17", true))); + try testing.expectEqualStrings("20261117_000000", &(try parseDateBound("20261117", false))); + try testing.expectEqualStrings("20261117_235959", &(try parseDateBound("20261117", true))); + try testing.expectEqualStrings("20261117_040615", &(try parseDateBound("20261117_040615", false))); + try testing.expectEqualStrings("20261117_040615", &(try parseDateBound("20261117_040615", true))); + try testing.expectEqualStrings("20240229_000000", &(try parseDateBound("2024-02-29", false))); + + try testing.expectError(error.InvalidDate, parseDateBound("", false)); + try testing.expectError(error.InvalidDate, parseDateBound("2026-02-29", false)); + try testing.expectError(error.InvalidDate, parseDateBound("2026-13-01", false)); + try testing.expectError(error.InvalidDate, parseDateBound("2026-11-00", false)); + try testing.expectError(error.InvalidDate, parseDateBound("20261117_240000", false)); + try testing.expectError(error.InvalidDate, parseDateBound("20261117_126000", false)); + try testing.expectError(error.InvalidDate, parseDateBound("20261117_120060", false)); + try testing.expectError(error.InvalidDate, parseDateBound("not-a-date", false)); +} + +test "DateFilter matches runs within inclusive bounds" { + const f: DateFilter = .{ + .since_bound = try parseDateBound("2026-11-17", false), + .until_bound = try parseDateBound("20261117_041222", true), + }; + try testing.expect(!f.matches("20261116_235959")); + try testing.expect(f.matches("20261117_000000")); + try testing.expect(f.matches("20261117_040615")); + try testing.expect(f.matches("20261117_041222")); + try testing.expect(!f.matches("20261117_041223")); +} diff --git a/tests/differential.sh b/tests/differential.sh index 9c2066b..aa77599 100644 --- a/tests/differential.sh +++ b/tests/differential.sh @@ -23,6 +23,7 @@ declare -A deliberate=( ["rotate, lock is a symbolic link"]="\`rotate\`'s lock file cannot be opened, or is a symbolic link or not a regular file" ["why"]="\`why\`" ["history"]="\`history\`" + ["failures"]="\`failures\`" ["loads"]="\`loads\`" ["runs"]="\`runs\`" ["used"]="\`used\`" @@ -165,6 +166,7 @@ compare "rotate, a run still written into" rotate "${work}/live" --compress-afte kill "${holder}"; wait "${holder}" 2>/dev/null compare "why" why --latest --root "${repo_root}/tests/fixtures/build-logs" compare "history" history hello --root "${repo_root}/tests/fixtures/build-logs" +compare "failures" failures --root "${repo_root}/tests/fixtures/build-logs" printf 'user=a module=M path=/p host=h time=1\n' > "${work}/loads.log" compare "loads" loads --module M "${work}/loads.log" printf '%s\n' '{"userT":{"user":"a","exec_path":"/bin/gmx","syshost":"h","start_time":1}}' > "${work}/runs.json" diff --git a/tests/test_failures.sh b/tests/test_failures.sh new file mode 100755 index 0000000..0a075b4 --- /dev/null +++ b/tests/test_failures.sh @@ -0,0 +1,140 @@ +#!/bin/bash +# Tests for opt-in recipe and date filtering in failures, why, and history: +# TOOL_BIN=zig-out/bin/eb-keep-logs bash tests/test_failures.sh +set -u + +repo_root=$(cd "$(dirname "$0")/.." && pwd) +tool="${TOOL_BIN:?TOOL_BIN names the Zig build}" +case "${tool}" in /*) ;; *) tool="${PWD}/${tool}" ;; esac +cd "${repo_root}" || exit 1 +logs=tests/fixtures/build-logs + +fails=0 +check() { + if [ "$2" = "$3" ]; then + echo "ok $1" + else + { + echo "FAIL $1" + echo " expected: $3" + echo " got: $2" + } >&2 + fails=$((fails + 1)) + fi +} + +work=$(mktemp -d "${TMPDIR:-/tmp}/eb-keep-logs-failures.XXXXXX") +trap 'rm -rf -- "${work}"' EXIT + +# failures: all failed builds across the fixture corpus. +out=$("${tool}" failures --root "${logs}") +check "failures: header count" "$(printf '%s\n' "${out}" | head -n 1)" \ + "${logs}: 8 failed builds in 7 runs" +check "failures: includes fetch and test steps" "$(printf '%s\n' "${out}" | awk 'NR >= 4 { print $3 }' | tr '\n' ' ')" \ + "fetch patch - configure build build sanitycheck test " + +# Date filtering: exact timestamp bounds and YYYY-MM-DD / YYYYMMDD bounds. +out=$("${tool}" failures --root "${logs}" --since 20261117_040615 --until 20261117_041222) +check "failures: timestamp range header" "$(printf '%s\n' "${out}" | head -n 1)" \ + "${logs}: 4 failed builds in 3 runs" +check "failures: timestamp range runs" "$(printf '%s\n' "${out}" | awk 'NR >= 4 { print $1 }' | tr '\n' ' ')" \ + "20261117_040615 20261117_040928 20261117_041222 20261117_041222 " + +# Recipe filtering: GCCcore-13.3.0 vs module name hello/2.12.1-GCCcore-13.3.0. +out=$("${tool}" failures --root "${logs}" --package hello-2.12.1-GCCcore-13.3.0) +check "failures: package filter header" "$(printf '%s\n' "${out}" | head -n 1)" \ + "${logs}: 1 failed build in 1 run" +check "failures: package filter row" "$(printf '%s\n' "${out}" | tail -n 1)" \ + "20261117_041222 sapphirerapids sanitycheck 2m54s hello-2.12.1-GCCcore-13.3.0" +check "failures: module slash form matches" \ + "$("${tool}" failures --root "${logs}" --package hello/2.12.1-GCCcore-13.3.0)" "${out}" + +# Target + date filtering with zero matching failures. +check "failures: target generic has 0 failures" \ + "$("${tool}" failures --root "${logs}" --target generic)" \ + "${logs}: 0 failed builds in 0 runs" +check "failures: future date range has 0 failures" \ + "$("${tool}" failures --root "${logs}" --since 2026-11-18)" \ + "${logs}: 0 failed builds in 0 runs" + +# Synthetic multi-recipe tree: filter by recipe or date across different packages. +mkdir -p \ + "${work}/tree/20261110_010000/zen4" \ + "${work}/tree/20261112_120000/zen4" \ + "${work}/tree/20261115_080000/zen4" +printf '== FAILED: Installation ended unsuccessfully: bad configure (took 10 secs)\n' \ + > "${work}/tree/20261110_010000/zen4/OpenMPI-5.0.stdout.log" +printf '== COMPLETED: Installation ended successfully (took 5 secs)\n' \ + > "${work}/tree/20261112_120000/zen4/OpenMPI-5.0.stdout.log" +printf '== FAILED: Installation ended unsuccessfully: bad test (took 20 secs)\n' \ + > "${work}/tree/20261112_120000/zen4/GROMACS-2024.1.stdout.log" +printf '== FAILED: Installation ended unsuccessfully: bad build (took 30 secs)\n' \ + > "${work}/tree/20261115_080000/zen4/OpenMPI-5.0.stdout.log" + +check "failures: multi-recipe date filter" \ + "$("${tool}" failures --root "${work}/tree" --since 20261111 --until 2026-11-14 | awk 'NR >= 4 { print $1, $5 }')" \ + "20261112_120000 GROMACS-2024.1" +check "failures: multi-recipe package filter" \ + "$("${tool}" failures --root "${work}/tree" --package OpenMPI | awk 'NR >= 4 { print $1, $5 }' | tr '\n' ';')" \ + "20261110_010000 OpenMPI-5.0;20261115_080000 OpenMPI-5.0;" + +# failures --json structure. +fjson=$("${tool}" failures --root "${work}/tree" --package OpenMPI --since 2026-11-12 --json) +check "failures --json fields" "$(python3 -c ' +import json, sys +d = json.loads(sys.stdin.read()) +assert d["schema_version"] == 1 and d["kind"] == "failures" +assert d["package"] == "OpenMPI" and d["since"] == "2026-11-12" and d["until"] is None +assert d["counts"] == {"runs": 1, "failed": 1} +assert len(d["failures"]) == 1 +f = d["failures"][0] +assert f["run"] == "20261115_080000" and f["target"] == "zen4" and f["name"] == "OpenMPI-5.0" +assert f["duration_s"] == 30 +print("ok") +' <<< "${fjson}")" "ok" + +# why opt-in filters: --package, --since, --until. +why_pkg=$("${tool}" why --root "${logs}" 20261117_041222 --package hello-2.12.1-GCCcore-14.2.0 --json) +check "why --package filters failures and counts" "$(python3 -c ' +import json, sys +d = json.loads(sys.stdin.read()) +assert d["counts"] == {"success": 1, "failed": 1, "unknown": 0} +assert len(d["failures"]) == 1 +assert d["failures"][0]["name"] == "hello-2.12.1-GCCcore-14.2.0" +print("ok") +' <<< "${why_pkg}")" "ok" + +why_date=$("${tool}" why --root "${logs}" --until 20261117_040615) +check "why --until picks newest run in range" "$(printf '%s\n' "${why_date}" | head -n 1)" \ + "${logs}/20261117_040615: 1 failed build of 1" + +# history opt-in filters: --since, --until, --failed. +hist_f=$("${tool}" history hello --root "${logs}" --since 20261117_040615 --until 20261117_042207 --failed) +check "history --since/--until/--failed header" "$(printf '%s\n' "${hist_f}" | head -n 1)" \ + "hello: 4 builds under ${logs}: 0 succeeded, 4 failed" +check "history --since/--until/--failed rows" "$(printf '%s\n' "${hist_f}" | awk 'NR >= 4 { print $1, $3 }' | tr '\n' ';')" \ + "20261117_040615 failed;20261117_040928 failed;20261117_041222 failed;20261117_041222 failed;" + +# Refusals for invalid filter arguments. +refused() { + local label=$1 message=$2; shift 2 + "${tool}" "$@" > /dev/null 2> "${work}/err" + check "refused, ${label}: exit 2" "$?" "2" + check "refused, ${label}: message" "$(cat "${work}/err")" "eb-keep-logs: ${message}" +} + +refused "failures bad date" "--since needs YYYY-MM-DD, YYYYMMDD, or YYYYMMDD_HHMMSS" failures --root "${logs}" --since 2026-02-29 +refused "failures inverted range" "--until must not be earlier than --since" failures --root "${logs}" --since 2026-11-18 --until 2026-11-17 +refused "failures empty package" "--package needs a package" failures --root "${logs}" --package= +refused "failures positional arg" "failures takes no positional argument, got 'extra'" failures extra +refused "why RUN with --since" "why takes RUN or --since/--until, not both" why "${logs}/20261117_040615" --since 2026-11-17 +refused "why date range no match" "no run under ${logs} matches the date range" why --root "${logs}" --since 2026-11-18 +refused "history inverted range" "--until must not be earlier than --since" history hello --root "${logs}" --since 2026-11-18 --until 2026-11-17 + +"${tool}" failures --help > "${work}/fhelp" 2> "${work}/ferr" +check "failures --help exit 0" "$?" "0" +check "failures --help stderr empty" "$(cat "${work}/ferr")" "" +check "failures --help names YYYYMMDD_HHMMSS" "$(grep -c 'YYYYMMDD_HHMMSS' "${work}/fhelp")" "1" + +if [ "${fails}" -ne 0 ]; then echo "${fails} failed"; exit 1; fi +echo "all passed" diff --git a/tests/test_json_schemas.sh b/tests/test_json_schemas.sh index a847ee5..8e14f62 100755 --- a/tests/test_json_schemas.sh +++ b/tests/test_json_schemas.sh @@ -66,6 +66,7 @@ run() { run "why" validate why why --json "${logs}/20261117_041222" run "history" validate history history --json --root "${logs}" hello-2.12.1-GCCcore-13.3.0 +run "failures" validate failures failures --json --root "${logs}" --package hello --since 2026-11-17 run "loads" validate loads loads --module GROMACS/2024.1 --json "${work}/moduleUsage.log" run "runs" validate runs runs --exec gmx --json "${work}/runs.json" run "pending dry-run" validate pending pending --json "${work}/dry.txt" diff --git a/tests/test_output_stability.sh b/tests/test_output_stability.sh index 4f346cb..76ae948 100755 --- a/tests/test_output_stability.sh +++ b/tests/test_output_stability.sh @@ -97,6 +97,24 @@ check "why default output unchanged" "$(printf '%s\n' "${why_out}" | head -n 7)" hist_out=$(cd "${repo_root}" && "${tool}" history --root tests/fixtures/build-logs hello) check "history default output unchanged" "${hist_out}" $'hello: 11 builds under tests/fixtures/build-logs: 3 succeeded, 8 failed\n\nRUN TARGET OUTCOME DURATION NAME\n20261117_035825 sapphirerapids failed 48s hello-2.12.1-GCCcore-14.2.0\n20261117_040032 sapphirerapids failed 41s hello-2.12.1-GCCcore-14.2.0\n20261117_040119 sapphirerapids failed - hello-2.12.1-GCCcore-14.2.0\n20261117_040311 sapphirerapids failed 2m24s hello-2.12.1-GCCcore-14.2.0\n20261117_040615 sapphirerapids failed 3m06s hello-2.12.1-GCCcore-14.2.0\n20261117_040928 sapphirerapids failed 2m30s hello-2.12.1-GCCcore-14.2.0\n20261117_041222 generic success 2m50s hello-2.12.1-GCCcore-14.2.0\n20261117_041222 sapphirerapids failed 2m54s hello-2.12.1-GCCcore-13.3.0\n20261117_041222 sapphirerapids failed 2m33s hello-2.12.1-GCCcore-14.2.0\n20261117_042207 sapphirerapids success 2m40s hello-2.12.1-GCCcore-13.3.0\n20261117_042207 sapphirerapids success 2m52s hello-2.12.1-GCCcore-14.2.0' +expect_fixture() { + local label=$1 fixture=$2 + shift 2 + (cd "${repo_root}" && "$@") > "${work}/fix.out" 2> "${work}/fix.err" + status=$? + check "${label}: exit 0" "${status}" "0" + check "${label}: stdout byte-identical" "$(cmp -s "${repo_root}/${fixture}" "${work}/fix.out" && echo ok)" "ok" + check "${label}: stderr empty" "$([ ! -s "${work}/fix.err" ] && echo ok)" "ok" +} +expect_fixture "why text fixture" "tests/fixtures/why/20261117_040615.txt" \ + "${tool}" why --root tests/fixtures/build-logs 20261117_040615 +expect_fixture "why json fixture" "tests/fixtures/why/20261117_041222.json" \ + "${tool}" why --json --root tests/fixtures/build-logs 20261117_041222 +expect_fixture "history text fixture" "tests/fixtures/why/history-hello.txt" \ + "${tool}" history --root tests/fixtures/build-logs hello +expect_fixture "history json fixture" "tests/fixtures/why/history-hello-2.12.1-GCCcore-13.3.0.json" \ + "${tool}" history --json --root tests/fixtures/build-logs hello-2.12.1-GCCcore-13.3.0 + printf 'user=alice module=GROMACS/2024.1 path=/cvmfs/g.lua host=login01 time=1\n' > "${work}/loads.log" check "loads default output unchanged" "$("${tool}" loads --module GROMACS/2024.1 "${work}/loads.log")" \ "user=alice host=login01 time=1 path=/cvmfs/g.lua"