Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ jobs:
with:
version: 0.17.1
- run: zig fmt --check build.zig build.zig.zon src
- run: shellcheck -S warning tests/differential.sh tests/test_why.sh
- run: shellcheck -S warning tests/differential.sh tests/test_why.sh tests/test_json_schemas.sh
- run: pip install jsonschema
- run: zig build test --summary all
- run: zig build -Drelease=true -Dcommit=${{ github.sha }} && zig-out/bin/eb-keep-logs --version
6 changes: 6 additions & 0 deletions build.zig
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,11 @@ pub fn build(b: *std.Build) void {
pending_test.setCwd(b.path("."));
pending_test.expectExitCode(0);
pending_test.has_side_effects = true;
const json_schema_test = b.addSystemCommand(&.{ "bash", "-c", "TOOL_BIN=\"$0\" exec bash tests/test_json_schemas.sh" });
json_schema_test.addArtifactArg(exe);
json_schema_test.setCwd(b.path("."));
json_schema_test.expectExitCode(0);
json_schema_test.has_side_effects = true;

const test_step = b.step("test", "Run the unit tests, the command-line suites and the differential run");
test_step.dependOn(&b.addRunArtifact(unit).step);
Expand All @@ -104,4 +109,5 @@ pub fn build(b: *std.Build) void {
test_step.dependOn(&used_test.step);
test_step.dependOn(&usage_test.step);
test_step.dependOn(&pending_test.step);
test_step.dependOn(&json_schema_test.step);
}
1 change: 1 addition & 0 deletions changelog.d/+json-schemas.feature.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
`loads`, `runs` and `pending` accept opt-in `--json` with `schema_version` 1. JSON Schema files under `docs/orgmode/reference/json/` describe every `--json` document (`why`, `history`, `loads`, `runs`, `used`, `usage`, `pending`). `tests/test_json_schemas.sh` validates command output against those schemas in CI.
41 changes: 28 additions & 13 deletions docs/orgmode/reference/cli.org
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@ eb-keep-logs failed [--syslog TARGET] STATUS NAME LOGDIR [TMP_LOGDIR]
eb-keep-logs rotate ROOT --compress-after DAYS [--delete-after DAYS] [--lock-timeout SECONDS] [--syslog TARGET]
eb-keep-logs why [RUN | --latest] [--target TARGET] [--root ROOT] [--json]
eb-keep-logs history PACKAGE [--target TARGET] [--root ROOT] [--json]
eb-keep-logs loads --module NAME FILE
eb-keep-logs runs --exec PATH FILE
eb-keep-logs loads --module NAME [--json] FILE
eb-keep-logs runs --exec PATH [--json] FILE
eb-keep-logs used --prefix PREFIX [--json] [--require-none] FILE...
eb-keep-logs usage --software DIR --modules DIR [--since EPOCH] [--unused | --active] [--json] FILE...
eb-keep-logs pending FILE
eb-keep-logs pending [--json] FILE
eb-keep-logs -h | --help
eb-keep-logs --version
#+end_example
Expand Down Expand Up @@ -46,7 +46,7 @@ eb-keep-logs --version
| =--latest= | =why= | Read the newest run under =--root=, by its name; with =--target=, the newest that has that target. The default without =RUN=. |
| =--target TARGET=, =--target=TARGET= | =why=, =history= | Read only the builds in the run's directory =TARGET=; for =history=, in each run's, a run without one passed over. Without it, those in the run's own directory and then in each directory in it, by name. |
| =--root ROOT=, =--root=ROOT= | =why=, =history= | The directory that holds the runs, directories named =YYYYMMDD_HHMMSS=, not symbolic links. Without it, the current directory. |
| =--json= | =why=, =history= | Print one line of JSON instead of the report; its keys are under Files below. |
| =--json= | =why=, =history=, =loads=, =runs=, =pending= | Print one line of JSON instead of the report; its keys are under [[file:json-schemas.org][JSON schemas]] and below. |
| =--software DIR=, =--software=DIR= | =usage= | Required. Read installed software from the direct =Name/Version= directories below =DIR=. |
| =--modules DIR=, =--modules=DIR= | =usage= | Required. Count an Lmod record only when its module-file path is =DIR= or below it. |
| =--since EPOCH=, =--since=EPOCH= | =usage= | Count records at or after the non-negative epoch time; leave out records with no numeric time. |
Expand Down Expand Up @@ -310,16 +310,19 @@ What =history= cannot read it reports on stderr as =why= does, a run that cannot

** loads

=loads --module NAME FILE= prints who loaded =NAME=, from Lmod's ModuleUsageTracking lines in =FILE=.
=loads --module NAME [--json] FILE= prints who loaded =NAME=, from Lmod's ModuleUsageTracking lines in =FILE=.
A record is =user=USER module=NAME path=PATH host=HOST time=TIME=, in any order, with a syslog prefix left in place.
Each load is one line, =user=USER host=HOST time=TIME path=PATH=, in the file's order.
A file with no load of =NAME= prints =no load of NAME=.
A file with no load of =NAME= prints =no load of NAME=; with =--json= the =loads= array is empty.
A line that is not a record is skipped.
The shell script has no =loads=.

With =--json= the document has =schema_version= 1, =kind= ="loads"=, =module=, and a =loads= array in file order. Each element has =user=, =host=, =time= and =path= as the text lines print them.

| Argument | Meaning |
|---|---|
| =NAME= | The module's full name, as Lmod recorded it, such as =GROMACS/2024.1=. Required, as =--module NAME= or =--module=NAME=. |
| =--json= | Print one line of JSON instead of the text lines. |
| =FILE= | The log. A file that cannot be read is refused. |


Expand Down Expand Up @@ -387,32 +390,38 @@ The shell script has no =usage=.

** runs

=runs --exec PATH FILE= prints who ran =PATH=, from XALT run records in =FILE=.
=runs --exec PATH [--json] FILE= prints who ran =PATH=, from XALT run records in =FILE=.
A run record is a JSON object whose =userT= holds =user=, =exec_path= and, when present, =syshost= and =start_time=.
A link record is not a run and is skipped.
=PATH= matches =exec_path= in full or as its last component.
Each run is one line, =user=USER host=HOST time=TIME path=PATH=, in the file's order.
A file with no run of =PATH= prints =no run of PATH=.
A file with no run of =PATH= prints =no run of PATH=; with =--json= the =runs= array is empty.
The shell script has no =runs=. It does not record runs.

With =--json= the document has =schema_version= 1, =kind= ="runs"=, =exec= as given, and a =runs= array in file order. Each element has =user=, =host=, =time= and =path= as the text lines print them; fields XALT omitted are empty strings.

| Argument | Meaning |
|---|---|
| =PATH= | The executable, as a full path or its last component. Required, as =--exec PATH= or =--exec=PATH=. |
| =--json= | Print one line of JSON instead of the text lines. |
| =FILE= | One or more JSON objects. An object that is not a run record is skipped. A file that cannot be read, or that is not JSON, is refused. |


** pending

=pending FILE= prints what an EasyBuild dry-run or missing-modules report still has to build.
=pending [--json] FILE= prints what an EasyBuild dry-run or missing-modules report still has to build.
A dry-run line is =* [ ] PATH (module: NAME)=, and =[F]= and =[R]= still build. =[x]= is already installed and is not printed.
=--terse= prints =x PATH= or =- PATH=.
A missing-modules line is =* NAME (FILE.eb)=, or with =--terse= one easyconfig path on each line.
Each is one line, =build NAME= or =missing NAME=, in the file's order.
A report with nothing left prints =nothing to build=.
A report with nothing left prints =nothing to build=; with =--json= the =entries= array is empty.
The shell script has no =pending=. It does not run =eb=, and it does not resolve dependencies.

With =--json= the document has =schema_version= 1, =kind= ="pending"=, and an =entries= array in file order. Each element has =action= (="build"= or ="missing"=) and =name= as the text lines print them.

| Argument | Meaning |
|---|---|
| =--json= | Print one line of JSON instead of the text lines. |
| =FILE= | The report, as =eb --dry-run= or =eb --missing-modules= printed it. A file that cannot be read, or that is neither report, is refused. |

*** --help
Expand Down Expand Up @@ -493,7 +502,7 @@ otherwise lose, and keep the pipeline's log tree bounded.
directory.
--json
Print one line of JSON instead, with schema_version 1.
eb-keep-logs loads --module NAME FILE
eb-keep-logs loads --module NAME [--json] FILE
Print who loaded NAME, from Lmod's ModuleUsageTracking lines in FILE.
Each record is user=USER module=NAME path=PATH host=HOST time=TIME,
in any order, and a syslog prefix is left in place. Each load is
Expand All @@ -503,6 +512,8 @@ otherwise lose, and keep the pipeline's log tree bounded.
--module NAME, --module=NAME
The module's full name, as Lmod recorded it, such as
GROMACS/2024.1. Required.
--json
Print one line of JSON instead, with schema_version 1.
FILE
The log that holds the records. A line that is not one is
skipped.
Expand Down Expand Up @@ -554,7 +565,7 @@ otherwise lose, and keep the pipeline's log tree bounded.
FILE
One or more files of concatenated XALT JSON objects and/or Lmod
ModuleUsageTracking lines. Required.
eb-keep-logs runs --exec PATH FILE
eb-keep-logs runs --exec PATH [--json] FILE
Print who ran PATH, from XALT run records in FILE. A run record is a
JSON object whose userT holds user, exec_path and, when present,
syshost and start_time. A link record is not a run and is skipped.
Expand All @@ -564,10 +575,12 @@ otherwise lose, and keep the pipeline's log tree bounded.
only. It does not record runs.
--exec PATH, --exec=PATH
The executable, as a full path or its last component. Required.
--json
Print one line of JSON instead, with schema_version 1.
FILE
One or more JSON objects. An object that is not a run record is
skipped.
eb-keep-logs pending FILE
eb-keep-logs pending [--json] FILE
Print what an EasyBuild dry-run or missing-modules report still has
to build. A dry-run line is "* [ ] PATH (module: NAME)", and [F] and
[R] count as still to build; [x] is already installed and is not
Expand All @@ -576,6 +589,8 @@ otherwise lose, and keep the pipeline's log tree bounded.
line. Each is one line, build NAME or missing NAME, in the file's
order. A report with nothing left prints nothing to build. Zig build
only. It does not run eb, and it does not resolve dependencies.
--json
Print one line of JSON instead, with schema_version 1.
FILE
The report, as eb --dry-run or eb --missing-modules printed it.
Required.
Expand Down
1 change: 1 addition & 0 deletions docs/orgmode/reference/index.org
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ Each page states facts in the order the thing itself has them.
:maxdepth: 1

cli
json-schemas
compatibility
api
#+end_export
27 changes: 27 additions & 0 deletions docs/orgmode/reference/json-schemas.org
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#+TITLE: JSON output schemas
#+OPTIONS: toc:nil todo:nil num:nil author:nil title:nil
#+STARTUP: content

* JSON output schemas

The Zig build's =--json= on =why=, =history=, =loads=, =runs=, =used=, =usage= and =pending= prints one line of JSON with =schema_version= 1.
The shell script refuses those commands; see [[file:compatibility.org][Compatibility]].
Each document is validated in CI against a JSON Schema file in the repository tree under =docs/orgmode/reference/json/=.
The schemas use draft 2020-12; a new =schema_version= may drop or rename a key, or change what one means.

| Command | Schema file in the repository |
|----------+-------------------------------|
| =why= | =docs/orgmode/reference/json/why.schema.json= |
| =history= | =docs/orgmode/reference/json/history.schema.json= |
| =loads= | =docs/orgmode/reference/json/loads.schema.json= |
| =runs= | =docs/orgmode/reference/json/runs.schema.json= |
| =used= | =docs/orgmode/reference/json/used.schema.json= |
| =usage= | =docs/orgmode/reference/json/usage.schema.json= |
| =pending= | =docs/orgmode/reference/json/pending.schema.json= |

Field names and semantics for =why=, =history=, =used= and =usage= are under [[file:cli.org][The eb-keep-logs command]].
=loads=, =runs= and =pending= document their JSON shape in the same page.

** See also

- [[file:cli.org][The eb-keep-logs command]]
43 changes: 43 additions & 0 deletions docs/orgmode/reference/json/history.schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://stackhands.github.io/eb-keep-logs/reference/json/history.schema.json",
"title": "eb-keep-logs history --json",
"type": "object",
"additionalProperties": false,
"required": ["schema_version", "kind", "package", "root", "counts", "builds"],
"properties": {
"schema_version": { "type": "integer", "const": 1 },
"kind": { "type": "string", "const": "history" },
"package": { "type": "string" },
"root": { "type": "string" },
"counts": {
"type": "object",
"additionalProperties": false,
"required": ["success", "failed", "unknown"],
"properties": {
"success": { "type": "integer", "minimum": 0 },
"failed": { "type": "integer", "minimum": 0 },
"unknown": { "type": "integer", "minimum": 0 }
}
},
"builds": {
"type": "array",
"items": { "$ref": "#/$defs/build" }
}
},
"$defs": {
"build": {
"type": "object",
"additionalProperties": false,
"required": ["run", "target", "name", "outcome", "duration_s", "log"],
"properties": {
"run": { "type": "string" },
"target": { "type": ["string", "null"] },
"name": { "type": "string" },
"outcome": { "type": "string", "enum": ["success", "failed", "unknown"] },
"duration_s": { "type": ["integer", "null"], "minimum": 0 },
"log": { "type": "string" }
}
}
}
}
30 changes: 30 additions & 0 deletions docs/orgmode/reference/json/loads.schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://stackhands.github.io/eb-keep-logs/reference/json/loads.schema.json",
"title": "eb-keep-logs loads --json",
"type": "object",
"additionalProperties": false,
"required": ["schema_version", "kind", "module", "loads"],
"properties": {
"schema_version": { "type": "integer", "const": 1 },
"kind": { "type": "string", "const": "loads" },
"module": { "type": "string" },
"loads": {
"type": "array",
"items": { "$ref": "#/$defs/load" }
}
},
"$defs": {
"load": {
"type": "object",
"additionalProperties": false,
"required": ["user", "host", "time", "path"],
"properties": {
"user": { "type": "string" },
"host": { "type": "string" },
"time": { "type": "string" },
"path": { "type": "string" }
}
}
}
}
27 changes: 27 additions & 0 deletions docs/orgmode/reference/json/pending.schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://stackhands.github.io/eb-keep-logs/reference/json/pending.schema.json",
"title": "eb-keep-logs pending --json",
"type": "object",
"additionalProperties": false,
"required": ["schema_version", "kind", "entries"],
"properties": {
"schema_version": { "type": "integer", "const": 1 },
"kind": { "type": "string", "const": "pending" },
"entries": {
"type": "array",
"items": { "$ref": "#/$defs/entry" }
}
},
"$defs": {
"entry": {
"type": "object",
"additionalProperties": false,
"required": ["action", "name"],
"properties": {
"action": { "type": "string", "enum": ["build", "missing"] },
"name": { "type": "string" }
}
}
}
}
30 changes: 30 additions & 0 deletions docs/orgmode/reference/json/runs.schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://stackhands.github.io/eb-keep-logs/reference/json/runs.schema.json",
"title": "eb-keep-logs runs --json",
"type": "object",
"additionalProperties": false,
"required": ["schema_version", "kind", "exec", "runs"],
"properties": {
"schema_version": { "type": "integer", "const": 1 },
"kind": { "type": "string", "const": "runs" },
"exec": { "type": "string" },
"runs": {
"type": "array",
"items": { "$ref": "#/$defs/run" }
}
},
"$defs": {
"run": {
"type": "object",
"additionalProperties": false,
"required": ["user", "host", "time", "path"],
"properties": {
"user": { "type": "string" },
"host": { "type": "string" },
"time": { "type": "string" },
"path": { "type": "string" }
}
}
}
}
35 changes: 35 additions & 0 deletions docs/orgmode/reference/json/usage.schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://stackhands.github.io/eb-keep-logs/reference/json/usage.schema.json",
"title": "eb-keep-logs usage --json",
"type": "object",
"additionalProperties": false,
"required": ["schema_version", "kind", "software", "modules", "since", "selection", "packages"],
"properties": {
"schema_version": { "type": "integer", "const": 1 },
"kind": { "type": "string", "const": "usage" },
"software": { "type": "string" },
"modules": { "type": "string" },
"since": { "type": ["string", "null"] },
"selection": { "type": "string", "enum": ["all", "unused", "active"] },
"packages": {
"type": "array",
"items": { "$ref": "#/$defs/package" }
}
},
"$defs": {
"package": {
"type": "object",
"additionalProperties": false,
"required": ["name", "version", "module", "active", "hits", "last"],
"properties": {
"name": { "type": "string" },
"version": { "type": "string" },
"module": { "type": "string" },
"active": { "type": "boolean" },
"hits": { "type": "integer", "minimum": 0 },
"last": { "type": ["string", "null"] }
}
}
}
}
Loading
Loading