Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions COMPAT.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ on the same invocations.
| `used` | refused with `eb-keep-logs: used is in the Zig build only`, exit 2 | lists paths under a prefix in XALT and Lmod records |
| `usage` | refused with `eb-keep-logs: usage is in the Zig build only`, exit 2 | classifies installed software by XALT and Lmod usage |
| `pending` | refused with `eb-keep-logs: pending is in the Zig build only`, exit 2 | prints what a dry-run or missing-modules report still has to build |
| `lifecycle` | refused with `eb-keep-logs: lifecycle is in the Zig build only`, exit 2 | emits a reversible lifecycle plan and `.modulerc.lua` warnings/forbids from usage reports |
| `--compress-after` or `--delete-after` with no value after it | never returns | refused, exit 2 |
| `EB_KEEP_LOGS_NOW` not empty and not an integer | evaluated as shell arithmetic | refused, exit 2 |
| `rotate`'s lock file cannot be opened, or is a symbolic link or not a regular file | `eb-keep-logs: cannot lock PATH`, exit 2, from a look at the name just before it is opened | `eb-keep-logs: cannot lock PATH: ERROR`, exit 2, from the open itself |
Expand Down
5 changes: 5 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,11 @@ Both are in the Zig build only.

Pass `--syslog TARGET` (or set `EB_KEEP_LOGS_SYSLOG`) to `failed` or `rotate` to emit structured RFC 5424 messages (`ebKeepLogs@<PEN>`, with `EB_KEEP_LOGS_SYSLOG_PEN` defaulting to `32473`, the IANA RFC 5398 documentation placeholder until a site or project PEN is assigned) or native `systemd-journald` fields (`journald`, `syslog`, `unix:///path`, `journal+unix:///path`, `udp://host:port`, `tcp://host:port`) with stable `MSGID` values (`EB_KEEP_FAILED`, `EB_KEEP_NONE`, `EB_ROTATE_DONE`) without copying log file contents into syslog.

## Plan module lifecycle from usage

`eb-keep-logs usage` compares installed `Name/Version` directories against XALT and Lmod records, and `eb-keep-logs lifecycle --policy policy.toml usage.json` turns that report into a reviewable `keep`, `warn`, `forbid-after`, and `candidate-remove` plan plus Lmod `.modulerc.lua` warnings and forbids.
It never deletes software or modulefiles, enforces the policy's `min_observation_days`, and calls `eb-stack rebuild-set` when given `--easyconfigs` to identify installed dependents before permitting `candidate-remove`.

## Tests

```bash
Expand Down
6 changes: 6 additions & 0 deletions build.zig
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,11 @@ pub fn build(b: *std.Build) void {
test_bundle_test.setCwd(b.path("."));
test_bundle_test.expectExitCode(0);
test_bundle_test.has_side_effects = true;
const lifecycle_test = b.addSystemCommand(&.{ "bash", "-c", "TOOL_BIN=\"$0\" exec bash tests/test_lifecycle.sh" });
lifecycle_test.addArtifactArg(exe);
lifecycle_test.setCwd(b.path("."));
lifecycle_test.expectExitCode(0);
lifecycle_test.has_side_effects = true;
const stable_output_test = b.addSystemCommand(&.{ "bash", "-c", "TOOL_BIN=\"$0\" exec bash tests/test_output_stability.sh" });
stable_output_test.addArtifactArg(exe);
stable_output_test.setCwd(b.path("."));
Expand All @@ -121,5 +126,6 @@ pub fn build(b: *std.Build) void {
test_step.dependOn(&pending_test.step);
test_step.dependOn(&json_schema_test.step);
test_step.dependOn(&test_bundle_test.step);
test_step.dependOn(&lifecycle_test.step);
test_step.dependOn(&stable_output_test.step);
}
1 change: 1 addition & 0 deletions changelog.d/+lifecycle-docs.doc.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Document `eb-keep-logs lifecycle` in the CLI reference, JSON schema reference, and a Diataxis how-to guide for planning reversible module deprecations.
1 change: 1 addition & 0 deletions changelog.d/+lifecycle.feature.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
`eb-keep-logs lifecycle` reads `usage --json` output and a TOML policy to emit a reviewable `keep`, `warn`, `forbid-after`, and `candidate-remove` plan plus Lmod `.modulerc.lua` warnings and forbids without deleting files, enforcing a minimum observation window and checking installed dependents via `eb-stack rebuild-set`.
1 change: 1 addition & 0 deletions docs/orgmode/howto/index.org
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ The title names the task.
check-prefix-use
review-unused-software
compare-flaky-test-bundles
plan-module-lifecycle
what-a-dry-run-would-build
rotate-a-log-tree
mark-licensed-logs
Expand Down
50 changes: 50 additions & 0 deletions docs/orgmode/howto/plan-module-lifecycle.org
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
#+TITLE: Plan module lifecycle and deprecation
#+OPTIONS: toc:nil todo:nil num:nil author:nil title:nil
#+STARTUP: content

* Plan module lifecycle and deprecation

Use =lifecycle= to turn an =eb-keep-logs usage --json= inventory into a reviewable deprecation plan and an Lmod =.modulerc.lua= file.
It never deletes software or modulefiles.

1. Collect usage over a bounded observation window with =--since=.
=lifecycle= refuses to promote unused modules to =candidate-remove= when =usage= ran without =--since= or over a shorter span than =min_observation_days=:

#+begin_src bash
since=$(date -d '180 days ago' +%s)
eb-keep-logs usage \
--software /srv/easybuild/software \
--modules /srv/easybuild/modules/all \
--since "$since" --json \
/var/log/xalt/run.json /var/log/lmod/module-usage.log \
> usage.json
#+end_src

2. Write a TOML policy file (=policy.toml=).
Set the minimum observation window, the inactivity thresholds for =warn=, =forbid-after= and =candidate-remove=, the grace period before a forbid date takes effect, and any module patterns to keep unconditionally:

#+begin_src toml
[policy]
min_observation_days = 90
warn_after_days = 60
forbid_after_days = 180
remove_after_days = 365
forbid_grace_days = 30
min_hits = 1
keep = ["GCCcore/*", "OpenMPI/*"]
#+end_src

3. Run =lifecycle= with your easyconfig tree so =eb-stack rebuild-set= checks installed dependents before any module is marked =candidate-remove=:

#+begin_src bash
eb-keep-logs lifecycle \
--policy policy.toml \
--easyconfigs /srv/easybuild/easyconfigs \
--modulerc-out /tmp/review.modulerc.lua \
usage.json
#+end_src

If a candidate still has installed reverse dependents (runtime, build-time or bundle extensions), =lifecycle= holds it at =warn= and lists its dependents instead of marking it =candidate-remove= or emitting a =forbid= directive that could break dependent modules.

4. Review the plan and generated =/tmp/review.modulerc.lua= before installing the =.modulerc.lua= into your module tree or retiring =candidate-remove= entries by hand.
Pass =--json= to inspect counts, =observed_days=, =dependents= and =blocked_reason= fields programmatically.
86 changes: 82 additions & 4 deletions docs/orgmode/reference/cli.org
Original file line number Diff line number Diff line change
Expand Up @@ -404,6 +404,50 @@ The shell script has no =usage=.
| =FILE= | One or more concatenated-XALT-JSON or Lmod usage-log files. A file that cannot be read, or truncated JSON, is refused. |


** lifecycle

=lifecycle --policy FILE [--modulerc-out FILE] [--easyconfigs DIR ...] [--easystack FILE ...] [--eb-stack PATH] [--rebuild-set FILE] [--now EPOCH] [--json] FILE= turns a =usage --json= report (generated with =selection= ="all"=) and a TOML policy into a reviewable deprecation plan and Lmod =.modulerc.lua= warnings and forbids.
It never deletes software or modulefiles.

The TOML policy accepts an optional =[policy]= or =[lifecycle]= header and these keys:

- =min_observation_days= (required, integer >= 1): minimum span in days between =since= in the usage report and =now= before =candidate-remove= is permitted.
- =warn_after_days= (required, integer >= 0): idle age threshold for =warn=.
- =forbid_after_days= (required, integer >= =warn_after_days=): idle age threshold for =forbid-after=.
- =remove_after_days= (required, integer >= =forbid_after_days=): idle age threshold for =candidate-remove=.
- =forbid_grace_days= (optional, default 30): days added after =now= (or after reaching =forbid_after_days=) when computing the =after= date in Lmod =forbid{}= entries.
- =min_hits= (optional, default 1): minimum recorded hits required to treat an undated active hit as active.
- =keep= (optional, array of strings): module names or =*= / =?= glob patterns (such as ="GCCcore/*"=) that stay =keep=.
- =warn_message= and =forbid_message= (optional strings): templates supporting ={module}= and ={after}= placeholders for generated Lmod messages.

Before allowing =candidate-remove=, =lifecycle= enforces two guards:

1. The usage report must carry a numeric =since= covering at least =min_observation_days= up to =now=.
2. Installed reverse dependents must be checked with =eb-stack rebuild-set= (via =--easyconfigs=, or a precomputed =--rebuild-set= report). Any candidate absent from the easyconfig graph or with installed dependents is held back from =candidate-remove=; candidates with installed dependents are held at =warn= without emitting a =forbid= directive so dependent modules continue to load.

Without =--json=, text output prints one sorted line per installed module:

#+begin_example
ACTION Name/Version hits=N idle_days=D [forbid_after=YYYY-MM-DD] [dependents=M1,M2] reason="..."
#+end_example

With =--json=, the document has =schema_version= 1, =kind= ="lifecycle"=, =software=, =modules=, =policy=, =now=, =since=, =observed_days=, =min_observation_days=, =observation_satisfied=, =dependents_checked=, =counts= (with keys =keep=, =warn=, =forbid-after=, =candidate-remove=), =entries= and =modulerc_lua=.
With =--modulerc-out FILE=, the generated =.modulerc.lua= text is written atomically via a temporary file in the destination directory; symlinks and directories at =FILE= are refused.
The shell script has no =lifecycle=.

| Option / argument | Meaning |
|---|---|
| =--policy FILE= | TOML policy file. Required. |
| =--modulerc-out FILE= | Write generated =.modulerc.lua= warnings and forbids to =FILE=. |
| =--easyconfigs DIR= | Easyconfig tree passed to =eb-stack rebuild-set=. Repeatable. |
| =--easystack FILE= | Deployed easystack YAML passed to =eb-stack rebuild-set=. Repeatable. |
| =--eb-stack PATH= | Explicit path to =eb-stack=. Default: =eb-stack= on =PATH=. |
| =--rebuild-set FILE= | Precomputed =rebuild-set.json= from =eb-stack rebuild-set=. Mutually exclusive with =--easyconfigs=. |
| =--now EPOCH= | Reference epoch time in seconds. Default: =EB_KEEP_LOGS_NOW=, else current time. |
| =--json= | Print one JSON document instead of text. |
| =FILE= | JSON output from =eb-keep-logs usage --json= with =selection= ="all"=. Required. |


** runs

=runs --exec PATH [--json] FILE= prints who ran =PATH=, from XALT run records in =FILE=.
Expand Down Expand Up @@ -670,6 +714,40 @@ otherwise lose, and keep the pipeline's log tree bounded.
Print one line of JSON instead of text. Default: text.
BUNDLE
Two or more test-bundle JSON files. Required.
eb-keep-logs lifecycle --policy FILE [--modulerc-out FILE]
[--easyconfigs DIR ...] [--easystack FILE ...]
[--eb-stack PATH] [--rebuild-set FILE]
[--now EPOCH] [--json] FILE
Generate a reversible lifecycle plan and Lmod .modulerc.lua
warnings/forbids from usage --json output and a TOML policy. Never
deletes software or modulefiles. Requires a minimum observation
window and checks installed dependents via eb-stack rebuild-set
before allowing candidate-remove. Zig build only.
--policy FILE, --policy=FILE
TOML lifecycle policy file. Required.
--modulerc-out FILE, --modulerc-out=FILE
Write the generated Lmod .modulerc.lua warnings and forbids
durably to FILE. Default: none; no file is written.
--easyconfigs DIR, --easyconfigs=DIR
Easyconfig tree passed to eb-stack rebuild-set to identify
dependents of candidate removals. Repeatable. Default: none.
--easystack FILE, --easystack=FILE
Deployed easystack YAML passed to eb-stack rebuild-set.
Repeatable. Default: none.
--eb-stack PATH, --eb-stack=PATH
Path to the eb-stack executable. Default: eb-stack on PATH.
--rebuild-set FILE, --rebuild-set=FILE
Precomputed eb-stack rebuild-set.json report instead of
running eb-stack. Default: none.
--now EPOCH, --now=EPOCH
Reference epoch time in seconds for idle age and observation
window checks. Default: the environment time override, else
the current time.
--json
Print one line of JSON instead, with schema_version 1.
FILE
JSON output produced by eb-keep-logs usage --json (with
selection "all"). Required.
eb-keep-logs -h | --help
Print this help on stdout and exit 0.
eb-keep-logs --version
Expand All @@ -679,8 +757,8 @@ Environment:
PATH Where failed looks up eb.
EASYBUILD_TMPDIR failed copies a command's output only from below it;
unset or empty, it copies none.
EB_KEEP_LOGS_NOW The time rotate takes as now, in seconds since the
epoch; unset or empty, the current time.
EB_KEEP_LOGS_NOW The time rotate and lifecycle take as now, in seconds
since the epoch; unset or empty, the current time.
EB_PIPELINE_LOG_DIR rotate leaves alone the run that holds this path.
EB_KEEP_LOGS_SYSLOG Default --syslog target (journald, syslog, unix:///...,
journal+unix:///..., udp://host:port, tcp://host:port).
Expand All @@ -691,7 +769,7 @@ Environment:
directory of zone files.

Exit status:
0 -h, --help or --version, or a failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle or test-compare that
0 -h, --help or --version, or a failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle, test-compare or lifecycle that
accepted its arguments, whatever it could keep, compress, remove or
read: what it could not do is reported on stderr. used with
--require-none also exits 2 when any path was listed.
Expand All @@ -713,7 +791,7 @@ eb-keep-logs 0.2.1 (x86_64-linux-musl, safe, zig 0.17.1, 5ceaead)
** See also

- [[file:../tutorials/keep-a-failed-build-log.org][Keep a failed build's logs]]
- [[file:../howto/call-from-a-pipeline.org][Call eb-keep-logs from a build pipeline]], [[file:../howto/rotate-a-log-tree.org][Rotate a log tree]] and [[file:../howto/mark-licensed-logs.org][Mark logs that may carry a licence number]]
- [[file:../howto/call-from-a-pipeline.org][Call eb-keep-logs from a build pipeline]], [[file:../howto/rotate-a-log-tree.org][Rotate a log tree]], [[file:../howto/mark-licensed-logs.org][Mark logs that may carry a licence number]], [[file:../howto/review-unused-software.org][Review unused installed software]] and [[file:../howto/plan-module-lifecycle.org][Plan module lifecycle and deprecation]]
- [[file:../explanation/why-eb-last-log.org][Finding a failed build's log]], [[file:../explanation/local-time.org][Run ages and local time]] and [[file:../explanation/rotate-in-use.org][Rotating a tree in use]]
- [[file:compatibility.org][Compatibility]], the differences from the shell script.
- [[file:api/zig/eb-keep-logs.md][The Zig API]], from the doc comments of the source.
9 changes: 5 additions & 4 deletions docs/orgmode/reference/json-schemas.org
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

* JSON output schemas

The Zig build's =--json= on =why=, =history=, =loads=, =runs=, =used=, =usage=, =pending= and =test-compare= prints one line of JSON with =schema_version= 1.
The Zig build's =--json= on =why=, =history=, =loads=, =runs=, =used=, =usage=, =pending=, =test-compare= and =lifecycle= prints one line of JSON with =schema_version= 1.
The =test-bundle= command always writes one JSON document with =schema_version= 1.
The shell script refuses those commands; see [[file:compatibility.org][Compatibility]].
Each document is validated in CI against a JSON Schema file in the repository tree under =docs/orgmode/reference/json/=.
Expand All @@ -17,12 +17,13 @@ The schemas use draft 2020-12; a new =schema_version= may drop or rename a key,
| =loads= | =docs/orgmode/reference/json/loads.schema.json= |
| =runs= | =docs/orgmode/reference/json/runs.schema.json= |
| =used= | =docs/orgmode/reference/json/used.schema.json= |
| =usage= | =docs/orgmode/reference/json/usage.schema.json= |
| =pending= | =docs/orgmode/reference/json/pending.schema.json= |
| =usage= | =docs/orgmode/reference/json/usage.schema.json= |
| =pending= | =docs/orgmode/reference/json/pending.schema.json= |
| =test-bundle= | =docs/orgmode/reference/json/test-bundle.schema.json= |
| =test-compare= | =docs/orgmode/reference/json/test-compare.schema.json= |
| =lifecycle= | =docs/orgmode/reference/json/lifecycle.schema.json= |

Field names and semantics for =why=, =history=, =used= and =usage= are under [[file:cli.org][The eb-keep-logs command]].
Field names and semantics for =why=, =history=, =used=, =usage= and =lifecycle= are under [[file:cli.org][The eb-keep-logs command]].
=loads=, =runs=, =pending=, =test-bundle= and =test-compare= document their JSON shape in the same page.

** See also
Expand Down
93 changes: 93 additions & 0 deletions docs/orgmode/reference/json/lifecycle.schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://stackhands.github.io/eb-keep-logs/reference/json/lifecycle.schema.json",
"title": "eb-keep-logs lifecycle --json",
"type": "object",
"additionalProperties": false,
"required": [
"schema_version",
"kind",
"software",
"modules",
"policy",
"now",
"since",
"observed_days",
"min_observation_days",
"observation_satisfied",
"dependents_checked",
"counts",
"entries",
"modulerc_lua"
],
"properties": {
"schema_version": { "type": "integer", "const": 1 },
"kind": { "type": "string", "const": "lifecycle" },
"software": { "type": "string" },
"modules": { "type": "string" },
"policy": { "type": "string" },
"now": { "type": "string" },
"since": { "type": ["string", "null"] },
"observed_days": { "type": ["integer", "null"], "minimum": 0 },
"min_observation_days": { "type": "integer", "minimum": 1 },
"observation_satisfied": { "type": "boolean" },
"dependents_checked": { "type": "boolean" },
"counts": { "$ref": "#/$defs/counts" },
"entries": {
"type": "array",
"items": { "$ref": "#/$defs/entry" }
},
"modulerc_lua": { "type": "string" }
},
"$defs": {
"counts": {
"type": "object",
"additionalProperties": false,
"required": ["keep", "warn", "forbid-after", "candidate-remove"],
"properties": {
"keep": { "type": "integer", "minimum": 0 },
"warn": { "type": "integer", "minimum": 0 },
"forbid-after": { "type": "integer", "minimum": 0 },
"candidate-remove": { "type": "integer", "minimum": 0 }
}
},
"entry": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"version",
"module",
"easyconfig",
"action",
"hits",
"last",
"idle_days",
"forbid_after",
"dependents",
"blocked_reason",
"reason"
],
"properties": {
"name": { "type": "string" },
"version": { "type": "string" },
"module": { "type": "string" },
"easyconfig": { "type": "string" },
"action": {
"type": "string",
"enum": ["keep", "warn", "forbid-after", "candidate-remove"]
},
"hits": { "type": "integer", "minimum": 0 },
"last": { "type": ["string", "null"] },
"idle_days": { "type": ["integer", "null"], "minimum": 0 },
"forbid_after": { "type": ["string", "null"] },
"dependents": {
"type": "array",
"items": { "type": "string" }
},
"blocked_reason": { "type": ["string", "null"] },
"reason": { "type": "string" }
}
}
}
}
Loading
Loading