diff --git a/COMPAT.md b/COMPAT.md index 5a1c3bb..7857039 100644 --- a/COMPAT.md +++ b/COMPAT.md @@ -33,6 +33,7 @@ on the same invocations. | `used` | refused with `eb-keep-logs: used is in the Zig build only`, exit 2 | lists paths under a prefix in XALT and Lmod records | | `usage` | refused with `eb-keep-logs: usage is in the Zig build only`, exit 2 | classifies installed software by XALT and Lmod usage | | `pending` | refused with `eb-keep-logs: pending is in the Zig build only`, exit 2 | prints what a dry-run or missing-modules report still has to build | +| `lifecycle` | refused with `eb-keep-logs: lifecycle is in the Zig build only`, exit 2 | emits a reversible lifecycle plan and `.modulerc.lua` warnings/forbids from usage reports | | `--compress-after` or `--delete-after` with no value after it | never returns | refused, exit 2 | | `EB_KEEP_LOGS_NOW` not empty and not an integer | evaluated as shell arithmetic | refused, exit 2 | | `rotate`'s lock file cannot be opened, or is a symbolic link or not a regular file | `eb-keep-logs: cannot lock PATH`, exit 2, from a look at the name just before it is opened | `eb-keep-logs: cannot lock PATH: ERROR`, exit 2, from the open itself | diff --git a/README.md b/README.md index b65a7a9..f7f9cb1 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,11 @@ Both are in the Zig build only. Pass `--syslog TARGET` (or set `EB_KEEP_LOGS_SYSLOG`) to `failed` or `rotate` to emit structured RFC 5424 messages (`ebKeepLogs@`, with `EB_KEEP_LOGS_SYSLOG_PEN` defaulting to `32473`, the IANA RFC 5398 documentation placeholder until a site or project PEN is assigned) or native `systemd-journald` fields (`journald`, `syslog`, `unix:///path`, `journal+unix:///path`, `udp://host:port`, `tcp://host:port`) with stable `MSGID` values (`EB_KEEP_FAILED`, `EB_KEEP_NONE`, `EB_ROTATE_DONE`) without copying log file contents into syslog. +## Plan module lifecycle from usage + +`eb-keep-logs usage` compares installed `Name/Version` directories against XALT and Lmod records, and `eb-keep-logs lifecycle --policy policy.toml usage.json` turns that report into a reviewable `keep`, `warn`, `forbid-after`, and `candidate-remove` plan plus Lmod `.modulerc.lua` warnings and forbids. +It never deletes software or modulefiles, enforces the policy's `min_observation_days`, and calls `eb-stack rebuild-set` when given `--easyconfigs` to identify installed dependents before permitting `candidate-remove`. + ## Tests ```bash diff --git a/build.zig b/build.zig index 7f36e40..44e9ce1 100644 --- a/build.zig +++ b/build.zig @@ -102,6 +102,11 @@ pub fn build(b: *std.Build) void { test_bundle_test.setCwd(b.path(".")); test_bundle_test.expectExitCode(0); test_bundle_test.has_side_effects = true; + const lifecycle_test = b.addSystemCommand(&.{ "bash", "-c", "TOOL_BIN=\"$0\" exec bash tests/test_lifecycle.sh" }); + lifecycle_test.addArtifactArg(exe); + lifecycle_test.setCwd(b.path(".")); + lifecycle_test.expectExitCode(0); + lifecycle_test.has_side_effects = true; const stable_output_test = b.addSystemCommand(&.{ "bash", "-c", "TOOL_BIN=\"$0\" exec bash tests/test_output_stability.sh" }); stable_output_test.addArtifactArg(exe); stable_output_test.setCwd(b.path(".")); @@ -121,5 +126,6 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&pending_test.step); test_step.dependOn(&json_schema_test.step); test_step.dependOn(&test_bundle_test.step); + test_step.dependOn(&lifecycle_test.step); test_step.dependOn(&stable_output_test.step); } diff --git a/changelog.d/+lifecycle-docs.doc.md b/changelog.d/+lifecycle-docs.doc.md new file mode 100644 index 0000000..c9efacd --- /dev/null +++ b/changelog.d/+lifecycle-docs.doc.md @@ -0,0 +1 @@ +Document `eb-keep-logs lifecycle` in the CLI reference, JSON schema reference, and a Diataxis how-to guide for planning reversible module deprecations. diff --git a/changelog.d/+lifecycle.feature.md b/changelog.d/+lifecycle.feature.md new file mode 100644 index 0000000..3283472 --- /dev/null +++ b/changelog.d/+lifecycle.feature.md @@ -0,0 +1 @@ +`eb-keep-logs lifecycle` reads `usage --json` output and a TOML policy to emit a reviewable `keep`, `warn`, `forbid-after`, and `candidate-remove` plan plus Lmod `.modulerc.lua` warnings and forbids without deleting files, enforcing a minimum observation window and checking installed dependents via `eb-stack rebuild-set`. diff --git a/docs/orgmode/howto/index.org b/docs/orgmode/howto/index.org index e988485..06861a0 100644 --- a/docs/orgmode/howto/index.org +++ b/docs/orgmode/howto/index.org @@ -19,6 +19,7 @@ The title names the task. check-prefix-use review-unused-software compare-flaky-test-bundles + plan-module-lifecycle what-a-dry-run-would-build rotate-a-log-tree mark-licensed-logs diff --git a/docs/orgmode/howto/plan-module-lifecycle.org b/docs/orgmode/howto/plan-module-lifecycle.org new file mode 100644 index 0000000..075f297 --- /dev/null +++ b/docs/orgmode/howto/plan-module-lifecycle.org @@ -0,0 +1,50 @@ +#+TITLE: Plan module lifecycle and deprecation +#+OPTIONS: toc:nil todo:nil num:nil author:nil title:nil +#+STARTUP: content + +* Plan module lifecycle and deprecation + +Use =lifecycle= to turn an =eb-keep-logs usage --json= inventory into a reviewable deprecation plan and an Lmod =.modulerc.lua= file. +It never deletes software or modulefiles. + +1. Collect usage over a bounded observation window with =--since=. + =lifecycle= refuses to promote unused modules to =candidate-remove= when =usage= ran without =--since= or over a shorter span than =min_observation_days=: + + #+begin_src bash + since=$(date -d '180 days ago' +%s) + eb-keep-logs usage \ + --software /srv/easybuild/software \ + --modules /srv/easybuild/modules/all \ + --since "$since" --json \ + /var/log/xalt/run.json /var/log/lmod/module-usage.log \ + > usage.json + #+end_src + +2. Write a TOML policy file (=policy.toml=). + Set the minimum observation window, the inactivity thresholds for =warn=, =forbid-after= and =candidate-remove=, the grace period before a forbid date takes effect, and any module patterns to keep unconditionally: + + #+begin_src toml + [policy] + min_observation_days = 90 + warn_after_days = 60 + forbid_after_days = 180 + remove_after_days = 365 + forbid_grace_days = 30 + min_hits = 1 + keep = ["GCCcore/*", "OpenMPI/*"] + #+end_src + +3. Run =lifecycle= with your easyconfig tree so =eb-stack rebuild-set= checks installed dependents before any module is marked =candidate-remove=: + + #+begin_src bash + eb-keep-logs lifecycle \ + --policy policy.toml \ + --easyconfigs /srv/easybuild/easyconfigs \ + --modulerc-out /tmp/review.modulerc.lua \ + usage.json + #+end_src + + If a candidate still has installed reverse dependents (runtime, build-time or bundle extensions), =lifecycle= holds it at =warn= and lists its dependents instead of marking it =candidate-remove= or emitting a =forbid= directive that could break dependent modules. + +4. Review the plan and generated =/tmp/review.modulerc.lua= before installing the =.modulerc.lua= into your module tree or retiring =candidate-remove= entries by hand. + Pass =--json= to inspect counts, =observed_days=, =dependents= and =blocked_reason= fields programmatically. diff --git a/docs/orgmode/reference/cli.org b/docs/orgmode/reference/cli.org index d8c9db8..4b3f15c 100644 --- a/docs/orgmode/reference/cli.org +++ b/docs/orgmode/reference/cli.org @@ -404,6 +404,50 @@ The shell script has no =usage=. | =FILE= | One or more concatenated-XALT-JSON or Lmod usage-log files. A file that cannot be read, or truncated JSON, is refused. | +** lifecycle + +=lifecycle --policy FILE [--modulerc-out FILE] [--easyconfigs DIR ...] [--easystack FILE ...] [--eb-stack PATH] [--rebuild-set FILE] [--now EPOCH] [--json] FILE= turns a =usage --json= report (generated with =selection= ="all"=) and a TOML policy into a reviewable deprecation plan and Lmod =.modulerc.lua= warnings and forbids. +It never deletes software or modulefiles. + +The TOML policy accepts an optional =[policy]= or =[lifecycle]= header and these keys: + +- =min_observation_days= (required, integer >= 1): minimum span in days between =since= in the usage report and =now= before =candidate-remove= is permitted. +- =warn_after_days= (required, integer >= 0): idle age threshold for =warn=. +- =forbid_after_days= (required, integer >= =warn_after_days=): idle age threshold for =forbid-after=. +- =remove_after_days= (required, integer >= =forbid_after_days=): idle age threshold for =candidate-remove=. +- =forbid_grace_days= (optional, default 30): days added after =now= (or after reaching =forbid_after_days=) when computing the =after= date in Lmod =forbid{}= entries. +- =min_hits= (optional, default 1): minimum recorded hits required to treat an undated active hit as active. +- =keep= (optional, array of strings): module names or =*= / =?= glob patterns (such as ="GCCcore/*"=) that stay =keep=. +- =warn_message= and =forbid_message= (optional strings): templates supporting ={module}= and ={after}= placeholders for generated Lmod messages. + +Before allowing =candidate-remove=, =lifecycle= enforces two guards: + +1. The usage report must carry a numeric =since= covering at least =min_observation_days= up to =now=. +2. Installed reverse dependents must be checked with =eb-stack rebuild-set= (via =--easyconfigs=, or a precomputed =--rebuild-set= report). Any candidate absent from the easyconfig graph or with installed dependents is held back from =candidate-remove=; candidates with installed dependents are held at =warn= without emitting a =forbid= directive so dependent modules continue to load. + +Without =--json=, text output prints one sorted line per installed module: + +#+begin_example +ACTION Name/Version hits=N idle_days=D [forbid_after=YYYY-MM-DD] [dependents=M1,M2] reason="..." +#+end_example + +With =--json=, the document has =schema_version= 1, =kind= ="lifecycle"=, =software=, =modules=, =policy=, =now=, =since=, =observed_days=, =min_observation_days=, =observation_satisfied=, =dependents_checked=, =counts= (with keys =keep=, =warn=, =forbid-after=, =candidate-remove=), =entries= and =modulerc_lua=. +With =--modulerc-out FILE=, the generated =.modulerc.lua= text is written atomically via a temporary file in the destination directory; symlinks and directories at =FILE= are refused. +The shell script has no =lifecycle=. + +| Option / argument | Meaning | +|---|---| +| =--policy FILE= | TOML policy file. Required. | +| =--modulerc-out FILE= | Write generated =.modulerc.lua= warnings and forbids to =FILE=. | +| =--easyconfigs DIR= | Easyconfig tree passed to =eb-stack rebuild-set=. Repeatable. | +| =--easystack FILE= | Deployed easystack YAML passed to =eb-stack rebuild-set=. Repeatable. | +| =--eb-stack PATH= | Explicit path to =eb-stack=. Default: =eb-stack= on =PATH=. | +| =--rebuild-set FILE= | Precomputed =rebuild-set.json= from =eb-stack rebuild-set=. Mutually exclusive with =--easyconfigs=. | +| =--now EPOCH= | Reference epoch time in seconds. Default: =EB_KEEP_LOGS_NOW=, else current time. | +| =--json= | Print one JSON document instead of text. | +| =FILE= | JSON output from =eb-keep-logs usage --json= with =selection= ="all"=. Required. | + + ** runs =runs --exec PATH [--json] FILE= prints who ran =PATH=, from XALT run records in =FILE=. @@ -670,6 +714,40 @@ otherwise lose, and keep the pipeline's log tree bounded. Print one line of JSON instead of text. Default: text. BUNDLE Two or more test-bundle JSON files. Required. + eb-keep-logs lifecycle --policy FILE [--modulerc-out FILE] + [--easyconfigs DIR ...] [--easystack FILE ...] + [--eb-stack PATH] [--rebuild-set FILE] + [--now EPOCH] [--json] FILE + Generate a reversible lifecycle plan and Lmod .modulerc.lua + warnings/forbids from usage --json output and a TOML policy. Never + deletes software or modulefiles. Requires a minimum observation + window and checks installed dependents via eb-stack rebuild-set + before allowing candidate-remove. Zig build only. + --policy FILE, --policy=FILE + TOML lifecycle policy file. Required. + --modulerc-out FILE, --modulerc-out=FILE + Write the generated Lmod .modulerc.lua warnings and forbids + durably to FILE. Default: none; no file is written. + --easyconfigs DIR, --easyconfigs=DIR + Easyconfig tree passed to eb-stack rebuild-set to identify + dependents of candidate removals. Repeatable. Default: none. + --easystack FILE, --easystack=FILE + Deployed easystack YAML passed to eb-stack rebuild-set. + Repeatable. Default: none. + --eb-stack PATH, --eb-stack=PATH + Path to the eb-stack executable. Default: eb-stack on PATH. + --rebuild-set FILE, --rebuild-set=FILE + Precomputed eb-stack rebuild-set.json report instead of + running eb-stack. Default: none. + --now EPOCH, --now=EPOCH + Reference epoch time in seconds for idle age and observation + window checks. Default: the environment time override, else + the current time. + --json + Print one line of JSON instead, with schema_version 1. + FILE + JSON output produced by eb-keep-logs usage --json (with + selection "all"). Required. eb-keep-logs -h | --help Print this help on stdout and exit 0. eb-keep-logs --version @@ -679,8 +757,8 @@ Environment: PATH Where failed looks up eb. EASYBUILD_TMPDIR failed copies a command's output only from below it; unset or empty, it copies none. - EB_KEEP_LOGS_NOW The time rotate takes as now, in seconds since the - epoch; unset or empty, the current time. + EB_KEEP_LOGS_NOW The time rotate and lifecycle take as now, in seconds + since the epoch; unset or empty, the current time. EB_PIPELINE_LOG_DIR rotate leaves alone the run that holds this path. EB_KEEP_LOGS_SYSLOG Default --syslog target (journald, syslog, unix:///..., journal+unix:///..., udp://host:port, tcp://host:port). @@ -691,7 +769,7 @@ Environment: directory of zone files. Exit status: - 0 -h, --help or --version, or a failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle or test-compare that + 0 -h, --help or --version, or a failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle, test-compare or lifecycle that accepted its arguments, whatever it could keep, compress, remove or read: what it could not do is reported on stderr. used with --require-none also exits 2 when any path was listed. @@ -713,7 +791,7 @@ eb-keep-logs 0.2.1 (x86_64-linux-musl, safe, zig 0.17.1, 5ceaead) ** See also - [[file:../tutorials/keep-a-failed-build-log.org][Keep a failed build's logs]] -- [[file:../howto/call-from-a-pipeline.org][Call eb-keep-logs from a build pipeline]], [[file:../howto/rotate-a-log-tree.org][Rotate a log tree]] and [[file:../howto/mark-licensed-logs.org][Mark logs that may carry a licence number]] +- [[file:../howto/call-from-a-pipeline.org][Call eb-keep-logs from a build pipeline]], [[file:../howto/rotate-a-log-tree.org][Rotate a log tree]], [[file:../howto/mark-licensed-logs.org][Mark logs that may carry a licence number]], [[file:../howto/review-unused-software.org][Review unused installed software]] and [[file:../howto/plan-module-lifecycle.org][Plan module lifecycle and deprecation]] - [[file:../explanation/why-eb-last-log.org][Finding a failed build's log]], [[file:../explanation/local-time.org][Run ages and local time]] and [[file:../explanation/rotate-in-use.org][Rotating a tree in use]] - [[file:compatibility.org][Compatibility]], the differences from the shell script. - [[file:api/zig/eb-keep-logs.md][The Zig API]], from the doc comments of the source. diff --git a/docs/orgmode/reference/json-schemas.org b/docs/orgmode/reference/json-schemas.org index 8f4aa9b..21f0ddb 100644 --- a/docs/orgmode/reference/json-schemas.org +++ b/docs/orgmode/reference/json-schemas.org @@ -4,7 +4,7 @@ * JSON output schemas -The Zig build's =--json= on =why=, =history=, =loads=, =runs=, =used=, =usage=, =pending= and =test-compare= prints one line of JSON with =schema_version= 1. +The Zig build's =--json= on =why=, =history=, =loads=, =runs=, =used=, =usage=, =pending=, =test-compare= and =lifecycle= prints one line of JSON with =schema_version= 1. The =test-bundle= command always writes one JSON document with =schema_version= 1. The shell script refuses those commands; see [[file:compatibility.org][Compatibility]]. Each document is validated in CI against a JSON Schema file in the repository tree under =docs/orgmode/reference/json/=. @@ -17,12 +17,13 @@ The schemas use draft 2020-12; a new =schema_version= may drop or rename a key, | =loads= | =docs/orgmode/reference/json/loads.schema.json= | | =runs= | =docs/orgmode/reference/json/runs.schema.json= | | =used= | =docs/orgmode/reference/json/used.schema.json= | -| =usage= | =docs/orgmode/reference/json/usage.schema.json= | -| =pending= | =docs/orgmode/reference/json/pending.schema.json= | +| =usage= | =docs/orgmode/reference/json/usage.schema.json= | +| =pending= | =docs/orgmode/reference/json/pending.schema.json= | | =test-bundle= | =docs/orgmode/reference/json/test-bundle.schema.json= | | =test-compare= | =docs/orgmode/reference/json/test-compare.schema.json= | +| =lifecycle= | =docs/orgmode/reference/json/lifecycle.schema.json= | -Field names and semantics for =why=, =history=, =used= and =usage= are under [[file:cli.org][The eb-keep-logs command]]. +Field names and semantics for =why=, =history=, =used=, =usage= and =lifecycle= are under [[file:cli.org][The eb-keep-logs command]]. =loads=, =runs=, =pending=, =test-bundle= and =test-compare= document their JSON shape in the same page. ** See also diff --git a/docs/orgmode/reference/json/lifecycle.schema.json b/docs/orgmode/reference/json/lifecycle.schema.json new file mode 100644 index 0000000..1ebfa7a --- /dev/null +++ b/docs/orgmode/reference/json/lifecycle.schema.json @@ -0,0 +1,93 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://stackhands.github.io/eb-keep-logs/reference/json/lifecycle.schema.json", + "title": "eb-keep-logs lifecycle --json", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "kind", + "software", + "modules", + "policy", + "now", + "since", + "observed_days", + "min_observation_days", + "observation_satisfied", + "dependents_checked", + "counts", + "entries", + "modulerc_lua" + ], + "properties": { + "schema_version": { "type": "integer", "const": 1 }, + "kind": { "type": "string", "const": "lifecycle" }, + "software": { "type": "string" }, + "modules": { "type": "string" }, + "policy": { "type": "string" }, + "now": { "type": "string" }, + "since": { "type": ["string", "null"] }, + "observed_days": { "type": ["integer", "null"], "minimum": 0 }, + "min_observation_days": { "type": "integer", "minimum": 1 }, + "observation_satisfied": { "type": "boolean" }, + "dependents_checked": { "type": "boolean" }, + "counts": { "$ref": "#/$defs/counts" }, + "entries": { + "type": "array", + "items": { "$ref": "#/$defs/entry" } + }, + "modulerc_lua": { "type": "string" } + }, + "$defs": { + "counts": { + "type": "object", + "additionalProperties": false, + "required": ["keep", "warn", "forbid-after", "candidate-remove"], + "properties": { + "keep": { "type": "integer", "minimum": 0 }, + "warn": { "type": "integer", "minimum": 0 }, + "forbid-after": { "type": "integer", "minimum": 0 }, + "candidate-remove": { "type": "integer", "minimum": 0 } + } + }, + "entry": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "version", + "module", + "easyconfig", + "action", + "hits", + "last", + "idle_days", + "forbid_after", + "dependents", + "blocked_reason", + "reason" + ], + "properties": { + "name": { "type": "string" }, + "version": { "type": "string" }, + "module": { "type": "string" }, + "easyconfig": { "type": "string" }, + "action": { + "type": "string", + "enum": ["keep", "warn", "forbid-after", "candidate-remove"] + }, + "hits": { "type": "integer", "minimum": 0 }, + "last": { "type": ["string", "null"] }, + "idle_days": { "type": ["integer", "null"], "minimum": 0 }, + "forbid_after": { "type": ["string", "null"] }, + "dependents": { + "type": "array", + "items": { "type": "string" } + }, + "blocked_reason": { "type": ["string", "null"] }, + "reason": { "type": "string" } + } + } + } +} diff --git a/eb-keep-logs b/eb-keep-logs index ed75f5b..d53c77f 100755 --- a/eb-keep-logs +++ b/eb-keep-logs @@ -224,6 +224,40 @@ otherwise lose, and keep the pipeline's log tree bounded. Print one line of JSON instead of text. Default: text. BUNDLE Two or more test-bundle JSON files. Required. + eb-keep-logs lifecycle --policy FILE [--modulerc-out FILE] + [--easyconfigs DIR ...] [--easystack FILE ...] + [--eb-stack PATH] [--rebuild-set FILE] + [--now EPOCH] [--json] FILE + Generate a reversible lifecycle plan and Lmod .modulerc.lua + warnings/forbids from usage --json output and a TOML policy. Never + deletes software or modulefiles. Requires a minimum observation + window and checks installed dependents via eb-stack rebuild-set + before allowing candidate-remove. Zig build only. + --policy FILE, --policy=FILE + TOML lifecycle policy file. Required. + --modulerc-out FILE, --modulerc-out=FILE + Write the generated Lmod .modulerc.lua warnings and forbids + durably to FILE. Default: none; no file is written. + --easyconfigs DIR, --easyconfigs=DIR + Easyconfig tree passed to eb-stack rebuild-set to identify + dependents of candidate removals. Repeatable. Default: none. + --easystack FILE, --easystack=FILE + Deployed easystack YAML passed to eb-stack rebuild-set. + Repeatable. Default: none. + --eb-stack PATH, --eb-stack=PATH + Path to the eb-stack executable. Default: eb-stack on PATH. + --rebuild-set FILE, --rebuild-set=FILE + Precomputed eb-stack rebuild-set.json report instead of + running eb-stack. Default: none. + --now EPOCH, --now=EPOCH + Reference epoch time in seconds for idle age and observation + window checks. Default: the environment time override, else + the current time. + --json + Print one line of JSON instead, with schema_version 1. + FILE + JSON output produced by eb-keep-logs usage --json (with + selection "all"). Required. eb-keep-logs -h | --help Print this help on stdout and exit 0. eb-keep-logs --version @@ -233,8 +267,8 @@ Environment: PATH Where failed looks up eb. EASYBUILD_TMPDIR failed copies a command's output only from below it; unset or empty, it copies none. - EB_KEEP_LOGS_NOW The time rotate takes as now, in seconds since the - epoch; unset or empty, the current time. + EB_KEEP_LOGS_NOW The time rotate and lifecycle take as now, in seconds + since the epoch; unset or empty, the current time. EB_PIPELINE_LOG_DIR rotate leaves alone the run that holds this path. EB_KEEP_LOGS_SYSLOG Default --syslog target (journald, syslog, unix:///..., journal+unix:///..., udp://host:port, tcp://host:port). @@ -245,7 +279,7 @@ Environment: directory of zone files. Exit status: - 0 -h, --help or --version, or a failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle or test-compare that + 0 -h, --help or --version, or a failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle, test-compare or lifecycle that accepted its arguments, whatever it could keep, compress, remove or read: what it could not do is reported on stderr. used with --require-none also exits 2 when any path was listed. @@ -813,7 +847,7 @@ case "${1:-}" in --version) echo "eb-keep-logs ${VERSION}" ;; failed) shift; cmd_failed "$@" ;; rotate) shift; cmd_rotate "$@" ;; - why | history | loads | runs | used | usage | pending | test-bundle | test-compare) die "$1 is in the Zig build only" ;; + why | history | loads | runs | used | usage | pending | test-bundle | test-compare | lifecycle) die "$1 is in the Zig build only" ;; "") usage; exit 2 ;; - *) die "unknown command '$1' (failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle, test-compare, --help)" ;; + *) die "unknown command '$1' (failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle, test-compare, lifecycle, --help)" ;; esac diff --git a/src/lifecycle.zig b/src/lifecycle.zig new file mode 100644 index 0000000..5455251 --- /dev/null +++ b/src/lifecycle.zig @@ -0,0 +1,1473 @@ +//! `lifecycle`: generate a reversible module lifecycle plan and Lmod +//! `.modulerc.lua` warnings/forbids from `eb-keep-logs usage --json` output. +//! +//! Reads a TOML policy, checks the minimum observation window, and calls +//! `eb-stack rebuild-set` for candidate removals so modules with installed +//! dependents are never cleared for removal. Never deletes anything. +const std = @import("std"); +const linux = std.os.linux; +const common = @import("zig_common"); +const sys = common.sys; +const fsx = common.fsx; +const proc = common.proc; +const cli = common.cli; +const localtime = @import("localtime.zig"); + +const Allocator = std.mem.Allocator; +const tool = "eb-keep-logs"; +/// The `schema_version` of what `--json` prints. +pub const schema_version: u32 = 1; +const read_limit: usize = 64 * 1024 * 1024; + +/// Lifecycle classification assigned to each installed module. +pub const Action = enum { + keep, + warn, + @"forbid-after", + @"candidate-remove", + + pub fn name(self: Action) []const u8 { + return switch (self) { + .keep => "keep", + .warn => "warn", + .@"forbid-after" => "forbid-after", + .@"candidate-remove" => "candidate-remove", + }; + } +}; + +/// Parsed lifecycle policy from TOML. +pub const Policy = struct { + min_observation_days: u64, + warn_after_days: u64, + forbid_after_days: u64, + remove_after_days: u64, + forbid_grace_days: u64 = 30, + min_hits: u64 = 1, + keep: []const []const u8 = &.{}, + warn_message: ?[]const u8 = null, + forbid_message: ?[]const u8 = null, +}; + +fn die(comptime fmt: []const u8, args: anytype) noreturn { + cli.die(tool, fmt, args); +} + +fn readRegularAll(gpa: Allocator, path: []const u8, label: []const u8) []u8 { + const opened = fsx.openRegular(fsx.cwd, path, true) catch |err| + die("cannot read {s} {s}: {s}", .{ label, path, @errorName(err) }); + defer sys.close(opened.fd); + return fsx.readAll(gpa, opened.fd, read_limit) catch |err| + die("cannot read {s} {s}: {s}", .{ label, path, @errorName(err) }); +} + +fn parseFiniteNonNeg(text: []const u8) ?f64 { + if (text.len == 0) return null; + const v = std.fmt.parseFloat(f64, text) catch return null; + if (!std.math.isFinite(v) or v < 0) return null; + return v; +} + +/// Match `text` against `pattern` where `*` matches any byte sequence and `?` +/// matches any single byte. +pub fn globMatch(pattern: []const u8, text: []const u8) bool { + var pi: usize = 0; + var ti: usize = 0; + var star_pi: ?usize = null; + var star_ti: usize = 0; + + while (ti < text.len) { + if (pi < pattern.len and (pattern[pi] == '?' or pattern[pi] == text[ti])) { + pi += 1; + ti += 1; + } else if (pi < pattern.len and pattern[pi] == '*') { + star_pi = pi; + star_ti = ti; + pi += 1; + } else if (star_pi) |sp| { + pi = sp + 1; + star_ti += 1; + ti = star_ti; + } else { + return false; + } + } + while (pi < pattern.len and pattern[pi] == '*') : (pi += 1) {} + return pi == pattern.len; +} + +fn matchesKeep(patterns: []const []const u8, module: []const u8) bool { + for (patterns) |pat| { + if (globMatch(pat, module)) return true; + } + return false; +} + +pub const PolicyError = Allocator.Error || error{BadPolicy}; + +const TomlParser = struct { + arena: Allocator, + src: []const u8, + pos: usize = 0, + err_msg: []const u8 = "invalid TOML policy", + + fn fail(self: *TomlParser, comptime fmt: []const u8, args: anytype) PolicyError { + self.err_msg = self.arena.print(fmt, args) catch "invalid TOML policy"; + return error.BadPolicy; + } + + fn skipHorizontalWs(self: *TomlParser) void { + while (self.pos < self.src.len and (self.src[self.pos] == ' ' or self.src[self.pos] == '\t')) : (self.pos += 1) {} + } + + fn skipWsAndComments(self: *TomlParser) PolicyError!void { + while (self.pos < self.src.len) { + const c = self.src[self.pos]; + if (c == ' ' or c == '\t' or c == '\r' or c == '\n') { + self.pos += 1; + } else if (c == '#') { + while (self.pos < self.src.len and self.src[self.pos] != '\n') : (self.pos += 1) { + if (self.src[self.pos] == 0) return self.fail("NUL byte in policy", .{}); + } + } else { + break; + } + } + } + + fn skipLineTail(self: *TomlParser) PolicyError!void { + self.skipHorizontalWs(); + if (self.pos < self.src.len and self.src[self.pos] == '#') { + while (self.pos < self.src.len and self.src[self.pos] != '\n') : (self.pos += 1) {} + } + if (self.pos < self.src.len) { + if (self.src[self.pos] == '\r') self.pos += 1; + if (self.pos < self.src.len) { + if (self.src[self.pos] != '\n') return self.fail("unexpected trailing characters in policy", .{}); + self.pos += 1; + } + } + } + + fn parseKey(self: *TomlParser) PolicyError![]const u8 { + const start = self.pos; + while (self.pos < self.src.len) { + const c = self.src[self.pos]; + if (std.ascii.isAlphanumeric(c) or c == '_' or c == '-') { + self.pos += 1; + } else { + break; + } + } + if (self.pos == start) return self.fail("expected key in policy", .{}); + return self.src[start..self.pos]; + } + + fn parseString(self: *TomlParser) PolicyError![]const u8 { + if (self.pos >= self.src.len) return self.fail("expected string in policy", .{}); + const quote = self.src[self.pos]; + if (quote != '"' and quote != '\'') return self.fail("expected string in policy", .{}); + self.pos += 1; + var out: std.ArrayList(u8) = .empty; + while (self.pos < self.src.len) { + const c = self.src[self.pos]; + if (c == 0) return self.fail("NUL byte in policy string", .{}); + if (c == '\n' or c == '\r') return self.fail("unterminated string in policy", .{}); + if (c == quote) { + self.pos += 1; + return try out.toOwnedSlice(self.arena); + } + if (quote == '"' and c == '\\') { + self.pos += 1; + if (self.pos >= self.src.len) return self.fail("unterminated escape in policy", .{}); + const esc = self.src[self.pos]; + self.pos += 1; + switch (esc) { + '"' => try out.append(self.arena, '"'), + '\\' => try out.append(self.arena, '\\'), + 'n' => try out.append(self.arena, '\n'), + 'r' => try out.append(self.arena, '\r'), + 't' => try out.append(self.arena, '\t'), + else => return self.fail("unsupported string escape '\\{c}' in policy", .{esc}), + } + } else { + try out.append(self.arena, c); + self.pos += 1; + } + } + return self.fail("unterminated string in policy", .{}); + } + + fn parseUInt(self: *TomlParser) PolicyError!u64 { + const start = self.pos; + var digits: std.ArrayList(u8) = .empty; + while (self.pos < self.src.len) { + const c = self.src[self.pos]; + if (std.ascii.isDigit(c)) { + try digits.append(self.arena, c); + self.pos += 1; + } else if (c == '_' and digits.items.len > 0 and self.pos + 1 < self.src.len and std.ascii.isDigit(self.src[self.pos + 1])) { + self.pos += 1; + } else { + break; + } + } + if (self.pos == start or digits.items.len == 0) return self.fail("expected non-negative integer in policy", .{}); + if (digits.items.len > 1 and digits.items[0] == '0') return self.fail("leading zero in integer in policy", .{}); + return std.fmt.parseInt(u64, digits.items, 10) catch return self.fail("integer overflow in policy", .{}); + } + + fn parseStringArray(self: *TomlParser) PolicyError![]const []const u8 { + if (self.pos >= self.src.len or self.src[self.pos] != '[') return self.fail("expected '[' in policy", .{}); + self.pos += 1; + var items: std.ArrayList([]const u8) = .empty; + while (true) { + try self.skipWsAndComments(); + if (self.pos >= self.src.len) return self.fail("unterminated array in policy", .{}); + if (self.src[self.pos] == ']') { + self.pos += 1; + return try items.toOwnedSlice(self.arena); + } + const s = try self.parseString(); + if (s.len == 0) return self.fail("policy keep entry must not be empty", .{}); + try items.append(self.arena, s); + try self.skipWsAndComments(); + if (self.pos >= self.src.len) return self.fail("unterminated array in policy", .{}); + if (self.src[self.pos] == ',') { + self.pos += 1; + } else if (self.src[self.pos] == ']') { + self.pos += 1; + return try items.toOwnedSlice(self.arena); + } else { + return self.fail("expected ',' or ']' in policy array", .{}); + } + } + } +}; + +/// Parse a lifecycle TOML policy document. +pub fn parsePolicy(arena: Allocator, text: []const u8, err_out: *[]const u8) PolicyError!Policy { + var p: TomlParser = .{ .arena = arena, .src = text }; + var min_obs: ?u64 = null; + var warn_days: ?u64 = null; + var forbid_days: ?u64 = null; + var remove_days: ?u64 = null; + var grace_days: ?u64 = null; + var min_hits: ?u64 = null; + var keep: ?[]const []const u8 = null; + var warn_msg: ?[]const u8 = null; + var forbid_msg: ?[]const u8 = null; + var seen_section = false; + + while (true) { + p.skipWsAndComments() catch |err| { + err_out.* = p.err_msg; + return err; + }; + if (p.pos >= p.src.len) break; + if (p.src[p.pos] == '[') { + p.pos += 1; + p.skipHorizontalWs(); + const sec = p.parseKey() catch |err| { + err_out.* = p.err_msg; + return err; + }; + p.skipHorizontalWs(); + if (p.pos >= p.src.len or p.src[p.pos] != ']') { + err_out.* = "unterminated section header in policy"; + return error.BadPolicy; + } + p.pos += 1; + p.skipLineTail() catch |err| { + err_out.* = p.err_msg; + return err; + }; + if (!std.mem.eql(u8, sec, "policy") and !std.mem.eql(u8, sec, "lifecycle")) { + err_out.* = arena.print("unknown policy section '[{s}]'", .{sec}) catch "unknown policy section"; + return error.BadPolicy; + } + if (seen_section) { + err_out.* = "duplicate policy section header"; + return error.BadPolicy; + } + seen_section = true; + continue; + } + const key = p.parseKey() catch |err| { + err_out.* = p.err_msg; + return err; + }; + p.skipHorizontalWs(); + if (p.pos >= p.src.len or p.src[p.pos] != '=') { + err_out.* = arena.print("expected '=' after policy key '{s}'", .{key}) catch "expected '=' in policy"; + return error.BadPolicy; + } + p.pos += 1; + p.skipHorizontalWs(); + + if (std.mem.eql(u8, key, "min_observation_days")) { + if (min_obs != null) { + err_out.* = "duplicate policy key 'min_observation_days'"; + return error.BadPolicy; + } + min_obs = p.parseUInt() catch |err| { + err_out.* = p.err_msg; + return err; + }; + } else if (std.mem.eql(u8, key, "warn_after_days")) { + if (warn_days != null) { + err_out.* = "duplicate policy key 'warn_after_days'"; + return error.BadPolicy; + } + warn_days = p.parseUInt() catch |err| { + err_out.* = p.err_msg; + return err; + }; + } else if (std.mem.eql(u8, key, "forbid_after_days")) { + if (forbid_days != null) { + err_out.* = "duplicate policy key 'forbid_after_days'"; + return error.BadPolicy; + } + forbid_days = p.parseUInt() catch |err| { + err_out.* = p.err_msg; + return err; + }; + } else if (std.mem.eql(u8, key, "remove_after_days")) { + if (remove_days != null) { + err_out.* = "duplicate policy key 'remove_after_days'"; + return error.BadPolicy; + } + remove_days = p.parseUInt() catch |err| { + err_out.* = p.err_msg; + return err; + }; + } else if (std.mem.eql(u8, key, "forbid_grace_days")) { + if (grace_days != null) { + err_out.* = "duplicate policy key 'forbid_grace_days'"; + return error.BadPolicy; + } + grace_days = p.parseUInt() catch |err| { + err_out.* = p.err_msg; + return err; + }; + } else if (std.mem.eql(u8, key, "min_hits")) { + if (min_hits != null) { + err_out.* = "duplicate policy key 'min_hits'"; + return error.BadPolicy; + } + min_hits = p.parseUInt() catch |err| { + err_out.* = p.err_msg; + return err; + }; + } else if (std.mem.eql(u8, key, "keep")) { + if (keep != null) { + err_out.* = "duplicate policy key 'keep'"; + return error.BadPolicy; + } + keep = p.parseStringArray() catch |err| { + err_out.* = p.err_msg; + return err; + }; + } else if (std.mem.eql(u8, key, "warn_message")) { + if (warn_msg != null) { + err_out.* = "duplicate policy key 'warn_message'"; + return error.BadPolicy; + } + warn_msg = p.parseString() catch |err| { + err_out.* = p.err_msg; + return err; + }; + } else if (std.mem.eql(u8, key, "forbid_message")) { + if (forbid_msg != null) { + err_out.* = "duplicate policy key 'forbid_message'"; + return error.BadPolicy; + } + forbid_msg = p.parseString() catch |err| { + err_out.* = p.err_msg; + return err; + }; + } else { + err_out.* = arena.print("unknown policy key '{s}'", .{key}) catch "unknown policy key"; + return error.BadPolicy; + } + + p.skipLineTail() catch |err| { + err_out.* = p.err_msg; + return err; + }; + } + + const mo = min_obs orelse { + err_out.* = "policy needs min_observation_days"; + return error.BadPolicy; + }; + const wd = warn_days orelse { + err_out.* = "policy needs warn_after_days"; + return error.BadPolicy; + }; + const fd = forbid_days orelse { + err_out.* = "policy needs forbid_after_days"; + return error.BadPolicy; + }; + const rd = remove_days orelse { + err_out.* = "policy needs remove_after_days"; + return error.BadPolicy; + }; + const mh = min_hits orelse 1; + + if (mo == 0) { + err_out.* = "min_observation_days must be at least 1"; + return error.BadPolicy; + } + if (mh == 0) { + err_out.* = "min_hits must be at least 1"; + return error.BadPolicy; + } + if (fd < wd) { + err_out.* = "forbid_after_days must be at least warn_after_days"; + return error.BadPolicy; + } + if (rd < fd) { + err_out.* = "remove_after_days must be at least forbid_after_days"; + return error.BadPolicy; + } + + return .{ + .min_observation_days = mo, + .warn_after_days = wd, + .forbid_after_days = fd, + .remove_after_days = rd, + .forbid_grace_days = grace_days orelse 30, + .min_hits = mh, + .keep = keep orelse &.{}, + .warn_message = warn_msg, + .forbid_message = forbid_msg, + }; +} + +const UsagePackage = struct { + name: []const u8, + version: []const u8, + module: []const u8, + hits: u64, + last: ?[]const u8, + last_epoch: ?f64, +}; + +const UsageInput = struct { + software: []const u8, + modules: []const u8, + since_text: ?[]const u8, + since_epoch: ?f64, + packages: []const UsagePackage, +}; + +fn requireObjectField(obj: std.json.ObjectMap, key: []const u8, label: []const u8) std.json.Value { + return obj.get(key) orelse die("{s} is missing '{s}'", .{ label, key }); +} + +fn requireStringField(obj: std.json.ObjectMap, key: []const u8, label: []const u8) []const u8 { + return switch (requireObjectField(obj, key, label)) { + .string => |s| s, + else => die("{s} '{s}' must be a string", .{ label, key }), + }; +} + +fn parseUsageJson(arena: Allocator, path: []const u8, text: []const u8) UsageInput { + const parsed = std.json.parseFromSlice(std.json.Value, arena, text, .{}) catch + die("{s} is not valid JSON", .{path}); + const root = switch (parsed.value) { + .object => |obj| obj, + else => die("{s} must be a JSON object", .{path}), + }; + const ver = switch (requireObjectField(root, "schema_version", path)) { + .integer => |n| n, + else => die("{s} schema_version must be 1", .{path}), + }; + if (ver != 1) die("{s} schema_version must be 1", .{path}); + const kind = requireStringField(root, "kind", path); + if (!std.mem.eql(u8, kind, "usage")) die("{s} kind must be \"usage\"", .{path}); + + const selection = requireStringField(root, "selection", path); + if (!std.mem.eql(u8, selection, "all")) + die("lifecycle requires usage JSON with selection \"all\", got \"{s}\"", .{selection}); + + const software = requireStringField(root, "software", path); + const modules = requireStringField(root, "modules", path); + + var since_text: ?[]const u8 = null; + var since_epoch: ?f64 = null; + switch (requireObjectField(root, "since", path)) { + .null => {}, + .string => |s| { + since_epoch = parseFiniteNonNeg(s) orelse die("{s} has invalid since '{s}'", .{ path, s }); + since_text = s; + }, + else => die("{s} 'since' must be a string or null", .{path}), + } + + const pkgs_val = requireObjectField(root, "packages", path); + const pkgs_arr = switch (pkgs_val) { + .array => |arr| arr, + else => die("{s} 'packages' must be an array", .{path}), + }; + + var out: std.ArrayList(UsagePackage) = .empty; + for (pkgs_arr.items) |item| { + const obj = switch (item) { + .object => |o| o, + else => die("{s} package entry must be an object", .{path}), + }; + const name = requireStringField(obj, "name", path); + const version = requireStringField(obj, "version", path); + const module = requireStringField(obj, "module", path); + if (name.len == 0 or version.len == 0 or module.len == 0) + die("{s} package entry has empty name, version or module", .{path}); + switch (requireObjectField(obj, "active", path)) { + .bool => {}, + else => die("{s} package 'active' must be a boolean", .{path}), + } + const hits_i = switch (requireObjectField(obj, "hits", path)) { + .integer => |n| n, + else => die("{s} package 'hits' must be an integer", .{path}), + }; + if (hits_i < 0) die("{s} package 'hits' must be non-negative", .{path}); + var last_text: ?[]const u8 = null; + var last_epoch: ?f64 = null; + switch (requireObjectField(obj, "last", path)) { + .null => {}, + .string => |s| if (s.len > 0) { + last_epoch = parseFiniteNonNeg(s) orelse die("{s} package '{s}' has invalid last '{s}'", .{ path, module, s }); + last_text = s; + }, + else => die("{s} package 'last' must be a string or null", .{path}), + } + out.append(arena, .{ + .name = name, + .version = version, + .module = module, + .hits = @intCast(hits_i), + .last = last_text, + .last_epoch = last_epoch, + }) catch die("out of memory", .{}); + } + + return .{ + .software = software, + .modules = modules, + .since_text = since_text, + .since_epoch = since_epoch, + .packages = out.items, + }; +} + +const Entry = struct { + name: []const u8, + version: []const u8, + module: []const u8, + easyconfig: []const u8, + action: Action, + hits: u64, + last: ?[]const u8, + idle_days: ?u64, + forbid_after: ?[]const u8 = null, + dependents: []const []const u8 = &.{}, + blocked_reason: ?[]const u8 = null, + reason: []const u8, + pending_candidate: bool = false, +}; + +fn lessEntry(_: void, a: Entry, b: Entry) bool { + return std.mem.lessThan(u8, a.module, b.module); +} + +fn formatUtcDate(arena: Allocator, epoch_sec: i64) Allocator.Error![]const u8 { + const days = @divFloor(epoch_sec, 86400); + const d = localtime.civilFromDays(days); + if (d.year >= 0) { + return try arena.print("{d:0>4}-{d:0>2}-{d:0>2}", .{ @as(u64, @intCast(d.year)), d.month, d.day }); + } + return try arena.print("{d}-{d:0>2}-{d:0>2}", .{ d.year, d.month, d.day }); +} + +fn optionValue( + args: []const []const u8, + i: *usize, + arg: []const u8, + comptime option: []const u8, + comptime message: []const u8, +) []const u8 { + const value = if (std.mem.startsWith(u8, arg, option ++ "=")) + arg[option.len + 1 ..] + else blk: { + i.* += 1; + if (i.* >= args.len) die(message, .{}); + break :blk args[i.*]; + }; + if (value.len == 0 or value[0] == '-') die(message, .{}); + return value; +} + +const RebuildInfo = struct { + changed: std.StringHashMapUnmanaged(void) = .empty, + missing: std.StringHashMapUnmanaged(void) = .empty, + dependents: std.StringHashMapUnmanaged(std.ArrayList([]const u8)) = .empty, +}; + +fn addDependent(arena: Allocator, map: *std.StringHashMapUnmanaged(std.ArrayList([]const u8)), ec: []const u8, dep: []const u8) !void { + const gop = try map.getOrPut(arena, ec); + if (!gop.found_existing) gop.value_ptr.* = .empty; + for (gop.value_ptr.items) |existing| { + if (std.mem.eql(u8, existing, dep)) return; + } + try gop.value_ptr.append(arena, try arena.dupe(u8, dep)); +} + +fn parseRebuildSetJson(arena: Allocator, path: []const u8, text: []const u8) RebuildInfo { + const parsed = std.json.parseFromSlice(std.json.Value, arena, text, .{}) catch + die("rebuild-set file {s} is not valid JSON", .{path}); + const root = switch (parsed.value) { + .object => |obj| obj, + else => die("rebuild-set file {s} must be a JSON object", .{path}), + }; + _ = switch (requireObjectField(root, "schema_version", path)) { + .integer => {}, + else => die("rebuild-set file {s} schema_version must be an integer", .{path}), + }; + + var info: RebuildInfo = .{}; + + const changed_arr = switch (requireObjectField(root, "changed", path)) { + .array => |a| a, + else => die("rebuild-set file {s} 'changed' must be an array", .{path}), + }; + for (changed_arr.items) |item| { + const s = switch (item) { + .string => |v| v, + else => die("rebuild-set file {s} 'changed' entries must be strings", .{path}), + }; + info.changed.put(arena, s, {}) catch die("out of memory", .{}); + } + + const missing_arr = switch (requireObjectField(root, "missing", path)) { + .array => |a| a, + else => die("rebuild-set file {s} 'missing' must be an array", .{path}), + }; + for (missing_arr.items) |item| { + const s = switch (item) { + .string => |v| v, + else => die("rebuild-set file {s} 'missing' entries must be strings", .{path}), + }; + info.missing.put(arena, s, {}) catch die("out of memory", .{}); + } + + const rows_arr = switch (requireObjectField(root, "rows", path)) { + .array => |a| a, + else => die("rebuild-set file {s} 'rows' must be an array", .{path}), + }; + for (rows_arr.items) |item| { + const row = switch (item) { + .object => |o| o, + else => die("rebuild-set file {s} row must be an object", .{path}), + }; + const row_changed = requireStringField(row, "changed", path); + const dependent = requireStringField(row, "dependent", path); + const edge_kind = requireStringField(row, "edge_kind", path); + if (std.mem.eql(u8, edge_kind, "changed")) continue; + + var parts = std.mem.splitSequence(u8, row_changed, ", "); + while (parts.next()) |ec| { + if (ec.len == 0) continue; + addDependent(arena, &info.dependents, ec, dependent) catch die("out of memory", .{}); + } + } + + return info; +} + +fn resolveExecutable(gpa: Allocator, explicit: ?[]const u8, path_env: []const u8) []u8 { + if (explicit) |raw| { + if (std.mem.indexOfScalar(u8, raw, '/') != null) { + var rb: [sys.path_max]u8 = undefined; + const resolved = fsx.realpath(fsx.cwd, raw, &rb) catch |err| + die("cannot resolve --eb-stack {s}: {t}", .{ raw, err }); + return gpa.dupe(u8, resolved) catch die("out of memory", .{}); + } + return (proc.which(gpa, raw, path_env) catch die("out of memory", .{})) orelse + die("cannot find '{s}' on PATH", .{raw}); + } + return (proc.which(gpa, "eb-stack", path_env) catch die("out of memory", .{})) orelse + die("cannot find eb-stack on PATH; pass --eb-stack or --rebuild-set", .{}); +} + +fn runEbStackRebuildSet( + gpa: Allocator, + arena: Allocator, + env: *const std.process.Environ.Map, + envp: proc.Envp, + eb_stack_exe: []const u8, + easyconfigs: []const []const u8, + easystacks: []const []const u8, + packages: []const UsagePackage, + candidates: []const []const u8, +) RebuildInfo { + const tmp_root = blk: { + const v = env.get("TMPDIR") orelse ""; + break :blk if (v.len > 0) std.mem.trimEnd(u8, v, "/") else "/tmp"; + }; + var rnd: [4]u8 = undefined; + _ = sys.call(linux.getrandom, .{ &rnd, rnd.len, 0 }) catch die("getrandom failed", .{}); + const work_dir = arena.print("{s}/eb-keep-logs-lifecycle-{d}-{x:0>8}", .{ + if (tmp_root.len > 0) tmp_root else "/tmp", + linux.getpid(), + std.mem.readInt(u32, &rnd, .little), + }) catch die("out of memory", .{}); + + var zb: [sys.path_max:0]u8 = undefined; + _ = sys.call(linux.mkdirat, .{ fsx.cwd, (sys.z(&zb, work_dir) catch die("path too long", .{})).ptr, 0o700 }) catch |err| + die("cannot create temporary directory {s}: {t}", .{ work_dir, err }); + defer fsx.deleteTree(gpa, fsx.cwd, work_dir) catch {}; + + const work_fd = fsx.openDir(fsx.cwd, work_dir, false) catch |err| + die("cannot open temporary directory {s}: {t}", .{ work_dir, err }); + defer sys.close(work_fd); + + const out_dir = arena.print("{s}/out", .{work_dir}) catch die("out of memory", .{}); + _ = sys.call(linux.mkdirat, .{ work_fd, "out", 0o700 }) catch |err| + die("cannot create {s}/out: {t}", .{ work_dir, err }); + + const installed_path = arena.print("{s}/installed.txt", .{work_dir}) catch die("out of memory", .{}); + { + var inst_buf: std.ArrayList(u8) = .empty; + for (packages) |pkg| { + inst_buf.print(arena, "{s}\n", .{pkg.module}) catch die("out of memory", .{}); + } + const fd = sys.openat(work_fd, "installed.txt", .{ .ACCMODE = .WRONLY, .CREAT = true, .EXCL = true, .NOFOLLOW = true }, 0o600) catch |err| + die("cannot write {s}: {t}", .{ installed_path, err }); + defer sys.close(fd); + sys.writeAll(fd, inst_buf.items) catch |err| die("cannot write {s}: {t}", .{ installed_path, err }); + } + + var argv: std.ArrayList([]const u8) = .empty; + argv.appendSlice(arena, &.{ eb_stack_exe, "rebuild-set" }) catch die("out of memory", .{}); + + if (candidates.len <= 32) { + for (candidates) |ec| { + argv.appendSlice(arena, &.{ "--changed", ec }) catch die("out of memory", .{}); + } + } else { + const changed_path = arena.print("{s}/changed.txt", .{work_dir}) catch die("out of memory", .{}); + var chg_buf: std.ArrayList(u8) = .empty; + for (candidates) |ec| { + chg_buf.print(arena, "{s}\n", .{ec}) catch die("out of memory", .{}); + } + const fd = sys.openat(work_fd, "changed.txt", .{ .ACCMODE = .WRONLY, .CREAT = true, .EXCL = true, .NOFOLLOW = true }, 0o600) catch |err| + die("cannot write {s}: {t}", .{ changed_path, err }); + defer sys.close(fd); + sys.writeAll(fd, chg_buf.items) catch |err| die("cannot write {s}: {t}", .{ changed_path, err }); + argv.appendSlice(arena, &.{ "--changed", changed_path }) catch die("out of memory", .{}); + } + + for (easyconfigs) |ec_dir| { + argv.appendSlice(arena, &.{ "--easyconfigs", ec_dir }) catch die("out of memory", .{}); + } + for (easystacks) |es| { + argv.appendSlice(arena, &.{ "--easystack", es }) catch die("out of memory", .{}); + } + argv.appendSlice(arena, &.{ + "--installed", installed_path, + "--out-dir", out_dir, + }) catch die("out of memory", .{}); + + const res = proc.run(gpa, eb_stack_exe, argv.items, envp, .{ + .stdout_limit = 1 << 20, + .stderr_limit = 64 * 1024, + }) catch |err| die("cannot run eb-stack rebuild-set: {s}", .{@errorName(err)}); + defer res.deinit(gpa); + + if (res.term.code() != 0) { + const stderr_trimmed = std.mem.trimEnd(u8, res.stderr, "\r\n"); + if (stderr_trimmed.len > 0) { + die("eb-stack rebuild-set failed (exit {d}): {s}", .{ res.term.code(), stderr_trimmed }); + } + die("eb-stack rebuild-set failed (exit {d})", .{res.term.code()}); + } + + const report_path = arena.print("{s}/rebuild-set.json", .{out_dir}) catch die("out of memory", .{}); + const report_text = readRegularAll(gpa, report_path, "rebuild-set output"); + defer gpa.free(report_text); + return parseRebuildSetJson(arena, report_path, report_text); +} + +fn escapeLua(arena: Allocator, s: []const u8) Allocator.Error![]const u8 { + var out: std.ArrayList(u8) = .empty; + for (s) |c| { + switch (c) { + '\\' => try out.appendSlice(arena, "\\\\"), + '"' => try out.appendSlice(arena, "\\\""), + '\n' => try out.appendSlice(arena, "\\n"), + '\r' => try out.appendSlice(arena, "\\r"), + '\t' => try out.appendSlice(arena, "\\t"), + else => if (c < 0x20 or c == 0x7f) { + try out.print(arena, "\\{d:0>3}", .{c}); + } else { + try out.append(arena, c); + }, + } + } + return try out.toOwnedSlice(arena); +} + +fn renderTemplate(arena: Allocator, tmpl: []const u8, module: []const u8, after: []const u8) Allocator.Error![]const u8 { + var out: std.ArrayList(u8) = .empty; + var i: usize = 0; + while (i < tmpl.len) { + if (std.mem.startsWith(u8, tmpl[i..], "{module}")) { + try out.appendSlice(arena, module); + i += "{module}".len; + } else if (std.mem.startsWith(u8, tmpl[i..], "{after}")) { + try out.appendSlice(arena, after); + i += "{after}".len; + } else { + try out.append(arena, tmpl[i]); + i += 1; + } + } + return try out.toOwnedSlice(arena); +} + +fn formatModulerc(arena: Allocator, policy: Policy, entries: []const Entry) Allocator.Error![]const u8 { + var out: std.ArrayList(u8) = .empty; + try out.appendSlice(arena, "-- Generated by eb-keep-logs lifecycle. Review before installing as .modulerc.lua.\n"); + try out.appendSlice(arena, "-- No software or modulefiles were modified or removed.\n"); + + for (entries) |ent| { + switch (ent.action) { + .keep => {}, + .warn => { + try out.print(arena, "\n-- warn: {s} ({s})\n", .{ ent.module, ent.reason }); + if (ent.forbid_after) |after| { + const name_q = try escapeLua(arena, ent.module); + const after_q = try escapeLua(arena, after); + const nearly_raw = if (policy.warn_message) |tmpl| + try renderTemplate(arena, tmpl, ent.module, after) + else + try arena.print("{s} has had no recorded usage recently and is scheduled to be forbidden after {s}.", .{ ent.module, after }); + const msg_raw = if (policy.forbid_message) |tmpl| + try renderTemplate(arena, tmpl, ent.module, after) + else + try arena.print("{s} is forbidden due to inactivity.", .{ent.module}); + try out.print(arena, "forbid{{\n name = \"{s}\",\n after = \"{s}\",\n nearly_message = \"{s}\",\n message = \"{s}\",\n}}\n", .{ + name_q, + after_q, + try escapeLua(arena, nearly_raw), + try escapeLua(arena, msg_raw), + }); + } + }, + .@"forbid-after" => { + const after = ent.forbid_after orelse ""; + try out.print(arena, "\n-- forbid-after: {s} ({s})\n", .{ ent.module, ent.reason }); + const name_q = try escapeLua(arena, ent.module); + const after_q = try escapeLua(arena, after); + const nearly_raw = if (policy.warn_message) |tmpl| + try renderTemplate(arena, tmpl, ent.module, after) + else + try arena.print("{s} is deprecated and will be forbidden after {s}.", .{ ent.module, after }); + const msg_raw = if (policy.forbid_message) |tmpl| + try renderTemplate(arena, tmpl, ent.module, after) + else + try arena.print("{s} is forbidden due to inactivity.", .{ent.module}); + try out.print(arena, "forbid{{\n name = \"{s}\",\n after = \"{s}\",\n nearly_message = \"{s}\",\n message = \"{s}\",\n}}\n", .{ + name_q, + after_q, + try escapeLua(arena, nearly_raw), + try escapeLua(arena, msg_raw), + }); + }, + .@"candidate-remove" => { + try out.print(arena, "\n-- candidate-remove: {s} ({s})\n", .{ ent.module, ent.reason }); + const name_q = try escapeLua(arena, ent.module); + const msg_raw = if (policy.forbid_message) |tmpl| + try renderTemplate(arena, tmpl, ent.module, "") + else + try arena.print("{s} is marked for removal; contact support if you still need it.", .{ent.module}); + try out.print(arena, "hide_version(\"{s}\")\n", .{name_q}); + try out.print(arena, "forbid{{\n name = \"{s}\",\n message = \"{s}\",\n}}\n", .{ + name_q, + try escapeLua(arena, msg_raw), + }); + }, + } + } + return try out.toOwnedSlice(arena); +} + +fn writeFileDurable(path: []const u8, bytes: []const u8) void { + const dir_path = std.fs.path.dirname(path) orelse "."; + const base = std.fs.path.basename(path); + if (base.len == 0 or std.mem.eql(u8, base, ".") or std.mem.eql(u8, base, "..")) + die("--modulerc-out '{s}' does not name a file", .{path}); + const dir = fsx.openDir(fsx.cwd, dir_path, true) catch |err| + die("cannot open --modulerc-out directory {s}: {t}", .{ dir_path, err }); + defer sys.close(dir); + + const existing: ?fsx.Stat = fsx.statAt(dir, base, false) catch |err| switch (err) { + error.NotFound => null, + else => |e| die("cannot stat --modulerc-out {s}: {t}", .{ path, e }), + }; + if (existing) |st| { + if (st.isLink()) die("--modulerc-out {s} is a symbolic link", .{path}); + if (st.isDir()) die("--modulerc-out {s} is a directory", .{path}); + if (!st.isReg()) die("--modulerc-out {s} is not a regular file", .{path}); + } + + var rnd: [4]u8 = undefined; + _ = sys.call(linux.getrandom, .{ &rnd, rnd.len, 0 }) catch die("getrandom failed", .{}); + var tmp_buf: [sys.path_max]u8 = undefined; + const tmp = std.fmt.bufPrint(&tmp_buf, ".{s}.{d}.{x:0>8}.tmp", .{ base, linux.getpid(), std.mem.readInt(u32, &rnd, .little) }) catch + die("--modulerc-out path too long: {s}", .{path}); + const fd = sys.openat(dir, tmp, .{ + .ACCMODE = .WRONLY, + .CREAT = true, + .EXCL = true, + .NOFOLLOW = true, + }, if (existing) |st| @intCast(st.perm()) else 0o644) catch |err| + die("cannot create temporary file for {s}: {t}", .{ path, err }); + var done = false; + defer { + sys.close(fd); + if (!done) sys.unlinkat(dir, tmp, 0) catch {}; + } + sys.writeAll(fd, bytes) catch |err| die("cannot write {s}: {t}", .{ path, err }); + if (existing) |st| { + sys.fchmod(fd, @intCast(st.perm())) catch |err| die("cannot set mode on {s}: {t}", .{ path, err }); + } + sys.fsync(fd) catch |err| die("cannot sync {s}: {t}", .{ path, err }); + sys.renameat(dir, tmp, dir, base) catch |err| die("cannot replace {s}: {t}", .{ path, err }); + done = true; + sys.fsync(dir) catch {}; +} + +const Counts = struct { + keep: usize = 0, + warn: usize = 0, + @"forbid-after": usize = 0, + @"candidate-remove": usize = 0, +}; + +fn formatText(arena: Allocator, entries: []const Entry) Allocator.Error![]u8 { + var out: std.ArrayList(u8) = .empty; + for (entries) |ent| { + try out.print(arena, "{s} {s} hits={d} ", .{ ent.action.name(), ent.module, ent.hits }); + if (ent.idle_days) |d| { + try out.print(arena, "idle_days={d}", .{d}); + } else { + try out.appendSlice(arena, "idle_days=unknown"); + } + if (ent.forbid_after) |after| { + try out.print(arena, " forbid_after={s}", .{after}); + } + if (ent.dependents.len > 0) { + try out.appendSlice(arena, " dependents="); + for (ent.dependents, 0..) |dep, idx| { + if (idx > 0) try out.append(arena, ','); + try out.appendSlice(arena, dep); + } + } + try out.print(arena, " reason=\"{s}\"\n", .{ent.reason}); + } + return try out.toOwnedSlice(arena); +} + +const JsonEntry = struct { + name: []const u8, + version: []const u8, + module: []const u8, + easyconfig: []const u8, + action: []const u8, + hits: u64, + last: ?[]const u8, + idle_days: ?u64, + forbid_after: ?[]const u8, + dependents: []const []const u8, + blocked_reason: ?[]const u8, + reason: []const u8, +}; + +const JsonReport = struct { + schema_version: u32 = schema_version, + kind: []const u8 = "lifecycle", + software: []const u8, + modules: []const u8, + policy: []const u8, + now: []const u8, + since: ?[]const u8, + observed_days: ?u64, + min_observation_days: u64, + observation_satisfied: bool, + dependents_checked: bool, + counts: Counts, + entries: []const JsonEntry, + modulerc_lua: []const u8, +}; + +fn formatJson( + gpa: Allocator, + arena: Allocator, + usage_in: UsageInput, + policy_path: []const u8, + now_text: []const u8, + observed_days: ?u64, + policy: Policy, + observation_satisfied: bool, + dependents_checked: bool, + counts: Counts, + entries: []const Entry, + modulerc_lua: []const u8, +) ![]u8 { + var json_entries: std.ArrayList(JsonEntry) = .empty; + for (entries) |ent| { + try json_entries.append(arena, .{ + .name = ent.name, + .version = ent.version, + .module = ent.module, + .easyconfig = ent.easyconfig, + .action = ent.action.name(), + .hits = ent.hits, + .last = ent.last, + .idle_days = ent.idle_days, + .forbid_after = ent.forbid_after, + .dependents = ent.dependents, + .blocked_reason = ent.blocked_reason, + .reason = ent.reason, + }); + } + const report: JsonReport = .{ + .software = usage_in.software, + .modules = usage_in.modules, + .policy = policy_path, + .now = now_text, + .since = usage_in.since_text, + .observed_days = observed_days, + .min_observation_days = policy.min_observation_days, + .observation_satisfied = observation_satisfied, + .dependents_checked = dependents_checked, + .counts = counts, + .entries = json_entries.items, + .modulerc_lua = modulerc_lua, + }; + var out: std.Io.Writer.Allocating = .init(gpa); + defer out.deinit(); + try std.json.Stringify.value(report, .{}, &out.writer); + try out.writer.writeAll("\n"); + return try out.toOwnedSlice(); +} + +/// `lifecycle --policy FILE [--modulerc-out FILE] [--easyconfigs DIR ...] +/// [--easystack FILE ...] [--eb-stack PATH] [--rebuild-set FILE] +/// [--now EPOCH] [--json] FILE`. +pub fn cmdLifecycle( + gpa: Allocator, + env: *const std.process.Environ.Map, + envp: proc.Envp, + args: []const []const u8, +) u8 { + var policy_path: []const u8 = ""; + var modulerc_out: ?[]const u8 = null; + var eb_stack_opt: ?[]const u8 = null; + var rebuild_set_opt: ?[]const u8 = null; + var now_opt: ?[]const u8 = null; + var json_out = false; + var usage_path: []const u8 = ""; + + var easyconfigs: std.ArrayList([]const u8) = .empty; + defer easyconfigs.deinit(gpa); + var easystacks: std.ArrayList([]const u8) = .empty; + defer easystacks.deinit(gpa); + + var i: usize = 0; + while (i < args.len) : (i += 1) { + const arg = args[i]; + if (std.mem.eql(u8, arg, "--policy") or std.mem.startsWith(u8, arg, "--policy=")) { + policy_path = optionValue(args, &i, arg, "--policy", "lifecycle needs --policy"); + } else if (std.mem.eql(u8, arg, "--modulerc-out") or std.mem.startsWith(u8, arg, "--modulerc-out=")) { + modulerc_out = optionValue(args, &i, arg, "--modulerc-out", "--modulerc-out needs a path"); + } else if (std.mem.eql(u8, arg, "--easyconfigs") or std.mem.startsWith(u8, arg, "--easyconfigs=")) { + const v = optionValue(args, &i, arg, "--easyconfigs", "--easyconfigs needs a directory"); + easyconfigs.append(gpa, v) catch die("out of memory", .{}); + } else if (std.mem.eql(u8, arg, "--easystack") or std.mem.startsWith(u8, arg, "--easystack=")) { + const v = optionValue(args, &i, arg, "--easystack", "--easystack needs a file"); + easystacks.append(gpa, v) catch die("out of memory", .{}); + } else if (std.mem.eql(u8, arg, "--eb-stack") or std.mem.startsWith(u8, arg, "--eb-stack=")) { + eb_stack_opt = optionValue(args, &i, arg, "--eb-stack", "--eb-stack needs a path"); + } else if (std.mem.eql(u8, arg, "--rebuild-set") or std.mem.startsWith(u8, arg, "--rebuild-set=")) { + rebuild_set_opt = optionValue(args, &i, arg, "--rebuild-set", "--rebuild-set needs a file"); + } else if (std.mem.eql(u8, arg, "--now") or std.mem.startsWith(u8, arg, "--now=")) { + now_opt = optionValue(args, &i, arg, "--now", "--now needs an epoch"); + } else if (std.mem.eql(u8, arg, "--json")) { + json_out = true; + } else if (arg.len > 0 and arg[0] == '-') { + die("lifecycle does not know '{s}'", .{arg}); + } else if (usage_path.len == 0) { + usage_path = arg; + } else { + die("lifecycle does not know '{s}'", .{arg}); + } + } + + if (policy_path.len == 0) die("lifecycle needs --policy", .{}); + if (usage_path.len == 0) die("lifecycle needs a usage JSON file", .{}); + if (rebuild_set_opt != null and easyconfigs.items.len > 0) + die("lifecycle takes --easyconfigs or --rebuild-set, not both", .{}); + if (easystacks.items.len > 0 and easyconfigs.items.len == 0) + die("--easystack requires --easyconfigs", .{}); + if (modulerc_out) |out_p| { + if (std.mem.indexOfScalar(u8, out_p, '\n') != null or std.mem.indexOfScalar(u8, out_p, 0) != null) + die("--modulerc-out must not contain a newline", .{}); + } + for (easyconfigs.items) |ec_dir| { + const d = fsx.openDir(fsx.cwd, ec_dir, true) catch + die("easyconfigs directory {s} is not a directory", .{ec_dir}); + sys.close(d); + } + + var arena_state = std.heap.ArenaAllocator.init(gpa); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const now_raw: []const u8 = if (now_opt) |n| n else (env.get("EB_KEEP_LOGS_NOW") orelse ""); + const now_epoch: f64 = if (now_raw.len > 0) + (parseFiniteNonNeg(now_raw) orelse die("reference epoch must be a non-negative number, not '{s}'", .{now_raw})) + else blk: { + var ts: linux.timespec = undefined; + _ = linux.clock_gettime(.REALTIME, &ts); + break :blk @floatFromInt(ts.sec); + }; + const now_text: []const u8 = if (now_raw.len > 0) + now_raw + else + (arena.print("{d}", .{@as(i64, @intFromFloat(@floor(now_epoch)))}) catch die("out of memory", .{})); + + const policy_text = readRegularAll(gpa, policy_path, "policy"); + defer gpa.free(policy_text); + var policy_err: []const u8 = ""; + const policy = parsePolicy(arena, policy_text, &policy_err) catch |err| switch (err) { + error.BadPolicy => die("{s}: {s}", .{ policy_path, policy_err }), + error.OutOfMemory => die("out of memory", .{}), + }; + + const usage_text = readRegularAll(gpa, usage_path, "usage report"); + defer gpa.free(usage_text); + const usage_in = parseUsageJson(arena, usage_path, usage_text); + + const observed_days: ?u64 = if (usage_in.since_epoch) |s| + (if (now_epoch >= s) @as(u64, @intFromFloat(@floor((now_epoch - s) / 86400.0))) else 0) + else + null; + const observation_satisfied = if (observed_days) |d| d >= policy.min_observation_days else false; + + var entries: std.ArrayList(Entry) = .empty; + var candidates: std.ArrayList([]const u8) = .empty; + + for (usage_in.packages) |pkg| { + const easyconfig = arena.print("{s}-{s}.eb", .{ pkg.name, pkg.version }) catch die("out of memory", .{}); + const idle_days: ?u64 = if (pkg.last_epoch) |le| + (if (now_epoch >= le) @as(u64, @intFromFloat(@floor((now_epoch - le) / 86400.0))) else 0) + else if (pkg.hits >= policy.min_hits) + null + else + observed_days; + + var ent: Entry = .{ + .name = pkg.name, + .version = pkg.version, + .module = pkg.module, + .easyconfig = easyconfig, + .action = .keep, + .hits = pkg.hits, + .last = pkg.last, + .idle_days = idle_days, + .reason = "", + }; + + if (matchesKeep(policy.keep, pkg.module)) { + ent.action = .keep; + ent.reason = "protected by policy keep rule"; + } else if (pkg.hits >= policy.min_hits and pkg.last_epoch == null) { + ent.action = .keep; + ent.reason = "active hit has no timestamp"; + } else if (idle_days == null) { + ent.action = .keep; + ent.reason = "observation window unknown (usage --since not set)"; + } else { + const d = idle_days.?; + if (d < policy.warn_after_days) { + ent.action = .keep; + ent.reason = arena.print("idle {d}d < warn_after_days {d}", .{ d, policy.warn_after_days }) catch die("out of memory", .{}); + } else if (d < policy.forbid_after_days) { + ent.action = .warn; + ent.reason = arena.print("idle {d}d >= warn_after_days {d}", .{ d, policy.warn_after_days }) catch die("out of memory", .{}); + } else if (d < policy.remove_after_days) { + ent.action = .@"forbid-after"; + ent.reason = arena.print("idle {d}d >= forbid_after_days {d}", .{ d, policy.forbid_after_days }) catch die("out of memory", .{}); + } else if (!observation_satisfied) { + ent.action = .@"forbid-after"; + ent.blocked_reason = if (observed_days) |od| + (arena.print("observation window {d}d < min_observation_days {d}", .{ od, policy.min_observation_days }) catch die("out of memory", .{})) + else + "observation window unknown (usage --since not set)"; + ent.reason = ent.blocked_reason.?; + } else { + ent.action = .@"candidate-remove"; + ent.pending_candidate = true; + candidates.append(arena, easyconfig) catch die("out of memory", .{}); + } + } + entries.append(arena, ent) catch die("out of memory", .{}); + } + + std.mem.sort(Entry, entries.items, {}, lessEntry); + + var dependents_checked = false; + var rebuild_info: RebuildInfo = .{}; + + if (rebuild_set_opt) |rs_path| { + const rs_text = readRegularAll(gpa, rs_path, "rebuild-set"); + defer gpa.free(rs_text); + rebuild_info = parseRebuildSetJson(arena, rs_path, rs_text); + dependents_checked = true; + } else if (candidates.items.len > 0 and easyconfigs.items.len > 0) { + const path_env = env.get("PATH") orelse ""; + const eb_stack_exe = resolveExecutable(gpa, eb_stack_opt, path_env); + defer gpa.free(eb_stack_exe); + rebuild_info = runEbStackRebuildSet( + gpa, + arena, + env, + envp, + eb_stack_exe, + easyconfigs.items, + easystacks.items, + usage_in.packages, + candidates.items, + ); + dependents_checked = true; + } + + const now_sec: i64 = @intFromFloat(@floor(now_epoch)); + for (entries.items) |*ent| { + if (ent.pending_candidate) { + if (!dependents_checked) { + ent.action = .@"forbid-after"; + ent.blocked_reason = "dependent modules not checked (pass --easyconfigs or --rebuild-set)"; + ent.reason = ent.blocked_reason.?; + } else if (!rebuild_info.changed.contains(ent.easyconfig)) { + ent.action = .@"forbid-after"; + ent.blocked_reason = "candidate easyconfig absent from rebuild-set.json"; + ent.reason = ent.blocked_reason.?; + } else if (rebuild_info.missing.contains(ent.easyconfig)) { + ent.action = .@"forbid-after"; + ent.blocked_reason = "easyconfig missing from eb-stack graph"; + ent.reason = ent.blocked_reason.?; + } else if (rebuild_info.dependents.getPtr(ent.easyconfig)) |deps_list| { + var filtered: std.ArrayList([]const u8) = .empty; + for (deps_list.items) |dep| { + if (!std.mem.eql(u8, dep, ent.module)) { + filtered.append(arena, dep) catch die("out of memory", .{}); + } + } + if (filtered.items.len > 0) { + std.mem.sort([]const u8, filtered.items, {}, struct { + fn lt(_: void, a: []const u8, b: []const u8) bool { + return std.mem.lessThan(u8, a, b); + } + }.lt); + ent.dependents = filtered.items; + var joined: std.ArrayList(u8) = .empty; + for (filtered.items, 0..) |dep, idx| { + if (idx > 0) joined.appendSlice(arena, ", ") catch die("out of memory", .{}); + joined.appendSlice(arena, dep) catch die("out of memory", .{}); + } + ent.action = .warn; + ent.blocked_reason = arena.print("has installed dependents: {s}", .{joined.items}) catch die("out of memory", .{}); + ent.reason = ent.blocked_reason.?; + } else { + ent.action = .@"candidate-remove"; + ent.reason = arena.print("idle {d}d >= remove_after_days {d}; 0 dependents", .{ + ent.idle_days.?, + policy.remove_after_days, + }) catch die("out of memory", .{}); + } + } else { + ent.action = .@"candidate-remove"; + ent.reason = arena.print("idle {d}d >= remove_after_days {d}; 0 dependents", .{ + ent.idle_days.?, + policy.remove_after_days, + }) catch die("out of memory", .{}); + } + } + + if (ent.action == .warn and ent.blocked_reason == null) { + const d = ent.idle_days.?; + const until_forbid = (policy.forbid_after_days - d) + policy.forbid_grace_days; + const target_sec = now_sec + @as(i64, @intCast(until_forbid)) * 86400; + ent.forbid_after = formatUtcDate(arena, target_sec) catch die("out of memory", .{}); + } else if (ent.action == .@"forbid-after") { + const target_sec = now_sec + @as(i64, @intCast(policy.forbid_grace_days)) * 86400; + ent.forbid_after = formatUtcDate(arena, target_sec) catch die("out of memory", .{}); + } + } + + var counts: Counts = .{}; + for (entries.items) |ent| { + switch (ent.action) { + .keep => counts.keep += 1, + .warn => counts.warn += 1, + .@"forbid-after" => counts.@"forbid-after" += 1, + .@"candidate-remove" => counts.@"candidate-remove" += 1, + } + } + + const modulerc_lua = formatModulerc(arena, policy, entries.items) catch die("out of memory", .{}); + if (modulerc_out) |out_p| { + writeFileDurable(out_p, modulerc_lua); + } + + const body = if (json_out) + formatJson( + gpa, + arena, + usage_in, + policy_path, + now_text, + observed_days, + policy, + observation_satisfied, + dependents_checked, + counts, + entries.items, + modulerc_lua, + ) catch die("out of memory", .{}) + else + formatText(arena, entries.items) catch die("out of memory", .{}); + defer if (json_out) gpa.free(body); + + cli.out("{s}", .{body}); + return 0; +} + +test "parsePolicy reads thresholds, optional fields and keep globs" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var err_msg: []const u8 = ""; + const text = + \\[policy] + \\min_observation_days = 90 + \\warn_after_days = 60 + \\forbid_after_days = 180 + \\remove_after_days = 365 + \\forbid_grace_days = 14 + \\min_hits = 2 + \\keep = [ + \\ "GCCcore/*", + \\ "OpenMPI/5.0*", + \\] + \\warn_message = "Module {module} will be forbidden on {after}." + ; + const pol = try parsePolicy(arena, text, &err_msg); + try std.testing.expectEqual(@as(u64, 90), pol.min_observation_days); + try std.testing.expectEqual(@as(u64, 60), pol.warn_after_days); + try std.testing.expectEqual(@as(u64, 180), pol.forbid_after_days); + try std.testing.expectEqual(@as(u64, 365), pol.remove_after_days); + try std.testing.expectEqual(@as(u64, 14), pol.forbid_grace_days); + try std.testing.expectEqual(@as(u64, 2), pol.min_hits); + try std.testing.expectEqual(@as(usize, 2), pol.keep.len); + try std.testing.expect(matchesKeep(pol.keep, "GCCcore/15.2.0")); + try std.testing.expect(!matchesKeep(pol.keep, "GROMACS/2026.1")); +} + +test "parsePolicy refuses bad order or unknown keys" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var err_msg: []const u8 = ""; + + const bad_order = + \\min_observation_days = 90 + \\warn_after_days = 200 + \\forbid_after_days = 100 + \\remove_after_days = 365 + ; + try std.testing.expectError(error.BadPolicy, parsePolicy(arena, bad_order, &err_msg)); + + const unknown_key = + \\min_observation_days = 90 + \\warn_after_days = 60 + \\forbid_after_days = 180 + \\remove_after_days = 365 + \\bogus = 1 + ; + try std.testing.expectError(error.BadPolicy, parsePolicy(arena, unknown_key, &err_msg)); +} + +test "globMatch handles stars and question marks" { + try std.testing.expect(globMatch("GCCcore/*", "GCCcore/15.2.0")); + try std.testing.expect(!globMatch("GCCcore/*", "GCC/15.2.0")); + try std.testing.expect(globMatch("Foo/1.?-*", "Foo/1.0-foss")); + try std.testing.expect(!globMatch("Foo/1.?-*", "Foo/10.0-foss")); +} + +test "formatModulerc escapes Lua strings and emits warn, forbid and hide_version" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const pol: Policy = .{ + .min_observation_days = 90, + .warn_after_days = 60, + .forbid_after_days = 180, + .remove_after_days = 365, + }; + const entries = [_]Entry{ + .{ + .name = "KeepMe", + .version = "1.0", + .module = "KeepMe/1.0", + .easyconfig = "KeepMe-1.0.eb", + .action = .keep, + .hits = 10, + .last = "1700000000", + .idle_days = 1, + .reason = "idle 1d < warn_after_days 60", + }, + .{ + .name = "OldSim", + .version = "2.1", + .module = "OldSim/2.1", + .easyconfig = "OldSim-2.1.eb", + .action = .@"forbid-after", + .hits = 1, + .last = "1700000000", + .idle_days = 200, + .forbid_after = "2026-12-17", + .reason = "idle 200d >= forbid_after_days 180", + }, + .{ + .name = "DeadPkg", + .version = "0.9", + .module = "DeadPkg/0.9", + .easyconfig = "DeadPkg-0.9.eb", + .action = .@"candidate-remove", + .hits = 0, + .last = null, + .idle_days = 400, + .reason = "idle 400d >= remove_after_days 365; 0 dependents", + }, + }; + const lua = try formatModulerc(arena, pol, &entries); + try std.testing.expect(std.mem.indexOf(u8, lua, "KeepMe/1.0") == null); + try std.testing.expect(std.mem.indexOf(u8, lua, "name = \"OldSim/2.1\"") != null); + try std.testing.expect(std.mem.indexOf(u8, lua, "after = \"2026-12-17\"") != null); + try std.testing.expect(std.mem.indexOf(u8, lua, "hide_version(\"DeadPkg/0.9\")") != null); +} diff --git a/src/localtime.zig b/src/localtime.zig index 9885901..a995b95 100644 --- a/src/localtime.zig +++ b/src/localtime.zig @@ -102,6 +102,28 @@ pub const Zone = struct { } }; +/// A calendar date in the proleptic Gregorian calendar. +pub const CivilDate = struct { + year: i64, + month: u8, + day: u8, +}; + +/// Calendar date `days` days after 1970-01-01 in the proleptic Gregorian +/// calendar (Howard Hinnant's civil_from_days). +pub fn civilFromDays(days: i64) CivilDate { + const z = days + 719468; + const era = @divFloor(z, 146097); + const doe = z - era * 146097; + const yoe = @divFloor(doe - @divFloor(doe, 1460) + @divFloor(doe, 36524) - @divFloor(doe, 146096), 365); + const doy = doe - (365 * yoe + @divFloor(yoe, 4) - @divFloor(yoe, 100)); + const mp = @divFloor(5 * doy + 2, 153); + const d: u8 = @intCast(doy - @divFloor(153 * mp + 2, 5) + 1); + const m: u8 = @intCast(if (mp < 10) mp + 3 else mp - 9); + const y: i64 = yoe + era * 400 + @as(i64, if (m <= 2) 1 else 0); + return .{ .year = y, .month = m, .day = d }; +} + /// Days from 1970-01-01 to `y_in`-`m`-`d` in the proleptic Gregorian /// calendar, negative before it (Howard Hinnant's days_from_civil). `m` is /// 1..12; neither it nor `d` is checked. @@ -501,3 +523,16 @@ test "a rule string after a leading ':' is read, as glibc reads it" { // `TZ=: date -d 20261101 +%s` prints 1793491200, the UTC midnight. try std.testing.expectEqual(@as(?i64, 1793491200), colon.fromLocal(daysFromCivil(2026, 11, 1) * 86400)); } + +test "civilFromDays inverts daysFromCivil" { + const cases = [_]CivilDate{ + .{ .year = 1970, .month = 1, .day = 1 }, + .{ .year = 2024, .month = 2, .day = 29 }, + .{ .year = 2026, .month = 11, .day = 17 }, + .{ .year = 2027, .month = 12, .day = 31 }, + }; + for (cases) |c| { + const d = daysFromCivil(c.year, c.month, c.day); + try std.testing.expectEqual(c, civilFromDays(d)); + } +} diff --git a/src/main.zig b/src/main.zig index e5e0a42..9b97a71 100644 --- a/src/main.zig +++ b/src/main.zig @@ -18,6 +18,7 @@ const pending = @import("pending.zig"); const used_cmd = @import("used.zig"); const usage_cmd = @import("usage.zig"); const test_bundle = @import("test_bundle.zig"); +const lifecycle = @import("lifecycle.zig"); const syslog = @import("syslog.zig"); const Allocator = std.mem.Allocator; @@ -239,6 +240,40 @@ const usage_text = \\ Print one line of JSON instead of text. Default: text. \\ BUNDLE \\ Two or more test-bundle JSON files. Required. + \\ eb-keep-logs lifecycle --policy FILE [--modulerc-out FILE] + \\ [--easyconfigs DIR ...] [--easystack FILE ...] + \\ [--eb-stack PATH] [--rebuild-set FILE] + \\ [--now EPOCH] [--json] FILE + \\ Generate a reversible lifecycle plan and Lmod .modulerc.lua + \\ warnings/forbids from usage --json output and a TOML policy. Never + \\ deletes software or modulefiles. Requires a minimum observation + \\ window and checks installed dependents via eb-stack rebuild-set + \\ before allowing candidate-remove. Zig build only. + \\ --policy FILE, --policy=FILE + \\ TOML lifecycle policy file. Required. + \\ --modulerc-out FILE, --modulerc-out=FILE + \\ Write the generated Lmod .modulerc.lua warnings and forbids + \\ durably to FILE. Default: none; no file is written. + \\ --easyconfigs DIR, --easyconfigs=DIR + \\ Easyconfig tree passed to eb-stack rebuild-set to identify + \\ dependents of candidate removals. Repeatable. Default: none. + \\ --easystack FILE, --easystack=FILE + \\ Deployed easystack YAML passed to eb-stack rebuild-set. + \\ Repeatable. Default: none. + \\ --eb-stack PATH, --eb-stack=PATH + \\ Path to the eb-stack executable. Default: eb-stack on PATH. + \\ --rebuild-set FILE, --rebuild-set=FILE + \\ Precomputed eb-stack rebuild-set.json report instead of + \\ running eb-stack. Default: none. + \\ --now EPOCH, --now=EPOCH + \\ Reference epoch time in seconds for idle age and observation + \\ window checks. Default: the environment time override, else + \\ the current time. + \\ --json + \\ Print one line of JSON instead, with schema_version 1. + \\ FILE + \\ JSON output produced by eb-keep-logs usage --json (with + \\ selection "all"). Required. \\ eb-keep-logs -h | --help \\ Print this help on stdout and exit 0. \\ eb-keep-logs --version @@ -248,8 +283,8 @@ const usage_text = \\ PATH Where failed looks up eb. \\ EASYBUILD_TMPDIR failed copies a command's output only from below it; \\ unset or empty, it copies none. - \\ EB_KEEP_LOGS_NOW The time rotate takes as now, in seconds since the - \\ epoch; unset or empty, the current time. + \\ EB_KEEP_LOGS_NOW The time rotate and lifecycle take as now, in seconds + \\ since the epoch; unset or empty, the current time. \\ EB_PIPELINE_LOG_DIR rotate leaves alone the run that holds this path. \\ EB_KEEP_LOGS_SYSLOG Default --syslog target (journald, syslog, unix:///..., \\ journal+unix:///..., udp://host:port, tcp://host:port). @@ -260,7 +295,7 @@ const usage_text = \\ directory of zone files. \\ \\Exit status: - \\ 0 -h, --help or --version, or a failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle or test-compare that + \\ 0 -h, --help or --version, or a failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle, test-compare or lifecycle that \\ accepted its arguments, whatever it could keep, compress, remove or \\ read: what it could not do is reported on stderr. used with \\ --require-none also exits 2 when any path was listed. @@ -1189,11 +1224,12 @@ pub fn main(init: std.process.Init) u8 { if (std.mem.eql(u8, cmd, "pending")) return pending.cmdPending(gpa, args[2..]); if (std.mem.eql(u8, cmd, "test-bundle")) return test_bundle.cmdTestBundle(gpa, args[2..]); if (std.mem.eql(u8, cmd, "test-compare")) return test_bundle.cmdTestCompare(gpa, args[2..]); + if (std.mem.eql(u8, cmd, "lifecycle")) return lifecycle.cmdLifecycle(c.gpa, c.env, c.envp, args[2..]); if (cmd.len == 0) { cli.put(1, usage_text); return cli.usage_exit; } - die("unknown command '{s}' (failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle, test-compare, --help)", .{cmd}); + die("unknown command '{s}' (failed, rotate, why, history, loads, runs, used, usage, pending, test-bundle, test-compare, lifecycle, --help)", .{cmd}); } test { @@ -1206,6 +1242,7 @@ test { _ = used_cmd; _ = usage_cmd; _ = test_bundle; + _ = lifecycle; _ = syslog; } diff --git a/tests/differential.sh b/tests/differential.sh index 70f5fac..1e01035 100644 --- a/tests/differential.sh +++ b/tests/differential.sh @@ -28,6 +28,7 @@ declare -A deliberate=( ["used"]="\`used\`" ["usage"]="\`usage\`" ["pending"]="\`pending\`" + ["lifecycle"]="\`lifecycle\`" ) compat_deliberate=$(sed -n '/^## Deliberate differences$/,/^## /p' "${repo_root}/COMPAT.md") # The environment both runs get, as env(1) takes it. @@ -163,6 +164,9 @@ mkdir -p "${work}/software/GROMACS/1.0" "${work}/modules" compare "usage" usage --software "${work}/software" --modules "${work}/modules" "${work}/used.json" printf '%s\n' ' * [ ] flex-2.6.4.eb (module: flex/2.6.4)' > "${work}/dry.txt" compare "pending" pending "${work}/dry.txt" +printf '%s\n' 'min_observation_days = 90' 'warn_after_days = 60' 'forbid_after_days = 180' 'remove_after_days = 365' > "${work}/policy.toml" +"${TOOL_BIN}" usage --software "${work}/software" --modules "${work}/modules" --json "${work}/used.json" > "${work}/usage.json" +compare "lifecycle" lifecycle --policy "${work}/policy.toml" "${work}/usage.json" echo "${diffs} differences" [ "${diffs}" -eq 0 ] diff --git a/tests/test_json_schemas.sh b/tests/test_json_schemas.sh index 506bf22..a847ee5 100755 --- a/tests/test_json_schemas.sh +++ b/tests/test_json_schemas.sh @@ -44,6 +44,13 @@ cat > "${work}/xalt.json" < "${work}/lmod.log" +"${tool}" usage --software "${software}" --modules "${modules}" --since 100 --json "${work}/xalt.json" "${work}/lmod.log" > "${work}/usage.json" +cat > "${work}/policy.toml" <<'EOF' +min_observation_days = 90 +warn_after_days = 60 +forbid_after_days = 180 +remove_after_days = 365 +EOF fails=0 run() { @@ -125,6 +132,7 @@ run "test-bundle" validate test-bundle test-bundle \ --gpu "none" \ --out "${work}/bundle-b.json" run "test-compare" bash -c "\"${tool}\" test-compare --json \"${work}/bundle-a.json\" \"${work}/bundle-b.json\" | python3 \"${repo_root}/tests/json_schema_validate.py\" \"${schema_dir}/test-compare.schema.json\"" +run "lifecycle" validate lifecycle lifecycle --policy "${work}/policy.toml" --now 1794873600 --json "${work}/usage.json" echo "${fails} failed" [ "${fails}" -eq 0 ] \ No newline at end of file diff --git a/tests/test_lifecycle.sh b/tests/test_lifecycle.sh new file mode 100755 index 0000000..f47e6a8 --- /dev/null +++ b/tests/test_lifecycle.sh @@ -0,0 +1,323 @@ +#!/bin/bash +# Reversible module lifecycle planning and .modulerc.lua generation. +# TOOL_BIN=zig-out/bin/eb-keep-logs bash tests/test_lifecycle.sh +set -u + +tool="${TOOL_BIN:?TOOL_BIN names the Zig build}" +case "${tool}" in /*) ;; *) tool="${PWD}/${tool}" ;; esac + +fails=0 +check() { + if [ "$2" = "$3" ]; then + echo "ok $1" + else + { + echo "FAIL $1" + echo "expected: $3" + echo "got: $2" + } >&2 + fails=$((fails + 1)) + fi +} + +work=$(mktemp -d "${TMPDIR:-/tmp}/eb-keep-logs-lifecycle.XXXXXX") +trap 'rm -rf -- "${work}"' EXIT + +software="${work}/software" +modules="${work}/modules" +ec_dir="${work}/easyconfigs" +mkdir -p "${ec_dir}" +for mod in \ + "GCCcore/15.2.0" \ + "ActiveApp/2.0" \ + "WarnApp/1.0" \ + "ForbidApp/1.0" \ + "DeadApp/1.0" \ + "LibDep/1.0" \ + "MissingApp/1.0" +do + mkdir -p "${software}/${mod}" + pkg="${mod%%/*}" + mkdir -p "${modules}/${pkg}" + touch "${modules}/${mod}.lua" +done + +# Now = 2026-11-17T00:00:00Z = 1794873600 +now=1794873600 +day=86400 +since=$((now - 400 * day)) +t_active=$((now - 10 * day)) +t_warn=$((now - 100 * day)) +t_forbid=$((now - 200 * day)) +t_dead=$((now - 380 * day)) + +cat > "${work}/usage.log" < "${work}/usage.json" +"${tool}" usage --software "${software}" --modules "${modules}" --json "${work}/usage.log" > "${work}/usage-nosince.json" +"${tool}" usage --software "${software}" --modules "${modules}" --since "$((now - 30 * day))" --json "${work}/usage.log" > "${work}/usage-short.json" +"${tool}" usage --software "${software}" --modules "${modules}" --since "${since}" --unused --json "${work}/usage.log" > "${work}/usage-unused.json" + +cat > "${work}/policy.toml" <<'EOF' +[policy] +min_observation_days = 90 +warn_after_days = 60 +forbid_after_days = 180 +remove_after_days = 365 +forbid_grace_days = 30 +min_hits = 1 +keep = ["GCCcore/*"] +EOF + +cat > "${work}/easystack.yml" <<'EOF' +easyconfigs: + - ActiveApp-2.0.eb +EOF + +# Mock eb-stack that records its argv and returns rebuild-set.json: +# - DeadApp-1.0.eb: 0 dependents -> candidate-remove +# - LibDep-1.0.eb: runtime dependent ActiveApp/2.0 -> blocked (warn) +# - MissingApp-1.0.eb: listed in missing -> blocked (forbid-after) +mkdir -p "${work}/bin" +cat > "${work}/bin/eb-stack" <<'EOF' +#!/bin/bash +set -euo pipefail +cmd=$1; shift +[ "${cmd}" = "rebuild-set" ] +out_dir="" +installed="" +ec_dirs=0 +es_files=0 +changed_args=() +while [ $# -gt 0 ]; do + case "$1" in + --changed) changed_args+=("$2"); shift 2 ;; + --easyconfigs) [ -d "$2" ]; ec_dirs=$((ec_dirs + 1)); shift 2 ;; + --easystack) [ -f "$2" ]; es_files=$((es_files + 1)); shift 2 ;; + --installed) installed="$2"; shift 2 ;; + --out-dir) out_dir="$2"; shift 2 ;; + *) echo "unexpected arg: $1" >&2; exit 2 ;; + esac +done +[ -n "${out_dir}" ] && [ -n "${installed}" ] && [ "${ec_dirs}" -ge 1 ] +printf '%s\n' "${changed_args[@]}" > "${out_dir}/../recorded-changed.txt" +cp "${installed}" "${out_dir}/../recorded-installed.txt" +python3 - "${out_dir}/rebuild-set.json" "${changed_args[@]}" <<'PY' +import json, sys, os +out_path = sys.argv[1] +raw_args = sys.argv[2:] +changed = [] +for a in raw_args: + if a.endswith(".eb"): + changed.append(os.path.basename(a)) + else: + with open(a) as f: + for line in f: + s = line.strip() + if s: + changed.append(os.path.basename(s)) +changed = sorted(set(changed)) +missing = [c for c in changed if c == "MissingApp-1.0.eb"] +rows = [] +for c in changed: + if c in missing: + continue + mod = c[:-3].replace("-", "/", 1) + rows.append({ + "changed": c, + "dependent": mod, + "dependent_easyconfig": c, + "edge_kind": "changed", + "class": "must-rebuild", + "reason": "patched easyconfig", + }) + if c == "LibDep-1.0.eb": + rows.append({ + "changed": c, + "dependent": "ActiveApp/2.0", + "dependent_easyconfig": "ActiveApp-2.0.eb", + "edge_kind": "runtime", + "class": "no-rebuild", + "reason": "runtime dependency", + }) +doc = { + "schema_version": 5, + "generated_at": "2026-11-17T00:00:00Z", + "installed_source": "", + "symbols_checked": False, + "easystacks": [], + "changed": changed, + "reach": {}, + "missing": missing, + "unresolved": [], + "notes": [], + "counts": {"no-rebuild": 0, "review": 0, "must-rebuild": 0}, + "rows": rows, +} +with open(out_path, "w") as f: + json.dump(doc, f) +PY +EOF +chmod +x "${work}/bin/eb-stack" + +plan_json=$("${tool}" lifecycle \ + --policy "${work}/policy.toml" \ + --easyconfigs "${ec_dir}" \ + --easystack "${work}/easystack.yml" \ + --eb-stack "${work}/bin/eb-stack" \ + --modulerc-out "${work}/modulerc.lua" \ + --now "${now}" \ + --json "${work}/usage.json") + +check "lifecycle json classification and guards" "$(python3 -c ' +import json, sys +d = json.load(sys.stdin) +assert d["schema_version"] == 1 +assert d["kind"] == "lifecycle" +assert d["observed_days"] == 400 +assert d["observation_satisfied"] is True +assert d["dependents_checked"] is True +assert d["counts"] == {"keep": 2, "warn": 2, "forbid-after": 2, "candidate-remove": 1}, d["counts"] +by_mod = {e["module"]: e for e in d["entries"]} +assert by_mod["ActiveApp/2.0"]["action"] == "keep" +assert by_mod["GCCcore/15.2.0"]["action"] == "keep" +assert by_mod["WarnApp/1.0"]["action"] == "warn" +assert by_mod["WarnApp/1.0"]["forbid_after"] == "2027-03-07" +assert by_mod["ForbidApp/1.0"]["action"] == "forbid-after" +assert by_mod["ForbidApp/1.0"]["forbid_after"] == "2026-12-17" +assert by_mod["DeadApp/1.0"]["action"] == "candidate-remove" +assert by_mod["DeadApp/1.0"]["dependents"] == [] +assert by_mod["LibDep/1.0"]["action"] == "warn" +assert by_mod["LibDep/1.0"]["dependents"] == ["ActiveApp/2.0"] +assert by_mod["LibDep/1.0"]["forbid_after"] is None +assert by_mod["MissingApp/1.0"]["action"] == "forbid-after" +assert "missing from eb-stack graph" in by_mod["MissingApp/1.0"]["blocked_reason"] +print("ok") +' <<<"${plan_json}")" "ok" + +check "modulerc.lua written and matches json" "$(cmp -s "${work}/modulerc.lua" <(python3 -c 'import json,sys; sys.stdout.write(json.load(sys.stdin)["modulerc_lua"])' <<<"${plan_json}") && echo ok)" "ok" +check "modulerc.lua warns WarnApp" "$(grep -c 'name = "WarnApp/1.0"' "${work}/modulerc.lua")" "1" +check "modulerc.lua forbids ForbidApp" "$(grep -c 'name = "ForbidApp/1.0"' "${work}/modulerc.lua")" "1" +check "modulerc.lua hides and forbids DeadApp" "$(grep -c 'hide_version("DeadApp/1.0")' "${work}/modulerc.lua")" "1" +check "modulerc.lua does not forbid GCCcore" "$(grep -c 'GCCcore' "${work}/modulerc.lua")" "0" +check "modulerc.lua does not emit forbid for LibDep with dependents" "$(grep -c 'name = "LibDep/1.0"' "${work}/modulerc.lua")" "0" + +# Never deletes any software or module files +check "software tree intact" "$(find "${software}" -mindepth 2 -maxdepth 2 -type d | wc -l)" "7" +check "module tree intact" "$(find "${modules}" -name '*.lua' | wc -l)" "7" + +# Default text output starts with action token +plan_text=$(EB_KEEP_LOGS_NOW="${now}" "${tool}" lifecycle \ + --policy "${work}/policy.toml" \ + --easyconfigs "${ec_dir}" \ + --eb-stack "${work}/bin/eb-stack" \ + "${work}/usage.json") +check "text output lists candidate-remove DeadApp" "$(grep -c '^candidate-remove DeadApp/1.0 ' <<<"${plan_text}")" "1" +check "text output lists dependents on LibDep" "$(grep -c '^warn LibDep/1.0 .* dependents=ActiveApp/2.0 ' <<<"${plan_text}")" "1" + +# Observation window guard: missing --since blocks candidate-remove +nosince_json=$("${tool}" lifecycle --policy "${work}/policy.toml" --easyconfigs "${ec_dir}" --eb-stack "${work}/bin/eb-stack" --now "${now}" --json "${work}/usage-nosince.json") +check "usage without --since blocks candidate-remove" "$(python3 -c ' +import json, sys +d = json.load(sys.stdin) +assert d["observation_satisfied"] is False +assert d["counts"]["candidate-remove"] == 0 +print("ok") +' <<<"${nosince_json}")" "ok" + +# Observation window guard: short window (< min_observation_days) blocks candidate-remove +short_json=$("${tool}" lifecycle --policy "${work}/policy.toml" --easyconfigs "${ec_dir}" --eb-stack "${work}/bin/eb-stack" --now "${now}" --json "${work}/usage-short.json") +check "short observation window blocks candidate-remove" "$(python3 -c ' +import json, sys +d = json.load(sys.stdin) +assert d["observed_days"] == 30 +assert d["observation_satisfied"] is False +assert d["counts"]["candidate-remove"] == 0 +print("ok") +' <<<"${short_json}")" "ok" + +# Without --easyconfigs or --rebuild-set, candidate-remove is blocked +nocheck_json=$("${tool}" lifecycle --policy "${work}/policy.toml" --now "${now}" --json "${work}/usage.json") +check "omitting --easyconfigs blocks candidate-remove" "$(python3 -c ' +import json, sys +d = json.load(sys.stdin) +assert d["dependents_checked"] is False +assert d["counts"]["candidate-remove"] == 0 +by_mod = {e["module"]: e for e in d["entries"]} +assert by_mod["DeadApp/1.0"]["action"] == "forbid-after" +assert "dependent modules not checked" in by_mod["DeadApp/1.0"]["blocked_reason"] +print("ok") +' <<<"${nocheck_json}")" "ok" + +# Precomputed --rebuild-set JSON works without invoking eb-stack +cat > "${work}/precomputed.json" <<'EOF' +{ + "schema_version": 5, + "changed": ["DeadApp-1.0.eb", "LibDep-1.0.eb"], + "missing": [], + "rows": [ + {"changed": "DeadApp-1.0.eb", "dependent": "DeadApp/1.0", "dependent_easyconfig": "DeadApp-1.0.eb", "edge_kind": "changed", "class": "must-rebuild", "reason": "patched"}, + {"changed": "LibDep-1.0.eb", "dependent": "LibDep/1.0", "dependent_easyconfig": "LibDep-1.0.eb", "edge_kind": "changed", "class": "must-rebuild", "reason": "patched"}, + {"changed": "LibDep-1.0.eb", "dependent": "ActiveApp/2.0", "dependent_easyconfig": "ActiveApp-2.0.eb", "edge_kind": "builddependency", "class": "review", "reason": "builddep"} + ] +} +EOF +pre_json=$("${tool}" lifecycle --policy "${work}/policy.toml" --rebuild-set "${work}/precomputed.json" --now "${now}" --json "${work}/usage.json") +check "precomputed --rebuild-set checks dependents and blocks unlisted" "$(python3 -c ' +import json, sys +d = json.load(sys.stdin) +assert d["dependents_checked"] is True +by_mod = {e["module"]: e for e in d["entries"]} +assert by_mod["DeadApp/1.0"]["action"] == "candidate-remove" +assert by_mod["LibDep/1.0"]["action"] == "warn" +assert by_mod["MissingApp/1.0"]["action"] == "forbid-after" +assert "absent from rebuild-set.json" in by_mod["MissingApp/1.0"]["blocked_reason"] +print("ok") +' <<<"${pre_json}")" "ok" + +# Batch > 32 candidates uses changed.txt path with eb-stack +many_soft="${work}/many_soft" +many_mod="${work}/many_mod" +mkdir -p "${many_soft}" "${many_mod}" +for idx in $(seq -w 1 40); do + mkdir -p "${many_soft}/Pkg${idx}/1.0" "${many_mod}/Pkg${idx}" + touch "${many_mod}/Pkg${idx}/1.0.lua" +done +: > "${work}/empty.log" +"${tool}" usage --software "${many_soft}" --modules "${many_mod}" --since "${since}" --json "${work}/empty.log" > "${work}/many-usage.json" +many_json=$(PATH="${work}/bin:${PATH}" "${tool}" lifecycle --policy "${work}/policy.toml" --easyconfigs "${ec_dir}" --now "${now}" --json "${work}/many-usage.json") +check "more than 32 candidates batch through changed.txt" "$(python3 -c ' +import json, sys +d = json.load(sys.stdin) +assert d["counts"]["candidate-remove"] == 40 +print("ok") +' <<<"${many_json}")" "ok" + +# Refusal cases +"${tool}" lifecycle --policy "${work}/policy.toml" --now "${now}" "${work}/usage-unused.json" >/dev/null 2>"${work}/err" +check "partial usage selection refused" "$?" "2" + +ln -s "${work}/nowhere.lua" "${work}/sym-modulerc.lua" +"${tool}" lifecycle --policy "${work}/policy.toml" --modulerc-out "${work}/sym-modulerc.lua" --now "${now}" "${work}/usage.json" >/dev/null 2>"${work}/err" +check "symlink --modulerc-out refused" "$?" "2" + +cat > "${work}/bad-policy.toml" <<'EOF' +min_observation_days = 90 +warn_after_days = 200 +forbid_after_days = 100 +remove_after_days = 365 +EOF +"${tool}" lifecycle --policy "${work}/bad-policy.toml" --modulerc-out "${work}/should-not-exist.lua" --now "${now}" "${work}/usage.json" >/dev/null 2>"${work}/err" +check "invalid policy order refused" "$?" "2" +check "no modulerc written on error" "$(test -e "${work}/should-not-exist.lua" && echo bad || echo ok)" "ok" + +PATH="/bin:/usr/bin" "${tool}" lifecycle --policy "${work}/policy.toml" --easyconfigs "${ec_dir}" --now "${now}" "${work}/usage.json" >/dev/null 2>"${work}/err" +check "missing eb-stack on PATH refused" "$?" "2" + +echo "${fails} failed" +[ "${fails}" -eq 0 ] diff --git a/tests/test_output_stability.sh b/tests/test_output_stability.sh index 747d554..4f346cb 100755 --- a/tests/test_output_stability.sh +++ b/tests/test_output_stability.sh @@ -1,10 +1,26 @@ #!/bin/bash -# Assert selected existing command outputs stay byte-identical. +# Assert existing default command outputs stay byte-identical after opt-in additions. # TOOL_BIN=zig-out/bin/eb-keep-logs bash tests/test_output_stability.sh set -u +repo_root=$(cd "$(dirname "$0")/.." && pwd) tool="${TOOL_BIN:?TOOL_BIN names the Zig build}" case "${tool}" in /*) ;; *) tool="${PWD}/${tool}" ;; esac +old_bin="${repo_root}/eb-keep-logs" + +fails=0 +check() { + if [ "$2" = "$3" ]; then + echo "ok $1" + else + { + echo "FAIL $1" + echo "expected: $3" + echo "got: $2" + } >&2 + fails=$((fails + 1)) + fi +} work=$(mktemp -d "${TMPDIR:-/tmp}/eb-keep-logs-stability.XXXXXX") trap 'rm -rf -- "${work}"' EXIT @@ -17,6 +33,7 @@ expect() { status=$? if [ "${status}" -ne "${expected_status}" ]; then echo "FAIL ${label}: status ${status} != ${expected_status}" >&2 + fails=$((fails + 1)) return 1 fi printf '%s' "${expected_out}" > "${work}/exp.out" @@ -24,11 +41,13 @@ expect() { if ! /usr/bin/cmp -s "${work}/exp.out" "${work}/out"; then echo "FAIL ${label}: stdout differs" >&2 /usr/bin/diff -u "${work}/exp.out" "${work}/out" >&2 || true + fails=$((fails + 1)) return 1 fi if ! /usr/bin/cmp -s "${work}/exp.err" "${work}/err"; then echo "FAIL ${label}: stderr differs" >&2 /usr/bin/diff -u "${work}/exp.err" "${work}/err" >&2 || true + fails=$((fails + 1)) return 1 fi echo "ok ${label}" @@ -42,3 +61,62 @@ expect() { PATH="${work}/empty" expect "failed no eb on PATH" 0 "No EasyBuild log to keep for Foo: eb is not on PATH. " "" "${tool}" failed 3 Foo logs ) + +# failed and rotate match the reference shell implementation byte-for-byte. +mkdir -p "${work}/bin" "${work}/logs-o" "${work}/logs-n" +printf '== processing EasyBuild easyconfig /ec/Foo-1.0.eb\nerror: bad\n' > "${work}/eb.log" +cat > "${work}/bin/eb" < "${work}/failed-o.out" 2> "${work}/failed-o.err" +PATH="${work}/bin:${PATH}" "${tool}" failed 1 Foo-1.0 "${work}/logs-n" > "${work}/failed-n.out" 2> "${work}/failed-n.err" +sed "s|${work}/logs-o|LOGDIR|g" "${work}/failed-o.out" > "${work}/failed-o.norm" +sed "s|${work}/logs-n|LOGDIR|g" "${work}/failed-n.out" > "${work}/failed-n.norm" +check "failed stdout byte-identical" "$(cmp -s "${work}/failed-o.norm" "${work}/failed-n.norm" && echo ok)" "ok" +check "failed stderr byte-identical" "$(cmp -s "${work}/failed-o.err" "${work}/failed-n.err" && echo ok)" "ok" +check "failed kept log byte-identical" "$(cmp -s "${work}/logs-o/Foo-1.0.failed.ebout.log" "${work}/logs-n/Foo-1.0.failed.ebout.log" && echo ok)" "ok" + +mkdir -p "${work}/rot-o/20261001_120000" "${work}/rot-n/20261001_120000" +printf 'log line\n' > "${work}/rot-o/20261001_120000/a.log" +printf 'log line\n' > "${work}/rot-n/20261001_120000/a.log" +EB_KEEP_LOGS_NOW=1794873600 "${old_bin}" rotate "${work}/rot-o" --compress-after 5 > "${work}/rot-o.out" 2> "${work}/rot-o.err" +EB_KEEP_LOGS_NOW=1794873600 "${tool}" rotate "${work}/rot-n" --compress-after 5 > "${work}/rot-n.out" 2> "${work}/rot-n.err" +sed "s|${work}/rot-o|ROOT|g" "${work}/rot-o.out" > "${work}/rot-o.norm" +sed "s|${work}/rot-n|ROOT|g" "${work}/rot-n.out" > "${work}/rot-n.norm" +check "rotate stdout byte-identical" "$(cmp -s "${work}/rot-o.norm" "${work}/rot-n.norm" && echo ok)" "ok" +check "rotate stderr byte-identical" "$(cmp -s "${work}/rot-o.err" "${work}/rot-n.err" && echo ok)" "ok" + +# Zig-only subcommands produce exact unchanged outputs. +why_out=$(cd "${repo_root}" && "${tool}" why --root tests/fixtures/build-logs 20261117_041222) +check "why default output unchanged" "$(printf '%s\n' "${why_out}" | head -n 7)" $'tests/fixtures/build-logs/20261117_041222: 2 failed builds of 3\n\nsapphirerapids/hello-2.12.1-GCCcore-13.3.0 failed in the sanitycheck step, after 2m54s\n package: hello/2.12.1-GCCcore-13.3.0\n easyconfig: /home/alice/corpus/cases/sanity/hello-2.12.1-GCCcore-13.3.0.eb\n EasyBuild: Sanity check failed: no file found at '\''lib/libhello.a'\'' in /home/alice/eessi/versions/2025.06/software/linux/x86_64/intel/sapphirerapids/software/hello/2.12.1-GCCcore-13.3.0 + log: tests/fixtures/build-logs/20261117_041222/sapphirerapids/hello-2.12.1-GCCcore-13.3.0.failed.ebout.log:258' + +hist_out=$(cd "${repo_root}" && "${tool}" history --root tests/fixtures/build-logs hello) +check "history default output unchanged" "${hist_out}" $'hello: 11 builds under tests/fixtures/build-logs: 3 succeeded, 8 failed\n\nRUN TARGET OUTCOME DURATION NAME\n20261117_035825 sapphirerapids failed 48s hello-2.12.1-GCCcore-14.2.0\n20261117_040032 sapphirerapids failed 41s hello-2.12.1-GCCcore-14.2.0\n20261117_040119 sapphirerapids failed - hello-2.12.1-GCCcore-14.2.0\n20261117_040311 sapphirerapids failed 2m24s hello-2.12.1-GCCcore-14.2.0\n20261117_040615 sapphirerapids failed 3m06s hello-2.12.1-GCCcore-14.2.0\n20261117_040928 sapphirerapids failed 2m30s hello-2.12.1-GCCcore-14.2.0\n20261117_041222 generic success 2m50s hello-2.12.1-GCCcore-14.2.0\n20261117_041222 sapphirerapids failed 2m54s hello-2.12.1-GCCcore-13.3.0\n20261117_041222 sapphirerapids failed 2m33s hello-2.12.1-GCCcore-14.2.0\n20261117_042207 sapphirerapids success 2m40s hello-2.12.1-GCCcore-13.3.0\n20261117_042207 sapphirerapids success 2m52s hello-2.12.1-GCCcore-14.2.0' + +printf 'user=alice module=GROMACS/2024.1 path=/cvmfs/g.lua host=login01 time=1\n' > "${work}/loads.log" +check "loads default output unchanged" "$("${tool}" loads --module GROMACS/2024.1 "${work}/loads.log")" \ + "user=alice host=login01 time=1 path=/cvmfs/g.lua" + +printf '%s\n' '{"userT":{"user":"ada","exec_path":"/opt/site/bin/gmx","syshost":"login01","start_time":"1428767000"}}' > "${work}/runs.json" +check "runs default output unchanged" "$("${tool}" runs --exec gmx "${work}/runs.json")" \ + "user=ada host=login01 time=1428767000 path=/opt/site/bin/gmx" + +mkdir -p "${work}/soft/Foo/1.0" "${work}/mods/Foo" +touch "${work}/mods/Foo/1.0.lua" +printf '%s\n' "{\"userT\":{\"user\":\"u\",\"exec_path\":\"${work}/soft/Foo/1.0/bin/foo\",\"start_time\":\"200\"}}" > "${work}/used.json" +check "used default output unchanged" "$("${tool}" used --prefix "${work}/soft/Foo/1.0" "${work}/used.json")" \ + "path=${work}/soft/Foo/1.0/bin/foo kind=run last=200 count=1" + +check "usage default output unchanged" "$("${tool}" usage --software "${work}/soft" --modules "${work}/mods" "${work}/used.json")" \ + "active Foo/1.0 hits=1 last=200" + +printf ' * [ ] $CFGS/f/flex/flex-2.6.4.eb (module: flex/2.6.4)\n' > "${work}/dry.txt" +check "pending default output unchanged" "$("${tool}" pending "${work}/dry.txt")" \ + "build flex/2.6.4" + +echo "${fails} failed" +[ "${fails}" -eq 0 ]