diff --git a/changelog.d/+dev-envrc.feature.md b/changelog.d/+dev-envrc.feature.md new file mode 100644 index 0000000..592c659 --- /dev/null +++ b/changelog.d/+dev-envrc.feature.md @@ -0,0 +1 @@ +`eb-local dev --envrc MODULE...` writes the line that evaluates that `dev` to `.envrc` in the working directory, between marker lines it owns, and updates it on a later run without touching other lines. With direnv the build environment then follows the directory, in every shell direnv has a hook for. diff --git a/changelog.d/+dev-lock.feature.md b/changelog.d/+dev-lock.feature.md new file mode 100644 index 0000000..b82a9cf --- /dev/null +++ b/changelog.d/+dev-lock.feature.md @@ -0,0 +1 @@ +`eb-local dev --lock MODULE...` writes `eb-local.lock`: the stack, the EESSI version, the CPU and accelerator target and every loaded module with its version, in load order. `eb-local dev --locked` loads exactly that list from a purged shell, refuses a lock of another stack, and loads none of it on a different CPU target. Node types that share a CPU target share a lock. diff --git a/changelog.d/+dev-pure.feature.md b/changelog.d/+dev-pure.feature.md new file mode 100644 index 0000000..d552a05 --- /dev/null +++ b/changelog.d/+dev-pure.feature.md @@ -0,0 +1 @@ +`eb-local dev --pure` runs `module purge` and unsets `LD_LIBRARY_PATH`, `LD_PRELOAD` and `LD_AUDIT` before the stack loads, so modules and loader variables the shell already had do not reach the build environment. diff --git a/docs/orgmode/reference/cli.org b/docs/orgmode/reference/cli.org index df75f1c..e889809 100644 --- a/docs/orgmode/reference/cli.org +++ b/docs/orgmode/reference/cli.org @@ -12,7 +12,7 @@ eb-local [OPTION]... --explain [eb options] eb-local eb [--] [eb options] eb-local --explain eb [--] [eb options] eb-local [OPTION]... env [--strict] [--json] -eb-local [OPTION]... dev [--strict] [--json] [--buildenv MODULE] [MODULE...] +eb-local [OPTION]... dev [--strict] [--pure] [--json] [--buildenv MODULE] [--envrc] [--lock | --locked] [MODULE...] eb-local rpath-wrappers [--for FAMILY | --unwrapped] eb-local check [--recursive] [--quiet] [--json] FILE... eb-local check (--module NAME/VERSION... | --dev | --prefix DIR) [--recursive] [--quiet] [--json] @@ -60,13 +60,17 @@ directory you own. eb-local env [--strict] [--json] lines that make your installed modules visible; --strict exits when the EESSI init file is missing (also when EB_LOCAL_ENV_STRICT=1) - eb-local dev [--strict] [--json] [--buildenv MODULE] [MODULE...] + eb-local dev [--strict] [--pure] [--json] [--buildenv MODULE] [--envrc] [--lock | --locked] [MODULE...] the same, plus MODULEs and a build environment, so gcc, make, cmake and meson just work; --strict exits when the EESSI init file is missing (also when EB_LOCAL_ENV_STRICT=1); --buildenv names the build environment to load; - --json prints the change as JSON, for env too + --json prints the change as JSON, for env too; + --envrc writes that line to .envrc for direnv; + --pure purges loaded modules and LD_* first; + --lock records what loads in eb-local.lock, + --locked loads exactly that again eb-local rpath-wrappers [--for FAMILY | --unwrapped] lines that put RPATH wrappers for the compilers on PATH first; dev uses them when no build @@ -319,6 +323,26 @@ Print the lines =env= prints, then the lines that load the modules =dev.modules= | =--strict= | Check that the EESSI init file exists in the emitted lines and exit 2 if absent (also when =EB_LOCAL_ENV_STRICT=1=). | | =--json= | Print the change the lines make to the environment as one line of JSON, as =env --json= does, with =kind= =dev=. | | =--buildenv MODULE= | Load MODULE as the build environment whatever the compilers, as =dev.buildenv= does. | +| =--envrc= | Write the line that evaluates this =dev= to =.envrc= in the working directory, for direnv, and print nothing on stdout. | + +=dev --envrc MODULE...= writes three lines to =.envrc=: + +#+begin_src sh +# >>> eb-local dev >>> rerun `eb-local dev --envrc` to change these lines +eval "$(/path/to/eb-local dev MODULE...)" +# <<< eb-local dev <<< +#+end_src + +The program is named by its own path, and =--config=, =--profile=, =--project= and =--buildenv= are repeated on the line. A block already in the file is replaced and every other line is kept; a file without one gets it appended. A begin line without its end line is refused. Run =direnv allow= once per change. direnv evaluates the file in bash and gives the calling shell the change, so the environment follows the directory in bash, zsh, fish, nushell and the other shells direnv has a hook for, and is taken back on leaving it. As with =--json=, only variables reach the calling shell. +| =--pure= | Before anything loads, run =module purge= and unset =LD_LIBRARY_PATH=, =LD_PRELOAD= and =LD_AUDIT=. | +| =--lock= | Run the lines in a bash child and write what loaded to =eb-local.lock= in the working directory; print nothing on stdout. | +| =--locked= | Print the lines that load exactly the modules of =eb-local.lock=, from a purged shell. Takes no MODULE. | + +=dev --lock MODULE...= records what a dev environment resolved to: =stack.module=, the EESSI version, the CPU target (=EESSI_SOFTWARE_SUBDIR=), the accelerator target, and every loaded module with its version, in load order, the build environment among them. The file is one line of JSON with =kind= =dev-lock=. It is not written when the lines load no module. + +=dev --locked= loads that list and nothing else: it purges first, as =--pure= does, and names every module in full, so Lmod resolves nothing. A lock written with another =stack.module= is refused. When the shell's CPU target is not the lock's, the lines say so and load none of its modules; node types with the same CPU target share their binaries, so a lock from one loads on the other. =--json= and =--envrc= work with =--locked=: =dev --envrc --locked= writes the locked line to =.envrc=. + +Without =--pure=, the modules the shell already has stay loaded beside the ones =dev= names, and the loader variables stay as they are, with a warning for =LD_LIBRARY_PATH=. With it, the build environment is the stack, the named modules and the build environment only. Sticky modules survive =module purge=. A site that preloads a library for every process (XALT through =LD_PRELOAD=) loses that in this shell. A =MODULE= starts with a letter or digit and holds letters, digits, =.=, =_=, =+=, =-= and single slashes, with none at the end. With =LD_LIBRARY_PATH= set in the shell, =dev= warns on stderr. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index a8c2eae..55a995b 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -14,7 +14,7 @@ eb-local [OPTION]... --explain [eb options] eb-local eb [--] [eb options] eb-local --explain eb [--] [eb options] eb-local [OPTION]... env [--strict] [--json] -eb-local [OPTION]... dev [--strict] [--json] [--buildenv MODULE] [MODULE...] +eb-local [OPTION]... dev [--strict] [--pure] [--json] [--buildenv MODULE] [--envrc] [--lock | --locked] [MODULE...] eb-local rpath-wrappers [--for FAMILY | --unwrapped] eb-local check [--recursive] [--quiet] [--json] FILE... eb-local check (--module NAME/VERSION... | --dev | --prefix DIR) [--recursive] [--quiet] [--json] @@ -62,13 +62,17 @@ directory you own. eb-local env [--strict] [--json] lines that make your installed modules visible; --strict exits when the EESSI init file is missing (also when EB_LOCAL_ENV_STRICT=1) - eb-local dev [--strict] [--json] [--buildenv MODULE] [MODULE...] + eb-local dev [--strict] [--pure] [--json] [--buildenv MODULE] [--envrc] [--lock | --locked] [MODULE...] the same, plus MODULEs and a build environment, so gcc, make, cmake and meson just work; --strict exits when the EESSI init file is missing (also when EB_LOCAL_ENV_STRICT=1); --buildenv names the build environment to load; - --json prints the change as JSON, for env too + --json prints the change as JSON, for env too; + --envrc writes that line to .envrc for direnv; + --pure purges loaded modules and LD_* first; + --lock records what loads in eb-local.lock, + --locked loads exactly that again eb-local rpath-wrappers [--for FAMILY | --unwrapped] lines that put RPATH wrappers for the compilers on PATH first; dev uses them when no build @@ -321,6 +325,28 @@ Print the lines `env` prints, then the lines that load the modules `dev.modules` | `--strict` | Check that the EESSI init file exists in the emitted lines and exit 2 if absent (also when `EB_LOCAL_ENV_STRICT=1`). | | `--json` | Print the change the lines make to the environment as one line of JSON, as `env --json` does, with `kind` `dev`. | | `--buildenv MODULE` | Load MODULE as the build environment whatever the compilers, as `dev.buildenv` does. | +| `--envrc` | Write the line that evaluates this `dev` to `.envrc` in the working directory, for direnv, and print nothing on stdout. | + +`dev --envrc MODULE...` writes three lines to `.envrc`: + +```sh +# >>> eb-local dev >>> rerun `eb-local dev --envrc` to change these lines +eval "$(/path/to/eb-local dev MODULE...)" +# <<< eb-local dev <<< +``` + +The program is named by its own path, and `--config`, `--profile`, `--project` and `--buildenv` are repeated on the line. A block already in the file is replaced and every other line is kept; a file without one gets it appended. A begin line without its end line is refused. Run `direnv allow` once per change. direnv evaluates the file in bash and gives the calling shell the change, so the environment follows the directory in bash, zsh, fish, nushell and the other shells direnv has a hook for, and is taken back on leaving it. As with `--json`, only variables reach the calling shell. + +| `--pure` | Before anything loads, run `module purge` and unset `LD_LIBRARY_PATH`, `LD_PRELOAD` and `LD_AUDIT`. | +|---|---| +| `--lock` | Run the lines in a bash child and write what loaded to `eb-local.lock` in the working directory; print nothing on stdout. | +| `--locked` | Print the lines that load exactly the modules of `eb-local.lock`, from a purged shell. Takes no MODULE. | + +`dev --lock MODULE...` records what a dev environment resolved to: `stack.module`, the EESSI version, the CPU target (`EESSI_SOFTWARE_SUBDIR`), the accelerator target, and every loaded module with its version, in load order, the build environment among them. The file is one line of JSON with `kind` `dev-lock`. It is not written when the lines load no module. + +`dev --locked` loads that list and nothing else: it purges first, as `--pure` does, and names every module in full, so Lmod resolves nothing. A lock written with another `stack.module` is refused. When the shell's CPU target is not the lock's, the lines say so and load none of its modules; node types with the same CPU target share their binaries, so a lock from one loads on the other. `--json` and `--envrc` work with `--locked`: `dev --envrc --locked` writes the locked line to `.envrc`. + +Without `--pure`, the modules the shell already has stay loaded beside the ones `dev` names, and the loader variables stay as they are, with a warning for `LD_LIBRARY_PATH`. With it, the build environment is the stack, the named modules and the build environment only. Sticky modules survive `module purge`. A site that preloads a library for every process (XALT through `LD_PRELOAD`) loses that in this shell. A `MODULE` starts with a letter or digit and holds letters, digits, `.`, `_`, `+`, `-` and single slashes, with none at the end. With `LD_LIBRARY_PATH` set in the shell, `dev` warns on stderr. diff --git a/src/main.zig b/src/main.zig index 74f2ac8..40c2458 100644 --- a/src/main.zig +++ b/src/main.zig @@ -65,13 +65,17 @@ const usage_text = \\ eb-local env [--strict] [--json] lines that make your installed modules visible; \\ --strict exits when the EESSI init file is missing \\ (also when EB_LOCAL_ENV_STRICT=1) - \\ eb-local dev [--strict] [--json] [--buildenv MODULE] [MODULE...] + \\ eb-local dev [--strict] [--pure] [--json] [--buildenv MODULE] [--envrc] [--lock | --locked] [MODULE...] \\ the same, plus MODULEs and a build environment, \\ so gcc, make, cmake and meson just work; \\ --strict exits when the EESSI init file is missing \\ (also when EB_LOCAL_ENV_STRICT=1); \\ --buildenv names the build environment to load; - \\ --json prints the change as JSON, for env too + \\ --json prints the change as JSON, for env too; + \\ --envrc writes that line to .envrc for direnv; + \\ --pure purges loaded modules and LD_* first; + \\ --lock records what loads in eb-local.lock, + \\ --locked loads exactly that again \\ eb-local rpath-wrappers [--for FAMILY | --unwrapped] \\ lines that put RPATH wrappers for the compilers \\ on PATH first; dev uses them when no build @@ -289,7 +293,16 @@ const env_strict_prelude = \\ ; -fn environmentLines(c: Ctx, s: config.Settings, p: Place, strict: bool) []const u8 { +/// `dev --pure`: start from no loaded modules and no loader variables, so +/// what the shell had before does not reach the build environment. Sticky +/// modules stay, as `module purge` leaves them. +const pure_prelude = + \\module purge >/dev/null 2>&1 || true + \\unset LD_LIBRARY_PATH LD_PRELOAD LD_AUDIT + \\ +; + +fn environmentLines(c: Ctx, s: config.Settings, p: Place, strict: bool, pure: bool) []const u8 { const a = c.a; var out: std.ArrayList(u8) = .empty; const add = struct { @@ -299,6 +312,7 @@ fn environmentLines(c: Ctx, s: config.Settings, p: Place, strict: bool) []const }.f; if (strict) add(a, &out, env_strict_prelude); add(a, &out, lmod_init); + if (pure) add(a, &out, pure_prelude); add(a, &out, str(a, "module unload {s} >/dev/null 2>&1 || true\n", .{q(a, s.extend_module.value)})); add(a, &out, \\unset EESSI_CVMFS_INSTALL EESSI_SITE_INSTALL EESSI_USER_INSTALL EESSI_PROJECT_INSTALL @@ -366,6 +380,8 @@ fn replay(a: Allocator, command: []const u8, s: config.Settings, g: Global, cano if (s.project.value != null) parts.appendSlice(a, &.{ "--project", q(a, canonical_prefix) }) catch oom(); parts.append(a, command) catch oom(); if (std.mem.eql(u8, command, "dev")) { + if (g.pure) parts.append(a, "--pure") catch oom(); + if (g.locked) parts.append(a, "--locked") catch oom(); if (g.buildenv) |p| parts.appendSlice(a, &.{ "--buildenv", q(a, p) }) catch oom(); for (extra) |m| parts.append(a, q(a, m)) catch oom(); } @@ -477,6 +493,14 @@ const Global = struct { env_strict: bool = false, /// One line of JSON on stdout instead of the human report. json_out: bool = false, + /// `dev --envrc`: write the dev line to `.envrc` here, for direnv. + envrc: bool = false, + /// `dev --pure`: purge the loaded modules and the loader variables first. + pure: bool = false, + /// `dev --lock`: write what the dev lines load to `eb-local.lock`. + lock: bool = false, + /// `dev --locked`: load the modules of `eb-local.lock`, and only those. + locked: bool = false, }; fn nonEmpty(v: ?[]const u8) ?[]const u8 { @@ -635,7 +659,10 @@ fn envVarLess(_: void, x: EnvVar, y: EnvVar) bool { /// Run `lines` in a bash child and print, as one JSON document, how its /// environment then differs from this process's. The child's own output goes /// to stderr, so stdout carries the document alone. -fn emitEnvChange(c: Ctx, kind: []const u8, lines: []const u8) u8 { +/// Run `lines` in a bash child and return its environment afterwards, as +/// `env -0` prints it. What the lines print goes to stderr. Null, with a +/// message, when the child cannot run or the lines fail. +fn runLines(c: Ctx, kind: []const u8, lines: []const u8) ?[]const u8 { const a = c.a; const bash = (proc.which(a, "bash", c.env.get("PATH") orelse "/usr/bin:/bin") catch null) orelse "/bin/bash"; const script = str(a, "exec 3>&1 1>&2\n{s}\nenv -0 >&3\n", .{lines}); @@ -645,16 +672,32 @@ fn emitEnvChange(c: Ctx, kind: []const u8, lines: []const u8) u8 { .timeout_ms = 300_000, }) catch |e| { warn("could not run the {s} lines in bash: {s}", .{ kind, @errorName(e) }); - return 1; + return null; }; if (r.stderr.len > 0) cli.put(2, r.stderr); if (r.term.code() != 0) { - warn("the {s} lines failed in bash (status {d}); no change is printed", .{ kind, r.term.code() }); - return 1; + warn("the {s} lines failed in bash (status {d}); nothing is written", .{ kind, r.term.code() }); + return null; } + return r.stdout; +} + +/// The value of `name` in the output of `env -0`, or "" when it is not set. +fn env0Value(env0: []const u8, name: []const u8) []const u8 { + var it = std.mem.splitScalar(u8, env0, 0); + while (it.next()) |entry| { + const eq = std.mem.indexOfScalar(u8, entry, '=') orelse continue; + if (std.mem.eql(u8, entry[0..eq], name)) return entry[eq + 1 ..]; + } + return ""; +} + +fn emitEnvChange(c: Ctx, kind: []const u8, lines: []const u8) u8 { + const a = c.a; + const env0 = runLines(c, kind, lines) orelse return 1; var set: std.ArrayList(EnvVar) = .empty; var after: std.StringHashMapUnmanaged(void) = .empty; - var it = std.mem.splitScalar(u8, r.stdout, 0); + var it = std.mem.splitScalar(u8, env0, 0); while (it.next()) |entry| { const eq = std.mem.indexOfScalar(u8, entry, '=') orelse continue; const name = entry[0..eq]; @@ -681,13 +724,164 @@ fn emitEnvChange(c: Ctx, kind: []const u8, lines: []const u8) u8 { fn cmdEnv(c: Ctx, s: config.Settings, g: Global) u8 { const p = place(c, s); - const lines = environmentLines(c, s, p, envStrict(c, g)); + const lines = environmentLines(c, s, p, envStrict(c, g), false); if (g.json_out) return emitEnvChange(c, "env", lines); evalHint(replay(c.a, "env", s, g, p.prefix, &.{})); cli.put(1, lines); return 0; } +/// The first and last line of the lines `dev --envrc` owns in `.envrc`. +/// What stands outside them is the user's and is kept. +const envrc_begin = "# >>> eb-local dev >>>"; +const envrc_end = "# <<< eb-local dev <<<"; + +/// Write `eval "$(eb-local WORDS)"` between the markers of `.envrc` in the +/// working directory, for direnv: it evaluates the file in bash and gives the +/// calling shell the change, whatever that shell is, and takes it back on +/// leaving the directory. A block already there is replaced; other lines +/// stay. The program is named by its own path, since a shell that direnv +/// starts need not have eb-local on PATH. +fn writeEnvrc(c: Ctx, words: []const u8) u8 { + const a = c.a; + var buf: [sys.path_max]u8 = undefined; + const exe = selfExe(&buf) orelse "eb-local"; + const block = str(a, "{s} rerun `eb-local dev --envrc` to change these lines\neval \"$({s} {s})\"\n{s}\n", .{ envrc_begin, q(a, exe), words, envrc_end }); + var exists_already = false; + const old: []const u8 = if (fsx.openRegular(fsx.cwd, ".envrc", true)) |opened| blk: { + defer sys.close(opened.fd); + exists_already = true; + break :blk fsx.readAll(a, opened.fd, 1 << 20) catch |e| die("cannot read .envrc ({s})", .{@errorName(e)}); + } else |_| ""; + const text = if (std.mem.indexOf(u8, old, envrc_begin)) |at| blk: { + const rest = std.mem.indexOfPos(u8, old, at, envrc_end) orelse + die(".envrc has `{s}` without `{s}`; remove that line or add the end line, then run this again", .{ envrc_begin, envrc_end }); + var end = rest + envrc_end.len; + if (end < old.len and old[end] == '\n') end += 1; + break :blk str(a, "{s}{s}{s}", .{ old[0..at], block, old[end..] }); + } else if (old.len == 0) + block + else + str(a, "{s}{s}{s}", .{ old, if (old[old.len - 1] == '\n') "" else "\n", block }); + if (exists_already) { + // Written beside the old file and renamed over it, with its mode. + fsx.replaceContents(a, fsx.cwd, ".envrc", text) catch |e| die("cannot write .envrc ({s})", .{@errorName(e)}); + } else { + const fd = sys.openat(fsx.cwd, ".envrc", .{ .ACCMODE = .WRONLY, .CREAT = true, .EXCL = true, .NOFOLLOW = true }, 0o644) catch |e| + die("cannot create .envrc ({s})", .{@errorName(e)}); + defer sys.close(fd); + sys.writeAll(fd, text) catch |e| die("cannot write .envrc ({s})", .{@errorName(e)}); + } + cli.err("eb-local: wrote .envrc in {s}; run `direnv allow` there to load it\n", .{c.cwd}); + return 0; +} + +/// The file `dev --lock` writes and `dev --locked` reads, in the working directory. +const lock_name = "eb-local.lock"; + +/// What a dev environment resolved to: enough for a later shell to load the +/// same modules, and to notice that it cannot. +const DevLock = struct { + schema_version: u32 = jsonout.schema_version, + kind: []const u8 = "dev-lock", + /// `stack.module` when the lock was written. + stack: []const u8, + eessi_version: []const u8 = "", + /// EESSI_SOFTWARE_SUBDIR: the CPU target the modules were resolved for. + /// Nodes with the same CPU share it, whatever their accelerator. + cpu_target: []const u8, + accelerator_target: []const u8 = "", + /// LOADEDMODULES in load order: every module, each with its version. + modules: []const []const u8, +}; + +/// Create `name` in the working directory, or replace it atomically with its mode kept. +fn writeCwdFile(a: Allocator, name: []const u8, text: []const u8) void { + if (fsx.statAt(fsx.cwd, name, true)) |_| { + fsx.replaceContents(a, fsx.cwd, name, text) catch |e| die("cannot write {s} ({s})", .{ name, @errorName(e) }); + } else |_| { + const fd = sys.openat(fsx.cwd, name, .{ .ACCMODE = .WRONLY, .CREAT = true, .EXCL = true, .NOFOLLOW = true }, 0o644) catch |e| + die("cannot create {s} ({s})", .{ name, @errorName(e) }); + defer sys.close(fd); + sys.writeAll(fd, text) catch |e| die("cannot write {s} ({s})", .{ name, @errorName(e) }); + } +} + +/// `dev --lock`: run the dev lines in a bash child and record what loaded. +fn writeLock(c: Ctx, s: config.Settings, lines: []const u8) u8 { + const a = c.a; + const env0 = runLines(c, "dev", lines) orelse return 1; + var modules: std.ArrayList([]const u8) = .empty; + var it = std.mem.splitScalar(u8, env0Value(env0, "LOADEDMODULES"), ':'); + while (it.next()) |m| { + if (m.len > 0) modules.append(a, m) catch oom(); + } + const cpu = env0Value(env0, "EESSI_SOFTWARE_SUBDIR"); + if (cpu.len == 0 or modules.items.len == 0) { + warn("the dev lines loaded no module or set no EESSI_SOFTWARE_SUBDIR, so there is nothing to lock; {s} is not written", .{lock_name}); + return 1; + } + const text = jsonout.renderLine(a, DevLock{ + .stack = s.stack_module.value, + .eessi_version = env0Value(env0, "EESSI_VERSION"), + .cpu_target = cpu, + .accelerator_target = env0Value(env0, "EESSI_ACCELERATOR_TARGET"), + .modules = modules.items, + }) catch oom(); + writeCwdFile(a, lock_name, text); + cli.err("eb-local: wrote {s}: {d} modules for {s}; `eb-local dev --locked` loads them again\n", .{ lock_name, modules.items.len, cpu }); + return 0; +} + +/// A CPU target as EESSI names one: path parts of letters, digits, `_`, `.` and `-`. +fn validTarget(text: []const u8) bool { + if (text.len == 0) return false; + for (text) |ch| { + if (!(std.ascii.isAlphanumeric(ch) or ch == '/' or ch == '_' or ch == '.' or ch == '-')) return false; + } + return true; +} + +/// `dev --locked`: the lines that load exactly the modules of `eb-local.lock`, +/// from a purged shell, and that load nothing when this shell's CPU target is +/// not the lock's. The same binaries serve every node with that CPU target, +/// so a lock written on one node type loads on another that shares it. +fn lockedLines(c: Ctx, s: config.Settings, g: Global, installed: Place) u8 { + const a = c.a; + const opened = fsx.openRegular(fsx.cwd, lock_name, true) catch + die("there is no {s} here; write one with `eb-local dev --lock MODULE...`", .{lock_name}); + defer sys.close(opened.fd); + const text = fsx.readAll(a, opened.fd, 1 << 20) catch |e| die("cannot read {s} ({s})", .{ lock_name, @errorName(e) }); + const lock = std.json.parseFromSliceLeaky(DevLock, a, text, .{ .ignore_unknown_fields = true }) catch + die("{s} is not a lock eb-local wrote; write it again with `eb-local dev --lock MODULE...`", .{lock_name}); + if (!std.mem.eql(u8, lock.kind, "dev-lock") or lock.schema_version != jsonout.schema_version) + die("{s} is not a dev lock of this eb-local (kind {s}, schema {d})", .{ lock_name, lock.kind, lock.schema_version }); + if (!validTarget(lock.cpu_target)) die("{s} names no usable CPU target", .{lock_name}); + if (lock.modules.len == 0) die("{s} lists no modules", .{lock_name}); + for (lock.modules) |m| { + if (!config.validModule(m)) die("{s} lists '{s}', which is not a module name", .{ lock_name, m }); + } + if (!std.mem.eql(u8, lock.stack, s.stack_module.value)) + die("{s} was written with the stack {s}; this configuration loads {s}. Write the lock again, or select that stack", .{ lock_name, lock.stack, s.stack_module.value }); + var out: std.ArrayList(u8) = .empty; + out.appendSlice(a, environmentLines(c, s, installed, envStrict(c, g), true)) catch oom(); + out.appendSlice(a, str(a, + \\if [ "${{EESSI_SOFTWARE_SUBDIR-}}" != {s} ]; then + \\ echo "eb-local: {s} was written for the CPU target "{s}", and this shell has ${{EESSI_SOFTWARE_SUBDIR:-none}}; its modules are not loaded." >&2 + \\else + \\module load + , .{ q(a, lock.cpu_target), lock_name, q(a, lock.cpu_target) })) catch oom(); + for (lock.modules) |m| out.appendSlice(a, str(a, " {s}", .{q(a, m)})) catch oom(); + out.append(a, '\n') catch oom(); + out.appendSlice(a, host_ca_bundle) catch oom(); + const dev_root = std.fs.path.join(a, &.{ installed.prefix, "dev" }) catch oom(); + out.appendSlice(a, str(a, "export EB_LOCAL_DEV_PREFIX={s}\"/$EESSI_VERSION/$EESSI_SOFTWARE_SUBDIR${{EESSI_ACCELERATOR_TARGET:+/$EESSI_ACCELERATOR_TARGET}}\"\nfi\n", .{q(a, dev_root)})) catch oom(); + if (g.json_out) return emitEnvChange(c, "dev", out.items); + evalHint("dev --locked"); + cli.put(1, out.items); + return 0; +} + fn cmdDev(c: Ctx, s: config.Settings, g: Global, extra: []const []const u8) u8 { const a = c.a; // A module is loaded once, where it is first named; a chosen build @@ -702,9 +896,15 @@ fn cmdDev(c: Ctx, s: config.Settings, g: Global, extra: []const []const u8) u8 { } else mods.append(a, m) catch oom(); }; const installed = place(c, s); - if (!g.json_out) evalHint(replay(a, "dev", s, g, installed.prefix, extra)); + if (g.locked) { + if (extra.len > 0) die("dev --locked loads the modules of {s}; name none on the line", .{lock_name}); + if (g.envrc) return writeEnvrc(c, replay(a, "dev", s, g, installed.prefix, extra)); + return lockedLines(c, s, g, installed); + } + if (g.envrc) return writeEnvrc(c, replay(a, "dev", s, g, installed.prefix, extra)); + if (!g.json_out and !g.lock) evalHint(replay(a, "dev", s, g, installed.prefix, extra)); var out: std.ArrayList(u8) = .empty; - out.appendSlice(a, environmentLines(c, s, installed, envStrict(c, g))) catch oom(); + out.appendSlice(a, environmentLines(c, s, installed, envStrict(c, g), g.pure)) catch oom(); // The modules the program uses first, the build environment last, so // its wrappers come first on PATH. When the program links, they record // a run path for each directory on LIBRARY_PATH, which the modules set @@ -725,9 +925,10 @@ fn cmdDev(c: Ctx, s: config.Settings, g: Global, extra: []const []const u8) u8 { // has picked them. const dev_root = std.fs.path.join(a, &.{ installed.prefix, "dev" }) catch oom(); out.appendSlice(a, str(a, "export EB_LOCAL_DEV_PREFIX={s}\"/$EESSI_VERSION/$EESSI_SOFTWARE_SUBDIR${{EESSI_ACCELERATOR_TARGET:+/$EESSI_ACCELERATOR_TARGET}}\"\n", .{q(a, dev_root)})) catch oom(); + if (g.lock) return writeLock(c, s, out.items); if (g.json_out) return emitEnvChange(c, "dev", out.items); cli.put(1, out.items); - if (nonEmpty(c.env.get("LD_LIBRARY_PATH")) != null) { + if (!g.pure and nonEmpty(c.env.get("LD_LIBRARY_PATH")) != null) { warn("LD_LIBRARY_PATH is set in this shell; programs built here do not need it,", .{}); cli.err(" and it can mix host libraries into EESSI's. Unset it unless you know you need it.\n", .{}); } @@ -4818,6 +5019,14 @@ pub fn main(init: std.process.Init) u8 { g.env_strict = true; } else if (std.mem.eql(u8, x, "--json")) { g.json_out = true; + } else if (std.mem.eql(u8, x, "--envrc")) { + g.envrc = true; + } else if (std.mem.eql(u8, x, "--pure")) { + g.pure = true; + } else if (std.mem.eql(u8, x, "--lock")) { + g.lock = true; + } else if (std.mem.eql(u8, x, "--locked")) { + g.locked = true; } else if (std.mem.eql(u8, x, "--buildenv")) { g.buildenv = value(args, &i, "--buildenv"); } else if (joined(x, "--buildenv")) |v| { diff --git a/tests/test_dev_cli_modes.py b/tests/test_dev_cli_modes.py index 03f39d8..84390a3 100644 --- a/tests/test_dev_cli_modes.py +++ b/tests/test_dev_cli_modes.py @@ -56,6 +56,163 @@ def test_json_is_the_change_the_dev_lines_make(self): self.assertIn("EB_LOCAL_DEV_PREFIX", values) self.assertTrue(values["EB_LOCAL_DEV_PREFIX"].startswith(str(self.prefix) + "/dev/"), values) self.assertNotIn("module load", got.stdout) + def test_envrc_writes_the_dev_line_and_updates_only_its_own_block(self): + envrc = self.root / ".envrc" + got = self.run_dev(["--envrc", "FFTW/3.3.10-GCC-15.2.0"]) + self.assertEqual(got.returncode, 0, got.stderr) + self.assertEqual(got.stdout, "") + self.assertIn("direnv allow", got.stderr) + first = envrc.read_text() + lines = first.splitlines() + self.assertTrue(lines[0].startswith("# >>> eb-local dev >>>"), first) + self.assertEqual(lines[-1], "# <<< eb-local dev <<<") + self.assertEqual(len(lines), 3) + # the program is named by its own path, and the line replays this dev + self.assertTrue(lines[1].startswith('eval "$(' + binary.BIN + " "), lines[1]) + self.assertTrue(lines[1].endswith(' dev FFTW/3.3.10-GCC-15.2.0)"'), lines[1]) + + # the user's own lines around the block survive a rewrite with other modules + envrc.write_text("export MINE=1\n" + first + "layout python\n") + again = self.run_dev(["--envrc", "--buildenv", "buildenv/custom", "HDF5/1.14.6-gompi-2026.1"]) + self.assertEqual(again.returncode, 0, again.stderr) + second = envrc.read_text().splitlines() + self.assertEqual(second[0], "export MINE=1") + self.assertEqual(second[-1], "layout python") + self.assertEqual(len(second), 5) + self.assertIn("--buildenv buildenv/custom", second[2]) + self.assertIn("HDF5/1.14.6-gompi-2026.1", second[2]) + self.assertNotIn("FFTW", "\n".join(second)) + + # a file without the block gets it appended, after a newline + envrc.write_text("export MINE=1") + self.assertEqual(self.run_dev(["--envrc"]).returncode, 0) + third = envrc.read_text().splitlines() + self.assertEqual(third[0], "export MINE=1") + self.assertTrue(third[1].startswith("# >>> eb-local dev >>>")) + self.assertEqual(len(third), 4) + + # a begin line without its end line is refused and the file is left alone + broken = "# >>> eb-local dev >>> by hand\nexport MINE=1\n" + envrc.write_text(broken) + refused = self.run_dev(["--envrc"]) + self.assertNotEqual(refused.returncode, 0) + self.assertIn("without", refused.stderr) + self.assertEqual(envrc.read_text(), broken) + + def test_the_envrc_block_is_valid_bash_that_runs_the_dev_lines(self): + self.assertEqual(self.run_dev(["--envrc"]).returncode, 0) + # bash, as direnv runs it: the block evaluates this program's dev lines + run = subprocess.run( + ["bash", "-c", 'source ./.envrc >/dev/null 2>&1; printf %s "${EB_LOCAL_DEV_PREFIX-}"'], + cwd=self.root, + env={"PATH": "/usr/bin:/bin", "HOME": str(self.root), "EB_LOCAL_CONFIG": str(self.site), + "EB_LOCAL_PREFIX": str(self.prefix)}, + capture_output=True, text=True, + ) + self.assertTrue(run.stdout.startswith(str(self.prefix) + "/dev/"), run.stdout + run.stderr) + def test_pure_purges_modules_and_loader_variables_before_the_stack_loads(self): + plain = self.run_dev(["FFTW/3.3.10-GCC-15.2.0"], LD_LIBRARY_PATH="/host/lib") + pure = self.run_dev(["--pure", "FFTW/3.3.10-GCC-15.2.0"], LD_LIBRARY_PATH="/host/lib") + self.assertEqual(pure.returncode, 0, pure.stderr) + purge = "module purge >/dev/null 2>&1 || true\nunset LD_LIBRARY_PATH LD_PRELOAD LD_AUDIT\n" + self.assertNotIn("module purge", plain.stdout) + self.assertEqual(pure.stdout.count(purge), 1) + # the rest of the lines is what dev prints without it + self.assertEqual(pure.stdout.replace(purge, ""), plain.stdout) + # after a module command exists, before anything is loaded + self.assertLess(pure.stdout.index("type module"), pure.stdout.index(purge)) + self.assertLess(pure.stdout.index(purge), pure.stdout.index("module load")) + # the variable is unset by the lines, so there is nothing to warn about + self.assertIn("LD_LIBRARY_PATH is set", plain.stderr) + self.assertNotIn("LD_LIBRARY_PATH is set", pure.stderr) + + LOADED = "EESSI/2026.06:GCCcore/15.2.0:FFTW/3.3.10-GCC-15.2.0:buildenv/default-GCC-15.2.0" + RESOLVED = {"LOADEDMODULES": LOADED, "EESSI_SOFTWARE_SUBDIR": "x86_64/amd/zen4", "EESSI_VERSION": "2026.06"} + + def test_lock_records_what_the_dev_lines_loaded(self): + lock = self.root / "eb-local.lock" + got = self.run_dev(["--lock", "FFTW/3.3.10-GCC-15.2.0"], **self.RESOLVED) + self.assertEqual(got.returncode, 0, got.stderr) + self.assertEqual(got.stdout, "") + self.assertIn("wrote eb-local.lock: 4 modules for x86_64/amd/zen4", got.stderr) + text = lock.read_text() + self.assertEqual(text.count("\n"), 1) + record = json.loads(text) + self.assertEqual(record["kind"], "dev-lock") + self.assertEqual(record["schema_version"], 1) + self.assertEqual(record["cpu_target"], "x86_64/amd/zen4") + self.assertEqual(record["eessi_version"], "2026.06") + self.assertEqual(record["accelerator_target"], "") + self.assertEqual(record["modules"], self.LOADED.split(":")) + self.assertTrue(record["stack"]) + + # a second lock replaces the first + again = self.run_dev(["--lock"], **dict(self.RESOLVED, LOADEDMODULES="EESSI/2026.06")) + self.assertEqual(again.returncode, 0, again.stderr) + self.assertEqual(json.loads(lock.read_text())["modules"], ["EESSI/2026.06"]) + + # nothing loaded, nothing to lock: the file is left as it is + before = lock.read_text() + empty = self.run_dev(["--lock"], EESSI_SOFTWARE_SUBDIR="x86_64/amd/zen4") + self.assertEqual(empty.returncode, 1) + self.assertIn("nothing to lock", empty.stderr) + self.assertEqual(lock.read_text(), before) + + def test_locked_loads_exactly_the_lock_and_checks_the_cpu_target(self): + lock = self.root / "eb-local.lock" + missing = self.run_dev(["--locked"]) + self.assertNotEqual(missing.returncode, 0) + self.assertIn("there is no eb-local.lock here", missing.stderr) + + self.assertEqual(self.run_dev(["--lock"], **self.RESOLVED).returncode, 0) + got = self.run_dev(["--locked"]) + self.assertEqual(got.returncode, 0, got.stderr) + load = "module load " + " ".join(self.LOADED.split(":")) + "\n" + self.assertEqual(got.stdout.count(load), 1) + self.assertEqual(got.stdout.count("module load"), 2) # the stack, then the lock + # purged first, and the lock's modules only on its own CPU target + self.assertLess(got.stdout.index("module purge"), got.stdout.index(load)) + self.assertIn('if [ "${EESSI_SOFTWARE_SUBDIR-}" != x86_64/amd/zen4 ]; then', got.stdout) + self.assertIn("export EB_LOCAL_DEV_PREFIX=", got.stdout) + + def run_lines(subdir): + return subprocess.run( + ["bash", "-c", got.stdout + 'printf "%s" "${EB_LOCAL_DEV_PREFIX-unset}"'], + env={"PATH": "/usr/bin:/bin", "HOME": str(self.root), "EESSI_SOFTWARE_SUBDIR": subdir, + "EESSI_VERSION": "2026.06"}, + capture_output=True, text=True, + ) + + other = run_lines("x86_64/intel/icelake") + self.assertIn("was written for the CPU target", other.stderr) + self.assertIn("this shell has x86_64/intel/icelake", other.stderr) + self.assertEqual(other.stdout, "unset") + same = run_lines("x86_64/amd/zen4") + self.assertNotIn("was written for the CPU target", same.stderr) + self.assertTrue(same.stdout.endswith("/dev/2026.06/x86_64/amd/zen4"), same.stdout) + + # modules come from the lock, not from the line + named = self.run_dev(["--locked", "FFTW/3.3.10-GCC-15.2.0"]) + self.assertNotEqual(named.returncode, 0) + self.assertIn("name none on the line", named.stderr) + + # a lock of another stack, a lock that is not one, and a hostile module name are refused + record = json.loads(lock.read_text()) + for change, message in ( + ({"stack": "EESSI/2099.01"}, "was written with the stack EESSI/2099.01"), + ({"kind": "something-else"}, "is not a dev lock"), + ({"modules": ["ok/1.0", "bad;rm -rf /"]}, "is not a module name"), + ({"cpu_target": "x86_64/$(id)"}, "names no usable CPU target"), + ({"modules": []}, "lists no modules"), + ): + with self.subTest(change=change): + lock.write_text(json.dumps(dict(record, **change)) + "\n") + refused = self.run_dev(["--locked"]) + self.assertNotEqual(refused.returncode, 0) + self.assertEqual(refused.stdout, "") + self.assertIn(message, refused.stderr) + lock.write_text("not json\n") + self.assertIn("is not a lock eb-local wrote", self.run_dev(["--locked"]).stderr) def test_names_the_host_ca_bundle_without_replacing_one_the_shell_set(self): got = self.run_dev([])