Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/+usage-json-robustness.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
`usage` preserves `resultT` hits when `userT` is `null`, warns on unreadable `software` subdirectories, and refuses non-JSON non-Lmod files.
2 changes: 1 addition & 1 deletion docs/orgmode/reference/cli.org
Original file line number Diff line number Diff line change
Expand Up @@ -445,7 +445,7 @@ The shell script has no =usage=.
| =--unused= | List installed entries with no hit. |
| =--active= | List installed entries with at least one hit. |
| =--json= | Print one JSON document instead of text. |
| =FILE= | One or more concatenated-XALT-JSON or Lmod usage-log files. A file that cannot be read, or truncated JSON, is refused. |
| =FILE= | One or more concatenated-XALT-JSON or Lmod usage-log files. Unreadable files, truncated JSON, or files containing neither JSON objects nor Lmod records are refused. |


** lifecycle
Expand Down
19 changes: 13 additions & 6 deletions src/usage.zig
Original file line number Diff line number Diff line change
Expand Up @@ -68,14 +68,21 @@ fn inventory(gpa: Allocator, arena: Allocator, software: []const u8) !Table {
{
if (name_entry.name.len == 0 or name_entry.name[0] == '.') continue;
if ((fsx.entryType(root, name_entry) catch continue) != fsx.DT.DIR) continue;
const name_dir = fsx.openDir(root, name_entry.name, false) catch continue;
const name_dir = fsx.openDir(root, name_entry.name, false) catch |err| {
warn("cannot open SOFTWARE {s}/{s}: {t}", .{ software, name_entry.name, err });
continue;
};
{
defer sys.close(name_dir);
var versions = fsx.DirIter.init(gpa, name_dir) catch continue;
var versions = fsx.DirIter.init(gpa, name_dir) catch |err| {
warn("cannot read SOFTWARE {s}/{s}: {t}", .{ software, name_entry.name, err });
continue;
};
defer versions.deinit(gpa);
while (versions.next() catch |err|
die("cannot read SOFTWARE {s}/{s}: {t}", .{ software, name_entry.name, err })) |version_entry|
{
while (versions.next() catch |err| {
warn("cannot read SOFTWARE {s}/{s}: {t}", .{ software, name_entry.name, err });
break;
}) |version_entry| {
if (version_entry.name.len == 0 or version_entry.name[0] == '.') continue;
if ((fsx.entryType(name_dir, version_entry) catch continue) != fsx.DT.DIR) continue;
const name = try arena.dupe(u8, name_entry.name);
Expand Down Expand Up @@ -313,7 +320,7 @@ pub fn cmdUsage(gpa: Allocator, args: []const []const u8) u8 {
defer gpa.free(text);
const hits = used.parseHits(arena, text) catch |err| switch (err) {
error.BadJson => {
warn("usage: {s} is not XALT run JSON", .{path});
warn("usage: {s} is not XALT run JSON or Lmod usage log", .{path});
return 2;
},
error.OutOfMemory => die("out of memory", .{}),
Expand Down
83 changes: 50 additions & 33 deletions src/used.zig
Original file line number Diff line number Diff line change
Expand Up @@ -202,63 +202,65 @@ fn processObject(gpa: Allocator, hits: *std.ArrayList(Hit), root: std.json.Objec
const result_t = root.getPtr("resultT");

if (user_t) |ut_val| {
const user_t_obj = switch (ut_val.*) {
.object => |obj| obj,
else => return,
};
const time = timeFromUserT(user_t_obj);
const epoch = epochFromUserT(user_t_obj);
if (stringField(user_t_obj, "exec_path")) |path| {
try appendHit(gpa, hits, .{ .kind = .run, .path = path, .time = time, .epoch = epoch });
}
if (root.getPtr("libA")) |lib_a| {
const arr = switch (lib_a.*) {
.array => |a| a,
else => return,
};
try walkLibArray(gpa, hits, arr, .@"run-lib", time, epoch);
switch (ut_val.*) {
.object => |user_t_obj| {
const time = timeFromUserT(user_t_obj);
const epoch = epochFromUserT(user_t_obj);
if (stringField(user_t_obj, "exec_path")) |path| {
try appendHit(gpa, hits, .{ .kind = .run, .path = path, .time = time, .epoch = epoch });
}
if (root.getPtr("libA")) |lib_a| {
if (lib_a.* == .array) {
try walkLibArray(gpa, hits, lib_a.array, .@"run-lib", time, epoch);
}
}
},
else => {},
}
}

if (result_t) |rt_val| {
const result_t_obj = switch (rt_val.*) {
.object => |obj| obj,
else => return,
};
const time = timeFromResultT(result_t_obj);
const epoch = epochFromResultT(result_t_obj);
if (stringField(result_t_obj, "exec_path")) |path| {
try appendHit(gpa, hits, .{ .kind = .link, .path = path, .time = time, .epoch = epoch });
}
if (root.getPtr("linkA")) |link_a| {
const arr = switch (link_a.*) {
.array => |a| a,
else => return,
};
try walkLibArray(gpa, hits, arr, .link, time, epoch);
switch (rt_val.*) {
.object => |result_t_obj| {
const time = timeFromResultT(result_t_obj);
const epoch = epochFromResultT(result_t_obj);
if (stringField(result_t_obj, "exec_path")) |path| {
try appendHit(gpa, hits, .{ .kind = .link, .path = path, .time = time, .epoch = epoch });
}
if (root.getPtr("linkA")) |link_a| {
if (link_a.* == .array) {
try walkLibArray(gpa, hits, link_a.array, .link, time, epoch);
}
}
},
else => {},
}
}
}

fn processLmodLine(gpa: Allocator, hits: *std.ArrayList(Hit), line: []const u8) !void {
const rec = loads.parseRecord(line) orelse return;
fn processLmodLine(gpa: Allocator, hits: *std.ArrayList(Hit), line: []const u8) !bool {
const rec = loads.parseRecord(line) orelse return false;
try appendHit(gpa, hits, .{
.kind = .@"module-load",
.path = rec.path,
.module = rec.module,
.time = rec.time,
.epoch = parseEpoch(rec.time),
});
return true;
}

/// Parse every XALT and Lmod hit in `text` once, without filtering paths.
pub fn parseHits(gpa: Allocator, text: []const u8) (Allocator.Error || error{BadJson})![]Hit {
var hits: std.ArrayList(Hit) = .empty;
errdefer hits.deinit(gpa);
var at: usize = 0;
var has_non_ws = false;
var saw_record = false;
while (true) {
while (at < text.len and std.ascii.isWhitespace(text[at])) : (at += 1) {}
if (at >= text.len) break;
has_non_ws = true;
if (text[at] == '{') {
const item = (try runs.nextObject(text, at)) orelse break;
at = item.end;
Expand All @@ -271,14 +273,16 @@ pub fn parseHits(gpa: Allocator, text: []const u8) (Allocator.Error || error{Bad
.object => |obj| obj,
else => continue,
};
saw_record = true;
try processObject(gpa, &hits, root);
continue;
}
const end = std.mem.indexOfScalarPos(u8, text, at, '\n') orelse text.len;
const line = std.mem.trimEnd(u8, text[at..end], "\r");
try processLmodLine(gpa, &hits, line);
if (try processLmodLine(gpa, &hits, line)) saw_record = true;
at = if (end < text.len) end + 1 else end;
}
if (has_non_ws and !saw_record) return error.BadJson;
return try hits.toOwnedSlice(gpa);
}

Expand Down Expand Up @@ -506,3 +510,16 @@ test "truncated JSON is refused" {
defer table.deinit();
try std.testing.expectError(error.BadJson, scanText(a, &table, "/opt", text));
}

test "userT null preserves resultT hits and non-JSON text is refused" {
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena.deinit();
const a = arena.allocator();
const hits = try parseHits(
a,
"{\"userT\":null,\"resultT\":{\"exec_path\":\"/opt/a/bin/x\",\"build_epoch\":\"200\"}}",
);
try std.testing.expectEqual(@as(usize, 1), hits.len);
try std.testing.expectEqual(Kind.link, hits[0].kind);
try std.testing.expectError(error.BadJson, parseHits(a, "not json or lmod\n"));
}
32 changes: 32 additions & 0 deletions tests/test_usage.sh
Original file line number Diff line number Diff line change
Expand Up @@ -97,5 +97,37 @@ check "selection is exclusive" "$?" "2"
check "selection refusal" "$(cat "${work}/err")" \
"eb-keep-logs: usage takes --unused or --active, not both"

cat > "${work}/null-usert.json" <<EOF
{"userT":null,"resultT":{"exec_path":"${software}/Foo/1.0/bin/foo","build_epoch":"250"}}
EOF
null_usert=$("${tool}" usage --software "${software}" --modules "${modules}" --json "${work}/null-usert.json")
check "userT null keeps resultT hits" "$(python3 -c '
import json, sys
o = json.load(sys.stdin)
foo = next(p for p in o["packages"] if p["module"] == "Foo/1.0")
assert foo["active"] and foo["hits"] == 1 and foo["last"] == "250"
print("ok")
' <<<"${null_usert}")" "ok"

if [ "$(id -u)" -ne 0 ]; then
mkdir -p "${work}/sw-perm/Readable/1.0" "${work}/sw-perm/Blocked/1.0"
chmod 000 "${work}/sw-perm/Blocked"
"${tool}" usage --software "${work}/sw-perm" --modules "${modules}" --json "${work}/xalt.json" > "${work}/perm.out" 2> "${work}/perm.err"
perm_rc=$?
chmod 755 "${work}/sw-perm/Blocked"
check "unreadable software/Name warns on stderr" "$perm_rc:$(grep -c 'cannot open SOFTWARE .*/Blocked' "${work}/perm.err")" "0:1"
check "readable software/Name still reported" "$(python3 -c '
import json, sys
o = json.load(sys.stdin)
assert [p["module"] for p in o["packages"]] == ["Readable/1.0"]
print("ok")
' <<<"$(cat "${work}/perm.out")")" "ok"
fi

printf 'not json or lmod\n' > "${work}/garbage.txt"
"${tool}" usage --software "${software}" --modules "${modules}" --json "${work}/garbage.txt" >/dev/null 2>"${work}/garbage.err"
check "non-JSON non-Lmod file refused" "$?" "2"
check "non-JSON refusal message" "$(grep -c 'garbage.txt' "${work}/garbage.err")" "1"

echo "${fails} failed"
[ "${fails}" -eq 0 ]
Loading