Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion COMPAT.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ on the same invocations.
| run ages | days from the local midnight of the date in the run's name, as `date -d YYYYMMDD` computes it, including TZ rules, except for the zones under unintended differences; a run whose local midnight does not exist is skipped |
| the run in progress | `rotate` neither compresses nor deletes the run that holds its stdout or stderr, when either is a regular file, or `EB_PIPELINE_LOG_DIR` |
| runs still written into | after taking its lock, `rotate` reads the links `/proc/PID/fd/N` of every process whose descriptors it may read; a run that holds the path a link with its owner's write bit names, also when the path ends in ` (deleted)`, is neither compressed nor deleted; without `/proc`, no run is left alone for this |
| rotation preview | `rotate --dry-run` prints each `compress PATH` and `delete PATH` action followed by the counts, changes nothing, creates no lock file, emits no syslog or journal event and writes no metrics; because it takes no lock, the output is a snapshot |
| rotation preview | `rotate --dry-run` prints each `compress PATH` (sorted bytewise per run, skipping files with an existing `.gz` or other hard links) and `delete PATH` action followed by the counts, changes nothing, creates no lock file, emits no syslog or journal event and writes no metrics; because it takes no lock, the output is a snapshot |

## Deliberate differences

Expand Down
1 change: 1 addition & 0 deletions changelog.d/+rotate-dry-run-parity.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
`rotate --dry-run` matches `rotate` path order and skips files with existing `.gz` siblings or multiple hard links.
2 changes: 1 addition & 1 deletion docs/orgmode/reference/cli.org
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ What =failed= cannot do it reports on stderr as =eb-keep-logs: MESSAGE=, and goe
| =ROOT/RUN= | Removed with everything in it, when old enough for =--delete-after=, without being compressed first. A symbolic link in it is removed, not followed. Removal stops at the first error. |
| =/proc/PID/fd/N= | Read, for every process whose descriptors =rotate= may read, after it has the lock: a link with its owner's write bit names a file the process has open for writing, and the run that file is in is left alone. Without =/proc=, no run is left alone for it. |

With =--dry-run= these paths are only read; =ROOT/.eb-keep-logs-rotate.lock= is neither opened nor created, and =--metrics-file= neither reads nor writes any metrics or lock file.
With =--dry-run= these paths are only read; =ROOT/.eb-keep-logs-rotate.lock= is neither opened nor created, =--metrics-file= neither reads nor writes any metrics or lock file, and files that already have a =.gz= sibling or other hard links are skipped as =rotate= skips them.

A run that holds the file =rotate='s standard output or standard error goes to, or =EB_PIPELINE_LOG_DIR=, or a file another process has open for writing, as a build that is still going has its log, is left alone, neither compressed nor deleted; a later =rotate= takes it.
The path is what counts: the path the link in =/proc= names, or for a file removed since, that path with =" (deleted)"= after it, has to start with =ROOT='s, with symbolic links resolved.
Expand Down
4 changes: 3 additions & 1 deletion eb-keep-logs
Original file line number Diff line number Diff line change
Expand Up @@ -1033,9 +1033,11 @@ cmd_rotate() {
[ "${age}" -ge "${compress_after}" ] || continue
if [ "${dry_run}" -eq 1 ]; then
while IFS= read -r -d '' file; do
if [ -e "${file}.gz" ] || [ -L "${file}.gz" ]; then continue; fi
if [ "$(stat -c '%h' -- "${file}" 2>/dev/null || echo 2)" -gt 1 ]; then continue; fi
echo "compress $(realpath "${file%/*}")/${file##*/}"
compressed=$((compressed + 1))
done < <(find "${run}" -type f \( -name '*.log' -o -name '*.txt' \) -print0 | sort -z)
done < <(find "${run}" -type f \( -name '*.log' -o -name '*.txt' \) -print0 | LC_ALL=C sort -z)
continue
fi
while IFS= read -r -d '' file; do
Expand Down
39 changes: 25 additions & 14 deletions src/main.zig
Original file line number Diff line number Diff line change
Expand Up @@ -1050,9 +1050,10 @@ fn compressTree(c: Ctx, arena: Allocator, dir: fsx.fd_t, renames: *Renames, coun
}
}

/// Print every *.log and *.txt regular file under `dir` that a rotation would
/// try to compress, without following symbolic links or changing anything.
fn previewTree(c: Ctx, arena: Allocator, dir: fsx.fd_t, count: *usize) void {
/// Collect every *.log and *.txt regular file under `dir` that a rotation
/// would compress (single link and no existing `.gz`), without following
/// symbolic links or changing anything.
fn previewTree(c: Ctx, arena: Allocator, dir: fsx.fd_t, paths: *std.ArrayList([]const u8)) void {
var rb: [sys.path_max]u8 = undefined;
const here = arena.dupe(u8, fsx.realpathFd(dir, &rb) catch |err| return warn("cannot resolve a run directory: {t}", .{err})) catch return;
var it = fsx.DirIter.init(c.gpa, dir) catch return;
Expand All @@ -1068,24 +1069,24 @@ fn previewTree(c: Ctx, arena: Allocator, dir: fsx.fd_t, count: *usize) void {
files.append(arena, n) catch return;
}
}
const less = struct {
fn lt(_: void, a: []const u8, b: []const u8) bool {
return std.mem.lessThan(u8, a, b);
}
}.lt;
std.mem.sort([]const u8, files.items, {}, less);
std.mem.sort([]const u8, subdirs.items, {}, less);
for (files.items) |n| {
cli.out("compress {s}/{s}\n", .{ here, n });
count.* += 1;
const st = fsx.statAt(dir, n, false) catch continue;
if (!st.isReg() or st.nlink > 1) continue;
const gz = arena.print("{s}.gz", .{n}) catch return;
if (fsx.statAt(dir, gz, false)) |_| continue else |err| switch (err) {
error.NotFound => {},
else => continue,
}
const path = arena.print("{s}/{s}", .{ here, n }) catch return;
paths.append(arena, path) catch return;
}
for (subdirs.items) |n| {
const sub = fsx.openDir(dir, n, false) catch |err| {
warn("cannot open {s}/{s}: {t}", .{ here, n, err });
continue;
};
defer sys.close(sub);
previewTree(c, arena, sub, count);
previewTree(c, arena, sub, paths);
}
}

Expand Down Expand Up @@ -1335,7 +1336,17 @@ fn cmdRotate(c: Ctx, args: []const [:0]const u8) u8 {
};
defer sys.close(dir);
if (dry_run) {
previewTree(c, arena, dir, &compressed);
var paths: std.ArrayList([]const u8) = .empty;
previewTree(c, arena, dir, &paths);
std.mem.sort([]const u8, paths.items, {}, struct {
fn lt(_: void, a: []const u8, b: []const u8) bool {
return std.mem.lessThan(u8, a, b);
}
}.lt);
for (paths.items) |path| {
cli.out("compress {s}\n", .{path});
compressed += 1;
}
continue;
}
var renames: Renames = .empty;
Expand Down
23 changes: 21 additions & 2 deletions tests/differential.sh
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,14 @@ compare "rotate, bad syslog target" rotate "${work}/logs" --compress-after 5 --s
EB_KEEP_LOGS_SYSLOG_PEN=0 compare "failed, bad syslog pen" failed 1 Foo "${work}/logs"
EB_KEEP_LOGS_SYSLOG_PEN=abc compare "rotate, bad syslog pen" rotate "${work}/logs" --compress-after 5
EB_KEEP_LOGS_NOW='' compare "rotate, empty EB_KEEP_LOGS_NOW" rotate "${work}/logs" --compress-after 5
mkdir -p "${work}/preview/20261001_120000"
echo old > "${work}/preview/20261001_120000/Old.stdout.log"
mkdir -p "${work}/preview/20261001_120000/sub"
echo old > "${work}/preview/20261001_120000/a.log"
echo old > "${work}/preview/20261001_120000/sub/b.log"
echo old > "${work}/preview/20261001_120000/z.log"
echo old > "${work}/preview/20261001_120000/skip.log"
echo gz > "${work}/preview/20261001_120000/skip.log.gz"
echo link > "${work}/preview/20261001_120000/linked.log"
ln "${work}/preview/20261001_120000/linked.log" "${work}/preview/20261001_120000/linked.hardlink"
run_env=(PATH="${PATH}" EB_KEEP_LOGS_NOW="$(date -d 2026-11-13 +%s)")
compare "rotate, dry run" rotate "${work}/preview" --compress-after 5 --delete-after 90 --dry-run
compare "rotate, dry run with metrics-file" rotate "${work}/preview" --compress-after 5 --delete-after 90 --dry-run --metrics-file "${work}/dry.prom"
Expand All @@ -144,6 +150,19 @@ if [ -e "${work}/dry.prom" ] || [ -e "${work}/.dry.prom.lock" ]; then
diffs=$((diffs + 1))
fi
compare "rotate, dry run with empty metrics-file" rotate "${work}/preview" --compress-after 5 --dry-run --metrics-file ""
EB_KEEP_LOGS_NOW="$(date -d 2026-11-13 +%s)" "${OLD_BIN}" rotate "${work}/preview" --compress-after 5 --dry-run > "${work}/dry-parity-old.out" 2> "${work}/dry-parity-old.err"
EB_KEEP_LOGS_NOW="$(date -d 2026-11-13 +%s)" "${TOOL_BIN}" rotate "${work}/preview" --compress-after 5 --dry-run > "${work}/dry-parity-new.out" 2> "${work}/dry-parity-new.err"
if ! cmp -s "${work}/dry-parity-old.out" "${work}/dry-parity-new.out" || ! cmp -s "${work}/dry-parity-old.err" "${work}/dry-parity-new.err"; then
echo "DIFF rotate, dry-run raw bytes differ between shell and Zig" >&2
diffs=$((diffs + 1))
fi
dry_count=$(sed -nE 's/^eb-keep-logs rotate dry-run: ([0-9]+) files? would be compressed.*/\1/p' "${work}/dry-parity-old.out")
rot_out=$(EB_KEEP_LOGS_NOW="$(date -d 2026-11-13 +%s)" "${TOOL_BIN}" rotate "${work}/preview" --compress-after 5 2>/dev/null)
rot_count=$(sed -nE 's/^eb-keep-logs rotate: ([0-9]+) files? compressed.*/\1/p' <<< "${rot_out}")
if [ "${dry_count}" != "3" ] || [ "${dry_count}" != "${rot_count}" ]; then
echo "DIFF rotate, dry-run count (${dry_count}) != rotate count (${rot_count})" >&2
diffs=$((diffs + 1))
fi
run_env=(PATH="${PATH}")
compare "rotate, bad lock timeout" rotate "${work}/logs" --compress-after 5 --lock-timeout 1.5
compare "rotate, lock timeout without a value" rotate "${work}/logs" --compress-after 5 --lock-timeout
Expand Down
17 changes: 17 additions & 0 deletions tests/test_eb_keep_logs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,23 @@ lock_missing=1
test -e "${root}/.eb-keep-logs-rotate.lock" && lock_missing=0
check "dry run: creates no lock" "$lock_missing" "1"

skip_root="${work}/dry-skip"
mkdir -p "${skip_root}/20261001_120000/sub"
printf 'a\n' > "${skip_root}/20261001_120000/a.log"
printf 'b\n' > "${skip_root}/20261001_120000/sub/b.log"
printf 'z\n' > "${skip_root}/20261001_120000/z.log"
printf 's\n' > "${skip_root}/20261001_120000/skip.log"
printf 'gz\n' > "${skip_root}/20261001_120000/skip.log.gz"
printf 'l\n' > "${skip_root}/20261001_120000/link.log"
ln "${skip_root}/20261001_120000/link.log" "${skip_root}/20261001_120000/link.hard"
skip_real=$(realpath "${skip_root}/20261001_120000")
check "dry run: sorted across subdirs and skips .gz and hard links" \
"$("${tool}" rotate "${skip_root}" --compress-after 30 --dry-run)" \
"compress ${skip_real}/a.log
compress ${skip_real}/sub/b.log
compress ${skip_real}/z.log
eb-keep-logs rotate dry-run: 3 files would be compressed, 0 runs would be deleted under ${skip_root}"

"${tool}" rotate "${root}" --compress-after 30 >/dev/null
check "old run compressed" "$(ls "${root}/20261001_120000/site-zen4" | sort | tr '\n' ' ')" "A.stdout.log.gz B.failed.ebout.log.gz "
check "compressed content intact" "$(zcat "${root}/20261001_120000/site-zen4/A.stdout.log.gz")" "old"
Expand Down
Loading