diff --git a/COMPAT.md b/COMPAT.md index 7edd3d3..6b29dc9 100644 --- a/COMPAT.md +++ b/COMPAT.md @@ -20,7 +20,7 @@ on the same invocations. | run ages | days from the local midnight of the date in the run's name, as `date -d YYYYMMDD` computes it, including TZ rules, except for the zones under unintended differences; a run whose local midnight does not exist is skipped | | the run in progress | `rotate` neither compresses nor deletes the run that holds its stdout or stderr, when either is a regular file, or `EB_PIPELINE_LOG_DIR` | | runs still written into | after taking its lock, `rotate` reads the links `/proc/PID/fd/N` of every process whose descriptors it may read; a run that holds the path a link with its owner's write bit names, also when the path ends in ` (deleted)`, is neither compressed nor deleted; without `/proc`, no run is left alone for this | -| rotation preview | `rotate --dry-run` prints each `compress PATH` and `delete PATH` action followed by the counts, changes nothing, creates no lock file, emits no syslog or journal event and writes no metrics; because it takes no lock, the output is a snapshot | +| rotation preview | `rotate --dry-run` prints each `compress PATH` (sorted bytewise per run, skipping files with an existing `.gz` or other hard links) and `delete PATH` action followed by the counts, changes nothing, creates no lock file, emits no syslog or journal event and writes no metrics; because it takes no lock, the output is a snapshot | ## Deliberate differences diff --git a/changelog.d/+rotate-dry-run-parity.bugfix.md b/changelog.d/+rotate-dry-run-parity.bugfix.md new file mode 100644 index 0000000..a57a552 --- /dev/null +++ b/changelog.d/+rotate-dry-run-parity.bugfix.md @@ -0,0 +1 @@ +`rotate --dry-run` matches `rotate` path order and skips files with existing `.gz` siblings or multiple hard links. diff --git a/docs/orgmode/reference/cli.org b/docs/orgmode/reference/cli.org index 7bac753..653df3c 100644 --- a/docs/orgmode/reference/cli.org +++ b/docs/orgmode/reference/cli.org @@ -198,7 +198,7 @@ What =failed= cannot do it reports on stderr as =eb-keep-logs: MESSAGE=, and goe | =ROOT/RUN= | Removed with everything in it, when old enough for =--delete-after=, without being compressed first. A symbolic link in it is removed, not followed. Removal stops at the first error. | | =/proc/PID/fd/N= | Read, for every process whose descriptors =rotate= may read, after it has the lock: a link with its owner's write bit names a file the process has open for writing, and the run that file is in is left alone. Without =/proc=, no run is left alone for it. | -With =--dry-run= these paths are only read; =ROOT/.eb-keep-logs-rotate.lock= is neither opened nor created, and =--metrics-file= neither reads nor writes any metrics or lock file. +With =--dry-run= these paths are only read; =ROOT/.eb-keep-logs-rotate.lock= is neither opened nor created, =--metrics-file= neither reads nor writes any metrics or lock file, and files that already have a =.gz= sibling or other hard links are skipped as =rotate= skips them. A run that holds the file =rotate='s standard output or standard error goes to, or =EB_PIPELINE_LOG_DIR=, or a file another process has open for writing, as a build that is still going has its log, is left alone, neither compressed nor deleted; a later =rotate= takes it. The path is what counts: the path the link in =/proc= names, or for a file removed since, that path with =" (deleted)"= after it, has to start with =ROOT='s, with symbolic links resolved. diff --git a/eb-keep-logs b/eb-keep-logs index 4ec979f..45519ab 100755 --- a/eb-keep-logs +++ b/eb-keep-logs @@ -1033,9 +1033,11 @@ cmd_rotate() { [ "${age}" -ge "${compress_after}" ] || continue if [ "${dry_run}" -eq 1 ]; then while IFS= read -r -d '' file; do + if [ -e "${file}.gz" ] || [ -L "${file}.gz" ]; then continue; fi + if [ "$(stat -c '%h' -- "${file}" 2>/dev/null || echo 2)" -gt 1 ]; then continue; fi echo "compress $(realpath "${file%/*}")/${file##*/}" compressed=$((compressed + 1)) - done < <(find "${run}" -type f \( -name '*.log' -o -name '*.txt' \) -print0 | sort -z) + done < <(find "${run}" -type f \( -name '*.log' -o -name '*.txt' \) -print0 | LC_ALL=C sort -z) continue fi while IFS= read -r -d '' file; do diff --git a/src/main.zig b/src/main.zig index 2e8fab0..af4315c 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1050,9 +1050,10 @@ fn compressTree(c: Ctx, arena: Allocator, dir: fsx.fd_t, renames: *Renames, coun } } -/// Print every *.log and *.txt regular file under `dir` that a rotation would -/// try to compress, without following symbolic links or changing anything. -fn previewTree(c: Ctx, arena: Allocator, dir: fsx.fd_t, count: *usize) void { +/// Collect every *.log and *.txt regular file under `dir` that a rotation +/// would compress (single link and no existing `.gz`), without following +/// symbolic links or changing anything. +fn previewTree(c: Ctx, arena: Allocator, dir: fsx.fd_t, paths: *std.ArrayList([]const u8)) void { var rb: [sys.path_max]u8 = undefined; const here = arena.dupe(u8, fsx.realpathFd(dir, &rb) catch |err| return warn("cannot resolve a run directory: {t}", .{err})) catch return; var it = fsx.DirIter.init(c.gpa, dir) catch return; @@ -1068,16 +1069,16 @@ fn previewTree(c: Ctx, arena: Allocator, dir: fsx.fd_t, count: *usize) void { files.append(arena, n) catch return; } } - const less = struct { - fn lt(_: void, a: []const u8, b: []const u8) bool { - return std.mem.lessThan(u8, a, b); - } - }.lt; - std.mem.sort([]const u8, files.items, {}, less); - std.mem.sort([]const u8, subdirs.items, {}, less); for (files.items) |n| { - cli.out("compress {s}/{s}\n", .{ here, n }); - count.* += 1; + const st = fsx.statAt(dir, n, false) catch continue; + if (!st.isReg() or st.nlink > 1) continue; + const gz = arena.print("{s}.gz", .{n}) catch return; + if (fsx.statAt(dir, gz, false)) |_| continue else |err| switch (err) { + error.NotFound => {}, + else => continue, + } + const path = arena.print("{s}/{s}", .{ here, n }) catch return; + paths.append(arena, path) catch return; } for (subdirs.items) |n| { const sub = fsx.openDir(dir, n, false) catch |err| { @@ -1085,7 +1086,7 @@ fn previewTree(c: Ctx, arena: Allocator, dir: fsx.fd_t, count: *usize) void { continue; }; defer sys.close(sub); - previewTree(c, arena, sub, count); + previewTree(c, arena, sub, paths); } } @@ -1335,7 +1336,17 @@ fn cmdRotate(c: Ctx, args: []const [:0]const u8) u8 { }; defer sys.close(dir); if (dry_run) { - previewTree(c, arena, dir, &compressed); + var paths: std.ArrayList([]const u8) = .empty; + previewTree(c, arena, dir, &paths); + std.mem.sort([]const u8, paths.items, {}, struct { + fn lt(_: void, a: []const u8, b: []const u8) bool { + return std.mem.lessThan(u8, a, b); + } + }.lt); + for (paths.items) |path| { + cli.out("compress {s}\n", .{path}); + compressed += 1; + } continue; } var renames: Renames = .empty; diff --git a/tests/differential.sh b/tests/differential.sh index aa77599..c08a178 100644 --- a/tests/differential.sh +++ b/tests/differential.sh @@ -134,8 +134,14 @@ compare "rotate, bad syslog target" rotate "${work}/logs" --compress-after 5 --s EB_KEEP_LOGS_SYSLOG_PEN=0 compare "failed, bad syslog pen" failed 1 Foo "${work}/logs" EB_KEEP_LOGS_SYSLOG_PEN=abc compare "rotate, bad syslog pen" rotate "${work}/logs" --compress-after 5 EB_KEEP_LOGS_NOW='' compare "rotate, empty EB_KEEP_LOGS_NOW" rotate "${work}/logs" --compress-after 5 -mkdir -p "${work}/preview/20261001_120000" -echo old > "${work}/preview/20261001_120000/Old.stdout.log" +mkdir -p "${work}/preview/20261001_120000/sub" +echo old > "${work}/preview/20261001_120000/a.log" +echo old > "${work}/preview/20261001_120000/sub/b.log" +echo old > "${work}/preview/20261001_120000/z.log" +echo old > "${work}/preview/20261001_120000/skip.log" +echo gz > "${work}/preview/20261001_120000/skip.log.gz" +echo link > "${work}/preview/20261001_120000/linked.log" +ln "${work}/preview/20261001_120000/linked.log" "${work}/preview/20261001_120000/linked.hardlink" run_env=(PATH="${PATH}" EB_KEEP_LOGS_NOW="$(date -d 2026-11-13 +%s)") compare "rotate, dry run" rotate "${work}/preview" --compress-after 5 --delete-after 90 --dry-run compare "rotate, dry run with metrics-file" rotate "${work}/preview" --compress-after 5 --delete-after 90 --dry-run --metrics-file "${work}/dry.prom" @@ -144,6 +150,19 @@ if [ -e "${work}/dry.prom" ] || [ -e "${work}/.dry.prom.lock" ]; then diffs=$((diffs + 1)) fi compare "rotate, dry run with empty metrics-file" rotate "${work}/preview" --compress-after 5 --dry-run --metrics-file "" +EB_KEEP_LOGS_NOW="$(date -d 2026-11-13 +%s)" "${OLD_BIN}" rotate "${work}/preview" --compress-after 5 --dry-run > "${work}/dry-parity-old.out" 2> "${work}/dry-parity-old.err" +EB_KEEP_LOGS_NOW="$(date -d 2026-11-13 +%s)" "${TOOL_BIN}" rotate "${work}/preview" --compress-after 5 --dry-run > "${work}/dry-parity-new.out" 2> "${work}/dry-parity-new.err" +if ! cmp -s "${work}/dry-parity-old.out" "${work}/dry-parity-new.out" || ! cmp -s "${work}/dry-parity-old.err" "${work}/dry-parity-new.err"; then + echo "DIFF rotate, dry-run raw bytes differ between shell and Zig" >&2 + diffs=$((diffs + 1)) +fi +dry_count=$(sed -nE 's/^eb-keep-logs rotate dry-run: ([0-9]+) files? would be compressed.*/\1/p' "${work}/dry-parity-old.out") +rot_out=$(EB_KEEP_LOGS_NOW="$(date -d 2026-11-13 +%s)" "${TOOL_BIN}" rotate "${work}/preview" --compress-after 5 2>/dev/null) +rot_count=$(sed -nE 's/^eb-keep-logs rotate: ([0-9]+) files? compressed.*/\1/p' <<< "${rot_out}") +if [ "${dry_count}" != "3" ] || [ "${dry_count}" != "${rot_count}" ]; then + echo "DIFF rotate, dry-run count (${dry_count}) != rotate count (${rot_count})" >&2 + diffs=$((diffs + 1)) +fi run_env=(PATH="${PATH}") compare "rotate, bad lock timeout" rotate "${work}/logs" --compress-after 5 --lock-timeout 1.5 compare "rotate, lock timeout without a value" rotate "${work}/logs" --compress-after 5 --lock-timeout diff --git a/tests/test_eb_keep_logs.sh b/tests/test_eb_keep_logs.sh index 88c5614..196b37f 100755 --- a/tests/test_eb_keep_logs.sh +++ b/tests/test_eb_keep_logs.sh @@ -306,6 +306,23 @@ lock_missing=1 test -e "${root}/.eb-keep-logs-rotate.lock" && lock_missing=0 check "dry run: creates no lock" "$lock_missing" "1" +skip_root="${work}/dry-skip" +mkdir -p "${skip_root}/20261001_120000/sub" +printf 'a\n' > "${skip_root}/20261001_120000/a.log" +printf 'b\n' > "${skip_root}/20261001_120000/sub/b.log" +printf 'z\n' > "${skip_root}/20261001_120000/z.log" +printf 's\n' > "${skip_root}/20261001_120000/skip.log" +printf 'gz\n' > "${skip_root}/20261001_120000/skip.log.gz" +printf 'l\n' > "${skip_root}/20261001_120000/link.log" +ln "${skip_root}/20261001_120000/link.log" "${skip_root}/20261001_120000/link.hard" +skip_real=$(realpath "${skip_root}/20261001_120000") +check "dry run: sorted across subdirs and skips .gz and hard links" \ + "$("${tool}" rotate "${skip_root}" --compress-after 30 --dry-run)" \ + "compress ${skip_real}/a.log +compress ${skip_real}/sub/b.log +compress ${skip_real}/z.log +eb-keep-logs rotate dry-run: 3 files would be compressed, 0 runs would be deleted under ${skip_root}" + "${tool}" rotate "${root}" --compress-after 30 >/dev/null check "old run compressed" "$(ls "${root}/20261001_120000/site-zen4" | sort | tr '\n' ' ')" "A.stdout.log.gz B.failed.ebout.log.gz " check "compressed content intact" "$(zcat "${root}/20261001_120000/site-zen4/A.stdout.log.gz")" "old"