Add filesystem operation journal and extended test runs

.github/workflows/go-test.yml

@@ -30,12 +30,12 @@ jobs:
       - name: Reject production imports of test support
         run: bash scripts/check-production-imports.sh
       - name: Run race tests outside full-length gRPC sequences
-        run: go test -race ./... -skip "$GRPC_SEQUENCE_TESTS"
+        run: go test -race -timeout=30m ./... -skip "$GRPC_SEQUENCE_TESTS"
         env:
           # Fail, rather than skip, the kernel-mount tests when FUSE is unusable.
           ELASTICKV_REQUIRE_FUSE: "1"
           ELASTICKV_REQUIRE_SQLITE: "1"
       # These unchanged 10,000-iteration tests need a fresh package budget
       # after the serial migration/snapshot acceptance tests have finished.
       - name: Run full-length gRPC consistency sequences
-        run: go test -race ./adapter -run "$GRPC_SEQUENCE_TESTS"
+        run: go test -race -timeout=30m ./adapter -run "$GRPC_SEQUENCE_TESTS"

.github/workflows/jepsen-test.yml

@@ -287,15 +287,15 @@ jobs:
         # temp file + rename (namespace txn and remove-while-open). Both run
         # even if one fails so the log shows which mode broke.
         # See jepsen/src/elastickv/filesystem_workload.clj.
-        timeout-minutes: 6
+        timeout-minutes: 10
         run: |
           declare -A RESULT
           FAILED=()
           for mode in overwrite rename; do
             echo "::group::write-mode=${mode}"
             set +e
-            timeout 150 ~/lein run -m elastickv.filesystem-workload --local \
-              --time-limit 5 --rate 5 --concurrency 6 \
+            timeout 240 ~/lein run -m elastickv.filesystem-workload --local \
+              --time-limit 30 --rate 5 --concurrency 6 \
               --write-mode "${mode}" \
               --mount-dirs /tmp/elastickv-ci/fs-n1,/tmp/elastickv-ci/fs-n2,/tmp/elastickv-ci/fs-n3
             rc=$?
@@ -319,7 +319,7 @@ jobs:
           fi
       - name: Run SQLite Jepsen workload on FUSE
         working-directory: jepsen
-        timeout-minutes: 3
+        timeout-minutes: 6
         run: |
           probe=/tmp/elastickv-ci/fs-n1/sqlite-probe.db
           if ! strace -f -o /tmp/elastickv-sqlite.strace sqlite3 "$probe" \
@@ -345,9 +345,9 @@ jobs:
               exit 1
             fi
           done
-          timeout 150 ~/lein run -m elastickv.sqlite-workload --local \
-            --time-limit 5 --rate 5 --concurrency 6 \
-            --key-count 6 --max-writes-per-key 16 \
+          timeout 240 ~/lein run -m elastickv.sqlite-workload --local \
+            --time-limit 30 --rate 5 --concurrency 6 \
+            --key-count 12 --max-writes-per-key 16 \
             --mount-dirs /tmp/elastickv-ci/fs-n1,/tmp/elastickv-ci/fs-n2,/tmp/elastickv-ci/fs-n3
       - name: Archive Jepsen store on filesystem failure
         if: failure()

Dockerfile.redis-proxy

@@ -2,6 +2,7 @@ FROM golang:latest AS build
 
 WORKDIR $GOPATH/src/app
 COPY go.mod go.sum ./
+COPY third_party/go-fuse/go.mod third_party/go-fuse/go.mod
 RUN go mod download
 COPY . .
 

docs/design/2026_10_25_partial_fuse_conflict_semantics.md

@@ -438,13 +438,57 @@ regression tests.
 
 ### Remaining work after the bounded legacy-path changes
 
-The current branch adds typed forwarding of definitive write conflicts, bounded
-retries for those conflicts in several filesystem service operations, a local
-FUSE predecessor barrier, and mounted SQLite PERSIST/DELETE test coverage. These
-changes improve the legacy path but do not route FUSE mutations through the
-operation journal. In particular, `AdvanceGeneration` still fails closed, and
-`Service.Fsync` still uses an attribute read rather than a durable receipt.
-The tests for the legacy path cannot satisfy the strict-mode gates below.
+The merged baseline forwards definitive write conflicts, retries several
+filesystem service operations within bounds, tracks local FUSE predecessors,
+and tests mounted SQLite PERSIST/DELETE workloads.
+This branch adds a transaction-primary generation fence and optional
+journal-backed paths for fixed-offset FUSE writes, setattr including size changes,
+create/mknod, mkdir, open, release, lease renewal, file unlink,
+empty-directory removal, and same-directory rename.
+The adapter retains their successful operation IDs per inode.
+Its flush and fsync paths resolve those IDs and check participant proofs.
+Directory fsync checks create, mkdir, removal and rename receipts recorded for
+that parent. File fsync also checks create, open and release receipts for its
+inode when those optional paths are selected.
+The default mount path still uses the legacy mutations.
+The optional lease renewal path serializes foreground and background refreshes
+with other inode mutations and blocks later fsync after an unknown renewal.
+It journals at most one renewal per handle within a quarter of the configured
+lease lifetime, capped at one minute. Other reads validate the current lease
+without extending it. A later renewal verifies the preceding lease receipt
+before removing that receipt from the local fsync set; failed verification
+keeps the older receipt.
+`Service.Fsync` still uses an attribute read after the adapter checks the
+receipts for its supported journaled operations.
+The optional receipt tracker is local to one mount and retains only successful
+operations in memory. Successful replies now trigger receipt verification and
+session acknowledgement after the FUSE device write attempt.
+Write planning failures before publication cancel their pending attempt with a
+typed result so a duplicate observes the same failure and the session can
+close. An uncertain cancellation remains unknown until resolution.
+The local go-fuse bridge now notifies the adapter after `Server.handleRequest`
+attempts the kernel reply. The adapter verifies participant receipts before
+acknowledging terminal operations and retries acknowledgements that could not
+complete. Suppressed replies do not trigger acknowledgement. The explicit
+`--filesystemStrict` mount option selects all supported journaled mutation
+paths. Recovery of a previous incarnation requires the separate
+`--filesystemStrictRecover` flag after its process has stopped.
+The explicit strict mount is available for validation, but the default remains
+the legacy path until the strict-mode acceptance gates below pass.
+
+The current Raft path persists Ready entries through `persistReadyToWAL`
+before applying them.
+The pinned etcd WAL calls sync for a Ready containing entries.
+The Pebble FSM apply path defaults to `pebble.Sync`; its optional no-sync mode
+depends on replay from the synced Raft WAL.
+Local Pebble close/reopen tests recover committed write, metadata, namespace,
+create and open results with their receipts. A separate process-kill test
+recovers create, write, truncate and rename receipts from the Pebble WAL
+without closing the store. Neither test exercises a Raft process or node
+crash.
+This code audit supports the receipt approach but does not close the M4 gate.
+Crash tests must cover failure between WAL persistence, FSM apply, proof
+visibility, FUSE reply, and the next fsync for every participant group.
 
 Complete the remaining work in dependency order:
 

docs/filesystem.md

@@ -79,6 +79,8 @@ flags:
 |---|---|---|
 | `--filesystemMount` | empty (disabled) | Directory to mount the filesystem on. |
 | `--filesystemClientID` | value of `--raftId` | Identity recorded in open-handle lease keys and in the mount log line, for attribution. Does not have to be unique: lease keys also carry a per-open handle ID. |
+| `--filesystemStrict` | `false` | Use a registered operation session for supported FUSE mutations. This validation mode requires the reply-aware go-fuse bridge. |
+| `--filesystemStrictRecover` | `false` | Fence and recover a prior strict session for the same client ID before registering a new one. Set this only after confirming the prior mount process has stopped. Requires `--filesystemStrict`. |
 | `--filesystemRootUID` | `0` | Owner UID of the root directory. Applied only when the root is first created. |
 | `--filesystemRootGID` | `0` | Owner GID of the root directory. Applied only when the root is first created. |
 | `--filesystemRootMode` | `0755` | Permission bits of the root directory. Applied only when the root is first created. |

go.mod

@@ -1,5 +1,7 @@
 module github.com/bootjp/elastickv
 
+replace github.com/hanwen/go-fuse/v2 => ./third_party/go-fuse
+
 go 1.26.0
 
 toolchain go1.27.2

internal/filesystem/fstest/memory.go

@@ -4,6 +4,7 @@ package fstest
 
 import (
 	"context"
+	"encoding/binary"
 	"sync"
 
 	"github.com/bootjp/elastickv/internal/filesystem"
@@ -20,6 +21,8 @@ type MemoryDispatcher struct {
 	store store.MVCCStore
 }
 
+const commitTimestampBytes = 8
+
 // NewMemoryDispatcher wraps st in a dispatcher that commits every group at
 // the next commit timestamp.
 func NewMemoryDispatcher(st store.MVCCStore) *MemoryDispatcher {
@@ -48,12 +51,19 @@ func (d *MemoryDispatcher) Dispatch(
 	}
 	mutations := make([]*store.KVPairMutation, 0, len(req.Elems))
 	for _, elem := range req.Elems {
+		value := append([]byte(nil), elem.Value...)
+		if elem.CommitTSValueOffset != 0 {
+			if len(value) < commitTimestampBytes || elem.CommitTSValueOffset > uint64(len(value)-commitTimestampBytes) {
+				return nil, errors.WithStack(filesystem.ErrInvalid)
+			}
+			binary.BigEndian.PutUint64(value[elem.CommitTSValueOffset:], commitTS)
+		}
 		switch elem.Op {
 		case kv.Put:
 			mutations = append(mutations, &store.KVPairMutation{
 				Op:    store.OpTypePut,
 				Key:   append([]byte(nil), elem.Key...),
-				Value: append([]byte(nil), elem.Value...),
+				Value: value,
 			})
 		case kv.Del:
 			mutations = append(mutations, &store.KVPairMutation{