diff --git a/docs/design/2026_04_26_implemented_raft_learner.md b/docs/design/2026_04_26_implemented_raft_learner.md index 9f2caa3f1..b01d33f6d 100644 --- a/docs/design/2026_04_26_implemented_raft_learner.md +++ b/docs/design/2026_04_26_implemented_raft_learner.md @@ -867,8 +867,9 @@ single-process 3-node demo cluster, attaches a learner via ### Milestone 3 — Hardening -- Jepsen workload that exercises learner attach during partition and - promote after heal. +- ~~Jepsen workload that exercises learner attach during partition and + promote after heal.~~ **Implemented** as + `jepsen/src/elastickv/learner_workload.clj`. - Monitoring: `suffrage` label on per-peer Prometheus labels (already exists in `monitoring/raft.go:355`; verify it survives the engine changes). @@ -890,10 +891,29 @@ The shipped learner primitive includes the engine API, etcd backend learner handling, v2 peers-file suffrage persistence, admin RPC/CLI surface, join-as-learner alarm, monitoring suffrage labels, promotion precondition checks against leader `Progress.Match`, and the operator runbook -(`docs/raft_learner_operations.md`). Remaining Milestone 3 hardening is not -claimed shipped here: the learner attach/promote-under-partition Jepsen -workload is still open, and follower-served read routing remains a separate -proposal. The `Status.PerPeer` progress field has since landed (see §6). +(`docs/raft_learner_operations.md`). The `Status.PerPeer` progress field has +also landed (see §6); exposing it over the RaftAdmin RPC is a follow-up, and +follower-served read routing remains a separate proposal. The learner +attach/promote-under-partition Jepsen workload has landed; its checker pins +three properties — promotion never outruns catch-up, no acknowledged write lost +across a promotion, and a learner never counted in the voter quorum. + +Note on the first: catch-up is measured against the LEADER's commit index, not +against `min-applied-index`. The engine's own test is +`Match >= min-applied-index`, so an operator who reads the learner's current +`Match` and passes it back satisfies it by construction — and both that broken +call and the correct one (pick a target, wait for `Match` to reach it) end with +`min-applied-index == Match`. That equality distinguishes nothing; only the +leader's position does. + +The managed workload also accepts `--encryption`. Setup starts the reserved +candidate with the same KEK and encryption capability flags as the voters, +but excludes it from initial voter joins, bootstrap writer discovery, and +initial envelope-cutover checks. The `AddLearner` membership interceptor +registers its writer identity for both active DEKs before attachment. The +workload then waits for both cutovers on the candidate before reporting +attachment successful. Catch-up and promotion still use the leader's commit +target; encryption does not change the learner reservation or quorum rules. ## 7. Risks diff --git a/jepsen/src/elastickv/db.clj b/jepsen/src/elastickv/db.clj index fe4d46aaf..751e38e93 100644 --- a/jepsen/src/elastickv/db.clj +++ b/jepsen/src/elastickv/db.clj @@ -2,6 +2,7 @@ "Jepsen DB adapter that builds, deploys, and manages elastickv nodes." (:require [clojure.java.io :as io] [clojure.java.shell :as sh] + [clojure.string :as str] [clojure.tools.logging :refer [info warn]] [jepsen [db :as db] [util :as util]] @@ -74,6 +75,73 @@ (c/upload (str build-dir "/" bin) (str bin-dir "/" bin)) (c/exec :chmod "755" (str bin-dir "/" bin)))))) +(defn raftadmin-binary + "Path to the uploaded raftadmin helper. Exposed so workloads that drive + membership changes do not each hardcode it." + [] + raftadmin-bin) + +(defn parse-raft-status + "Parses `raftadmin status` output into a keyword map. + + Numeric fields come back as longs and quoted strings unquoted, so a caller + can ask for :commit_index or :applied_index without re-deriving the format." + [out] + (->> (str/split-lines (or out "")) + (keep (fn [line] + (when-let [[_ k v] (re-matches #"\s*([a-z_]+):\s+(.*)" line)] + (let [v (str/trim v)] + [(keyword k) + (cond + (re-matches #"-?\d+" v) (Long/parseLong v) + (and (> (count v) 1) + (str/starts-with? v "\"") + (str/ends-with? v "\"")) + (subs v 1 (dec (count v))) + :else v)])))) + (into {}))) + +(defn raft-status + "Runs `raftadmin status` from node against addr and returns the parsed map." + [node addr] + (parse-raft-status + (c/on node (c/su (c/exec :env "RAFTADMIN_ALLOW_INSECURE=true" + raftadmin-bin addr "status"))))) + +(def ^:private metrics-port 9090) + +(defn parse-lease-read-counters + "Reads lease outcomes independently of const-label order." + [out] + (let [samples (keep (fn [line] + (when-let [[_ labels value] + (re-matches #"elastickv_lease_read_total\{(.*)\}\s+([0-9.eE+-]+)\s*" line)] + (when-let [[_ outcome] + (re-find #"(?:^|,)outcome=\"(hit|miss)\"(?:,|$)" labels)] + [(keyword outcome) (long (Double/parseDouble value))]))) + (str/split-lines (or out ""))) + counts (reduce (fn [acc [outcome n]] (update acc outcome (fnil + 0) n)) + {} samples)] + (when (contains? counts :hit) + (merge {:miss 0} counts)))) + +(defn lease-read-counters + "Reads elastickv_lease_read_total{outcome=...} from a node's metrics + endpoint, as {:hit n :miss n}. + + This is the only direct evidence of which read path ran. A lease read that + loses the fast path falls back to LinearizableRead instead of failing, so + latency alone cannot identify the regression. + + Returns nil when the endpoint is unreachable, so callers can distinguish no + samples from zero misses." + [node] + (try + (let [out (c/on node + (c/su (c/exec :curl :--connect-timeout 2 :--max-time 5 :-fsS + (str "http://127.0.0.1:" metrics-port "/metrics"))))] + (parse-lease-read-counters out)) + (catch Exception _ nil))) (defn- provision-kek! "Writes the test-only file KEK before the encryption-capable server starts." [node] @@ -138,7 +206,7 @@ "Builds the production server argv for one managed Jepsen node." [{:keys [node grpc redis dynamo s3 sqs sqs-region data-dir raft-engine raft-redis-map raft-dynamo-map raft-s3-map raft-sqs-map raft-groups shard-ranges - filesystem-mount encryption bootstrap?]}] + filesystem-mount encryption bootstrap? join-as-learner join-members]}] (cond-> ["--address" grpc "--redisAddress" redis "--raftId" (name node) @@ -156,7 +224,9 @@ encryption (conj "--encryptionSidecarPath" sidecar-file "--encryption-enabled" "--kekFile" kek-file) - bootstrap? (conj "--raftBootstrap"))) + bootstrap? (conj "--raftBootstrap") + join-as-learner (conj "--raftJoinAsLearner") + (and join-as-learner join-members) (conj "--raftJoinMembers" join-members))) (defn- filesystem-mount-dir "Resolves the FUSE mount directory for node from the :filesystem-mount @@ -236,7 +306,7 @@ (c/exec :bash "-c" (unmount-filesystem-cmd mount-dir)))))) (defn- start-node! - [test node {:keys [bootstrap-node grpc-port redis-port dynamo-port s3-port sqs-port sqs-region data-dir raft-groups shard-ranges raft-engine server-env filesystem-mount encryption]}] + [test node {:keys [bootstrap-node grpc-port redis-port dynamo-port s3-port sqs-port sqs-region data-dir raft-groups shard-ranges raft-engine server-env filesystem-mount encryption join-as-learner join-members]}] (when (and (seq raft-groups) (> (count raft-groups) 1) (nil? shard-ranges)) @@ -262,7 +332,8 @@ :raft-s3-map raft-s3-map :raft-sqs-map raft-sqs-map :raft-groups raft-groups :shard-ranges shard-ranges :filesystem-mount mount-dir :encryption encryption - :bootstrap? bootstrap?}) + :bootstrap? bootstrap? :join-as-learner join-as-learner + :join-members join-members}) daemon-opts (cond-> {:chdir bin-dir :logfile log-file :pidfile pid-file @@ -313,6 +384,28 @@ (c/exec :env "RAFTADMIN_ALLOW_INSECURE=true" raftadmin-bin leader-addr "add_voter" peer-id peer-addr "0")))) +(defn join-members-arg + "The --raftJoinMembers value for a learner candidate: raftID=host:port pairs. + + The candidate is included at its own listener address. resolveJoinServers + rejects a list that omits the local --raftId or gives it another address, so + the candidate must remain present even though setup does not add it as a + voter." + [nodes reserved grpc-port] + (let [reserved (when reserved (name reserved)) + members (->> nodes + (map name) + (remove #(= reserved %)))] + (->> (if reserved (conj (vec members) reserved) (vec members)) + (map #(str % "=" % ":" grpc-port)) + (clojure.string/join ",")))) + +(defn voter-peers + "The peers setup joins as voters: every node after the bootstrap one, minus + any reserved learner candidate." + [nodes reserved] + (let [reserved (when reserved (name reserved))] + (vec (remove #(= reserved (name %)) (rest nodes))))) (defn encryption-endpoint "Returns the default Raft group's gRPC address for EncryptionAdmin." [node grpc-port raft-groups] @@ -400,6 +493,20 @@ {:endpoint endpoint}))) (encryption-command! node (args-fn endpoint full-node-id local-epoch)))))) +(defn await-node-encryption! + "Waits for an attached member to apply both envelope cutovers." + [controller-node endpoint] + (util/await-fn + (fn [] + (let [state (encryption-status controller-node endpoint)] + (when-not (and (:storage-envelope-active state) + (:raft-envelope-active state)) + (throw (ex-info "encryption envelopes are not active on member" + {:endpoint endpoint :state state}))) + true)) + {:timeout 60000 + :log-message (str "waiting for both envelope cutovers on " endpoint)})) + (defn- activate-encryption! "Bootstraps both DEKs, activates both envelopes, and verifies every member." [test node grpc-port raft-groups] @@ -413,26 +520,22 @@ (cutover-encryption! node peers enable-storage-envelope-args) (cutover-encryption! node peers enable-raft-envelope-args) (doseq [peer (:nodes test)] - (let [peer-endpoint (encryption-endpoint peer grpc-port raft-groups)] - (util/await-fn - (fn [] - (let [state (encryption-status node peer-endpoint)] - (when-not (and (:storage-envelope-active state) - (:raft-envelope-active state)) - (throw (ex-info "encryption envelopes are not active on member" - {:endpoint peer-endpoint :state state}))) - true)) - {:timeout 60000 - :log-message (str "waiting for both envelope cutovers on " peer)}))) + (await-node-encryption! node (encryption-endpoint peer grpc-port raft-groups))) (info "encryption active on every node"))) (defn- node-start-opts - [test opts] - (merge {:data-dir data-dir - :grpc-port (or (:grpc-port opts) 50051) - :redis-port (or (:redis-port opts) 6379) - :bootstrap-node (first (:nodes test))} - opts)) + [test node opts] + (let [grpc-port (or (:grpc-port opts) 50051) + reserved (:reserve-learner opts) + learner? (and reserved (= (name node) (name reserved)))] + (merge {:data-dir data-dir + :grpc-port grpc-port + :redis-port (or (:redis-port opts) 6379) + :bootstrap-node (first (:nodes test))} + opts + (when learner? + {:join-as-learner true + :join-members (join-members-arg (:nodes test) reserved grpc-port)})))) (defn- wait-for-node-grpc! [node opts] @@ -474,7 +577,7 @@ [test peer opts] (when-let [mount-dir (filesystem-mount-dir (:filesystem-mount opts) peer)] (stop-node! peer) - (start-node! test peer (node-start-opts test opts)) + (start-node! test peer (node-start-opts test peer opts)) (wait-for-node-grpc! peer opts) (wait-for-mount! peer mount-dir) (info "filesystem mounted on joined peer" peer))) @@ -484,11 +587,15 @@ [test node opts] (wait-for-node-grpc! node opts) (wait-for-mount! node (filesystem-mount-dir (:filesystem-mount opts) node)) - (doseq [peer (rest (:nodes test))] + (doseq [peer (voter-peers (:nodes test) (:reserve-learner opts))] (join-peer! node peer opts) (enable-peer-mount! test peer opts)) + (when-let [candidate (:reserve-learner opts)] + (wait-for-node-grpc! candidate opts)) (when (:encryption opts) - (activate-encryption! test node (or (:grpc-port opts) 50051) + (activate-encryption! (assoc test :nodes (into [(first (:nodes test))] + (voter-peers (:nodes test) (:reserve-learner opts)))) + node (or (:grpc-port opts) 50051) (:raft-groups opts)))) (defrecord ElastickvDB [opts] @@ -503,7 +610,7 @@ (when (:encryption opts) (provision-kek! node)) (let [bootstrap? (= node (first (:nodes test))) - start-opts (node-start-opts test opts)] + start-opts (node-start-opts test node opts)] ;; Only the bootstrap node mounts immediately; peers are restarted with ;; their mount by enable-peer-mount! once they have joined. (start-node! test node (if bootstrap? @@ -537,7 +644,7 @@ db/Kill (start! [this test node] - (start-node! test node (node-start-opts test opts)) + (start-node! test node (node-start-opts test node opts)) (wait-for-node-grpc! node opts) (wait-for-mount! node (filesystem-mount-dir (:filesystem-mount opts) node)) (info "node started" node) diff --git a/jepsen/src/elastickv/jepsen_test.clj b/jepsen/src/elastickv/jepsen_test.clj index fc2d2bcb1..ff97a1c68 100644 --- a/jepsen/src/elastickv/jepsen_test.clj +++ b/jepsen/src/elastickv/jepsen_test.clj @@ -1,6 +1,7 @@ (ns elastickv.jepsen-test (:gen-class) - (:require [elastickv.redis-workload :as redis-workload] + (:require [elastickv.learner-workload :as learner-workload] + [elastickv.redis-workload :as redis-workload] [elastickv.redis-zset-safety-workload :as zset-safety-workload] [elastickv.dynamodb-workload :as dynamodb-workload] [elastickv.dynamodb-types-workload :as dynamodb-types-workload] @@ -29,6 +30,10 @@ ([] (elastickv-filesystem-test {})) ([opts] (filesystem-workload/elastickv-filesystem-test opts))) +(defn elastickv-learner-test + ([] (elastickv-learner-test {})) + ([opts] (learner-workload/elastickv-learner-test opts))) + (defn elastickv-zset-safety-test ([] (elastickv-zset-safety-test {})) ([opts] (zset-safety-workload/elastickv-zset-safety-test opts))) @@ -42,7 +47,8 @@ "elastickv-zset-safety-test" elastickv-zset-safety-test "elastickv-dynamodb-test" elastickv-dynamodb-test "elastickv-s3-test" elastickv-s3-test - "elastickv-filesystem-test" elastickv-filesystem-test}) + "elastickv-filesystem-test" elastickv-filesystem-test + "elastickv-learner-test" elastickv-learner-test}) (defn elastickv-sqs-htfifo-test "HT-FIFO Jepsen test (PR 7b). Run via the workload's own -main: @@ -58,8 +64,9 @@ lein run -m elastickv.jepsen-test [jepsen-subcmd] [jepsen-opts ...] Supported s: elastickv-test, elastickv-zset-safety-test, - elastickv-dynamodb-test, elastickv-s3-test, elastickv-filesystem-test. When the first positional - arg is not a known test name, we default to `elastickv-test` for + elastickv-dynamodb-test, elastickv-s3-test, elastickv-filesystem-test, + elastickv-learner-test. When the first positional arg is not a known test + name, we default to `elastickv-test` for backward compatibility and forward ALL args to jepsen.cli/run!. The jepsen subcommand (`test` or `analyze`) is auto-prepended when diff --git a/jepsen/src/elastickv/learner_workload.clj b/jepsen/src/elastickv/learner_workload.clj new file mode 100644 index 000000000..9c5fe30fa --- /dev/null +++ b/jepsen/src/elastickv/learner_workload.clj @@ -0,0 +1,907 @@ +(ns elastickv.learner-workload + "Jepsen workload for the Raft learner primitive: attach a learner, promote + it under partition, and assert the safety properties the learner design + leaves as Milestone 3 hardening. + + Operations: + + {:f :write :value n} write n to the register + {:f :read :value {:lease? b + :value n}} read the register + {:f :add-learner :value node} attach node as a learner + {:f :promote-learner :value {:node n + :catch-up-target t + :target-source :leader-commit-index + :match m}} + promote, recording the + target the operator + SAMPLED, where it came + from, and the learner's + Match when the promotion + committed + + Properties checked (see `learner-safety-checker`): + + 1. **Promotion never outruns catch-up** — `match >= catch-up-target`, + against the immutable sampled target, plus two guards: a promotion with + no evidence fails closed, and a target not sampled from the leader is a + procedure violation. See `premature-promotions` for why neither + min-applied-index nor the leader's current commit index works alone. + + 2. **No acknowledged write is lost across a promotion** — established by + temporal ordering, not set subtraction. See + `lost-writes-across-promotions`. + + 3. **A learner never counts toward the lease** — isolating a non-voter must + not fail lease reads. Expressed on reads rather than writes because + `quorumAckTracker` gates `LastQuorumAck` and the lease-read fast path, + not the write-commit quorum. See `learner-partition-read-failures`." + (:gen-class) + (:require [clojure.tools.logging :refer [warn]] + [elastickv.cli :as cli] + [elastickv.db :as ekdb] + [jepsen.control :as c] + [jepsen.db :as jdb] + [jepsen.os.debian :as debian] + [jepsen [checker :as checker] + [client :as client] + [control :as control] + [generator :as gen] + [nemesis :as nemesis] + [net :as net] + [os :as os]] + [taoensso.carmine :as car :refer [wcar]])) + +;; FOUR nodes, not five: three voters plus the reserved candidate. +;; +;; With five, the run starts at four voters and promotion takes it to five -- +;; but majority(4) and majority(5) are both 3, and followerQuorumForClusterSize +;; is 2 for both, so the quorum-denominator transition this workload is named +;; for never actually happens and a bug that only appears when promotion raises +;; the threshold cannot surface. Three voters going to four moves the majority +;; from 2 to 3 and the follower quorum from 1 to 2. +(def default-nodes ["n1" "n2" "n3" "n4"]) + +;; The last node is reserved as the learner candidate: ElastickvDB's setup +;; otherwise runs `raftadmin add_voter` for every node after the bootstrap +;; one, which leaves no non-member to attach. See ekdb/db :reserve-learner. +(defn learner-candidate + "The node held out of the initial voter set, so :add-learner has something + to attach. Without one, every node is already a voter before the workload + starts and the operation the test is named for cannot run at all." + [nodes] + (last nodes)) + +(defn voter-nodes + "The nodes that join as voters during setup." + [nodes] + (vec (butlast nodes))) + +(defn- raft-majority + "floor(N/2)+1 -- the voters a proposal needs, including the leader." + [n] + (inc (quot n 2))) + +(defn promotion-changes-quorum? + "Whether promoting the candidate actually raises the Raft majority. + + This is the workload's reason to exist, and it is not true of every + topology: four voters going to five needs three either way. A run on such a + topology exercises the promotion mechanics but not the transition, so the + constructor warns rather than letting a green result be read as evidence + about quorum handling." + [nodes] + (let [voters (count (voter-nodes nodes))] + (> (raft-majority (inc voters)) (raft-majority voters)))) +;; --------------------------------------------------------------------------- +;; Pure history analysis +;; --------------------------------------------------------------------------- + +(defn successful-promotions + "Promotions that reported :ok. + + The coverage gate counts these, not every completion: a history with a + :fail or :info promotion had a positive promotion count while every safety + predicate -- which all filter to :ok -- saw nothing, so a run in which no + learner was ever promoted was reported valid." + [history] + (->> history + (filter #(and (= :promote-learner (:f %)) (= :ok (:type %)))) + vec)) + +(defn completed-promotions + "Every :promote-learner COMPLETION. + + Completions only, not invocations: a Jepsen operation appears twice in a + history (an :invoke plus an :ok / :fail / :info), so selecting both counted + each promotion twice and reported an invocation with no completion as a + promotion that happened." + [history] + (->> history + (filter #(= :promote-learner (:f %))) + (remove #(= :invoke (:type %))) + vec)) + +(defn unmeasurable-promotions + "Promotions that reported :ok without the evidence needed to judge them. + + Fails closed. A successful promotion whose completion is missing :match or + :catch-up-target -- status collection failed, say -- used to be filtered out + by the numeric guards, so the checker could return :valid? true having + verified catch-up for nothing at all. An unmeasurable promotion is not a + safe one; it is one we cannot vouch for, and it must show up." + [history] + (->> (completed-promotions history) + (filter #(= :ok (:type %))) + (remove (fn [op] + (let [{:keys [match catch-up-target]} (:value op)] + (and (number? match) (number? catch-up-target))))) + vec)) + +(defn premature-promotions + "Promotions that committed while the learner was behind its catch-up target. + + Measured against the target the operator SAMPLED and passed as + min_applied_index, not against the leader's commit index at promotion time. + + Both alternatives are wrong in opposite directions: + + - Comparing min-applied-index with Match alone proves nothing, because the + engine's own test IS `Match >= min_applied_index`. An operator who reads + the learner's current Match and passes it back satisfies that by + construction, so the correct and the broken procedure are + indistinguishable from the pair. + - Comparing Match with the leader's CURRENT commit index rejects the + documented safe workflow. An operator samples commit index T, waits for + the learner to reach T, and promotes; if the leader commits more entries + while that happens the promotion legitimately has `match >= T` but + `match < leader-commit-index`, and the healthy run is marked invalid. + That also contradicts the \"within N entries\" policy in + docs/raft_learner_operations.md. + + The immutable sampled target is the only reference that distinguishes the + two procedures without rejecting the safe one, so the workload records where + its target came from and `promotions-without-a-sampled-target` rejects a + target derived from the learner." + [history] + (->> (completed-promotions history) + (filter #(= :ok (:type %))) + (filter (fn [op] + (let [{:keys [match catch-up-target]} (:value op)] + (and (number? match) + (number? catch-up-target) + (< match catch-up-target))))) + vec)) + +(defn unenforced-min-applied-index + "Deliberately premature promotions the server ACCEPTED. + + premature-promotions cannot fail on its own. The live workflow samples the + target, waits for the learner to reach it, and only then promotes, recording + that already-qualified applied index as :match -- so match >= target holds by + construction and the property stays empty even if the engine dropped its + min_applied_index test entirely. + + The only way to show the enforcement exists is to ask for something it must + refuse. This op promotes with a min_applied_index far above the leader's + commit index, which no learner can have reached, and expects rejection. + A success here is the defect premature-promotions was meant to catch." + [history] + (->> history + (filter #(and (= :promote-learner-early (:f %)) (= :ok (:type %)))) + vec)) + +(defn unattempted-min-applied-index-probe + "True when the run never issued the premature-promotion probe at all. + + A probe that did not run is not evidence of enforcement, and an empty + unenforced-min-applied-index would otherwise look identical to a passing + one." + [history] + (empty? (filter #(and (= :promote-learner-early (:f %)) + (contains? #{:ok :fail :info} (:type %))) + history))) + +(defn promotions-without-a-sampled-target + "Promotions whose catch-up target did not come from the leader. + + This is what closes the loophole the Match comparison could not: a target + read off the learner's own Match makes the engine's check vacuous, so the + workload records :target-source and anything other than + :leader-commit-index is a procedure violation regardless of the outcome." + [history] + (->> (completed-promotions history) + (filter #(= :ok (:type %))) + (remove #(= :leader-commit-index (:target-source (:value %)))) + vec)) + +(defn- write-intervals + "Each write as [invoke-time complete-time op], for overlap tests. + + A write still in flight has no completion; it is treated as running to the + end of the history, because a write that never completed can still have + taken effect." + [history] + (let [writes (->> history (filter #(= :write (:f %))) (sort-by :time)) + end (or (some->> history (map :time) (reduce max)) 0)] + (->> writes + (reduce (fn [{:keys [pending done]} op] + (case (:type op) + :invoke {:pending (assoc pending (:process op) op) + :done done} + (:ok :fail :info) + (if-let [inv (get pending (:process op))] + {:pending (dissoc pending (:process op)) + :done (conj done [(:time inv) (:time op) op])} + {:pending pending :done done}) + {:pending pending :done done})) + {:pending {} :done []}) + ((fn [{:keys [pending done]}] + (into done (map (fn [[_ inv]] [(:time inv) end inv]) + pending))))))) + +(defn- concurrent-writes + "Writes whose execution interval overlaps [from to], excluding the write + that defines it. + + Completion order alone does not identify a register's final value. Write 1 + running t=10..40 and write 2 running t=20..30 both linearize either way, so + a later read of 2 is legal even though 1 completed last and no write was + INVOKED after 40. Looking only at invocations in (from, to) misses write 2 + entirely and reports a loss that never happened. + + So the ambiguity test is overlap with the selected write's own interval, not + just the gap between it and the read." + [history from to selected] + (->> (write-intervals history) + (remove (fn [[_ _ op]] (identical? op selected))) + (filter (fn [[start stop _]] (and (< start to) (> stop from)))) + (mapv (fn [[_ _ op]] op)))) + +(defn- last-ok-write-interval-before + "The last write acknowledged before t, as its [invoke complete op] triple. + + The triple, not the op: the caller needs the write's own INVOCATION time to + test what overlapped it, and it needs the identical op so that write can be + excluded from its own overlap set. Returning a copy with the invocation time + attached breaks the second -- the copy is not identical to the entry in the + interval list, so the selected write counts as overlapping itself and every + history looks ambiguous." + [history t] + (->> (write-intervals history) + (filter (fn [[_ stop op]] (and (= :ok (:type op)) (< stop t)))) + (sort-by (fn [[_ stop _]] stop)) + last)) + +(defn- read-invocation-times + "Each completed read paired with the time it was actually invoked. + + A process set is not enough. If process P invokes a read before t, completes + it after t, and then invokes another read after t, P is in the set of + processes that invoked after t -- so the EARLIER, pre-t read is selected as + though it had been invoked after t. That admits exactly the stale evidence + the invocation filter exists to exclude. + + Jepsen runs one operation at a time per process, so the invocation a + completion belongs to is the last invoke on that process before it." + [history] + (let [reads (->> history + (filter #(= :read (:f %))) + (sort-by :time))] + (->> reads + (reduce (fn [{:keys [pending done]} op] + (case (:type op) + :invoke {:pending (assoc pending (:process op) (:time op)) + :done done} + (:ok :fail :info) + {:pending (dissoc pending (:process op)) + :done (if (contains? pending (:process op)) + (conj done (assoc op ::invoked-at + (get pending (:process op)))) + done)} + {:pending pending :done done})) + {:pending {} :done []}) + :done))) + +(defn- first-ok-read-invoked-after + "The first successful read whose own INVOCATION happened after t. + + Invocation, not completion: with concurrent workers a read invoked before the + promotion can linearize against the old value and return after it. Selecting + on completion time picks up exactly that read and reports the acknowledged + write as lost, so the pairing has to start from the invoke event -- and from + THIS completion's invoke event, not from any invoke the same process made." + [history t] + (->> (read-invocation-times history) + (filter #(= :ok (:type %))) + (filter #(> (::invoked-at %) t)) + (sort-by :time) + first)) + +(defn lost-writes-across-promotions + "Acknowledged writes that a promotion lost, by TEMPORAL ordering. + + Set subtraction cannot establish this. `write 1 :ok, write 2 :ok, promote + :ok, read 2 :ok` is a legal register history, but subtracting observed + values from acknowledged ones reports 1 as lost merely because it was + overwritten before anyone read it; and a read of 1 taken BEFORE its write + would mask a genuine later loss. + + So this pairs each promotion with the last write acknowledged before it and + the first read that succeeded after it, and reports a loss only when no + other write was in flight in between -- the case where the register's value + is pinned and the read is obliged to return it. Concurrency makes the + expected value ambiguous rather than wrong, so those cases are skipped + instead of guessed at." + [history] + (->> (completed-promotions history) + (filter #(= :ok (:type %))) + (keep (fn [promotion] + (let [t (:time promotion) + interval (last-ok-write-interval-before history t) + [w-start _ write] interval + read (first-ok-read-invoked-after history t)] + ;; A read's :value is the map {:lease? b :value n}, so the + ;; register value has to be unwrapped before comparing it + ;; with the write's scalar. + (let [observed (get-in read [:value :value])] + (when (and write read + ;; From the selected write's own INVOCATION, so a + ;; write that overlapped it counts as ambiguity + ;; rather than being skipped. + (empty? (concurrent-writes + history w-start (:time read) write)) + (not= (:value write) observed)) + {:promotion (:value promotion) + :acked-write (:value write) + :observed-after observed}))))) + vec)) + +(defn promotions-without-post-read-evidence + "Successful promotions with no unambiguous successful read after them. + + lost-writes-across-promotions SKIPS a promotion it cannot pair with a later + read, which happens when catch-up finishes near the time limit. Counting + reads anywhere in the history hid that: the run passed having observed no + post-promotion state at all." + [history] + (->> (successful-promotions history) + (remove #(some? (first-ok-read-invoked-after history (:time %)))) + vec)) + +(defn learner-partition-windows + "Every learners-only partition interval, as [start stop] time pairs. + + Each start is paired with its OWN stop. Taking the first start and the + first later stop examined one window and silently ignored every subsequent + learner-isolation period, so a regression in the second or later window + could not fail the check. Only the completion acknowledging a successful + heal ends the window; invoking stop does not restore the network." + [history] + (let [nemesis (->> history (filter #(= :nemesis (:process %))) (sort-by :time)) + starts (->> nemesis + (filter #(= :start-partition (:f %))) + (filter #(= :learners-only (get-in % [:value :scope])))) + stops (->> nemesis + (filter #(and (= :stop-partition (:f %)) + (= :info (:type %)) + (true? (get-in % [:value :healed])))) + (map :time) vec)] + (->> starts + (map (fn [start] + (let [t (:time start)] + [t (or (first (filter #(> % t) stops)) Long/MAX_VALUE)]))) + vec))) + +(defn- partition-edge-counters + "The {:hit n :miss n} samples the nemesis took at each window's edges." + [history] + (let [nemesis (->> history (filter #(= :nemesis (:process %))) (sort-by :time)) + starts (->> nemesis + (filter #(and (= :start-partition (:f %)) + (= :learners-only (get-in % [:value :scope])) + (= :info (:type %)))) + vec) + stops (->> nemesis + (filter #(and (= :stop-partition (:f %)) + (= :info (:type %)) + (true? (get-in % [:value :healed])))) + vec)] + (->> starts + (keep (fn [start] + (when-let [stop (->> stops + (filter #(> (:time %) (:time start))) + first)] + {:start start + :stop stop + :before (get-in start [:value :lease-counters]) + :after (get-in stop [:value :lease-counters])}))) + vec))) + +(defn lease-fast-path-losses + "Windows in which a lease read FELL BACK to the linearizable path. + + Neither failures nor latency can establish this. A Redis GET calls + LeaseReadForKeyThrough, and when the lease is unavailable kv/raft_engine.go + transparently falls back to LinearizableRead -- so a failure-only check stays + empty through exactly the regression it claims to catch. Latency does not + separate them either: LinearizableRead issues a ReadIndex immediately and, + with every voter connected on one host, its quorum round trip completes in a + few milliseconds. Any budget loose enough not to flag ordinary jitter is far + above the fallback's actual cost, so a run can turn every GET into a + successful slow-path read and still report valid. + + elastickv_lease_read_total separates them by construction: the metric's own + help text defines miss as \"fell back to LinearizableRead\". So the property + is a counter delta across the window, not a time." + [history] + (->> (partition-edge-counters history) + (keep (fn [{:keys [before after start]}] + (when (and before after) + (let [delta (- (:miss after) (:miss before))] + (when (pos? delta) + {:window-start (:time start) + :miss-delta delta}))))) + vec)) + +(defn unmeasured-partition-windows + "Windows that produced no evidence either way. + + Two ways a window proves nothing, and both used to read as a pass: + + - the counters could not be sampled at all (metrics endpoint unreachable), + so zero misses is indistinguishable from no data; + - the lease hit counter did not move, meaning no read was served from the + leader's lease while the candidate was attached and isolated. The read + coverage gate counted successful reads ANYWHERE in the run, so reads taken + before attachment or after the heal satisfied it while the quorum-ack + property was never exercised." + [history] + (->> (partition-edge-counters history) + (keep (fn [{:keys [before after start]}] + (cond + (or (nil? before) (nil? after)) + {:window-start (:time start) :reason :counters-unavailable} + + (not (pos? (- (:hit after) (:hit before)))) + {:window-start (:time start) :reason :no-lease-read-served}))) + vec)) + +(defn learner-partition-read-failures + "Failed lease reads invoked while only learners were partitioned. + + Reads, not writes. The learner is excluded from the write-commit quorum by + the voter set, but `quorumAckTracker` feeds `LastQuorumAck`, which gates the + leader-local LEASE-READ fast path. A learner wrongly counted there does not + stop writes committing -- so a write-failure check stays green through the + exact regression it claims to catch -- it stalls the lease the leader serves + fast reads from. + + Attribute by invocation, not completion: a pre-isolation timeout is not + evidence about the fault, and a read begun during isolation still counts + when it remains stalled until after healing." + [history] + (let [windows (learner-partition-windows history)] + (if (empty? windows) + [] + (->> (read-invocation-times history) + (filter #(and (= :read (:f %)) + (= :fail (:type %)) + (true? (:lease? (:value %))))) + (filter (fn [op] + (some (fn [[start stop]] + (and (>= (::invoked-at op) start) + (<= (::invoked-at op) stop))) + windows))) + (map #(dissoc % ::invoked-at)) + vec)))) + +;; --------------------------------------------------------------------------- +;; Checker +;; --------------------------------------------------------------------------- + +(defn learner-safety-checker + "Checks the learner safety properties over a completed history. + + An EMPTY history is invalid. A run that emitted no operations proves + nothing, and reporting it valid is how a workload that cannot actually + drive the cluster still passes -- which is exactly what this workload did + before it had a client, a nemesis, or a generator that produced its + documented operations." + [] + (reify checker/Checker + (check [_ _test history _opts] + (let [promotions (successful-promotions history) + premature (premature-promotions history) + unmeasurable (unmeasurable-promotions history) + unsampled (promotions-without-a-sampled-target history) + no-evidence (promotions-without-post-read-evidence history) + lost (lost-writes-across-promotions history) + stalls (learner-partition-read-failures history) + fallbacks (lease-fast-path-losses history) + unmeasured (unmeasured-partition-windows history) + unenforced (unenforced-min-applied-index history) + no-probe (unattempted-min-applied-index-probe history) + writes (count (filter #(and (= :write (:f %)) (= :ok (:type %))) history)) + reads (count (filter #(and (= :read (:f %)) (= :ok (:type %))) history))] + (when (seq premature) + (warn "learner promoted before reaching its sampled target:" premature)) + (when (seq unmeasurable) + (warn "promotion reported ok without catch-up evidence:" unmeasurable)) + (when (seq unenforced) + (warn "server accepted a promotion below min_applied_index:" unenforced)) + (when (seq unmeasured) + (warn "partition window produced no lease evidence:" unmeasured)) + {:valid? (and (pos? (count promotions)) + (pos? writes) + (pos? reads) + (not no-probe) + (empty? premature) + (empty? unmeasurable) + (empty? unsampled) + (empty? no-evidence) + (empty? lost) + (empty? stalls) + (empty? fallbacks) + (empty? unmeasured) + (empty? unenforced)) + :promotions (count promotions) + :ok-writes writes + :ok-reads reads + :premature-promotions premature + :unmeasurable-promotions unmeasurable + :unsampled-targets unsampled + :promotions-without-post-read-evidence no-evidence + :lost-writes lost + :learner-read-failures stalls + :lease-fast-path-losses fallbacks + :unmeasured-partition-windows unmeasured + :unenforced-min-applied-index unenforced + :min-applied-index-probe-ran (not no-probe)})))) + +;; --------------------------------------------------------------------------- +;; Client +;; --------------------------------------------------------------------------- + +(def ^:private register-key "learner-register") + +(defn- raftadmin! + "Runs raftadmin on node against the leader address." + [node & args] + (c/on node (c/su (apply c/exec :env "RAFTADMIN_ALLOW_INSECURE=true" + (ekdb/raftadmin-binary) args)))) + +(defn- leader-commit-index + "Samples the leader's commit index: the catch-up target. + + Sampled from the LEADER because a target read off the learner is what makes + the engine's `Match >= min_applied_index` test vacuous." + [node leader-addr] + (:commit_index (ekdb/raft-status node leader-addr))) + +(defn- learner-applied-index + "The learner's OWN applied index, read from the learner. + + Deliberately not the leader's Match for that peer: an independently observed + measure cannot be satisfied by the leader's own bookkeeping, so it is the + stronger evidence of catch-up. (Per-peer Match is not available from + `raftadmin status` on this branch in any case.)" + [learner-node learner-addr] + (:applied_index (ekdb/raft-status learner-node learner-addr))) + +(def ^:private catch-up-poll-ms 200) +(def ^:private catch-up-timeout-ms 60000) + +(defn- await-catch-up! + "Polls the learner until its applied index reaches target." + [learner-node learner-addr target] + (let [deadline (+ (System/currentTimeMillis) catch-up-timeout-ms)] + (loop [] + (let [applied (or (learner-applied-index learner-node learner-addr) 0)] + (cond + (>= applied target) applied + (> (System/currentTimeMillis) deadline) + (throw (ex-info "learner did not reach the catch-up target" + {:target target :applied applied})) + :else (do (Thread/sleep (long catch-up-poll-ms)) (recur))))))) + +(defn- redis-conn + [node->port test node] + (let [port (get node->port node 6379) + host (or (:redis-host test) (name node))] + {:pool {} :spec {:host host :port port :timeout-ms 10000}})) + +(defrecord LearnerClient [node->port leader-addr conn leaseConn] + client/Client + (open! [this test node] + (assoc this + :conn (redis-conn node->port test node) + ;; Lease probes go to a CONNECTED VOTER, not to this worker's own + ;; node. A GET issued through the isolated learner has to proxy to + ;; the leader across the partition and can fail even when the + ;; leader's lease is perfectly healthy -- the checker would then + ;; read client placement as a quorum-ack regression. Ordinary + ;; register traffic stays distributed; only the lease measurement + ;; is pinned. + :leaseConn (redis-conn node->port test (first (:nodes test))))) + + (close! [this _test] this) + (setup! [_this _test]) + (teardown! [_this _test]) + + (invoke! [this test op] + (let [conn (:conn this) + nodes (:nodes test) + leader (first nodes) + addr (or leader-addr (str leader ":50051"))] + (try + (case (:f op) + ;; The reply is CHECKED. Carmine returns nil or a Throwable value on + ;; missing, protocol and some error-reply paths, and treating those + ;; as acknowledged would let the checker call an unconfirmed write + ;; durable -- producing either a false loss or a false preservation + ;; later. Anything that is not "OK" is :info (indeterminate), which + ;; is what an unacknowledged write actually is. + :write (let [reply (wcar conn (car/set register-key (:value op)))] + (if (= "OK" (some-> reply str)) + (assoc op :type :ok) + (assoc op :type :info :error {:unexpected-reply reply}))) + + ;; :lease? marks the read as one the leader may serve from its lease, + ;; which is the path a learner wrongly counted in quorumAckTracker + ;; would break. :latency-ms is recorded because losing the fast path + ;; does not fail the read -- the engine falls back to + ;; LinearizableRead -- it makes it take a Raft round trip. + :read (let [started (System/nanoTime) + v (wcar (:leaseConn this) (car/get register-key)) + elapsed (/ (double (- (System/nanoTime) started)) 1e6)] + (assoc op :type :ok + :value {:lease? true + :latency-ms elapsed + :value (when v (Long/parseLong (str v)))})) + + :add-learner + (let [candidate (name (:value op))] + (raftadmin! leader addr "add_learner" candidate + (str candidate ":" (:grpc-port test 50051)) "0") + (when (:encryption test) + (ekdb/await-node-encryption! leader + (str candidate ":" (:grpc-port test 50051)))) + (assoc op :type :ok)) + + ;; Asks the server for something it must refuse: promotion at a + ;; min_applied_index far above the leader's own commit index, which + ;; no learner can have reached. Rejection is the pass. This is the + ;; only way the enforcement can be observed -- the live promotion + ;; path waits for catch-up first, so it can never present the server + ;; with a violation to reject. + :promote-learner-early + (let [candidate (name (:value op)) + candidate-addr (str candidate ":" (:grpc-port test 50051)) + commit (leader-commit-index leader addr) + unreachable (+ (or commit 0) 1000000)] + (raftadmin! leader addr "promote_learner" candidate + "0" (str unreachable)) + ;; Reached only when the server ACCEPTED it -- the defect. + (assoc op :type :ok + :value {:node candidate + :addr candidate-addr + :leader-commit-index commit + :min-applied-index unreachable})) + + :promote-learner + (let [candidate (name (:value op)) + candidate-addr (str candidate ":" (:grpc-port test 50051)) + ;; Sample the target FIRST, then wait for the learner to reach + ;; it, then promote against that same immutable value. + ;; Recording where the target came from is what lets the + ;; checker reject the vacuous procedure. + target (leader-commit-index leader addr) + applied (await-catch-up! candidate candidate-addr target)] + (raftadmin! leader addr "promote_learner" candidate + "0" (str target)) + (assoc op :type :ok + :value {:node candidate + :catch-up-target target + :target-source :leader-commit-index + :match applied}))) + (catch Exception e + ;; A failed READ keeps its :lease? marker. Returning the bare op left + ;; every real read exception without it, so + ;; learner-partition-read-failures filtered them all out and the + ;; lease regression could never invalidate a run. + ;; :promote-learner-early lands here on the happy path -- its + ;; rejection is the property holding, not a workload failure, which + ;; is why only an :ok from it is reported as a defect. + (if (= :read (:f op)) + (assoc op :type :fail + :value {:lease? true} + :error (.getMessage e)) + (assoc op :type :fail :error (.getMessage e)))))))) + +;; --------------------------------------------------------------------------- +;; Nemesis +;; --------------------------------------------------------------------------- + +(defn learner-partition-nemesis + "Isolates ONLY the learner candidate, leaving every voter connected. + + That is the shape the quorum property needs: voters retain quorum among + themselves, so anything that degrades must be attributable to the learner + being counted where it should not be. + + Implemented directly rather than via nemesis/partitioner, which dispatches on + :start and :stop. Feeding it :start-partition / :stop-partition matched no + branch, so the learner was never isolated and the partition workload ran with + no fault at all -- and its :value would have been taken as the grudge itself, + which is not one." + [nodes] + (let [learner (learner-candidate nodes) + grudge (nemesis/complete-grudge [[learner] (voter-nodes nodes)])] + (reify nemesis/Nemesis + (setup! [this test] + (net/heal! (:net test) test) + this) + + (invoke! [_this test op] + ;; The lease counters are sampled on the node the client sends its + ;; lease probes to, at both edges of the window. The DELTA across the + ;; window is the evidence: misses are reads that fell back to + ;; LinearizableRead, which is precisely the regression, and hits prove + ;; reads were actually served in the window rather than the window + ;; being empty. + (let [probe-node (first (:nodes test))] + (case (:f op) + :start-partition (do + ;; The interval begins only after the network + ;; fault is installed. Sampling before drop-all! + ;; would admit pre-partition reads into the delta. + (net/drop-all! test grudge) + (let [before (ekdb/lease-read-counters probe-node)] + (assoc op :value {:scope :learners-only + :isolated [learner] + :lease-counters before}))) + :stop-partition (let [after (try + ;; Sample while isolation is still + ;; active; finally guarantees that a + ;; failed scrape cannot strand it. + (ekdb/lease-read-counters probe-node) + (finally + (net/heal! (:net test) test)))] + (assoc op :value {:scope :learners-only + :healed true + :lease-counters after})) + (assoc op :value :unsupported)))) + + (teardown! [_this test] + (net/heal! (:net test) test))))) + +(defn learner-nemesis-generator + "ONE start-partition / stop-partition pair, tagged :learners-only. + + Not a cycle. The promotion converts the candidate into a voter, so a later + window carrying the same :learners-only label is isolating a node that now + counts toward quorum: later failures or latency would be attributed to + learner handling, and on a small cluster the isolation could remove a + legitimate voter quorum outright. The single window is opened before the + promotion phase and closed before it runs." + [] + ;; A seq is a generator in Jepsen 0.3.x; gen/seq was removed. + ;; + ;; The window opens at 6s, AFTER :add-learner at ~5s. At 2s it opened before + ;; the candidate was attached, leaving barely a second in which the node was + ;; both attached and isolated -- and at rate 5 with stagger that interval + ;; could contain no read at all, so the quorum-ack property went unexercised + ;; while reads elsewhere in the run satisfied the coverage gate. + [(gen/sleep 6) + {:type :info :f :start-partition} + (gen/sleep 4) + {:type :info :f :stop-partition}]) + +;; --------------------------------------------------------------------------- +;; Generator +;; --------------------------------------------------------------------------- + +(defn client-generator + "Register traffic plus one attach/promote cycle for the reserved candidate. + + The previous generator was `gen/nemesis` applied to nil with no :client at + all, so a run emitted NOTHING: none of the documented :write, :read, + :add-learner or :promote-learner operations could appear, and the checker + reported the resulting empty history as valid." + [nodes] + (let [candidate (learner-candidate nodes) + register (gen/mix [(fn [] {:f :write :value (rand-int 1000000)}) + (fn [] {:f :read})])] + (gen/phases + ;; Some traffic first, so the promotion has acknowledged writes to + ;; preserve across it. + (gen/time-limit 5 register) + (gen/once {:f :add-learner :value candidate}) + ;; Spans the 6s..10s partition window, so the reads the lease counters + ;; measure happen with the candidate attached and isolated. + (gen/time-limit 6 register) + ;; While it is still a learner: ask for a promotion the server must + ;; refuse, which is the only observation that its min_applied_index test + ;; is doing anything. + (gen/once {:f :promote-learner-early :value candidate}) + (gen/time-limit 2 register) + (gen/once {:f :promote-learner :value candidate}) + register))) + +(defn elastickv-learner-test + "Builds a Jepsen test map exercising learner attach and promotion." + ([] (elastickv-learner-test {})) + ([opts] + (when (or (some? (:raft-groups opts)) (some? (:shard-ranges opts))) + (throw (ex-info "learner workload requires the default single Raft group; --raft-groups and --shard-ranges are unsupported" + (select-keys opts [:raft-groups :shard-ranges])))) + (let [nodes (or (:nodes opts) default-nodes) + local? (:local opts) + grpc-port (or (:grpc-port opts) 50051) + redis-port (or (:redis-port opts) 6379) + db (if local? + jdb/noop + (ekdb/db {:grpc-port grpc-port + :redis-port redis-port + :encryption (:encryption opts) + ;; Held out of the voter set so :add-learner + ;; has a non-member to attach. + :reserve-learner (learner-candidate nodes)})) + time-limit (or (:time-limit opts) 30) + ports (or (:node->port opts) + (cli/ports->node-map + (repeat (count nodes) redis-port) nodes))] + {:name "elastickv-learner" + :nodes nodes + :encryption (boolean (:encryption opts)) + ;; Carried into the test map so LearnerClient.open! can honour it. + ;; Without this a programmatic caller could not override the host, and + ;; local or port-mapped runs tried to resolve each logical node name. + :redis-host (:redis-host opts) + :db db + :os (if local? os/noop debian/os) + :net (if local? net/noop net/iptables) + :ssh (merge {:username "vagrant" + :private-key-path "/home/vagrant/.ssh/id_rsa" + :strict-host-key-checking false} + (when local? {:dummy true}) + (:ssh opts)) + :remote control/ssh + :client (->LearnerClient ports nil nil nil) + :nemesis (if local? nemesis/noop (learner-partition-nemesis nodes)) + ;; Jepsen 0.3.x cannot fressian-serialize some final generators. + :final-generator nil + :concurrency (or (:concurrency opts) 10) + :time-limit time-limit + :rate (double (or (:rate opts) 5)) + :checker (learner-safety-checker) + :generator (->> (client-generator nodes) + (gen/nemesis (if local? + (gen/once {:type :info :f :noop}) + (learner-nemesis-generator))) + (gen/stagger (/ (double (or (:rate opts) 5)))) + (gen/time-limit time-limit)) + :grpc-port grpc-port + :ports ports}))) + +(defn prepare-learner-opts + "Normalise parsed CLI options for elastickv-learner-test. + + Not `identity`. run-workload! hands the raw parsed map straight through, and + common-cli-opts leaves :nodes as the comma-separated STRING it came from -- so + the constructor treated a string as a node collection: learner-candidate + returned its last character, the port map was keyed by characters, and the + resulting Jepsen :nodes was invalid. parse-common-opts splits it, and the host + override is copied to :redis-host so the client can honour it." + [options] + (let [options (cli/parse-common-opts options nil)] + (assoc options :redis-host (:host options)))) + +(defn -main + "Runnable entry point. Without one, neither invocation form could select + this workload: the namespace had no -main, and the shared dispatcher in + elastickv.jepsen-test neither required it nor listed it, so passing its + name fell through to the Redis test." + [& args] + (cli/run-workload! args cli/common-cli-opts prepare-learner-opts elastickv-learner-test)) diff --git a/jepsen/test/elastickv/db_test.clj b/jepsen/test/elastickv/db_test.clj index 7fbf81df2..761394df2 100644 --- a/jepsen/test/elastickv/db_test.clj +++ b/jepsen/test/elastickv/db_test.clj @@ -9,6 +9,17 @@ (def ^:private awkward-dir "/mnt/elastickv test's dir") +(deftest lease-counters-accept-production-const-labels + (is (= {:hit 2 :miss 1} + (db/parse-lease-read-counters + (str "# TYPE elastickv_lease_read_total counter\n" + "elastickv_lease_read_total{node_address=\"10.0.0.1:50051\",node_id=\"n1\",outcome=\"hit\"} 2\n" + "elastickv_lease_read_total{outcome=\"miss\",node_id=\"n1\",node_address=\"10.0.0.1:50051\"} 1\n")))) + (is (= {:hit 1000 :miss 0} + (db/parse-lease-read-counters "elastickv_lease_read_total{outcome=\"hit\"} 1e3"))) + (is (nil? (db/parse-lease-read-counters "other_metric{outcome=\"hit\"} 7"))) + (is (nil? (db/parse-lease-read-counters "elastickv_lease_read_total{other_outcome=\"hit\"} 7")))) + (defn- bash-parses? [script] (zero? (:exit (sh/sh "bash" "-n" "-c" script)))) diff --git a/jepsen/test/elastickv/encrypted_cluster_test.clj b/jepsen/test/elastickv/encrypted_cluster_test.clj index 34b67ba0d..7d7e74077 100644 --- a/jepsen/test/elastickv/encrypted_cluster_test.clj +++ b/jepsen/test/elastickv/encrypted_cluster_test.clj @@ -151,6 +151,72 @@ (fn [] (#'ekdb/finish-bootstrap-setup! test "n1" opts))) (is (= [:grpc-ready :mount-checked :activate] @events)))) +(deftest encrypted-bootstrap-keeps-the-reserved-learner-out-of-membership + (let [test {:nodes ["n1" "n2" "n3" "n4"]} + opts {:encryption true :reserve-learner "n4"} + events (atom []) + discoveries (atom []) + verified (atom [])] + (with-redefs-fn + {#'ekdb/wait-for-node-grpc! (fn [node _] (swap! events conj [:ready node])) + #'ekdb/wait-for-mount! (fn [& _]) + #'ekdb/join-peer! (fn [_ peer _] (swap! events conj [:join peer])) + #'ekdb/enable-peer-mount! (fn [_ peer _] (swap! events conj [:mount peer])) + #'ekdb/wrap-fresh-dek! (constantly "wrapped") + #'ekdb/encryption-command! + (fn [_ args] + (swap! events conj [:admin (nth args 2)]) + (when (= "bootstrap" (nth args 2)) + (swap! discoveries into (filter #(.startsWith ^String % "--discover-from=") args)))) + #'ekdb/encryption-status + (fn [_ endpoint] + (swap! verified conj endpoint) + {:full-node-id 11 :local-epoch 1 + :storage-envelope-active true :raft-envelope-active true})} + (fn [] (#'ekdb/finish-bootstrap-setup! test "n1" opts))) + (is (= [[:ready "n1"] [:join "n2"] [:mount "n2"] + [:join "n3"] [:mount "n3"] [:ready "n4"] [:admin "bootstrap"] + [:admin "enable-storage-envelope"] [:admin "enable-raft-envelope"]] + @events)) + (is (= ["--discover-from=n1:50051" "--discover-from=n2:50051" + "--discover-from=n3:50051"] @discoveries)) + (is (= #{"n1:50051" "n2:50051" "n3:50051"} (set @verified))) + (is (= ["n1:50051" "n2:50051" "n3:50051"] (vec (take-last 3 @verified)))))) + +(deftest a-reserved-candidate-that-never-listens-fails-database-setup + (let [activated? (atom false)] + (with-redefs-fn + {#'ekdb/wait-for-node-grpc! (fn [node _] + (when (= "n4" node) + (throw (ex-info "candidate did not start" {})))) + #'ekdb/wait-for-mount! (fn [& _]) + #'ekdb/join-peer! (fn [& _]) + #'ekdb/enable-peer-mount! (fn [& _]) + #'ekdb/activate-encryption! (fn [& _] (reset! activated? true))} + (fn [] + (is (thrown-with-msg? clojure.lang.ExceptionInfo #"candidate did not start" + (#'ekdb/finish-bootstrap-setup! + {:nodes ["n1" "n2" "n3" "n4"]} "n1" + {:encryption true :reserve-learner "n4"}))))) + (is (false? @activated?)))) + +(deftest encrypted-candidate-keeps-join-flags-on-every-start + (let [test {:nodes ["n1" "n2" "n3" "n4"]} + opts {:encryption true :reserve-learner "n4" :filesystem-mount "/mnt/kv"} + candidate (#'ekdb/node-start-opts test "n4" opts) + voter (#'ekdb/node-start-opts test "n2" opts) + args (ekdb/server-args + (merge candidate {:node "n4" :grpc "n4:50051" :redis "n4:6379" + :raft-redis-map "n1=n1:6379"}))] + (is (true? (:encryption candidate))) + (is (true? (:join-as-learner candidate))) + (is (= "n1=n1:50051,n2=n2:50051,n3=n3:50051,n4=n4:50051" + (:join-members candidate))) + (is (nil? (:join-as-learner voter))) + (doseq [flag ["--encryptionSidecarPath" "--encryption-enabled" "--kekFile" + "--raftJoinAsLearner" "--raftJoinMembers" "--filesystemMount"]] + (is (some #{flag} args) flag)))) + (deftest activation-follows-leadership-between-each-operation (let [commands (atom []) status-reads (atom []) diff --git a/jepsen/test/elastickv/learner_workload_test.clj b/jepsen/test/elastickv/learner_workload_test.clj new file mode 100644 index 000000000..d050402a9 --- /dev/null +++ b/jepsen/test/elastickv/learner_workload_test.clj @@ -0,0 +1,654 @@ +(ns elastickv.learner-workload-test + "Unit tests for the learner workload's checker and wiring. + + The checker tests matter more than usual here: the workload's whole value is + that it can FAIL when the learner primitive misbehaves, and an earlier + revision could not — it had no client, no nemesis and a generator that + emitted nothing, so every run produced an empty history the checker called + valid. Several properties were also measuring the wrong thing. Each test + below names the way it used to pass wrongly." + (:require [clojure.test :refer :all] + [clojure.tools.cli :as tools.cli] + [elastickv.cli :as cli] + [elastickv.db :as ekdb] + [elastickv.jepsen-test :as jt] + [elastickv.learner-workload :as lw] + [jepsen.checker :as checker] + [jepsen.client :as client] + [jepsen.nemesis :as nemesis] + [jepsen.net :as net])) + +(defn- check [history] + (checker/check (lw/learner-safety-checker) {} history {})) + +(deftest encrypted-attachment-waits-for-the-candidates-cutovers + (let [test (lw/elastickv-learner-test {:nodes ["n1" "n2" "n3" "n4"] :encryption true}) + events (atom [])] + (is (true? (get-in test [:db :opts :encryption]))) + (is (= "n4" (get-in test [:db :opts :reserve-learner]))) + (with-redefs-fn + {#'lw/raftadmin! (fn [& args] (swap! events conj (vec args))) + #'ekdb/await-node-encryption! (fn [& args] (swap! events conj (vec args)))} + (fn [] + (is (= :ok (:type (client/invoke! (:client test) test + {:f :add-learner :value "n4"})))))) + (is (= [["n1" "n1:50051" "add_learner" "n4" "n4:50051" "0"] + ["n1" "n4:50051"]] @events)))) + +(deftest an-inactive-encrypted-candidate-is-not-a-successful-attachment + (let [test (lw/elastickv-learner-test {:nodes ["n1" "n2" "n3" "n4"] :encryption true})] + (with-redefs-fn + {#'lw/raftadmin! (fn [& _]) + #'ekdb/await-node-encryption! (fn [& _] (throw (ex-info "cutovers inactive" {})))} + (fn [] + (is (= :fail (:type (client/invoke! (:client test) test + {:f :add-learner :value "n4"})))))))) + +(def ^:private healthy-prefix + [{:type :invoke :f :write :value 1 :time 10 :process 0} + {:type :ok :f :write :value 1 :time 20 :process 0} + {:type :invoke :f :read :time 30 :process 0} + {:type :ok :f :read :value {:lease? true :value 1} :time 40 :process 0} + ;; The min_applied_index probe, REFUSED. A :fail here is the property + ;; holding: the live promotion path waits for catch-up first, so this is + ;; the only op that can present the server with a violation to reject. + {:type :invoke :f :promote-learner-early :value "n5" :time 42 :process 0} + {:type :fail :f :promote-learner-early :time 44 :process 0 + :error "min_applied_index not reached"}]) + +(def ^:private probe-refused + "The min_applied_index probe, refused. Histories that are not about the + probe still have to carry it: a run that never asked proves nothing about + enforcement, so the checker treats an absent probe as a failure." + [{:type :invoke :f :promote-learner-early :value "n5" :time 1 :process 9} + {:type :fail :f :promote-learner-early :time 2 :process 9 + :error "min_applied_index not reached"}]) + +(defn- promotion + [m time] + {:type :ok :f :promote-learner :time time + :value (merge {:node "n5" :catch-up-target 100 + :target-source :leader-commit-index :match 100} + m)}) + +(defn- healthy-history [] + (concat healthy-prefix + [{:type :invoke :f :promote-learner :value "n5" :time 50 :process 0} + (promotion {} 60) + ;; Invoked AFTER the promotion completed, which is what makes this + ;; read evidence about post-promotion state. + {:type :invoke :f :read :time 70 :process 0} + {:type :ok :f :read :value {:lease? true :value 1} :time 80 :process 0}])) + +;; A post-promotion read, invoke and completion, for histories whose subject is +;; some other property but which still have to satisfy the coverage gate. +(defn- post-promotion-read + ([t] (post-promotion-read t 1)) + ([t v] [{:type :invoke :f :read :time t :process 0} + {:type :ok :f :read :value {:lease? true :value v} :time (inc t) :process 0}])) + +(deftest healthy-history-is-valid + (is (:valid? (check (healthy-history))))) + +;; --------------------------------------------------------------------------- +;; 1. Promotion never outruns catch-up +;; --------------------------------------------------------------------------- + +(deftest premature-promotion-is-rejected + (let [r (check (concat healthy-prefix [(promotion {:match 90 :catch-up-target 100} 60)]))] + (is (false? (:valid? r))) + (is (= 1 (count (:premature-promotions r)))))) + +(deftest catch-up-is-measured-against-the-sampled-target-not-a-moving-leader + ;; The documented safe workflow: sample target T, wait for the learner to + ;; reach T, promote. The leader keeps committing, so at promotion time the + ;; learner is at T while the leader is well past it. Comparing Match with + ;; the leader's CURRENT commit index rejected this healthy run. + (let [r (check (concat healthy-prefix + [(promotion {:catch-up-target 100 + :match 100 + :leader-commit-index 100000} 60)] + (post-promotion-read 70)))] + (is (:valid? r) + "a learner that reached its sampled target is caught up, whatever the leader did since"))) + +(deftest a-target-read-off-the-learner-is-rejected + ;; The loophole the Match comparison cannot close: the engine's own test is + ;; Match >= min_applied_index, so passing the learner's current Match + ;; satisfies it by construction. The procedure, not the arithmetic, has to + ;; be checked. + (let [r (check (concat healthy-prefix + [(promotion {:target-source :learner-match} 60)]))] + (is (false? (:valid? r))) + (is (= 1 (count (:unsampled-targets r)))))) + +(deftest a-promotion-without-evidence-fails-closed + ;; Previously the numeric guards simply skipped these, so a successful but + ;; unmeasurable promotion left the checker reporting valid having verified + ;; nothing. + (doseq [missing [{:match nil} {:catch-up-target nil}]] + (let [r (check (concat healthy-prefix [(promotion missing 60)]))] + (is (false? (:valid? r)) (str "missing " (keys missing))) + (is (= 1 (count (:unmeasurable-promotions r))))))) + +(deftest promotions-are-counted-once-per-completed-call + ;; An op appears as :invoke plus a completion, so counting both reported two + ;; promotions per call and counted a bare invocation as one. + (let [r (check (concat healthy-prefix + [{:type :invoke :f :promote-learner :value "n5" :time 50} + (promotion {} 60)]))] + (is (= 1 (:promotions r))))) + +;; --------------------------------------------------------------------------- +;; 2. No acknowledged write is lost across a promotion +;; --------------------------------------------------------------------------- + +(deftest a-write-lost-across-a-promotion-is-rejected + (let [r (check (concat probe-refused + [{:type :invoke :f :write :value 7 :time 10} + {:type :ok :f :write :value 7 :time 20} + {:type :invoke :f :promote-learner :value "n5" :time 30} + (promotion {} 40) + {:type :invoke :f :read :time 50} + {:type :ok :f :read :value {:lease? true :value 3} :time 60}]))] + (is (false? (:valid? r))) + (is (= 1 (count (:lost-writes r)))))) + +(deftest an-overwritten-value-is-not-a-lost-write + ;; Set subtraction reported 1 as lost in this legal register history merely + ;; because it was overwritten before anyone read it. + (let [r (check (concat probe-refused + [{:type :invoke :f :write :value 1 :time 10} + {:type :ok :f :write :value 1 :time 20} + {:type :invoke :f :write :value 2 :time 30} + {:type :ok :f :write :value 2 :time 40} + {:type :invoke :f :promote-learner :value "n5" :time 50} + (promotion {} 60) + {:type :invoke :f :read :time 70} + {:type :ok :f :read :value {:lease? true :value 2} :time 80}]))] + (is (:valid? r) (str "lost-writes=" (:lost-writes r))))) + +(deftest a-concurrent-write-makes-the-expected-value-ambiguous-not-wrong + ;; With another write in flight between the acked write and the read, the + ;; register's value is not pinned, so no conclusion is drawn rather than a + ;; false loss being reported. + (let [r (check (concat probe-refused + [{:type :invoke :f :write :value 1 :time 10} + {:type :ok :f :write :value 1 :time 20} + {:type :invoke :f :promote-learner :value "n5" :time 30} + (promotion {} 40) + {:type :invoke :f :write :value 9 :time 50} + {:type :invoke :f :read :time 60} + {:type :ok :f :read :value {:lease? true :value 9} :time 70}]))] + (is (:valid? r) (str "lost-writes=" (:lost-writes r))))) + +;; --------------------------------------------------------------------------- +;; 3. A learner never counts toward the lease +;; --------------------------------------------------------------------------- + +(defn- partition-window + "A learners-only window carrying the lease counters the nemesis samples at + its edges. The default delta is a healthy one: reads were served from the + lease (hit moved) and none fell back (miss did not)." + ([start stop] (partition-window start stop {:hit 10 :miss 0} {:hit 14 :miss 0})) + ([start stop before after] + [{:type :info :process :nemesis :f :start-partition + :value {:scope :learners-only :lease-counters before} :time start} + {:type :info :process :nemesis :f :stop-partition + :value {:scope :learners-only :healed true :lease-counters after} :time stop}])) + +(deftest a-lease-read-failing-under-learner-isolation-is-rejected + ;; Reads, not writes: quorumAckTracker feeds LastQuorumAck and the + ;; lease-read fast path, not the write-commit quorum, so a write-failure + ;; check stayed green through the exact regression it claimed to detect. + (let [r (check (concat healthy-prefix + [(promotion {} 50)] + (partition-window 100 200) + [{:type :invoke :f :read :time 120} + {:type :fail :f :read :value {:lease? true} :time 130}]))] + (is (false? (:valid? r))) + (is (= 1 (count (:learner-read-failures r)))))) + +(deftest every-learner-partition-window-is-checked-not-just-the-first + ;; Taking the first start and the first later stop ignored every subsequent + ;; isolation window, so a regression in the second one could not fail. + (let [r (check (concat healthy-prefix + [(promotion {} 50)] + (partition-window 100 200) + (partition-window 300 400) + [{:type :invoke :f :read :time 320} + {:type :fail :f :read :value {:lease? true} :time 330}]))] + (is (false? (:valid? r))) + (is (= 1 (count (:learner-read-failures r))) + "a failure in the SECOND window must still be caught"))) + +(deftest a-read-failure-outside-any-window-is-not-attributed-to-the-learner + (let [r (check (concat healthy-prefix + [(promotion {} 50)] + (post-promotion-read 60) + (partition-window 100 200) + [{:type :invoke :f :read :time 500} + {:type :fail :f :read :value {:lease? true} :time 510}]))] + (is (:valid? r) (str "failures=" (:learner-read-failures r))))) + +(deftest failed-lease-reads-are-attributed-by-their-own-invocations + (let [history (concat (partition-window 100 200) + [{:type :invoke :f :read :time 90 :process 1} + {:type :fail :f :read :time 110 :process 1 + :value {:lease? true} :error "before isolation"} + {:type :invoke :f :read :time 120 :process 1} + {:type :fail :f :read :time 250 :process 1 + :value {:lease? true} :error "during isolation"} + {:type :invoke :f :read :time 300 :process 1} + {:type :fail :f :read :time 310 :process 1 + :value {:lease? true} :error "after isolation"}]) + failures (lw/learner-partition-read-failures history)] + (is (= ["during isolation"] (mapv :error failures))) + (is (= [250] (mapv :time failures))))) + +(deftest isolation-ends-at-the-confirmed-heal-not-the-stop-invocation + (doseq [invocation-type [:invoke :info]] + (let [history (concat (partition-window 100 300) + [{:type invocation-type :process :nemesis + :f :stop-partition :time 200} + {:type :invoke :f :read :process 1 :time 250} + {:type :fail :f :read :process 1 :time 350 + :value {:lease? true} :error "still isolated"} + {:type :invoke :f :read :process 1 :time 400} + {:type :fail :f :read :process 1 :time 410 + :value {:lease? true} :error "already healed"}])] + (is (= [[100 300]] (lw/learner-partition-windows history))) + (is (= ["still isolated"] + (mapv :error (lw/learner-partition-read-failures history)))) + (is (empty? (lw/unmeasured-partition-windows history)))))) + +;; --------------------------------------------------------------------------- +;; The checker must not pass a run that proved nothing +;; --------------------------------------------------------------------------- + +(deftest an-empty-history-is-invalid + ;; THE load-bearing test. Before the workload had a client, a nemesis and a + ;; generator that emitted its documented operations, a real run produced an + ;; empty history — and the checker called it valid, so the gate could never + ;; fail. + (is (false? (:valid? (check []))))) + +(deftest a-history-with-no-promotion-is-invalid + (is (false? (:valid? (check healthy-prefix))) + "a learner test that never promoted has not tested promotion")) + +(deftest a-history-with-no-reads-is-invalid + (is (false? (:valid? (check [{:type :invoke :f :write :value 1 :time 10} + {:type :ok :f :write :value 1 :time 20} + (promotion {} 30)]))) + "the lease property is unobservable without reads")) + +;; --------------------------------------------------------------------------- +;; Wiring: the workload has to be able to run at all +;; --------------------------------------------------------------------------- + +(deftest the-test-map-has-a-client-and-a-nemesis + ;; It had neither, so nothing could drive the cluster. + (let [t (lw/elastickv-learner-test {:nodes ["n1" "n2" "n3" "n4" "n5"]})] + (is (some? (:client t))) + (is (some? (:nemesis t))) + (is (some? (:generator t))))) + +(deftest the-generator-emits-every-documented-operation + ;; The old generator was gen/nemesis applied to nil, so none of :write, + ;; :read, :add-learner or :promote-learner could ever appear. + (let [ops (->> (lw/client-generator ["n1" "n2" "n3" "n4" "n5"]) + (tree-seq coll? seq) + (keep #(when (map? %) (:f %))) + set)] + (is (contains? ops :add-learner)) + (is (contains? ops :promote-learner)))) + +(deftest a-node-is-reserved-outside-the-initial-voter-set + ;; ElastickvDB's setup adds every node after the bootstrap one as a voter, + ;; so without a reservation there is no non-member left to attach and + ;; :add-learner cannot run. + (let [nodes ["n1" "n2" "n3" "n4" "n5"]] + (is (= "n5" (lw/learner-candidate nodes))) + (is (= ["n2" "n3" "n4"] (ekdb/voter-peers nodes "n5")) + "the reserved candidate must not be joined as a voter") + (is (= ["n2" "n3" "n4" "n5"] (ekdb/voter-peers nodes nil)) + "with no reservation the existing behaviour is unchanged") + (is (= "n5" (get-in (lw/elastickv-learner-test {:nodes nodes}) + [:db :opts :reserve-learner]))))) + +(deftest the-workload-is-reachable-from-the-shared-dispatcher + ;; Neither invocation form could select it: the namespace had no -main and + ;; the dispatcher neither required nor listed it, so the name fell through + ;; to the Redis test. + (is (fn? (deref (resolve 'elastickv.learner-workload/-main))) + "the namespace needs a -main to be runnable directly") + (is (= "elastickv-learner" + (:name (jt/elastickv-learner-test {:nodes ["n1" "n2" "n3"]}))))) + +(deftest raft-status-parsing-reads-the-indices-the-promotion-needs + (let [out (str "state: Leader\n" + "leader_id: \"n1\"\n" + "term: 3\n" + "commit_index: 4211\n" + "applied_index: 4207\n" + "pending_conf_change: false\n") + got (ekdb/parse-raft-status out)] + (is (= 4211 (:commit_index got))) + (is (= 4207 (:applied_index got))) + (is (= "n1" (:leader_id got))))) + +;; --------------------------------------------------------------------------- +;; Round 3: the workload has to be able to run, and each property has to be +;; able to fire. Every test below names the way the previous revision passed +;; wrongly. +;; --------------------------------------------------------------------------- + +(deftest coverage-requires-a-successful-promotion + ;; The gate counted every completion, so a history with good reads and writes + ;; plus a FAILED promotion had a positive count while every safety predicate + ;; — which all filter to :ok — saw nothing. A run in which no learner was + ;; ever promoted was reported valid. + (let [failed (check (concat healthy-prefix + [{:type :invoke :f :promote-learner :value "n5" :time 50} + {:type :fail :f :promote-learner :time 60}] + (post-promotion-read 70)))] + (is (false? (:valid? failed))) + (is (zero? (:promotions failed)) + "a failed promotion is not a promotion"))) + +(deftest a-promotion-with-no-read-after-it-is-invalid + ;; lost-writes-across-promotions SILENTLY SKIPS a promotion it cannot pair + ;; with a later read, which happens when catch-up finishes near the time + ;; limit. Counting reads anywhere in the history hid that: the run passed + ;; having observed no post-promotion state at all. + (let [r (check (concat healthy-prefix [(promotion {} 60)]))] + (is (false? (:valid? r))) + (is (= 1 (count (:promotions-without-post-read-evidence r)))))) + +(deftest a-read-invoked-before-the-promotion-is-not-post-promotion-evidence + ;; With concurrent workers a read invoked before the promotion can linearize + ;; against the old value and return after it. Selecting on completion time + ;; alone picked up exactly that read — both accepting it as evidence and + ;; reporting the acknowledged write as lost. + (let [r (check [{:type :invoke :f :write :value 7 :time 10} + {:type :ok :f :write :value 7 :time 20} + ;; Invoked BEFORE the promotion, returns after it. + {:type :invoke :f :read :time 30 :process 1} + {:type :invoke :f :promote-learner :value "n5" :time 40} + (promotion {} 50) + {:type :ok :f :read :value {:lease? true :value 7} :time 60 :process 1}])] + (is (false? (:valid? r))) + (is (= 1 (count (:promotions-without-post-read-evidence r))) + "a read that started before the promotion proves nothing about after it") + (is (empty? (:lost-writes r)) + "and it must not be mistaken for a lost write either"))) + +(deftest a-lease-read-that-fell-back-under-learner-isolation-is-rejected + ;; Neither failures nor latency can establish this. A Redis GET calls + ;; LeaseReadForKeyThrough and the engine falls back to LinearizableRead + ;; transparently, so the read SUCCEEDS — and on one host that fallback costs + ;; milliseconds, so any latency budget loose enough not to flag jitter sits + ;; far above it. elastickv_lease_read_total separates them by construction: + ;; its help text defines miss as "fell back to LinearizableRead". + (let [r (check (concat healthy-prefix + [(promotion {} 50)] + (post-promotion-read 60) + (partition-window 100 200 + {:hit 10 :miss 0} + {:hit 12 :miss 3})))] + (is (false? (:valid? r))) + (is (= 1 (count (:lease-fast-path-losses r))) + "a lease read that fell back to a Raft round trip must fail the run") + (is (= 3 (:miss-delta (first (:lease-fast-path-losses r))))))) + +(deftest a-fast-lease-read-under-learner-isolation-is-fine + (let [r (check (concat healthy-prefix + [(promotion {} 50)] + (post-promotion-read 60) + (partition-window 100 200)))] + (is (:valid? r) (str "losses=" (:lease-fast-path-losses r) + " unmeasured=" (:unmeasured-partition-windows r))))) + +(deftest a-fallback-outside-the-window-is-not-attributed-to-the-learner + ;; The counters are sampled at the window's own edges, so a miss that + ;; happened outside it cannot appear in the delta at all. + (let [r (check (concat healthy-prefix + [(promotion {} 50)] + (post-promotion-read 60) + (partition-window 100 200) + [{:type :invoke :f :read :time 500} + {:type :ok :f :read :time 510 + :value {:lease? true :latency-ms 900.0 :value 1}}]))] + (is (:valid? r) (str "losses=" (:lease-fast-path-losses r))))) + +(deftest a-window-with-no-lease-read-served-proves-nothing + ;; The read coverage gate counted successful reads ANYWHERE in the run, so + ;; reads taken before attachment or after the heal satisfied it while the + ;; quorum-ack property went unexercised. An unmoved hit counter is what that + ;; looks like from the metrics. + (let [r (check (concat healthy-prefix + [(promotion {} 50)] + (post-promotion-read 60) + (partition-window 100 200 + {:hit 10 :miss 0} + {:hit 10 :miss 0})))] + (is (false? (:valid? r))) + (is (= [:no-lease-read-served] + (map :reason (:unmeasured-partition-windows r)))))) + +(deftest a-window-whose-counters-could-not-be-sampled-proves-nothing + ;; Zero misses and no data are indistinguishable unless this is checked. + (let [r (check (concat healthy-prefix + [(promotion {} 50)] + (post-promotion-read 60) + (partition-window 100 200 nil nil)))] + (is (false? (:valid? r))) + (is (= [:counters-unavailable] + (map :reason (:unmeasured-partition-windows r)))))) + +(deftest a-server-that-accepts-a-premature-promotion-fails-the-run + ;; premature-promotions cannot fail on its own: the live path waits for + ;; catch-up and then records that already-qualified index as :match, so + ;; match >= target holds by construction even if the engine dropped the + ;; min_applied_index test entirely. An accepted probe is the observation. + (let [r (check (concat [{:type :invoke :f :write :value 1 :time 10 :process 0} + {:type :ok :f :write :value 1 :time 20 :process 0} + {:type :invoke :f :read :time 30 :process 0} + {:type :ok :f :read :value {:lease? true :value 1} + :time 40 :process 0} + {:type :invoke :f :promote-learner-early :value "n5" + :time 42 :process 0} + {:type :ok :f :promote-learner-early :time 44 :process 0 + :value {:node "n5" :leader-commit-index 100 + :min-applied-index 1000100}}] + [(promotion {} 50)] + (post-promotion-read 60)))] + (is (false? (:valid? r))) + (is (= 1 (count (:unenforced-min-applied-index r))) + "a promotion accepted above any reachable applied index is the defect"))) + +(deftest a-run-that-never-probed-min-applied-index-is-not-evidence + ;; Without this, "no accepted premature promotions" and "never asked" look + ;; identical in the result map. + (let [r (check (concat [{:type :invoke :f :write :value 1 :time 10 :process 0} + {:type :ok :f :write :value 1 :time 20 :process 0} + {:type :invoke :f :read :time 30 :process 0} + {:type :ok :f :read :value {:lease? true :value 1} + :time 40 :process 0}] + [(promotion {} 50)] + (post-promotion-read 60)))] + (is (false? (:valid? r))) + (is (false? (:min-applied-index-probe-ran r))))) + +(deftest overlapping-writes-are-ambiguous-rather-than-lost + ;; Completion order alone does not identify the register's final value. + ;; Write 1 runs t=10..40 and write 2 runs t=20..30: both linearization + ;; orders are legal, so a later read of 2 is valid. Looking only for writes + ;; INVOKED after write 1 completed misses write 2 and reports a loss that + ;; never happened. + (let [r (check (concat [{:type :invoke :f :write :value 1 :time 10 :process 0} + {:type :invoke :f :write :value 2 :time 20 :process 1} + {:type :ok :f :write :value 2 :time 30 :process 1} + {:type :ok :f :write :value 1 :time 40 :process 0} + {:type :invoke :f :promote-learner-early :value "n5" + :time 42 :process 2} + {:type :fail :f :promote-learner-early :time 44 :process 2}] + [(promotion {} 50)] + [{:type :invoke :f :read :time 60 :process 3} + {:type :ok :f :read :value {:lease? true :value 2} + :time 70 :process 3}]))] + (is (empty? (:lost-writes r)) + "a write that overlapped the last completion makes the value ambiguous") + (is (:valid? r) (str r)))) + +(deftest a-completion-is-paired-with-its-own-invocation + ;; Process membership is not a pairing. P invokes a read before the + ;; promotion, completes it after, then invokes another read after — P is in + ;; the set of processes that invoked after t, so the EARLIER, pre-promotion + ;; completion was selected as the first post-promotion read. + (let [r (check (concat healthy-prefix + [{:type :invoke :f :write :value 7 :time 45 :process 0} + {:type :ok :f :write :value 7 :time 46 :process 0} + ;; Invoked before the promotion, completes after. + {:type :invoke :f :read :time 48 :process 1} + (promotion {} 50) + {:type :ok :f :read :value {:lease? true :value 1} + :time 55 :process 1} + ;; The same process reads again, properly after. + {:type :invoke :f :read :time 60 :process 1} + {:type :ok :f :read :value {:lease? true :value 7} + :time 70 :process 1}]))] + (is (empty? (:lost-writes r)) + "the stale completion must not be read as the post-promotion value") + (is (:valid? r) (str r)))) + +(deftest promotion-must-actually-change-the-quorum-threshold + ;; Five nodes reserve one candidate, so the run starts at four voters and + ;; promotion takes it to five — but majority(4) and majority(5) are both 3, + ;; so the transition the workload is named for never happens. + (is (false? (lw/promotion-changes-quorum? ["n1" "n2" "n3" "n4" "n5"])) + "four voters to five needs three either way") + (is (lw/promotion-changes-quorum? ["n1" "n2" "n3" "n4"]) + "three voters to four moves the majority from 2 to 3") + (is (lw/promotion-changes-quorum? lw/default-nodes) + "the default topology must be one where promotion changes the threshold")) + +(deftest the-nemesis-handles-the-operations-the-generator-emits + ;; jepsen.nemesis/partitioner dispatches on :start and :stop. Feeding it + ;; :start-partition / :stop-partition matched no branch, so the learner was + ;; never isolated and the partition workload ran with no fault at all — and + ;; its :value would have been taken as the grudge, which is not one. + (let [nodes ["n1" "n2" "n3" "n4" "n5"] + emitted (->> (lw/learner-nemesis-generator) + (tree-seq coll? seq) + (keep #(when (map? %) (:f %))) + set)] + (is (= #{:start-partition :stop-partition} emitted)) + ;; And the nemesis must claim exactly those. + (let [n (lw/learner-partition-nemesis nodes)] + (is (some? n)) + (is (satisfies? nemesis/Nemesis n))))) + +(deftest partition-counters-are-sampled-inside-the-network-boundaries + (let [nodes ["n1" "n2" "n3" "n4"] + test {:nodes nodes :net :test-net} + n (lw/learner-partition-nemesis nodes) + events (atom [])] + (with-redefs [net/drop-all! (fn [_test _grudge] + (swap! events conj :partition-installed)) + net/heal! (fn [_net _test] + (swap! events conj :partition-healed)) + ekdb/lease-read-counters (fn [_node] + (swap! events conj :counter-sampled) + {:hit 1 :miss 0})] + (nemesis/invoke! n test {:type :info :f :start-partition}) + (is (= [:partition-installed :counter-sampled] @events)) + + (reset! events []) + (is (true? (get-in (nemesis/invoke! n test {:type :info :f :stop-partition}) + [:value :healed]))) + (is (= [:counter-sampled :partition-healed] @events))))) + +(deftest the-learner-partition-runs-once-not-in-a-cycle + ;; The promotion converts the candidate into a voter, so a later window + ;; carrying the same :learners-only label isolates a node that now counts + ;; toward quorum: failures and latency after that are not learner behaviour, + ;; and on a small cluster the isolation can remove a legitimate voter quorum. + (let [ops (->> (lw/learner-nemesis-generator) + (tree-seq coll? seq) + (keep #(when (map? %) (:f %))))] + (is (= 1 (count (filter #{:start-partition} ops)))) + (is (= 1 (count (filter #{:stop-partition} ops)))))) + +(deftest cli-options-are-parsed-before-the-workload-is-constructed + ;; run-workload! hands the raw parsed map straight through, and + ;; common-cli-opts leaves :nodes as the comma-separated STRING. With identity + ;; as the preparation fn the constructor treated that string as a node + ;; collection: learner-candidate returned its last character and the port map + ;; was keyed by characters. + (let [prepared (lw/prepare-learner-opts {:nodes "n1,n2,n3" :faults ""})] + (is (= ["n1" "n2" "n3"] (:nodes prepared)) + "a comma-separated string must be split before it reaches the constructor") + (is (= "n3" (lw/learner-candidate (:nodes prepared)))))) + +(deftest unsupported-topology-options-are-not-silently-ignored + (doseq [args [["--raft-groups" "1=50061"] + ["--raft-groups" "1=50061,2=50062"] + ["--shard-ranges" "a:z=1"]] + local? [false true]] + (let [{:keys [options errors]} (tools.cli/parse-opts args cli/common-cli-opts) + prepared (lw/prepare-learner-opts (assoc options :local local?))] + (is (empty? errors)) + (is (thrown-with-msg? clojure.lang.ExceptionInfo #"default single Raft group" + (lw/elastickv-learner-test prepared)) + (str args " local=" local?)))) + (let [{:keys [options errors]} (tools.cli/parse-opts [] cli/common-cli-opts)] + (is (empty? errors)) + (is (map? (lw/elastickv-learner-test (lw/prepare-learner-opts options)))))) + +(deftest the-redis-host-override-reaches-the-test-map + ;; LearnerClient.open! reads :redis-host from the completed test map, so the + ;; constructor has to carry it; otherwise a programmatic caller could not + ;; override the host and local runs tried to resolve each logical node name. + (let [t (lw/elastickv-learner-test {:nodes ["n1" "n2" "n3"] :redis-host "127.0.0.1"})] + (is (= "127.0.0.1" (:redis-host t))))) + +(deftest the-reserved-candidate-starts-with-a-join-configuration + ;; Reserving the candidate only kept it out of the add_voter loop; it still + ;; launched as a fresh non-bootstrap process with no peers, which the etcd + ;; engine refuses (errNoPeersConfigured), so it exited before :add-learner + ;; could attach it. + (let [args (set (ekdb/server-args {:node "n5" :grpc "n5:50051" :redis "n5:6379" + :data-dir "/var/lib/elastickv" + :raft-redis-map "n5=n5:6379" + :join-as-learner true + :join-members "n1=n1:50051,n2=n2:50051"}))] + (is (contains? args "--raftJoinAsLearner")) + (is (contains? args "--raftJoinMembers")) + (is (contains? args "n1=n1:50051,n2=n2:50051"))) + + ;; The candidate is INCLUDED, at its own listener address. resolveJoinServers + ;; (main.go) rejects a members list that omits the local --raftId with + ;; ErrJoinMembersMissingLocalNode, so a candidate given only the voters + ;; receives the join flags and then exits during startup validation. + (is (= "n1=n1:50051,n2=n2:50051,n5=n5:50051" + (ekdb/join-members-arg ["n1" "n2" "n5"] "n5" 50051))) + + ;; The address must be the candidate's OWN listener: resolveJoinServers also + ;; refuses when the local entry's address does not match, so a placeholder + ;; that merely carries the right ID would fail the same validation. + (is (clojure.string/includes? + (ekdb/join-members-arg ["n1" "n2" "n5"] "n5" 50051) + "n5=n5:50051")) + + ;; With no reserved candidate the list is just the nodes, unduplicated. + (is (= "n1=n1:50051,n2=n2:50051" + (ekdb/join-members-arg ["n1" "n2"] nil 50051))) + + ;; An ordinary voter gets neither flag. + (let [args (set (ekdb/server-args {:node "n2" :grpc "n2:50051" :redis "n2:6379" + :data-dir "/var/lib/elastickv" + :raft-redis-map "n2=n2:6379"}))] + (is (not (contains? args "--raftJoinAsLearner"))) + (is (not (contains? args "--raftJoinMembers")))))