Add bencher_replica: in-process SQLite replication to replace Litestream

Litestream 0.5.13 blocked all API writes for ~5.5 minutes in production
(2026-07-10): when it decides a full re-snapshot is needed, it copies the
entire database into a local LTX file while holding the SQLite write lock,
and no configuration reaches that code path. Its LTX compaction also churns
whole-database S3 transfers several times a day.

bencher_replica is an in-process replacement built around six invariants
(documented in src/lib.rs), the prime one being that the SQLite write lock
is only ever held for O(WAL-tail) work, never O(database):

- WAL parser with full salt and cumulative checksum-chain verification
- Local filesystem XOR S3-compatible storage behind one contract
- Step-driven sync engine; checkpoints are PASSIVE while the replicator
  itself holds BEGIN IMMEDIATE, closing the ship-vs-checkpoint race without
  ever needing RESTART or TRUNCATE checkpoints
- Generation-based snapshots via a single-step SQLite online backup into a
  scratch file (transactionally consistent under concurrent checkpoints),
  throttled zstd multipart upload, snapshot.json as the atomic commit marker
- Latest-only restore in the same startup handshake slot Litestream used,
  with chain pre-validation and checkpoint-consumption verification
- Restore-and-compare verification (default daily) and shadow mode: with
  both plus.litestream and plus.replica configured, Litestream keeps
  checkpoint ownership and restore precedence during the burn-in

Also included:
- Fix: standalone sweep connections (stats, credit grants) now disable
  wal_autocheckpoint when replication is configured; previously the credit
  sweep could checkpoint and restart the WAL behind Litestream's back
- plus.replica config (JsonReplication), otel Replica* counters, main.rs
  lifecycle wiring (restore precedence, fatal race arm, final ship inside
  the Fly kill budget), TestServer::new_with_replica, Dockerfile stubs
- JsonLitestream.metrics_port and [[metrics]] in the Fly configs so
  litestream_* Prometheus metrics are scraped during the shadow period

Testing: 275 crate tests (WAL fixtures cross-validated against SQLite
itself, a three-backend storage contract suite, 8 fault-injection
scenarios, 6 crash kill points, 8 seeded 200-op equivalence workloads,
ignored soak and live-S3 tiers) plus 4 server-level integration tests.
An adversarial multi-agent review confirmed 18 findings, all fixed with
regression tests, including a silent data-loss gap in resume (salt-match
resume now proves content against the replica tip, and the meta-verified
path requires salt1 continuity).

Author: epompeii

.github/workflows/test.yml

@@ -281,3 +281,76 @@ jobs:
         env:
           RUST_BACKTRACE: full
         run: cargo test-api smoke docker
+
+  replica_live_s3:
+    name: Replica Live S3 (${{ matrix.server.name }})
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        server:
+          # Two S3-compatible implementations: MinIO exercises the full API
+          # (including ListMultipartUploads for the orphan sweep); RustFS
+          # pins compatibility with a second, Rust-native implementation.
+          - name: minio
+            start: >-
+              docker run -d --name live-s3 -p 9000:9000
+              -e MINIO_ROOT_USER=bencher-test
+              -e MINIO_ROOT_PASSWORD=bencher-test-secret
+              minio/minio:RELEASE.2025-09-07T16-13-09Z server /data
+          - name: rustfs
+            start: >-
+              docker run -d --name live-s3 -p 9000:9000
+              -e RUSTFS_ACCESS_KEY=bencher-test
+              -e RUSTFS_SECRET_KEY=bencher-test-secret
+              rustfs/rustfs:1.0.0-beta.10
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/setup-rust
+        with:
+          cache-key: replica-live-s3
+          mold-version: ${{ inputs.mold-version }}
+      - name: Start S3 server
+        env:
+          START_SERVER: ${{ matrix.server.start }}
+        run: |
+          ${START_SERVER}
+          for _ in $(seq 1 30); do
+            if curl -s -o /dev/null http://127.0.0.1:9000/; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "S3 server did not come up" >&2
+          docker logs live-s3 >&2 || true
+          exit 1
+      - name: Create bucket
+        env:
+          AWS_ACCESS_KEY_ID: bencher-test
+          AWS_SECRET_ACCESS_KEY: bencher-test-secret
+          AWS_DEFAULT_REGION: us-east-1
+        run: |
+          for _ in $(seq 1 15); do
+            if aws --endpoint-url http://127.0.0.1:9000 s3api create-bucket \
+              --bucket bencher-replica-test; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "failed to create the test bucket" >&2
+          docker logs live-s3 >&2 || true
+          exit 1
+      - name: Run live S3 tier
+        env:
+          RUST_BACKTRACE: "1"
+          BENCHER_REPLICA_TEST_S3_BUCKET: bencher-replica-test
+          BENCHER_REPLICA_TEST_S3_ACCESS_KEY_ID: bencher-test
+          BENCHER_REPLICA_TEST_S3_SECRET_ACCESS_KEY: bencher-test-secret
+          BENCHER_REPLICA_TEST_S3_ENDPOINT: http://127.0.0.1:9000
+          BENCHER_REPLICA_TEST_S3_REGION: us-east-1
+          BENCHER_REPLICA_TEST_S3_PREFIX: ci-live-s3
+        run: |
+          cargo nextest run -p bencher_replica --features plus,testing \
+            --run-ignored ignored-only --no-fail-fast -E 'binary(live_s3)'

Cargo.lock

@@ -67,6 +67,7 @@ bencher_otel_provider = { path = "plus/bencher_otel_provider" }
 bencher_oci_storage = { path = "plus/bencher_oci_storage" }
 bencher_rate_limiter = { path = "plus/bencher_rate_limiter" }
 bencher_recaptcha = { path = "plus/bencher_recaptcha" }
+bencher_replica = { path = "plus/bencher_replica" }
 # plus - runner
 bencher_runner = { path = "plus/bencher_runner" }
 bencher_oci = { path = "plus/bencher_oci" }

Cargo.toml

@@ -61,6 +61,7 @@ RUN cargo init --lib bencher_otel
 RUN cargo init --lib bencher_otel_provider
 RUN cargo init --lib bencher_rate_limiter
 RUN cargo init --lib bencher_recaptcha
+RUN cargo init --lib bencher_replica
 RUN cargo init --lib bencher_rootfs
 RUN cargo init --lib bencher_runner
 RUN cargo init --lib bencher_init

docker/bench.Dockerfile

@@ -42,8 +42,12 @@ slog.workspace = true
 
 [dev-dependencies]
 bencher_api_tests.workspace = true
-bencher_json = { workspace = true, features = ["server", "schema"] }
+bencher_json = { workspace = true, features = ["server", "schema", "test-clock"] }
+bencher_replica = { workspace = true, features = ["plus", "testing"] }
+camino.workspace = true
 http.workspace = true
+rusqlite.workspace = true
+tempfile.workspace = true
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
 
 [lints]

lib/api_server/Cargo.toml

@@ -27,6 +27,7 @@ pub async fn server_config_options(
 /// View server configuration
 ///
 /// View the API server configuration.
+/// Secrets in the configuration are masked in the response.
 /// The user must be an admin on the server to use this route.
 #[endpoint {
     method = GET,
@@ -53,7 +54,7 @@ async fn get_one_inner(log: &Logger) -> Result<JsonConfig, HttpError> {
             )
         })?
         .unwrap_or_default()
-        .into())
+        .sanitized())
 }
 
 #[endpoint {

lib/api_server/src/config.rs

@@ -2,8 +2,16 @@
 #[cfg(test)]
 use bencher_api_tests as _;
 #[cfg(test)]
+use bencher_replica as _;
+#[cfg(test)]
+use camino as _;
+#[cfg(test)]
 use http as _;
 #[cfg(test)]
+use rusqlite as _;
+#[cfg(test)]
+use tempfile as _;
+#[cfg(test)]
 use tokio as _;
 
 mod backup;

lib/api_server/src/lib.rs

@@ -4,7 +4,7 @@ use bencher_json::{
 };
 use bencher_schema::{
     auth_conn,
-    context::{ApiContext, DbConnection},
+    context::{ApiContext, DbConnection, configure_standalone_connection},
     error::{forbidden_error, issue_error, not_found_error},
     model::{
         server::QueryServer,
@@ -49,8 +49,23 @@ pub async fn server_stats_get(
 
 async fn get_one_inner(log: &Logger, context: &ApiContext) -> Result<JsonServerStats, HttpError> {
     let query_server = QueryServer::get_server(auth_conn!(context))?;
-    let conn = DbConnection::establish(context.database.path.to_string_lossy().as_ref())
+    let mut conn = DbConnection::establish(context.database.path.to_string_lossy().as_ref())
         .map_err(not_found_error)?;
+    // Route this standalone connection through the shared configuration so it
+    // does not bypass the busy_timeout / autocheckpoint settings the pools and
+    // the writer use: under replication a stray write must never checkpoint.
+    configure_standalone_connection(
+        &mut conn,
+        context.database.busy_timeout,
+        context.database.replicated,
+    )
+    .map_err(|e| {
+        issue_error(
+            "Failed to configure server stats connection",
+            "Failed to configure the server stats database connection PRAGMAs",
+            e,
+        )
+    })?;
     query_server
         .get_stats(log.clone(), conn, context.is_bencher_cloud)
         .await