+ * Default {@code true} (on). WebSocket transport only. + */ + public LineSenderBuilder symbolDictReset(boolean enabled) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("symbol_dict_reset is only supported for WebSocket transport"); + } + this.symbolDictReset = enabled; + return this; + } + + /** + * Number of distinct symbols the sender's dictionary may accumulate before + * {@link #symbolDictReset(boolean)} triggers a recycle. Must be greater than + * {@code 0} and no larger than {@link QwpConstants#MAX_SYMBOL_DICTIONARY_SIZE}. + *
+ * Default {@code 100_000}. WebSocket transport only. + */ + public LineSenderBuilder symbolDictResetThreshold(int threshold) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("symbol_dict_reset_threshold is only supported for WebSocket transport"); + } + if (threshold <= 0 || threshold > QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE) { + throw new LineSenderException("symbol_dict_reset_threshold must be > 0 and <= ") + .put(QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE).put(": ").put(threshold); + } + this.symbolDictResetThreshold = threshold; + return this; + } + + /** + * Upper bound, in milliseconds, a triggered symbol-dictionary recycle waits + * for an opportunistic (idle) window before forcing the rebuild. {@code 0} + * means opportunistic-only: the recycle never forces, it only takes idle + * windows as they occur. + *
+ * Default {@code 30_000} (30 s). WebSocket transport only.
+ */
+ public LineSenderBuilder symbolDictResetMaxWaitMillis(long maxWaitMillis) {
+ if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) {
+ throw new LineSenderException("symbol_dict_reset_max_wait_millis is only supported for WebSocket transport");
+ }
+ if (maxWaitMillis < 0) {
+ throw new LineSenderException("symbol_dict_reset_max_wait_millis must be >= 0: ").put(maxWaitMillis);
+ }
+ this.symbolDictResetMaxWaitMillis = maxWaitMillis;
+ return this;
+ }
+
/**
* close() drain timeout in milliseconds. The sender's {@code close()}
* method blocks up to this many millis waiting for the server to ACK
@@ -3870,6 +3936,30 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) {
}
pos = getValue(configurationString, pos, sink, "catch_up_cap_gap_min_escalation_window_millis");
catchUpCapGapMinEscalationWindowMillis(parseLongValue(sink, "catch_up_cap_gap_min_escalation_window_millis"));
+ } else if (Chars.equals("symbol_dict_reset", sink)) {
+ if (protocol != PROTOCOL_WEBSOCKET) {
+ throw new LineSenderException("symbol_dict_reset is only supported for WebSocket transport");
+ }
+ pos = getValue(configurationString, pos, sink, "symbol_dict_reset");
+ if (Chars.equalsIgnoreCase("on", sink)) {
+ symbolDictReset(true);
+ } else if (Chars.equalsIgnoreCase("off", sink)) {
+ symbolDictReset(false);
+ } else {
+ throw new LineSenderException("invalid symbol_dict_reset [value=").put(sink).put(", allowed-values=[on, off]]");
+ }
+ } else if (Chars.equals("symbol_dict_reset_threshold", sink)) {
+ if (protocol != PROTOCOL_WEBSOCKET) {
+ throw new LineSenderException("symbol_dict_reset_threshold is only supported for WebSocket transport");
+ }
+ pos = getValue(configurationString, pos, sink, "symbol_dict_reset_threshold");
+ symbolDictResetThreshold(parseIntValue(sink, "symbol_dict_reset_threshold"));
+ } else if (Chars.equals("symbol_dict_reset_max_wait_millis", sink)) {
+ if (protocol != PROTOCOL_WEBSOCKET) {
+ throw new LineSenderException("symbol_dict_reset_max_wait_millis is only supported for WebSocket transport");
+ }
+ pos = getValue(configurationString, pos, sink, "symbol_dict_reset_max_wait_millis");
+ symbolDictResetMaxWaitMillis(parseLongValue(sink, "symbol_dict_reset_max_wait_millis"));
} else if (Chars.equals("initial_connect_retry", sink)) {
if (protocol != PROTOCOL_WEBSOCKET) {
throw new LineSenderException("initial_connect_retry is only supported for WebSocket transport");
@@ -4145,6 +4235,12 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString)
if (view.has("catch_up_cap_gap_min_escalation_window_millis")) {
catchUpCapGapMinEscalationWindowMillis(wsLong(view, v, "catch_up_cap_gap_min_escalation_window_millis"));
}
+ if (view.has("symbol_dict_reset_threshold")) {
+ symbolDictResetThreshold(wsInt(view, v, "symbol_dict_reset_threshold"));
+ }
+ if (view.has("symbol_dict_reset_max_wait_millis")) {
+ symbolDictResetMaxWaitMillis(wsLong(view, v, "symbol_dict_reset_max_wait_millis"));
+ }
if (view.has("sf_append_deadline_millis")) {
sfAppendDeadlineMillis(wsLong(view, v, "sf_append_deadline_millis"));
}
@@ -4214,6 +4310,16 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString)
throw new LineSenderException("invalid initial_connect_retry [value=").put(s).put(", allowed-values=[on, off, true, false, sync, async]]");
}
}
+ s = view.getStr("symbol_dict_reset");
+ if (s != null) {
+ if (s.equalsIgnoreCase("on")) {
+ symbolDictReset(true);
+ } else if (s.equalsIgnoreCase("off")) {
+ symbolDictReset(false);
+ } else {
+ throw new LineSenderException("invalid symbol_dict_reset [value=").put(s).put(", allowed-values=[on, off]]");
+ }
+ }
return this;
} catch (IllegalArgumentException e) {
throw new LineSenderException(e.getMessage());
@@ -4329,6 +4435,9 @@ public java.util.Map
+ * Called only from the tail of {@link #resetTableBuffersAfterFlush()} (a
+ * safe point: no row in progress, this flush's data already handed to the
+ * engine), never from the per-symbol registration path
+ * ({@link #getOrAddGlobalSymbol}) -- arming mid-row or mid-encode would
+ * observe a dictionary size that has not yet settled for this batch.
+ */
+ private void armIfEligible() {
+ boolean shouldArm = resetEnabled
+ && (globalSymbolDictionary.size() >= resetThresholdSymbols || manualResetRequested);
+ if (shouldArm && !resetArmed) {
+ armedSinceNanos = System.nanoTime();
+ starvationWaitDoneThisArm = false;
+ }
+ resetArmed = shouldArm;
}
/**
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleArmingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleArmingTest.java
new file mode 100644
index 00000000..009c6fb4
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleArmingTest.java
@@ -0,0 +1,273 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.DelegatingFilesFacade;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * Covers the arming half of the symbol-dictionary recycle feature:
+ * {@code QwpWebSocketSender.armIfEligible()}, called at the tail of
+ * {@code resetTableBuffersAfterFlush()}, and the manual advisory API
+ * {@link Sender#resetSymbolDictionary()}.
+ */
+public class SymbolDictRecycleArmingTest {
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ @Test
+ public void testArmsAtThreshold() throws Exception {
+ // threshold=3, send rows with symbols a,b -> flush -> not armed;
+ // add c -> flush -> armed
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = ackingServer()) {
+ try (Sender sender = Sender.fromConfig(cfg(server) + "symbol_dict_reset_threshold=3;")) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.table("t").symbol("s", "a").longColumn("v", 1).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1).atNow();
+ sender.flush();
+ Assert.assertFalse(ws.isResetArmed());
+ sender.table("t").symbol("s", "c").longColumn("v", 1).atNow();
+ sender.flush();
+ Assert.assertTrue(ws.isResetArmed());
+ }
+ }
+ });
+ }
+
+ /**
+ * Decision 5: arming ignores {@code deltaDictEnabled}. Proving this by
+ * crossing {@code symbol_dict_reset_threshold} while degraded (as
+ * {@link #testArmsAtThreshold} does for the default delta-dict mode) would
+ * additionally need a custom low threshold on a hand-built
+ * {@code CursorSendEngine} carrying the fault-injecting {@code FilesFacade}
+ * (fault injection requires bypassing {@code Sender.fromConfig}, which
+ * offers no {@code FilesFacade} seam) -- reaching both together needs
+ * {@code QwpWebSocketSender}'s widest 27-parameter {@code connect()}
+ * overload. Substituting the manual {@link Sender#resetSymbolDictionary()}
+ * advisory request for "cross the threshold" reaches the identical
+ * {@code armIfEligible()} branch -- the other arm of the same {@code ||} --
+ * through the same 9-parameter {@code connect()} overload
+ * {@code MmapFaultDegradesTest} itself uses, with no loss of coverage:
+ * {@code armIfEligible()} does not special-case either trigger on
+ * {@code deltaDictEnabled}.
+ */
+ @Test
+ public void testArmsInFullDictMode() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("arm-full-dict-sf").toString();
+ String slot = Paths.get(sfDir, "default").toString();
+ Assert.assertEquals(0, io.questdb.client.std.Files.mkdir(sfDir,
+ io.questdb.client.std.Files.DIR_MODE_DEFAULT));
+
+ try (TestWebSocketServer server = ackingServer()) {
+ int port = server.getPort();
+
+ MmapFaultDictFacade ff = new MmapFaultDictFacade();
+ CursorSendEngine engine = new CursorSendEngine(
+ slot, 4L * 1024 * 1024, 64L * 1024 * 1024,
+ CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, ff);
+ QwpWebSocketSender sender = QwpWebSocketSender.connect(
+ "localhost", port, null, 0, 0, 0L, null, false, engine);
+ try {
+ ff.armed = true; // next dictionary mmap growth raises a recognised fault
+ sender.table("m").symbol("s", "boom").longColumn("v", 1L).atNow();
+ try {
+ sender.flush();
+ Assert.fail("expected the injected mmap fault to fail this flush");
+ } catch (LineSenderException expected) {
+ // Same guard MmapFaultDegradesTest pins: the fault degrades the
+ // sender to full self-sufficient frames instead of propagating raw.
+ }
+ Assert.assertFalse("a recognised mmap access fault must degrade the sender "
+ + "to full-dict mode",
+ sender.isDeltaDictEnabledForTest());
+
+ // The fault facade disarms itself after firing once, so this retry
+ // succeeds and clears pendingRowCount back to 0.
+ sender.flush();
+ Assert.assertFalse("neither threshold nor manual request has fired yet",
+ sender.isResetArmed());
+
+ sender.resetSymbolDictionary();
+ Assert.assertTrue("manual reset request must arm even in full-dict mode",
+ sender.isResetArmed());
+ } finally {
+ sender.close();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testDoesNotArmWhenDisabled() throws Exception {
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = ackingServer()) {
+ try (Sender sender = Sender.fromConfig(
+ cfg(server) + "symbol_dict_reset=off;symbol_dict_reset_threshold=2;")) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ sender.flush();
+ Assert.assertFalse("symbol_dict_reset=off must never arm", ws.isResetArmed());
+ sender.table("t").symbol("s", "c").longColumn("v", 1L).atNow();
+ sender.flush();
+ Assert.assertFalse("symbol_dict_reset=off must never arm", ws.isResetArmed());
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testManualResetRequestArms() throws Exception {
+ assertMemoryLeak(() -> {
+ // pendingRowCount == 0: resetSymbolDictionary() arms immediately.
+ try (TestWebSocketServer server = ackingServer()) {
+ try (Sender sender = Sender.fromConfig(cfg(server))) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.resetSymbolDictionary();
+ Assert.assertTrue(ws.isResetArmed());
+ }
+ }
+
+ // Mid-batch: a request while a row is buffered (pendingRowCount != 0)
+ // only arms once the next flush runs armIfEligible() at its tail.
+ try (TestWebSocketServer server = ackingServer()) {
+ try (Sender sender = Sender.fromConfig(cfg(server))) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.resetSymbolDictionary();
+ Assert.assertFalse("a mid-batch request must not arm before the next flush",
+ ws.isResetArmed());
+ sender.flush();
+ Assert.assertTrue(ws.isResetArmed());
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testResetSymbolDictionaryOnNonWsSenderIsNoOp() throws Exception {
+ assertMemoryLeak(() -> {
+ // protocolVersion(2) skips the eager server-side settings detection
+ // connect that build() otherwise performs, so no live server is needed
+ // (see LineSenderBuilderTest.testCustomPemRootsDoNotRequirePassword).
+ try (Sender sender = Sender.builder(Sender.Transport.HTTP)
+ .address("localhost")
+ .protocolVersion(2)
+ .build()) {
+ sender.resetSymbolDictionary();
+ }
+ });
+ }
+
+ @Test
+ public void testSplitFlushPathArms() throws Exception {
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = ackingServer()) {
+ server.setAdvertisedMaxBatchSize(150); // forces the two-table batch to split
+ // Padding inflates each table past half the cap, so the combined
+ // two-table message exceeds it while each single-table split frame fits.
+ String pad = TestUtils.repeat("x", 60);
+ try (Sender sender = Sender.fromConfig(cfg(server) + "symbol_dict_reset_threshold=2;")) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.table("t1").symbol("s", "a").stringColumn("p", pad).longColumn("v", 1L).atNow();
+ sender.table("t2").symbol("s", "b").stringColumn("p", pad).longColumn("v", 2L).atNow();
+ sender.flush();
+ Assert.assertTrue("the split-flush path shares resetTableBuffersAfterFlush's tail",
+ ws.isResetArmed());
+ }
+ }
+ });
+ }
+
+ private static TestWebSocketServer ackingServer() throws Exception {
+ TestWebSocketServer server = new TestWebSocketServer(new AckAllHandler());
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ return server;
+ }
+
+ private static String cfg(TestWebSocketServer server) {
+ return "ws::addr=localhost:" + server.getPort() + ";";
+ }
+
+ /**
+ * ACKs every frame it receives; does not otherwise inspect the wire.
+ */
+ private static class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ try {
+ client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Raises a RECOGNISED mmap access fault out of the persisted dictionary's next
+ * mmap growth, once, when {@link #armed}. Copied from
+ * {@code MmapFaultDegradesTest.MmapFaultDictFacade}.
+ */
+ private static final class MmapFaultDictFacade extends DelegatingFilesFacade {
+ boolean armed;
+
+ @Override
+ public boolean isMmapAllowed() {
+ return true;
+ }
+
+ @Override
+ public long mmap(int fd, long len, long offset, int flags, int memoryTag) {
+ if (armed) {
+ armed = false;
+ throw new InternalError(
+ "a fault occurred in a recent unsafe memory access operation in compiled Java code");
+ }
+ return INSTANCE.mmap(fd, len, offset, flags, memoryTag);
+ }
+ }
+}
From 8f90f4559321d9de580305acedb6d11237665083 Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Mon, 17 Aug 2026 17:25:43 +0100
Subject: [PATCH 05/49] Prove threshold arming survives full-dict degradation
testArmsInFullDictMode previously substituted the manual
resetSymbolDictionary() advisory request for crossing
symbol_dict_reset_threshold, which never touches globalSymbolDictionary and
so left a future deltaDictEnabled-conditioned regression in threshold-based
arming undetected. Rewrite it to construct the sender through the widest
connect(List
+ * {@code externalFsnBase} is the additive offset this loop folds into every
+ * user-visible FSN it produces (progress-dispatcher advances and
+ * {@link SenderError} spans) -- see {@link #externalFsnBase}. Pass {@code 0L}
+ * unless the caller is replacing an engine a symbol-dict recycle rebuilt.
*/
public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
long fsnAtZero, long parkNanos,
@@ -934,13 +949,14 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
int maxHeadFrameRejections,
long poisonMinEscalationWindowMillis,
long catchUpCapGapMinEscalationWindowMillis,
- ReconnectPolicy reconnectPolicy) {
+ ReconnectPolicy reconnectPolicy,
+ long externalFsnBase) {
this(client, engine, fsnAtZero, parkNanos, reconnectFactory,
reconnectInitialBackoffMillis,
reconnectMaxBackoffMillis, durableAckMode,
durableAckKeepaliveIntervalMillis, maxHeadFrameRejections,
poisonMinEscalationWindowMillis, catchUpCapGapMinEscalationWindowMillis,
- catchUpPolicyFor(reconnectPolicy));
+ catchUpPolicyFor(reconnectPolicy), externalFsnBase);
}
private static CatchUpCapGapPolicy catchUpPolicyFor(ReconnectPolicy reconnectPolicy) {
@@ -1786,8 +1802,8 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM
// role rejects are classified into the transient branch below.
LOG.error("terminal upgrade error during {} -- won't retry: {}",
phase, e.getMessage());
- long fromFsn = engine.ackedFsn() + 1L;
- long toFsn = Math.max(fromFsn, engine.publishedFsn());
+ long fromFsn = externalFsnBase + engine.ackedFsn() + 1L;
+ long toFsn = Math.max(fromFsn, externalFsnBase + engine.publishedFsn());
SenderError err = new SenderError(
SenderError.Category.SECURITY_ERROR,
SenderError.Policy.TERMINAL,
@@ -1826,8 +1842,8 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM
// volatile first-writer-wins latch observed by the owner.
capabilityGapTerminal = e;
}
- long fromFsn = engine.ackedFsn() + 1L;
- long toFsn = Math.max(fromFsn, engine.publishedFsn());
+ long fromFsn = externalFsnBase + engine.ackedFsn() + 1L;
+ long toFsn = Math.max(fromFsn, externalFsnBase + engine.publishedFsn());
SenderError err = new SenderError(
SenderError.Category.PROTOCOL_VIOLATION,
SenderError.Policy.TERMINAL,
@@ -1958,7 +1974,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM
* producer stays alive and no data is at risk.
*/
private void dispatchRetriedEndpointPolicyFailure(SenderError.Category category, String message) {
- long fromFsn = engine.ackedFsn() + 1L;
+ long fromFsn = externalFsnBase + engine.ackedFsn() + 1L;
dispatchError(new SenderError(
category,
SenderError.Policy.RETRIABLE,
@@ -1966,7 +1982,7 @@ private void dispatchRetriedEndpointPolicyFailure(SenderError.Category category,
message,
SenderError.NO_MESSAGE_SEQUENCE,
fromFsn,
- Math.max(fromFsn, engine.publishedFsn()),
+ Math.max(fromFsn, externalFsnBase + engine.publishedFsn()),
null,
System.nanoTime()
));
@@ -2072,9 +2088,11 @@ private void haltOnPoisonedFrame(String lastRejection, long toFsnHint) {
// the operator at those bytes would misattribute the poison. The
// caller supplies the span end: a NACK names the exact frame, so the
// span is that single frame; a non-orderly close cannot single one
- // out, so it spans to publishedFsn.
- long fromFsn = poisonFsn;
- long toFsn = Math.max(fromFsn, toFsnHint);
+ // out, so it spans to publishedFsn. poisonFsn and toFsnHint are both
+ // raw internal FSNs (fsnAtZero already folded in by the caller where
+ // relevant); rebase both by externalFsnBase here.
+ long fromFsn = externalFsnBase + poisonFsn;
+ long toFsn = Math.max(fromFsn, externalFsnBase + toFsnHint);
String msg = "frame at fsn=" + fromFsn + " rejected " + poisonStrikes
+ " consecutive times with no acceptance at or beyond it -- poisoned frame, replay cannot succeed (last: "
+ lastRejection + ')';
@@ -2098,12 +2116,14 @@ private void haltOnPoisonedFrame(String lastRejection, long toFsnHint) {
* Notify the progress dispatcher that the ack watermark advanced to
* {@code ackedFsn}. Caller must already have observed the advance via
* {@link CursorSendEngine#acknowledge}'s boolean return; this method
- * does no further filtering.
+ * does no further filtering. {@code ackedFsn} is the engine-relative FSN
+ * (fsnAtZero already folded in by the caller); this rebases it by
+ * {@link #externalFsnBase} before it reaches the user-visible dispatcher.
*/
private void dispatchProgress(long ackedFsn) {
SenderProgressDispatcher d = progressDispatcher;
if (d != null) {
- d.offer(ackedFsn);
+ d.offer(externalFsnBase + ackedFsn);
}
}
@@ -3842,8 +3862,8 @@ private void handlePreSendRejection(long wireSeq, byte status,
// protocol-violation close path uses (see onClose above): there
// is no FSN we can attribute the rejection to, so we report
// the unacked range the producer can correlate against.
- long fromFsn = engine.ackedFsn() + 1L;
- long toFsn = Math.max(fromFsn, engine.publishedFsn());
+ long fromFsn = externalFsnBase + engine.ackedFsn() + 1L;
+ long toFsn = Math.max(fromFsn, externalFsnBase + engine.publishedFsn());
String tableName = response.getTableEntryCount() == 1
? response.getTableName(0)
: null;
@@ -3966,8 +3986,8 @@ private void handleServerRejection(long wireSeq) {
status & 0xFF,
response.getErrorMessage(),
wireSeq,
- fsn,
- fsn,
+ externalFsnBase + fsn,
+ externalFsnBase + fsn,
tableName,
System.nanoTime()
);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleFsnContinuityTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleFsnContinuityTest.java
new file mode 100644
index 00000000..da886eeb
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleFsnContinuityTest.java
@@ -0,0 +1,451 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.SenderError;
+import io.questdb.client.SenderProgressHandler;
+import io.questdb.client.LineSenderServerException;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.WebSocketResponse;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * FSN epoch-base continuity across a symbol-dict recycle. Task 5's engine
+ * rebuild restarts the internal cursor engine's raw FSNs at 0; every
+ * user-visible FSN surface must stay strictly monotone across that boundary
+ * by translating {@code external = fsnEpochBase + raw} (negative sentinels
+ * pass untranslated). The recycle machinery itself lands in a later task --
+ * these tests exercise the translation seam ({@code rollFsnEpochBaseForTest})
+ * directly, either on a single already-connected sender (for the sender-level
+ * surfaces, which read the live {@code fsnEpochBase} field regardless of when
+ * the I/O loop was built) or by rolling a fresh sender's base BEFORE its
+ * first connect (for the loop-level surfaces -- the progress dispatcher and
+ * {@link SenderError} spans -- whose {@code externalFsnBase} is frozen at
+ * loop construction).
+ */
+public class SymbolDictRecycleFsnContinuityTest {
+
+ @Test
+ public void testPreRollTargetAnswersTrueAfterRoll() throws Exception {
+ try (TestWebSocketServer server = ackingServer()) {
+ try (QwpWebSocketSender sender = (QwpWebSocketSender) Sender.fromConfig(cfg(server))) {
+ sender.table("t").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: the batch must actually be acked before the roll",
+ sender.drain(5_000));
+
+ sender.rollFsnEpochBaseForTest(fsn1);
+
+ long t0 = System.nanoTime();
+ Assert.assertTrue("a target FSN from a pre-recycle epoch must be reported acked "
+ + "immediately -- it was proven acked before the swap",
+ sender.awaitAckedFsn(fsn1, 0));
+ long elapsedMs = (System.nanoTime() - t0) / 1_000_000;
+ Assert.assertTrue("must short-circuit, not poll: took " + elapsedMs + "ms",
+ elapsedMs < 200);
+ }
+ }
+ }
+
+ @Test
+ public void testPostRollSequencesExceedAllPreRoll() throws Exception {
+ try (TestWebSocketServer server = ackingServer()) {
+ try (QwpWebSocketSender sender = (QwpWebSocketSender) Sender.fromConfig(cfg(server))) {
+ sender.table("t").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.drain(5_000));
+
+ sender.rollFsnEpochBaseForTest(fsn1);
+ long newBase = sender.getFsnEpochBaseForTest();
+ Assert.assertEquals(fsn1 + 1, newBase);
+
+ sender.table("t").longColumn("v", 2L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.drain(5_000));
+
+ Assert.assertTrue("post-roll FSN must exceed every pre-roll FSN: fsn2=" + fsn2
+ + " fsn1=" + fsn1,
+ fsn2 > fsn1);
+ // Raw FSNs increment by 1 per single-row flush, so the raw watermark right
+ // after this second flush is fsn1+1 (the same raw value the roll consumed
+ // plus one) -- fsn2 must be the epoch base plus that raw value, not just
+ // greater than fsn1 (a bug that dropped the translation entirely would still
+ // pass the ">" check above because the underlying raw engine never resets).
+ Assert.assertEquals(newBase + fsn1 + 1, fsn2);
+ }
+ }
+ }
+
+ @Test
+ public void testGetAckedFsnMonotoneAcrossRoll() throws Exception {
+ try (TestWebSocketServer server = ackingServer()) {
+ long w;
+ long lastPublishedFsn;
+ try (QwpWebSocketSender sender1 = (QwpWebSocketSender) Sender.fromConfig(cfg(server))) {
+ sender1.table("t").longColumn("v", 1L).atNow();
+ long fsn1 = sender1.flushAndGetSequence();
+ Assert.assertTrue(sender1.drain(5_000));
+ w = sender1.getAckedFsn();
+ lastPublishedFsn = fsn1;
+ Assert.assertEquals("sanity: single-batch acked watermark must match its own FSN",
+ fsn1, w);
+ }
+
+ // A fresh sender/engine models the post-recycle engine that restarts its
+ // internal FSNs at 0; rolling its epoch base by the outgoing epoch's last
+ // published FSN is exactly what the recycle swap does in production.
+ QwpWebSocketSender sender2 = createRolledSender(server, lastPublishedFsn);
+ try {
+ long newBase = sender2.getFsnEpochBaseForTest();
+ Assert.assertEquals(lastPublishedFsn + 1, newBase);
+
+ Assert.assertEquals("before any new ack, getAckedFsn must read the synthetic "
+ + "watermark: one past the last external FSN the outgoing epoch "
+ + "ever reported",
+ newBase - 1, sender2.getAckedFsn());
+ Assert.assertTrue(sender2.getAckedFsn() >= w);
+
+ sender2.table("t").longColumn("v", 2L).atNow();
+ sender2.flush();
+ Assert.assertTrue(sender2.drain(5_000));
+ Assert.assertTrue("a new ack must advance the watermark past the synthetic "
+ + "post-roll value",
+ sender2.getAckedFsn() > newBase - 1);
+ } finally {
+ sender2.close();
+ }
+ }
+ }
+
+ /**
+ * The raw-feed bug test: without the {@code drain()} fix, a rolled epoch base makes
+ * the raw {@code cursorEngine.publishedFsn()} target look like it belongs to a
+ * pre-recycle epoch (its raw value is smaller than the rolled base), so the fixed
+ * {@code awaitAckedFsn} would short-circuit {@code true} on an un-rebased target --
+ * even though the frame was never actually acked. Must fail (spurious true) before
+ * {@code drain()} translates its target by {@code fsnEpochBase}.
+ */
+ @Test
+ public void testDrainAfterRollWaitsForNewFrames() throws Exception {
+ GatedAckHandler handler = new GatedAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ QwpWebSocketSender sender = (QwpWebSocketSender) Sender.fromConfig(cfg(server));
+ try {
+ // Roll well past the raw FSNs this fresh engine will ever publish, so a
+ // missing translation in drain() would make its raw target look pre-roll.
+ sender.rollFsnEpochBaseForTest(999L);
+
+ sender.table("foo").longColumn("v", 1L).atNow();
+ boolean drainedEarly = sender.drain(200);
+ Assert.assertFalse("drain() must not spuriously report the new frame acked just "
+ + "because its raw FSN is smaller than the rolled epoch base",
+ drainedEarly);
+
+ handler.releaseAcks();
+ Assert.assertTrue("drain() must return true once the real ack arrives",
+ sender.drain(5_000));
+ } finally {
+ handler.releaseAcks();
+ sender.close();
+ }
+ }
+ }
+
+ /**
+ * {@link SenderError#getFromFsn()} / {@link SenderError#getToFsn()} surface synchronously
+ * via {@link LineSenderServerException#getServerError()}, unreachable by any
+ * dispatcher-side rebase -- the loop must rebase the span itself. Rolls the epoch base
+ * BEFORE the sender's first connect (the loop's {@code externalFsnBase} is frozen at
+ * construction) so the terminal NACK's span is built under a nonzero base.
+ */
+ @Test
+ public void testSenderErrorSpansCarryExternalFsns() throws Exception {
+ TerminalNackHandler handler = new TerminalNackHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ AtomicReference
+ * Precondition: {@code cursorSendLoop} must be {@code null}. A live loop's
+ * {@code externalFsnBase} is a construction-time snapshot -- it is never updated
+ * on an already-built loop -- so rolling while one is attached would silently
+ * desynchronize the two: {@link #getAckedFsn()} / {@link #flushAndGetSequence()}
+ * would report post-roll values while every {@code SenderProgressHandler} advance
+ * and {@link SenderError} span the loop emits would stay pinned at pre-roll
+ * values. The recycle swap must call this strictly between tearing the old loop
+ * down and constructing the new one.
*/
private void rollFsnEpochBase(long lastPublishedFsn) {
+ if (cursorSendLoop != null) {
+ throw new IllegalStateException("rollFsnEpochBase must run while cursorSendLoop"
+ + " is null -- the loop's externalFsnBase is a construction-time snapshot,"
+ + " never updated on a live loop; roll strictly between tearing the old"
+ + " loop down and building the new one");
+ }
fsnEpochBase += lastPublishedFsn + 1L;
}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleFsnContinuityTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleFsnContinuityTest.java
index da886eeb..afe5d21e 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleFsnContinuityTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleFsnContinuityTest.java
@@ -52,33 +52,54 @@
* by translating {@code external = fsnEpochBase + raw} (negative sentinels
* pass untranslated). The recycle machinery itself lands in a later task --
* these tests exercise the translation seam ({@code rollFsnEpochBaseForTest})
- * directly, either on a single already-connected sender (for the sender-level
- * surfaces, which read the live {@code fsnEpochBase} field regardless of when
- * the I/O loop was built) or by rolling a fresh sender's base BEFORE its
- * first connect (for the loop-level surfaces -- the progress dispatcher and
- * {@link SenderError} spans -- whose {@code externalFsnBase} is frozen at
- * loop construction).
+ * directly.
+ *
+ * Every test that rolls the base does so on a sender BEFORE its first connect
+ * (via {@link #createRolledSender}), never on an already-connected one:
+ * {@code rollFsnEpochBase}'s precondition forbids rolling while a live
+ * {@code CursorWebSocketSendLoop} is attached (its {@code externalFsnBase} is a
+ * construction-time snapshot, never updated on a live loop -- see that method's
+ * javadoc). Tests that need a realistic pre-roll FSN to roll by first drive a
+ * SEPARATE, ordinarily-connected sender against the same server to publish and
+ * ack a batch, close it, then hand that FSN to {@code createRolledSender} for a
+ * second, fresh sender/engine -- modelling the post-recycle engine that
+ * restarts its raw FSNs at 0.
*/
public class SymbolDictRecycleFsnContinuityTest {
+ /**
+ * Rolls a FRESH sender/engine (never published-to raw watermark starts at -1), not
+ * the already-connected one that produced {@code fsn1}: {@code rollFsnEpochBase}'s
+ * precondition forbids rolling while a live loop is attached (see its javadoc), and
+ * -- independent of that -- an already-connected sender's engine keeps its raw
+ * watermark across the roll, which would make this test pass even with the
+ * translation deleted (raw {@code ackedFsn() == fsn1 >= fsn1} regardless of any
+ * epoch math). Only a genuinely fresh engine (raw {@code ackedFsn() == -1}) makes
+ * the pre-roll short-circuit the ONLY way {@code awaitAckedFsn(fsn1, 0)} can return
+ * true here.
+ */
@Test
public void testPreRollTargetAnswersTrueAfterRoll() throws Exception {
try (TestWebSocketServer server = ackingServer()) {
- try (QwpWebSocketSender sender = (QwpWebSocketSender) Sender.fromConfig(cfg(server))) {
- sender.table("t").longColumn("v", 1L).atNow();
- long fsn1 = sender.flushAndGetSequence();
+ long fsn1;
+ try (QwpWebSocketSender sender1 = (QwpWebSocketSender) Sender.fromConfig(cfg(server))) {
+ sender1.table("t").longColumn("v", 1L).atNow();
+ fsn1 = sender1.flushAndGetSequence();
Assert.assertTrue("setup: the batch must actually be acked before the roll",
- sender.drain(5_000));
-
- sender.rollFsnEpochBaseForTest(fsn1);
+ sender1.drain(5_000));
+ }
+ QwpWebSocketSender sender2 = createRolledSender(server, fsn1);
+ try {
long t0 = System.nanoTime();
Assert.assertTrue("a target FSN from a pre-recycle epoch must be reported acked "
+ "immediately -- it was proven acked before the swap",
- sender.awaitAckedFsn(fsn1, 0));
+ sender2.awaitAckedFsn(fsn1, 0));
long elapsedMs = (System.nanoTime() - t0) / 1_000_000;
Assert.assertTrue("must short-circuit, not poll: took " + elapsedMs + "ms",
elapsedMs < 200);
+ } finally {
+ sender2.close();
}
}
}
@@ -86,28 +107,33 @@ public void testPreRollTargetAnswersTrueAfterRoll() throws Exception {
@Test
public void testPostRollSequencesExceedAllPreRoll() throws Exception {
try (TestWebSocketServer server = ackingServer()) {
- try (QwpWebSocketSender sender = (QwpWebSocketSender) Sender.fromConfig(cfg(server))) {
- sender.table("t").longColumn("v", 1L).atNow();
- long fsn1 = sender.flushAndGetSequence();
- Assert.assertTrue(sender.drain(5_000));
+ long fsn1;
+ try (QwpWebSocketSender sender1 = (QwpWebSocketSender) Sender.fromConfig(cfg(server))) {
+ sender1.table("t").longColumn("v", 1L).atNow();
+ fsn1 = sender1.flushAndGetSequence();
+ Assert.assertTrue(sender1.drain(5_000));
+ }
- sender.rollFsnEpochBaseForTest(fsn1);
- long newBase = sender.getFsnEpochBaseForTest();
+ QwpWebSocketSender sender2 = createRolledSender(server, fsn1);
+ try {
+ long newBase = sender2.getFsnEpochBaseForTest();
Assert.assertEquals(fsn1 + 1, newBase);
- sender.table("t").longColumn("v", 2L).atNow();
- long fsn2 = sender.flushAndGetSequence();
- Assert.assertTrue(sender.drain(5_000));
+ sender2.table("t").longColumn("v", 2L).atNow();
+ long fsn2 = sender2.flushAndGetSequence();
+ Assert.assertTrue(sender2.drain(5_000));
Assert.assertTrue("post-roll FSN must exceed every pre-roll FSN: fsn2=" + fsn2
+ " fsn1=" + fsn1,
fsn2 > fsn1);
- // Raw FSNs increment by 1 per single-row flush, so the raw watermark right
- // after this second flush is fsn1+1 (the same raw value the roll consumed
- // plus one) -- fsn2 must be the epoch base plus that raw value, not just
- // greater than fsn1 (a bug that dropped the translation entirely would still
- // pass the ">" check above because the underlying raw engine never resets).
- Assert.assertEquals(newBase + fsn1 + 1, fsn2);
+ // sender2's engine is genuinely fresh (raw publishedFsn() starts at -1), so
+ // its first-ever flush publishes raw 0. The exact-equality check is strictly
+ // stronger than ">" alone: it also catches an off-by-one in the roll formula
+ // (e.g. fsnEpochBase += lastPublishedFsn instead of + 1L), which the ">"
+ // check above would not.
+ Assert.assertEquals(newBase, fsn2);
+ } finally {
+ sender2.close();
}
}
}
@@ -168,12 +194,12 @@ public void testDrainAfterRollWaitsForNewFrames() throws Exception {
server.start();
Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
- QwpWebSocketSender sender = (QwpWebSocketSender) Sender.fromConfig(cfg(server));
+ // Roll well past the raw FSNs this fresh engine will ever publish, so a
+ // missing translation in drain() would make its raw target look pre-roll.
+ // Must roll before the sender's first connect (see rollFsnEpochBase's
+ // precondition: cursorSendLoop must be null).
+ QwpWebSocketSender sender = createRolledSender(server, 999L);
try {
- // Roll well past the raw FSNs this fresh engine will ever publish, so a
- // missing translation in drain() would make its raw target look pre-roll.
- sender.rollFsnEpochBaseForTest(999L);
-
sender.table("foo").longColumn("v", 1L).atNow();
boolean drainedEarly = sender.drain(200);
Assert.assertFalse("drain() must not spuriously report the new frame acked just "
@@ -289,26 +315,31 @@ public void testProgressStreamMonotoneAcrossRoll() throws Exception {
@Test
public void testLatchedErrorStillThrowsForOldEpochTarget() throws Exception {
+ // Acks the first frame it ever receives (from sender1, producing fsn1), then
+ // terminal-NACKs everything after (sender2's frame, on its own fresh connection).
AckFirstThenTerminalNackHandler handler = new AckFirstThenTerminalNackHandler();
try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
server.start();
Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
- QwpWebSocketSender sender = (QwpWebSocketSender) Sender.fromConfig(cfg(server));
- try {
- sender.table("foo").longColumn("v", 1L).atNow();
- long fsn1 = sender.flushAndGetSequence();
- Assert.assertTrue(sender.drain(5_000));
-
- sender.rollFsnEpochBaseForTest(fsn1);
+ long fsn1;
+ try (QwpWebSocketSender sender1 = (QwpWebSocketSender) Sender.fromConfig(cfg(server))) {
+ sender1.table("foo").longColumn("v", 1L).atNow();
+ fsn1 = sender1.flushAndGetSequence();
+ Assert.assertTrue(sender1.drain(5_000));
+ }
- sender.table("foo").longColumn("v", 2L).atNow();
- sender.flush();
- waitFor(() -> sender.getLastTerminalError() != null, 5_000);
+ // Roll before sender2's first connect (rollFsnEpochBase's precondition), then
+ // force sender2's own loop to latch a terminal via the handler's NACK.
+ QwpWebSocketSender sender2 = createRolledSender(server, fsn1);
+ try {
+ sender2.table("foo").longColumn("v", 2L).atNow();
+ sender2.flush();
+ waitFor(() -> sender2.getLastTerminalError() != null, 5_000);
LineSenderException thrown = null;
try {
- boolean acked = sender.awaitAckedFsn(fsn1, 0);
+ boolean acked = sender2.awaitAckedFsn(fsn1, 0);
Assert.fail("awaitAckedFsn must throw on a latched terminal error, but "
+ "returned " + acked);
} catch (LineSenderException e) {
@@ -319,7 +350,7 @@ public void testLatchedErrorStillThrowsForOldEpochTarget() throws Exception {
+ "pre-roll short-circuit", thrown);
} finally {
try {
- sender.close();
+ sender2.close();
} catch (LineSenderException ignored) {
}
}
From f9e986f0705c25e7439af4a15d69f915bb3f5291 Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Mon, 17 Aug 2026 18:43:29 +0100
Subject: [PATCH 08/49] Recycle the send stack at the empty-backlog barrier
Implement the table() barrier hook and recycleForDictReset() swap: once
the symbol-dictionary recycle is armed (Task 3) and the ring is proven
drained, table() tears the cursor I/O loop and engine down, rolls the
FSN epoch base (Task 4), replaces the producer's symbol dictionary, and
rebuilds the engine via the Task 1 EngineRebuildFactory before
reconnecting -- all synchronously inside a single call.
A failed rebuild latches recycleFailure as a terminal state: every frame
existed before the swap was already proven acked, so no data is at
risk, but the sender that observed the torn-down engine/loop refuses
further use. checkRecycleFailure() covers table() and the flush-family
entry points (flush, flushAndGetSequence, drain, awaitAckedFsn) --
deliberately not close(), which must still be able to tear down a
latched sender.
---
.../qwp/client/QwpWebSocketSender.java | 185 ++++++++
.../qwp/client/SymbolDictRecycleTest.java | 448 ++++++++++++++++++
2 files changed, 633 insertions(+)
create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index 25335076..bc4f2e45 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -431,6 +431,17 @@ public interface EngineRebuildFactory {
// opportunistic-wait step sets it once it has waited out its window for
// THIS arm cycle, so a subsequent forced-wait check does not re-wait.
private boolean starvationWaitDoneThisArm;
+ // Set (once) by recycleForDictReset's catch block when the recycle swap
+ // itself fails -- everything was acked before the swap tore the old
+ // engine down, so no data is at risk, but this sender can no longer make
+ // progress (no cursor engine, no I/O loop) and refuses further use.
+ // checkRecycleFailure() rethrows a fresh LineSenderException wrapping
+ // this cause on every later table()/flush-family call; close() still
+ // works normally.
+ private Throwable recycleFailure;
+ // Incremented once per completed symbol-dictionary recycle. 0 until the
+ // first recycle commits.
+ private long symbolDictEpoch;
private long reconnectInitialBackoffMillis =
CursorWebSocketSendLoop.DEFAULT_RECONNECT_INITIAL_BACKOFF_MILLIS;
private long reconnectMaxBackoffMillis =
@@ -1013,6 +1024,7 @@ public void atNow() {
@Override
public boolean awaitAckedFsn(long targetFsn, long timeoutMillis) {
checkNotClosed();
+ checkRecycleFailure();
if (cursorEngine == null) {
return targetFsn < 0L;
}
@@ -1692,6 +1704,7 @@ public QwpWebSocketSender floatColumn(CharSequence columnName, float value) {
*/
@Override
public void flush() {
+ checkRecycleFailure();
flushAndGetSequence();
}
@@ -1710,6 +1723,7 @@ public void flush() {
@Override
public long flushAndGetSequence() {
checkNotClosed();
+ checkRecycleFailure();
if (cursorEngine != null) {
cursorEngine.checkDurability();
}
@@ -1770,6 +1784,7 @@ public long flushAndGetSequence() {
*/
@Override
public boolean drain(long timeoutMillis) {
+ checkRecycleFailure();
flush();
long targetRaw = cursorEngine != null ? cursorEngine.publishedFsn() : -1L;
long targetFsn = targetRaw < 0 ? targetRaw : fsnEpochBase + targetRaw;
@@ -2108,6 +2123,15 @@ public long getFsnEpochBaseForTest() {
return fsnEpochBase;
}
+ /**
+ * Number of symbol-dictionary recycles this sender has completed.
+ * Incremented by {@link #recycleForDictReset()} once the swap commits.
+ */
+ @TestOnly
+ public long getSymbolDictEpochForTest() {
+ return symbolDictEpoch;
+ }
+
/**
* Test-only entry point for {@link #rollFsnEpochBase}, the same private
* roll the symbol-dict recycle swap calls in production once the engine
@@ -2920,6 +2944,10 @@ public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) {
@Override
public QwpWebSocketSender table(CharSequence tableName) {
checkNotClosed();
+ checkRecycleFailure();
+ if (resetArmed) {
+ maybeRecycleForDictReset();
+ }
// Fast path: if table name matches current, skip hashmap lookup
if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) {
return this;
@@ -3622,6 +3650,24 @@ private void checkNotClosed() {
checkConnectionError();
}
+ /**
+ * Terminal latch for a failed symbol-dictionary recycle swap
+ * ({@link #recycleForDictReset()}). Everything was acked before the swap
+ * tore the old engine down, so no data is at risk -- but the swap itself
+ * left this sender without a cursor engine or I/O loop, so it refuses
+ * further use. Checked by {@link #table(CharSequence)} and the
+ * flush-family entry points ({@link #flush()}, {@link #flushAndGetSequence()},
+ * {@link #drain(long)}, {@link #awaitAckedFsn(long, long)}) -- deliberately
+ * NOT by {@link #close()}, which must still be able to tear down a
+ * latched sender.
+ */
+ private void checkRecycleFailure() {
+ if (recycleFailure != null) {
+ throw new LineSenderException(recycleFailure)
+ .put("sender is terminal: symbol dictionary recycle failed");
+ }
+ }
+
private void checkTableSelected() {
if (currentTableBuffer == null) {
throw new LineSenderException("table() must be called before adding columns");
@@ -4494,6 +4540,145 @@ private void armIfEligible() {
resetArmed = shouldArm;
}
+ /**
+ * True once every FSN this engine has published has also been
+ * server-acknowledged (or nothing has been published yet). The barrier
+ * {@link #recycleForDictReset()} waits for: the swap tears the cursor
+ * engine down, so it must never run while a frame is still in flight.
+ *
+ * Read order matters: {@code publishedFsn} (producer-written, cannot move
+ * during this call -- we ARE the producer) first, then {@code ackedFsn}
+ * (monotone, I/O-thread-written) second. Reading them in the other order
+ * could observe a published advance without its matching ack and falsely
+ * report drained.
+ */
+ private boolean isRingDrained() {
+ long published = cursorEngine.publishedFsn();
+ return published < 0 || cursorEngine.ackedFsn() >= published;
+ }
+
+ /**
+ * Placeholder for the opportunistic/forced starvation-wait policy a later
+ * task fills in: when the ring is NOT drained at arming time, this
+ * decides whether to wait out an idle window before forcing the recycle
+ * regardless. No-op here -- an armed sender with a non-empty backlog
+ * simply stays armed and re-checks on the next {@link #table(CharSequence)}
+ * call.
+ */
+ private void maybeBlockForStarvedReset() {
+ }
+
+ /**
+ * Evaluates whether the barrier in {@link #table(CharSequence)} may run
+ * the symbol-dictionary recycle right now. Only ever called with
+ * {@link #resetArmed} true.
+ *
+ * Refuses when there is producer-side state the swap cannot safely tear
+ * down: no connection yet (a V4 sender that has never sent is never
+ * pre-connected here), a flush in flight ({@code pendingRowCount != 0}),
+ * or a row under construction. Otherwise proceeds to the ring-drained
+ * check: if the backlog is empty, recycle immediately; if not, defer to
+ * the (later-task) starvation-wait policy instead of blocking the caller
+ * indefinitely here.
+ */
+ private void maybeRecycleForDictReset() {
+ if (!connected
+ || pendingRowCount != 0
+ || (currentTableBuffer != null && currentTableBuffer.hasInProgressRow())) {
+ return;
+ }
+ if (isRingDrained()) {
+ recycleForDictReset();
+ } else {
+ maybeBlockForStarvedReset();
+ }
+ }
+
+ /**
+ * The symbol-dictionary recycle swap. Runs synchronously on the producer
+ * thread from the {@link #table(CharSequence)} barrier, once
+ * {@link #maybeRecycleForDictReset()} has proven the ring is drained.
+ * Eight steps, strictly ordered:
+ *
- * Refuses when there is producer-side state the swap cannot safely tear
- * down: no connection yet (a V4 sender that has never sent is never
+ * Refuses before any teardown when the rebuild itself is impossible or
+ * unsafe: no {@link #engineRebuildFactory} (every public
+ * {@code QwpWebSocketSender.connect(...)} overload leaves it null --
+ * only {@code Sender.build()} installs one -- and the recycle feature is
+ * default-on, so a connect()-built sender must simply stay unarmed rather
+ * than NPE at step 6 and latch terminal), or a cursor engine this sender
+ * does not own ({@code setCursorEngine(engine, false)}'s contract: the
+ * caller retains ownership, so closing it out from under them at step 3
+ * would be a use-after-free from the caller's point of view).
+ *
+ * Also refuses when there is producer-side state the swap cannot safely
+ * tear down: no connection yet (a V4 sender that has never sent is never
* pre-connected here), a flush in flight ({@code pendingRowCount != 0}),
* or a row under construction. Otherwise proceeds to the ring-drained
* check: if the backlog is empty, recycle immediately; if not, defer to
@@ -4582,6 +4595,9 @@ private void maybeBlockForStarvedReset() {
* indefinitely here.
*/
private void maybeRecycleForDictReset() {
+ if (engineRebuildFactory == null || !ownsCursorEngine) {
+ return;
+ }
if (!connected
|| pendingRowCount != 0
|| (currentTableBuffer != null && currentTableBuffer.hasInProgressRow())) {
@@ -4615,13 +4631,15 @@ private void maybeRecycleForDictReset() {
* precondition).
*
+ * The recycle swap's eight steps ({@code QwpWebSocketSender.recycleForDictReset()})
+ * were written against the store-and-forward slot lifecycle, but the factory's
+ * {@code slotPath == null} arm, {@code CursorSendEngine}'s file-less close, and the
+ * barrier itself are all mode-agnostic by construction -- nothing in
+ * {@code maybeRecycleForDictReset()} or the swap checks whether the sender is
+ * SF-backed. This suite pins that: every scenario {@code SymbolDictRecycleTest}
+ * proves for a disk-backed sender must hold identically for a {@code Sender.fromConfig}
+ * sender built with no {@code sf_dir} at all. No production change is expected to
+ * make these pass; a failure here means Task 5's swap accidentally gated something
+ * on store-and-forward being present.
+ */
+public class SymbolDictRecycleMemoryModeTest {
+
+ @Test
+ public void testRecycleAtEmptyBacklog() throws Exception {
+ assertMemoryLeak(() -> {
+ RecycleHandler handler = new RecycleHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ // No sf_dir: memory mode. Everything else mirrors
+ // SymbolDictRecycleTest#testRecycleAtEmptyBacklog exactly.
+ String cfg = "ws::addr=localhost:" + port + ";symbol_dict_reset_threshold=2;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: batch must be acked before the recycle",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+ Assert.assertEquals(1, handler.connectionsAccepted.get());
+ Assert.assertEquals(0, ws.getSymbolDictEpochForTest());
+
+ // Ring drained, no row in progress: this table() call must
+ // recycle synchronously, exactly as in SF mode.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertFalse("recycle must disarm", ws.isResetArmed());
+ Assert.assertEquals("recycle must open a fresh connection",
+ 2, server.handshakeCount());
+ Assert.assertEquals(1, ws.getSymbolDictEpochForTest());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-recycle batch must still get acked",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertTrue("post-recycle FSN must exceed pre-recycle FSN "
+ + "[fsn1=" + fsn1 + ", fsn2=" + fsn2 + ']',
+ fsn2 > fsn1);
+ }
+
+ Assert.assertEquals("exactly 2 connections total", 2, handler.connectionsAccepted.get());
+ Assert.assertEquals("connection 2's first data frame must carry deltaStart == 0 "
+ + "(a fresh, empty dictionary)",
+ 0, handler.conn2FirstFrameDeltaStart);
+ Assert.assertEquals("connection 2's dictionary must hold only the post-recycle "
+ + "symbols, not a, b",
+ Arrays.asList("c", "d"), handler.dictFor(2));
+ }
+ });
+ }
+
+ /**
+ * Strengthens {@link #testRecycleAtEmptyBacklog} into a content oracle: every
+ * row before and after the recycle carries a distinct symbol value, and this
+ * asserts the server observed the FULL, exact, gap-free, duplicate-free
+ * sequence across both connections -- not just a spot check of the boundary
+ * frame. Proves the epoch swap loses (and doesn't duplicate) nothing that was
+ * ever acked, in memory mode exactly as {@code testPostRecycleSlotContents}
+ * proves the persisted-dictionary shape in SF mode.
+ */
+ @Test
+ public void testRecycleLosesNothingAcked() throws Exception {
+ assertMemoryLeak(() -> {
+ RecycleHandler handler = new RecycleHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";symbol_dict_reset_threshold=2;";
+
+ List
+ * Otherwise waits (parked, {@code awaitAckedFsn}-shaped) until either the
+ * ring drains -- in which case the recycle runs synchronously before
+ * returning -- or {@code resetMaxWaitMillis} elapses from THIS call, in
+ * which case it gives up, counts the timeout, and leaves
+ * {@link #resetArmed} set so a later drained {@link #table(CharSequence)}
+ * call can still recycle opportunistically.
*/
private void maybeBlockForStarvedReset() {
+ if (resetMaxWaitMillis <= 0 || starvationWaitDoneThisArm) {
+ return;
+ }
+ if (hasDeferredMessages) {
+ return;
+ }
+ if (System.nanoTime() - armedSinceNanos < resetMaxWaitMillis * 1_000_000L) {
+ return;
+ }
+ starvationWaitDoneThisArm = true;
+ long deadlineNanos = System.nanoTime() + resetMaxWaitMillis * 1_000_000L;
+ while (!isRingDrained()) {
+ cursorEngine.checkDurability();
+ if (cursorSendLoop != null) {
+ cursorSendLoop.checkError();
+ }
+ checkConnectionError();
+ if (System.nanoTime() >= deadlineNanos) {
+ symbolDictResetStarvationTimeouts++;
+ LOG.warn("symbol dictionary reset starved: backlog not drained within {} ms; "
+ + "re-arming opportunistically", resetMaxWaitMillis);
+ return;
+ }
+ java.util.concurrent.locks.LockSupport.parkNanos(50_000L);
+ }
+ recycleForDictReset();
}
/**
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStarvationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStarvationTest.java
new file mode 100644
index 00000000..fd8b2113
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStarvationTest.java
@@ -0,0 +1,516 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.WebSocketResponse;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.FLAG_DEFER_COMMIT;
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * The bounded blocking wait {@code maybeBlockForStarvedReset()} runs from
+ * {@code maybeRecycleForDictReset()} when a symbol-dictionary recycle is
+ * armed but the ring is not yet drained: it opportunistically waits (parked,
+ * {@code awaitAckedFsn}-shaped) for the outstanding acks to arrive, up to
+ * {@code symbol_dict_reset_max_wait_millis}, before giving up for this armed
+ * window. {@code resetMaxWaitMillis <= 0} disables the wait entirely (never
+ * block); a deferred-commit group open at the time of the check must never
+ * be waited on, since the server withholds its acks by design until the
+ * closing commit lands (the one data-loss-adjacent path in this feature --
+ * forcing a wait there would just stall until the deadline, since only THIS
+ * thread can ever send the commit that unblocks it).
+ */
+public class SymbolDictRecycleStarvationTest {
+
+ /**
+ * {@code symbol_dict_reset_max_wait_millis=0} must disable the wait
+ * unconditionally, regardless of how long the recycle has been armed or
+ * how large the backlog is. Every {@code table()} call must return in
+ * (near) constant time, the sender must never recycle, and it must stay
+ * armed forever -- nothing ever consumes the arming.
+ */
+ @Test
+ public void testMaxWaitZeroNeverBlocks() throws Exception {
+ assertMemoryLeak(() -> {
+ GatedAckHandler handler = new GatedAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port
+ + ";symbol_dict_reset_threshold=2"
+ + ";symbol_dict_reset_max_wait_millis=0;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ sender.flush(); // unacked forever -- handler never releases
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+ Assert.assertEquals(0L, ws.getSymbolDictResetStarvationTimeoutsForTest());
+
+ // Repeated table() calls, spread over time, must every one of
+ // them return fast: resetMaxWaitMillis<=0 is checked BEFORE the
+ // armed-window elapsed check, so it never even looks at how long
+ // this arm has been outstanding.
+ for (int i = 0; i < 5; i++) {
+ long t0 = System.nanoTime();
+ sender.table("t");
+ long elapsedMs = (System.nanoTime() - t0) / 1_000_000;
+ Assert.assertTrue("table() call #" + i + " took " + elapsedMs
+ + "ms -- resetMaxWaitMillis=0 must never block",
+ elapsedMs < 100);
+ Thread.sleep(20);
+ }
+
+ Assert.assertTrue("armed forever -- nothing ever consumes it with no factory "
+ + "action taken",
+ ws.isResetArmed());
+ Assert.assertEquals("must never recycle -- the ring never drained and "
+ + "blocking is disabled",
+ 0L, ws.getSymbolDictEpochForTest());
+ Assert.assertEquals(0L, ws.getSymbolDictResetStarvationTimeoutsForTest());
+
+ // Release before the try-with-resources closes the sender below,
+ // or close()'s own drain would hang on this still-unacked batch --
+ // irrelevant to what this test is proving.
+ handler.releaseAcks();
+ }
+ }
+ });
+ }
+
+ /**
+ * With a non-zero max wait, a {@code table()} call made after the armed
+ * window has elapsed must block (parked) until the backlog drains, then
+ * recycle synchronously within that same call -- proving the wait
+ * actually parks rather than spinning or returning early, and that a
+ * drain arriving mid-wait is observed without needing a fresh
+ * {@code table()} call.
+ */
+ @Test
+ public void testBlocksThenRecyclesWhenAcksArrive() throws Exception {
+ assertMemoryLeak(() -> {
+ GatedAckHandler handler = new GatedAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ long maxWaitMillis = 400;
+ String cfg = "ws::addr=localhost:" + port
+ + ";symbol_dict_reset_threshold=2"
+ + ";symbol_dict_reset_max_wait_millis=" + maxWaitMillis + ";";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ sender.flush();
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+
+ // Let the armed-window guard elapse (armedSinceNanos check uses
+ // the SAME resetMaxWaitMillis) so the triggering table() call
+ // actually enters the blocking wait -- with a FRESH maxWaitMillis
+ // deadline of its own -- instead of returning immediately.
+ Thread.sleep(maxWaitMillis + 50);
+
+ // Release the acks from another thread, well before the fresh
+ // maxWaitMillis deadline elapses, while the main thread is
+ // parked in the wait loop.
+ long releaseDelayMs = 150;
+ Thread releaser = new Thread(() -> {
+ try {
+ Thread.sleep(releaseDelayMs);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return;
+ }
+ handler.releaseAcks();
+ });
+ releaser.start();
+
+ long t0 = System.nanoTime();
+ sender.table("t"); // blocks, then recycles once the ack lands
+ long elapsedMs = (System.nanoTime() - t0) / 1_000_000;
+ releaser.join();
+
+ Assert.assertTrue("must have actually blocked for roughly the release delay "
+ + "(" + releaseDelayMs + "ms), got " + elapsedMs + "ms",
+ elapsedMs >= releaseDelayMs - 50);
+ Assert.assertTrue("must not have run into the full max-wait deadline "
+ + "(" + maxWaitMillis + "ms), got " + elapsedMs + "ms",
+ elapsedMs < maxWaitMillis);
+ Assert.assertFalse("recycle must disarm", ws.isResetArmed());
+ Assert.assertEquals("recycle must have run exactly once",
+ 1L, ws.getSymbolDictEpochForTest());
+ Assert.assertEquals("a successful drain-and-recycle is not a timeout",
+ 0L, ws.getSymbolDictResetStarvationTimeoutsForTest());
+ }
+ }
+ });
+ }
+
+ /**
+ * Acks never arrive at all: the blocked {@code table()} call must give up
+ * at (roughly) {@code maxWaitMillis}, not hang indefinitely, and the
+ * sender must keep working afterwards -- ingest is not stuck. The
+ * starvation counter records the timeout, the recycle stays armed
+ * (nothing consumed it), and at most one blocking wait happens per armed
+ * window: an immediately-following table() call must NOT re-block. Once
+ * the backlog does drain later (acks release), the still-armed recycle
+ * fires on the next table() call -- opportunistically, without another
+ * blocking wait.
+ */
+ @Test
+ public void testTimeoutLogsAndReArms() throws Exception {
+ assertMemoryLeak(() -> {
+ GatedAckHandler handler = new GatedAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ long maxWaitMillis = 300;
+ String cfg = "ws::addr=localhost:" + port
+ + ";symbol_dict_reset_threshold=2"
+ + ";symbol_dict_reset_max_wait_millis=" + maxWaitMillis + ";";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+
+ // Let the armed-window guard elapse so the call below actually
+ // enters the wait instead of short-circuiting on arm recency.
+ Thread.sleep(maxWaitMillis + 50);
+
+ long t0 = System.nanoTime();
+ sender.table("t"); // acks never come -- must give up at ~maxWaitMillis
+ long elapsedMs = (System.nanoTime() - t0) / 1_000_000;
+
+ Assert.assertTrue("must have waited out roughly the full max-wait deadline "
+ + "(" + maxWaitMillis + "ms), got " + elapsedMs + "ms",
+ elapsedMs >= maxWaitMillis - 50);
+ Assert.assertTrue("must not hang far past its own deadline, got " + elapsedMs + "ms",
+ elapsedMs < maxWaitMillis + 5_000);
+ Assert.assertEquals("a timed-out wait must record exactly one starvation timeout",
+ 1L, ws.getSymbolDictResetStarvationTimeoutsForTest());
+ Assert.assertTrue("timing out must not consume the arming -- the recycle is "
+ + "still owed once the backlog eventually drains",
+ ws.isResetArmed());
+ Assert.assertEquals("no recycle happened -- only the wait gave up",
+ 0L, ws.getSymbolDictEpochForTest());
+
+ // Ingest continues: more rows can still be appended and flushed
+ // without the sender getting stuck.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+
+ // At most one blocking wait per armed window: this table() call
+ // must NOT re-block even though the backlog is still undrained.
+ long t1 = System.nanoTime();
+ sender.table("t");
+ long secondElapsedMs = (System.nanoTime() - t1) / 1_000_000;
+ Assert.assertTrue("a second table() call in the same armed window must not "
+ + "re-block, took " + secondElapsedMs + "ms",
+ secondElapsedMs < 100);
+ Assert.assertEquals("still just the one timeout from before",
+ 1L, ws.getSymbolDictResetStarvationTimeoutsForTest());
+
+ // Now let the backlog actually drain: the still-armed recycle
+ // must fire opportunistically on the next table() call, with no
+ // further blocking wait needed (isRingDrained() short-circuits
+ // straight to recycleForDictReset()). drain() uses watermark
+ // semantics -- it waits for publishedFsn(), covering BOTH the
+ // original arm batch and the ingest-continues row above it, not
+ // just fsn1's frame.
+ handler.releaseAcks();
+ Assert.assertTrue("setup: every pending frame must get acked once released",
+ sender.drain(5_000));
+ Assert.assertTrue("setup: the original arm batch must be covered too",
+ sender.awaitAckedFsn(fsn1, 0));
+
+ sender.table("t");
+ Assert.assertFalse("the still-armed recycle must fire now that the backlog "
+ + "has drained",
+ ws.isResetArmed());
+ Assert.assertEquals(1L, ws.getSymbolDictEpochForTest());
+ Assert.assertEquals("draining later must not add another timeout",
+ 1L, ws.getSymbolDictResetStarvationTimeoutsForTest());
+ }
+ }
+ });
+ }
+
+ /**
+ * A terminal error latched WHILE the wait is parked must interrupt it
+ * immediately -- the wait polls {@code cursorSendLoop.checkError()} /
+ * {@code checkConnectionError()} every park interval, exactly like
+ * {@code awaitAckedFsn}. Forces the poison detector to escalate to a
+ * terminal on the very first NACK ({@code max_frame_rejections=1},
+ * {@code poison_min_escalation_window_millis=0}) so the timing is
+ * deterministic instead of depending on the (much slower) defaults.
+ */
+ @Test
+ public void testLatchedErrorDuringWaitThrows() throws Exception {
+ assertMemoryLeak(() -> {
+ GatedThenPoisonHandler handler = new GatedThenPoisonHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ long maxWaitMillis = 400;
+ String cfg = "ws::addr=localhost:" + port
+ + ";symbol_dict_reset_threshold=2"
+ + ";symbol_dict_reset_max_wait_millis=" + maxWaitMillis
+ + ";max_frame_rejections=1;poison_min_escalation_window_millis=0;";
+
+ Sender sender = Sender.fromConfig(cfg);
+ try {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ sender.flush();
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+
+ // Let the armed-window guard elapse (armedSinceNanos check uses
+ // the SAME resetMaxWaitMillis) so the triggering table() call
+ // below actually enters the blocking wait with a FRESH deadline.
+ Thread.sleep(maxWaitMillis + 50);
+
+ long poisonDelayMs = 150;
+ Thread poisoner = new Thread(() -> {
+ try {
+ Thread.sleep(poisonDelayMs);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return;
+ }
+ handler.poison();
+ });
+ poisoner.start();
+
+ long t0 = System.nanoTime();
+ LineSenderException thrown = null;
+ try {
+ sender.table("t");
+ } catch (LineSenderException e) {
+ thrown = e;
+ }
+ long elapsedMs = (System.nanoTime() - t0) / 1_000_000;
+ poisoner.join();
+
+ Assert.assertNotNull("a terminal error latched during the wait must propagate "
+ + "out of table(), not be swallowed into an indefinite hang",
+ thrown);
+ Assert.assertTrue("must have thrown promptly once poisoned, not waited out the "
+ + "full max-wait deadline (" + maxWaitMillis + "ms), got "
+ + elapsedMs + "ms",
+ elapsedMs < maxWaitMillis);
+ Assert.assertTrue("the recycle must never have run -- the swap requires a "
+ + "healthy connection",
+ ws.isResetArmed());
+ Assert.assertEquals("a thrown wait is not a timeout",
+ 0L, ws.getSymbolDictResetStarvationTimeoutsForTest());
+ } finally {
+ try {
+ sender.close();
+ } catch (LineSenderException ignored) {
+ // close() may also observe the latched terminal -- irrelevant here
+ }
+ }
+ }
+ });
+ }
+
+ /**
+ * A deferred-commit group open past the armed window is the one
+ * data-safety-critical case: the server withholds acks for
+ * {@code FLAG_DEFER_COMMIT} frames by design, and this producer thread is
+ * the only one that can ever send the closing commit. Blocking here would
+ * therefore ALWAYS run out the clock -- worse, it would do so while
+ * holding up the very thread the caller needs free to actually close the
+ * group. {@code table()} must return immediately (the futility guard),
+ * and once the group is closed and its commit acked, the still-armed
+ * recycle must fire at the very next drained {@code table()} call.
+ */
+ @Test
+ public void testDeferredCommitGroupSkipsWait() throws Exception {
+ assertMemoryLeak(() -> {
+ DeferAwareAckHandler handler = new DeferAwareAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ long maxWaitMillis = 300;
+ String cfg = "ws::addr=localhost:" + port
+ + ";symbol_dict_reset_threshold=2"
+ + ";symbol_dict_reset_max_wait_millis=" + maxWaitMillis + ";";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ ws.setDeferCommit(true);
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ sender.flush(); // deferred frame -- server withholds its ack
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+
+ // Let the armed window elapse -- without the futility guard this
+ // is exactly when a naive implementation would start blocking.
+ Thread.sleep(maxWaitMillis + 50);
+
+ long t0 = System.nanoTime();
+ sender.table("t"); // futility guard: must return immediately
+ long elapsedMs = (System.nanoTime() - t0) / 1_000_000;
+
+ Assert.assertTrue("an open deferred-commit group must never be waited on "
+ + "(the server withholds its ack by design), took " + elapsedMs + "ms",
+ elapsedMs < 100);
+ Assert.assertTrue("must still be armed -- neither the wait nor the recycle ran",
+ ws.isResetArmed());
+ Assert.assertEquals("the futility guard is not a timeout",
+ 0L, ws.getSymbolDictResetStarvationTimeoutsForTest());
+ Assert.assertEquals(0L, ws.getSymbolDictEpochForTest());
+
+ // Close the deferred group: commit, and wait for its ack.
+ ws.setDeferCommit(false);
+ long commitFsn = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: the commit must get acked",
+ sender.awaitAckedFsn(commitFsn, 5_000));
+
+ // The ring is now drained -- the still-armed recycle must fire at
+ // the very next drained row start, with no wait needed at all.
+ sender.table("t");
+ Assert.assertFalse("the still-armed recycle must fire once the group is "
+ + "committed and acked",
+ ws.isResetArmed());
+ Assert.assertEquals(1L, ws.getSymbolDictEpochForTest());
+ Assert.assertEquals("no wait ever ran in this test",
+ 0L, ws.getSymbolDictResetStarvationTimeoutsForTest());
+ }
+ }
+ });
+ }
+
+ /**
+ * Receives frames but withholds every ack until {@link #releaseAcks()} is
+ * called, so a starvation wait provably has an unacknowledged target to
+ * wait on. Mirrors {@code CloseDrainTest.GatedAckHandler} /
+ * {@code SymbolDictRecycleFsnContinuityTest.GatedAckHandler}.
+ */
+ private static class GatedAckHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private final AtomicLong nextSeq = new AtomicLong(0);
+ private final CountDownLatch released = new CountDownLatch(1);
+
+ @Override
+ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ try {
+ if (!released.await(20, TimeUnit.SECONDS)) {
+ throw new AssertionError("starvation-wait witness never released the ack gate");
+ }
+ client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement()));
+ } catch (IOException | InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
+ }
+
+ void releaseAcks() {
+ released.countDown();
+ }
+ }
+
+ /**
+ * Withholds its response until {@link #poison()} is called, then replies
+ * with a terminal-worthy NACK (STATUS_PARSE_ERROR) instead of an ack --
+ * models a connection killed while a starvation wait is parked.
+ */
+ private static class GatedThenPoisonHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private final AtomicLong nextSeq = new AtomicLong(0);
+ private final CountDownLatch poisoned = new CountDownLatch(1);
+
+ @Override
+ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ try {
+ if (!poisoned.await(20, TimeUnit.SECONDS)) {
+ throw new AssertionError("starvation-wait witness never poisoned the connection");
+ }
+ client.sendBinary(QwpWireTestUtils.buildNack(
+ nextSeq.getAndIncrement(), WebSocketResponse.STATUS_PARSE_ERROR));
+ } catch (IOException | InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
+ }
+
+ void poison() {
+ poisoned.countDown();
+ }
+ }
+
+ /**
+ * Acks every non-deferred frame immediately (each connection's wire
+ * sequence restarts at 0), but withholds acks for any frame carrying
+ * {@code FLAG_DEFER_COMMIT} -- exactly the real server's ack contract for
+ * an open deferred-commit group (see
+ * {@code CloseDrainTest.AckFirstFrameOnlyHandler}, which models the same
+ * contract for a fixed one-committed-then-deferred shape). Acking the
+ * closing commit frame's wire sequence retroactively covers the whole
+ * group, since {@code ackedFsn} is a cumulative watermark.
+ */
+ private static class DeferAwareAckHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ long seq = nextSeq.getAndIncrement();
+ boolean deferred = data.length > 5 && (data[5] & FLAG_DEFER_COMMIT) != 0;
+ if (deferred) {
+ return; // withhold the ack -- the group is still open
+ }
+ try {
+ client.sendBinary(QwpWireTestUtils.buildAck(seq));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}
From 3e6867334728c8ba30aa48a439608e9187c3a1f7 Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Mon, 17 Aug 2026 20:12:25 +0100
Subject: [PATCH 12/49] Fix round 1: correct starvation-wait javadoc, tighten
tests
Javadoc for symbol_dict_reset_max_wait_millis (Sender.java builder
method, and the DEFAULT_.../field comments in QwpWebSocketSender.java)
said the knob controls how long the recycle waits "before forcing the
rebuild" and that 0 means "never forces". That is backwards: nothing
is ever forced through. On timeout the wait gives up, increments the
starvation counter, logs a warning, and stays armed -- the dictionary
threshold is the only actual backstop. Rewrite all three to state the
real policy: once the armed window exceeds the knob, the NEXT
table(...) call may block the calling thread for up to the knob's
value waiting for the backlog to drain; 0 disables blocking entirely.
WARN log wording "re-arming opportunistically" -> "staying armed":
the arm is never consumed on a timeout, so nothing re-arms.
Test fixes in SymbolDictRecycleStarvationTest:
- testTimeoutLogsAndReArms: the "second table() must not re-block"
probe was placed after a row had been queued (pendingRowCount==1),
so it short-circuited on maybeRecycleForDictReset()'s precondition
guard before ever reaching the wait -- vacuously true regardless of
starvationWaitDoneThisArm. Moved the probe earlier, to a table()
call with pendingRowCount==0, so it actually exercises the "at most
one blocking wait per armed window" guard.
- testBlocksThenRecyclesWhenAcksArrive: raised maxWaitMillis from 400
to 700ms (keeping the 150ms release delay) so the full recycle path
(I/O loop join, engine close, rebuild, fresh handshake) has real
headroom under the elapsedMs
+ * {@link SymbolDictRecycleTest} and {@link SymbolDictRecycleMemoryModeTest}
+ * pin the swap itself; {@link SymbolDictRecycleArmingTest} pins how
+ * {@code resetArmed} flips true; {@link SymbolDictRecycleStarvationTest} pins
+ * the bounded blocking wait for an unacked backlog. This suite pins the
+ * OPPOSITE: every condition under which an armed sender must keep working
+ * normally and NOT recycle, until that condition clears -- at which point
+ * the still-armed request fires as a positive control in the same test.
+ * Every test asserts both halves: no recycle (connection count and
+ * {@code getSymbolDictEpochForTest()} unchanged) AND that ingestion keeps
+ * working (a row lands and gets acked) both before and after the eventual
+ * recycle.
+ */
+public class SymbolDictRecycleRefusalTest {
+
+ /**
+ * The most basic refusal: a batch that itself crossed the arming
+ * threshold is still unacked when the very next {@code table()} call
+ * checks the barrier. {@code symbol_dict_reset_max_wait_millis=0}
+ * disables the (separately-pinned, {@link SymbolDictRecycleStarvationTest})
+ * blocking wait, so every refusal here is instant and this test stays
+ * purely about the ring-drained guard. Repeated {@code table()} calls
+ * spread over a real span of wall-clock time (not one instantaneous
+ * check) prove the recycle does not fire late, either -- the whole
+ * mechanism is synchronous and producer-thread-driven, but a bounded
+ * settle window is the only way a test can actually witness that rather
+ * than assume it.
+ */
+ @Test
+ public void testUnackedBacklogRefusesUntilAcked() throws Exception {
+ assertMemoryLeak(() -> {
+ GatedAckHandler handler = new GatedAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port
+ + ";symbol_dict_reset_threshold=2"
+ + ";symbol_dict_reset_max_wait_millis=0;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence(); // ack withheld by the handler
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+ Assert.assertEquals(1, server.handshakeCount());
+ Assert.assertEquals(0, ws.getSymbolDictEpochForTest());
+
+ for (int i = 0; i < 5; i++) {
+ sender.table("t");
+ Assert.assertTrue("recycle must not fire while the arming batch is unacked",
+ ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpochForTest());
+ Assert.assertEquals(1, server.handshakeCount());
+ Thread.sleep(30);
+ }
+
+ // Positive control: release the acks and prove the still-armed
+ // recycle fires on the very next drained table() call.
+ handler.releaseAcks();
+ Assert.assertTrue("setup: the arming batch must get acked once released",
+ sender.awaitAckedFsn(fsn1, 5_000));
+
+ sender.table("t");
+ Assert.assertFalse("recycle must fire once the backlog drains", ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpochForTest());
+ Assert.assertEquals("recycle must open a fresh connection",
+ 2, server.handshakeCount());
+
+ // Ingestion continues on the fresh epoch.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-recycle batch must still get acked",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertTrue(fsn2 > fsn1);
+ }
+ }
+ });
+ }
+
+ /**
+ * Isolates the {@code pendingRowCount != 0} guard from the ring-drained
+ * guard {@link #testUnackedBacklogRefusesUntilAcked} pins: the arming
+ * batch's ack is released and awaited WHILE a third row sits buffered
+ * (committed via {@code atNow()}, but never flushed -- {@code auto_flush_rows}
+ * is set well above 1 so it does not auto-flush). By the time the
+ * settle-window loop runs, the ring itself is fully drained, so any
+ * refusal it observes can only be this guard, not the earlier one.
+ */
+ @Test
+ public void testPendingRowCountRefuses() throws Exception {
+ assertMemoryLeak(() -> {
+ GatedAckHandler handler = new GatedAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port
+ + ";symbol_dict_reset_threshold=2"
+ + ";symbol_dict_reset_max_wait_millis=0"
+ + ";auto_flush_rows=10;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence(); // ack withheld by the handler
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+
+ // Ring not drained yet: refused by the OTHER guard, which just
+ // lets execution fall through so a new row can be buffered.
+ sender.table("t");
+ Assert.assertTrue(ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpochForTest());
+
+ // A third row, committed but never flushed: pendingRowCount=1,
+ // far under auto_flush_rows=10, so it stays buffered.
+ sender.symbol("s", "c").longColumn("v", 2L).atNow();
+
+ // Drain the arming batch -- from here on the ring itself is
+ // fully drained, isolating the pendingRowCount guard.
+ handler.releaseAcks();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+
+ for (int i = 0; i < 5; i++) {
+ sender.table("t");
+ Assert.assertTrue("recycle must not fire while a row is buffered "
+ + "unflushed, even with the ring otherwise drained",
+ ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpochForTest());
+ Assert.assertEquals(1, server.handshakeCount());
+ Thread.sleep(30);
+ }
+
+ // Positive control: flush the buffered row, then the
+ // still-armed recycle fires on the next table() call.
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+
+ sender.table("t");
+ Assert.assertFalse("recycle must fire once the buffered batch is flushed "
+ + "and acked",
+ ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpochForTest());
+ Assert.assertEquals(2, server.handshakeCount());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn3 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn3, 5_000));
+ Assert.assertTrue(fsn3 > fsn2);
+ }
+ }
+ });
+ }
+
+ /**
+ * A row under construction (columns set, {@code atNow()} not yet called)
+ * refuses the barrier two different ways depending on the next
+ * {@code table()} call's table name. The same-name case is the sharper
+ * proof: {@code table()}'s resetArmed check runs BEFORE the
+ * same-table-name fast path that would otherwise skip straight past
+ * everything, so this is the only way to prove the hook actually sits
+ * ahead of that shortcut. The different-name case falls through to the
+ * pre-existing "cannot switch tables while row is in progress" guard
+ * instead -- a thrown exception, not a recycle, and not a new failure
+ * mode this feature introduced.
+ */
+ @Test
+ public void testInProgressRowRefuses() throws Exception {
+ assertMemoryLeak(() -> {
+ AckAllHandler handler = new AckAllHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ // Start a row but do not commit it: symbol() registers "a"
+ // into the dictionary immediately, yet the row itself stays
+ // in progress until atNow() runs.
+ sender.table("t").symbol("s", "a");
+ Assert.assertEquals(0, ws.getSymbolDictEpochForTest());
+
+ // Arm WHILE the row is in progress: pendingRowCount is still
+ // 0 (an in-progress row is not counted as pending), so the
+ // manual request arms immediately even though a row is
+ // genuinely mid-flight.
+ sender.resetSymbolDictionary();
+ Assert.assertTrue(ws.isResetArmed());
+
+ for (int i = 0; i < 3; i++) {
+ sender.table("t"); // same name -- fast path would skip past everything
+ Assert.assertTrue("recycle must not fire while a row is in progress",
+ ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpochForTest());
+ Assert.assertEquals(1, server.handshakeCount());
+ Thread.sleep(20);
+ }
+
+ LineSenderException thrown = null;
+ try {
+ sender.table("other");
+ Assert.fail("expected 'cannot switch tables' while a row is in progress");
+ } catch (LineSenderException e) {
+ thrown = e;
+ }
+ Assert.assertNotNull(thrown);
+ Assert.assertTrue("unexpected message: " + thrown.getMessage(),
+ thrown.getMessage().contains("cannot switch tables while row is in progress"));
+ Assert.assertTrue("the failed table-switch attempt must not have consumed "
+ + "the arming",
+ ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpochForTest());
+ Assert.assertEquals(1, server.handshakeCount());
+
+ // Complete the row: ingestion still works after both refusals.
+ sender.longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("nothing yet consumed the arming", ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpochForTest());
+
+ // Positive control: with the row complete and the batch
+ // acked, the still-armed recycle fires on the next call.
+ sender.table("t");
+ Assert.assertFalse("recycle must fire once the row completes and the ring "
+ + "drains",
+ ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpochForTest());
+ Assert.assertEquals(2, server.handshakeCount());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 2L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertTrue(fsn2 > fsn1);
+ }
+ }
+ });
+ }
+
+ /**
+ * The one data-safety-critical refusal, mirroring
+ * {@code SymbolDictRecycleStarvationTest#testDeferredCommitGroupSkipsWait}
+ * but for the barrier itself rather than the blocking-wait futility
+ * guard: the server withholds acks for {@code FLAG_DEFER_COMMIT} frames
+ * by design until the closing commit lands, so {@code isRingDrained()}
+ * stays false for as long as the group is open, however long that is.
+ * {@code symbol_dict_reset_max_wait_millis=0} keeps this test orthogonal
+ * to the (separately-pinned) starvation-wait timing.
+ */
+ @Test
+ public void testDeferredCommitGroupRefusesUntilCommitAcked() throws Exception {
+ assertMemoryLeak(() -> {
+ DeferAwareAckHandler handler = new DeferAwareAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port
+ + ";symbol_dict_reset_threshold=2"
+ + ";symbol_dict_reset_max_wait_millis=0;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ ws.setDeferCommit(true);
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ sender.flush(); // deferred frame -- server withholds its ack by design
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+
+ for (int i = 0; i < 5; i++) {
+ sender.table("t");
+ Assert.assertTrue("an open deferred-commit group must never let the "
+ + "recycle fire -- the server withholds its ack until "
+ + "the closing commit",
+ ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpochForTest());
+ Assert.assertEquals(1, server.handshakeCount());
+ Thread.sleep(30);
+ }
+
+ // Positive control: close the group, wait for its
+ // (retroactive) ack, and prove the still-armed recycle
+ // fires next.
+ ws.setDeferCommit(false);
+ long commitFsn = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: the commit must get acked",
+ sender.awaitAckedFsn(commitFsn, 5_000));
+
+ sender.table("t");
+ Assert.assertFalse("recycle must fire once the group is committed and acked",
+ ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpochForTest());
+ Assert.assertEquals(2, server.handshakeCount());
+
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertTrue(fsn2 > commitFsn);
+ }
+ }
+ });
+ }
+
+ /**
+ * A manual {@code resetSymbolDictionary()} call arms {@code resetArmed}
+ * regardless of connection state ({@code armIfEligible()} touches only
+ * producer-side fields), so it can go through before the sender has ever
+ * connected -- modelled the same way
+ * {@code SymbolDictRecycleFsnContinuityTest} builds unconnected senders:
+ * {@link QwpWebSocketSender#createForTesting} plus a manually attached
+ * engine, with {@link QwpWebSocketSender#setEngineRebuildFactory} filled
+ * in (unlike {@code createForTesting}'s production counterparts, a
+ * connect()-built sender normally has none -- see
+ * {@code SymbolDictRecycleTest#testConnectBuiltSenderNeverRecyclesWithoutFactory})
+ * so the deferred request can actually execute once connected. The very
+ * next {@code table()} call -- still pre-connect -- must defer rather
+ * than NPE: {@code !connected} refuses the barrier before it ever
+ * touches the cursor engine or I/O loop.
+ */
+ @Test
+ public void testManualResetBeforeFirstConnectDeferred() throws Exception {
+ assertMemoryLeak(() -> {
+ AckAllHandler handler = new AckAllHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+
+ QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", port);
+ try {
+ CursorSendEngine engine = new CursorSendEngine(
+ null, 4L * 1024 * 1024, 128L * 1024 * 1024,
+ CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS);
+ sender.setCursorEngine(engine, true);
+ sender.setEngineRebuildFactory(() -> new CursorSendEngine(
+ null, 4L * 1024 * 1024, 128L * 1024 * 1024,
+ CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS));
+
+ // Manual request before the sender has ever connected.
+ sender.resetSymbolDictionary();
+ Assert.assertTrue("a manual request arms immediately, independent of "
+ + "connection state",
+ sender.isResetArmed());
+ Assert.assertEquals(0, sender.getSymbolDictEpochForTest());
+ Assert.assertEquals(0, server.handshakeCount());
+
+ // table()'s barrier check runs here while still pre-connect
+ // (ensureConnected() only runs later, inside atNow()'s
+ // sendRow()) -- must defer quietly, not NPE.
+ sender.table("t").longColumn("v", 1L).atNow();
+ Assert.assertTrue("still armed -- deferred, not consumed",
+ sender.isResetArmed());
+ Assert.assertEquals(0, sender.getSymbolDictEpochForTest());
+ Assert.assertEquals(1, server.handshakeCount());
+
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("flush alone does not consume the arming -- only table() "
+ + "does",
+ sender.isResetArmed());
+ Assert.assertEquals(0, sender.getSymbolDictEpochForTest());
+
+ // Positive control: now connected and drained, the
+ // deferred request executes on the next table() call.
+ sender.table("t");
+ Assert.assertFalse("the deferred request must execute once connected and "
+ + "drained",
+ sender.isResetArmed());
+ Assert.assertEquals(1, sender.getSymbolDictEpochForTest());
+ Assert.assertEquals(2, server.handshakeCount());
+
+ sender.table("t").longColumn("v", 2L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertTrue(fsn2 > fsn1);
+ } finally {
+ sender.close();
+ }
+ }
+ });
+ }
+
+ /**
+ * {@code reset()} discards a buffered-but-never-shipped row -- including
+ * reclaiming any symbol id it registered but never sent, via the same
+ * {@code truncateTo} mechanism the {@code BatchTooLargeForCapException}
+ * remedy documents. This proves that discard is compatible with an
+ * already-armed swap: after {@code reset()} clears the in-progress row
+ * that was the ONLY thing refusing the barrier, every guard is
+ * satisfied (connected, no pending row, no in-progress row, ring
+ * drained from the earlier shipped batch), so the next {@code table()}
+ * call recycles -- observed here to fire deterministically, not
+ * probabilistically, once those guards clear. It is compatible with
+ * {@code reset()}'s own reclaim: the swap replaces the whole dictionary
+ * object outright (step 5 of {@code recycleForDictReset()}), so
+ * whatever {@code truncateTo} did to the outgoing instance is moot --
+ * the swap subsumes it.
+ */
+ @Test
+ public void testResetDiscardsBufferedRowThenArmedSwapFires() throws Exception {
+ assertMemoryLeak(() -> {
+ AckAllHandler handler = new AckAllHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";symbol_dict_reset_threshold=3;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ // A real shipped batch: two distinct symbols, below the
+ // threshold of 3, so nothing arms yet.
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertFalse("dictionary has only 2 entries, below the threshold of 3",
+ ws.isResetArmed());
+
+ // Start (but never commit) a third row -- registers "c",
+ // crossing the threshold, but arming is only ever
+ // evaluated at a flush's tail or by resetSymbolDictionary(),
+ // neither of which has run yet.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L);
+ Assert.assertFalse(ws.isResetArmed());
+
+ // Arm explicitly while the row is still in progress --
+ // the in-progress-row guard refuses table(), exactly as
+ // testInProgressRowRefuses proves.
+ sender.resetSymbolDictionary();
+ Assert.assertTrue(ws.isResetArmed());
+ sender.table("t"); // refused: row "c" is in progress
+ Assert.assertTrue(ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpochForTest());
+ Assert.assertEquals(1, server.handshakeCount());
+
+ // Discard the buffered row -- reset() drops the
+ // in-progress row AND reclaims "c"'s never-shipped id.
+ sender.reset();
+
+ // Every barrier guard is now satisfied: connected, no
+ // pending row (reset cleared it), no in-progress row
+ // (reset cleared it), ring drained (a, b were acked
+ // before any of this). The armed swap fires here.
+ sender.table("t");
+ Assert.assertFalse("the armed swap fires once reset() clears the blocking "
+ + "in-progress row",
+ ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpochForTest());
+ Assert.assertEquals(2, server.handshakeCount());
+
+ // Ingestion continues correctly post-swap: a fresh row
+ // lands and gets acked with no exception.
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertTrue(fsn2 > fsn1);
+ }
+ }
+ });
+ }
+
+ /**
+ * ACKs every frame it receives; does not otherwise inspect the wire.
+ * Resets its wire sequence per new connection, mirroring
+ * {@code SymbolDictRecycleTest.RecycleHandler}, so post-recycle
+ * ingestion on the fresh connection acks correctly too.
+ */
+ private static class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private TestWebSocketServer.ClientHandler currentClient;
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ if (currentClient != client) {
+ currentClient = client;
+ nextSeq.set(0);
+ }
+ try {
+ client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Acks every non-deferred frame immediately, but withholds acks for any
+ * frame carrying {@code FLAG_DEFER_COMMIT} -- the real server's ack
+ * contract for an open deferred-commit group. Mirrors
+ * {@code SymbolDictRecycleStarvationTest.DeferAwareAckHandler}, plus a
+ * per-connection wire-sequence reset so ingestion on the post-recycle
+ * connection acks correctly too.
+ */
+ private static class DeferAwareAckHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private TestWebSocketServer.ClientHandler currentClient;
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ if (currentClient != client) {
+ currentClient = client;
+ nextSeq.set(0);
+ }
+ long seq = nextSeq.getAndIncrement();
+ boolean deferred = data.length > 5 && (data[5] & FLAG_DEFER_COMMIT) != 0;
+ if (deferred) {
+ return; // withhold the ack -- the group is still open
+ }
+ try {
+ client.sendBinary(QwpWireTestUtils.buildAck(seq));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Receives frames but withholds every ack until {@link #releaseAcks()}
+ * is called, so a refusal-guard test provably has an unacknowledged
+ * target to refuse on. Mirrors
+ * {@code SymbolDictRecycleFsnContinuityTest.GatedAckHandler} /
+ * {@code SymbolDictRecycleStarvationTest.GatedAckHandler}, plus a
+ * per-connection wire-sequence reset so ingestion on the post-recycle
+ * connection acks correctly too.
+ */
+ private static class GatedAckHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private final CountDownLatch released = new CountDownLatch(1);
+ private TestWebSocketServer.ClientHandler currentClient;
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ try {
+ if (!released.await(20, TimeUnit.SECONDS)) {
+ throw new AssertionError("refusal-guard witness never released the ack gate");
+ }
+ synchronized (this) {
+ if (currentClient != client) {
+ currentClient = client;
+ nextSeq.set(0);
+ }
+ client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement()));
+ }
+ } catch (IOException | InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
+ }
+
+ void releaseAcks() {
+ released.countDown();
+ }
+ }
+}
From 2841394beeda13705f6162d44ba28c227514809f Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Mon, 17 Aug 2026 21:41:26 +0100
Subject: [PATCH 14/49] Pin recycle behavior under outage and orphan drain
Adds SymbolDictRecycleOutageTest covering two interleavings between the
symbol-dictionary recycle swap and events outside the producer's own
control: a real connection outage on its own stream (recycle triggers
while the pre-recycle I/O thread is mid-reconnect against a killed
server; step 2's close() joins it, step 7's fresh connect recovers once
the endpoint accepts again), and a sibling orphan drainer mid-drain
(the recycle only tears down the foreground sender's own cursor
engine/I/O loop, leaving a concurrently-gated BackgroundDrainer
untouched and able to complete afterward).
The third scenario from the task brief -- OrphanScanner.isCandidateOrphan
rejecting an empty slot directory -- turned out to already be pinned by
OrphanScannerTest#testIsCandidateOrphanDirect and
#testEmptySlotDirIsNotAnOrphan, so no new test was added for it.
Test-only change; no production code touched.
---
.../client/SymbolDictRecycleOutageTest.java | 452 ++++++++++++++++++
1 file changed, 452 insertions(+)
create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleOutageTest.java
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleOutageTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleOutageTest.java
new file mode 100644
index 00000000..6c7b1aa5
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleOutageTest.java
@@ -0,0 +1,452 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE;
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * Interleavings between the symbol-dictionary recycle swap
+ * ({@code QwpWebSocketSender.recycleForDictReset()}) and two things outside
+ * the producer's own control: a real connection outage on its own stream, and
+ * a sibling {@code BackgroundDrainer} running against a co-located orphan
+ * slot.
+ *
+ * (a) proves the recycle's step 2 ({@code cursorSendLoop.close()}) correctly
+ * joins an I/O thread that is itself mid-reconnect (not idle, not yet given
+ * up), and that step 7's fresh {@code ensureConnected()} recovers once the
+ * endpoint accepts connections again -- exercising
+ * {@code CursorWebSocketSendLoop.close()}'s "handles both states" contract
+ * under a real outage rather than a synthetic one.
+ *
+ * (b) proves the swap only ever tears down the producer's OWN cursor
+ * engine/I/O loop: an orphan drainer's engine and loop are entirely separate
+ * objects owned by {@code BackgroundDrainerPool}, so a recycle firing while a
+ * drain is in flight must leave the drain untouched and able to complete
+ * afterward.
+ */
+public class SymbolDictRecycleOutageTest {
+
+ private static final String ORPHAN_MARKER_SYMBOL = "orphan-marker-1";
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ /**
+ * Kills the server out from under an armed, fully-drained sender, waits
+ * for the pre-recycle I/O thread to actually enter its own reconnect
+ * loop (not just assumed via a fixed sleep), then triggers the recycle
+ * from a background thread -- because step 7's fresh
+ * {@code ensureConnected()} blocks the calling {@code table()} call (sync
+ * initial-connect mode) until the endpoint accepts again or
+ * {@code reconnect_max_duration_millis} elapses. The main thread revives
+ * a fresh server on the same port shortly after, well inside that
+ * budget, mirroring {@code ReconnectTest}'s down-then-up realism.
+ */
+ @Test
+ public void testRecycleDuringOutageReconnectsAfresh() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("outage-recycle").toString();
+ AckAllHandler firstHandler = new AckAllHandler();
+ int port;
+ try (TestWebSocketServer server = new TestWebSocketServer(firstHandler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=2"
+ + ";reconnect_initial_backoff_millis=20"
+ + ";reconnect_max_backoff_millis=80"
+ + ";reconnect_max_duration_millis=6000;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: the arming batch must be acked before the outage",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpochForTest());
+
+ // Kill the connection AND the listener -- a real outage, not
+ // just a dropped socket the same server would re-accept
+ // instantly.
+ server.close();
+
+ // Confirm the pre-recycle I/O thread actually entered its
+ // own reconnect loop against the now-refused port before we
+ // trigger the recycle -- so step 2's close() below is
+ // provably joining a MID-reconnect thread, not one that
+ // simply hasn't noticed the drop yet.
+ long attemptDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+ while (ws.getTotalReconnectAttempts() == 0 && System.nanoTime() < attemptDeadline) {
+ Thread.sleep(5);
+ }
+ Assert.assertTrue("pre-recycle I/O thread must have entered reconnect before "
+ + "the triggering table() call",
+ ws.getTotalReconnectAttempts() > 0);
+
+ // Trigger the recycle off-thread: step 7's fresh
+ // ensureConnected() blocks the caller (sync initial-connect
+ // mode) until the endpoint accepts again.
+ AtomicReference
+ * The catch-up mechanism itself is {@link DeltaDictCatchUpTest}'s territory
+ * ({@code CursorWebSocketSendLoop.setWireBaselineWithCatchUp}'s gate:
+ * {@code client != null && sentDictCount > 0 && hasReplayDictionaryDependency}).
+ * This suite does not re-implement or re-verify that mechanism -- it only
+ * observes the ONE fact specific to the recycle: {@code sentDictCount} on the
+ * fresh loop starts at 0 because {@code recycleForDictReset()}'s step 6
+ * rebuilds the engine on a freshly-emptied slot, whose {@code
+ * PersistedSymbolDict.recoveredSize()} is 0 -- so the loop constructor's
+ * {@code pd.recoveredSize() > 0} seed never fires, and the gate stays false
+ * for the whole first post-recycle connection. A PLAIN (non-recycle)
+ * reconnect on that same connection, by contrast, reuses the SAME loop
+ * instance whose mirror has since grown from the frames it sent -- so it DOES
+ * trip the gate. Observing both back to back in one test is the only way to
+ * prove the zero count above is the recycle's fresh-mirror property and not a
+ * blind spot in how this suite's handler counts frames.
+ *
+ * No production change is expected to make these pass. A failure here means
+ * either the fresh-mirror seeding regressed (a post-recycle connection
+ * started paying for catch-up again) or {@code
+ * resetSymbolDictStateForNewConnection()} grew a second job (folding the
+ * recycle's own state reset into itself and clobbering {@code
+ * sentMaxSymbolId} on an ordinary reconnect).
+ */
+public class SymbolDictRecycleCatchUpSkipTest {
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ /**
+ * The core scenario, SF-disk mode. {@code symbol_dict_reset_threshold=3}
+ * is deliberately higher than the 2 symbols this test registers in the
+ * new epoch before forcing the unplanned drop: epoch 0 crosses the
+ * threshold on its own (a, b, x -- 3 distinct symbols), so the recycle
+ * fires exactly once, synchronously, on the "c" call. Epoch 1 then
+ * registers only c, d (2 symbols, below the threshold) before the drop,
+ * and only e (a 3rd) after it -- staying unarmed for the whole test so no
+ * SECOND recycle can sneak in and confound the "does a plain reconnect
+ * still catch up / preserve the baseline" assertions below. (A lower
+ * threshold that let epoch 1 re-arm on c, d would turn the later {@code
+ * table("e")} call into an unwanted second recycle, landing e on a 4th
+ * connection instead of a plain reconnect's 3rd -- exactly the
+ * confounder this threshold choice avoids.)
+ */
+ @Test
+ public void testRecycleSkipsCatchUpThenUnplannedReconnectBoundsCatchUpToNewEpoch() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("catchup-skip").toString();
+ SkipCatchUpHandler handler = new SkipCatchUpHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=3;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ // Epoch 0 (connection 1): 3 distinct symbols cross threshold=3 and arm.
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "x").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: the arming batch must be acked before the recycle",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("must be armed after crossing threshold=3", ws.isResetArmed());
+ Assert.assertEquals(1, handler.connectionsAccepted.get());
+ Assert.assertEquals(0, ws.getSymbolDictEpochForTest());
+
+ // Ring drained: this table() call recycles synchronously onto a fresh
+ // connection (2), a fresh (empty) engine/dictionary/epoch, and "c" is
+ // then the new epoch's own first symbol.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertFalse("recycle must disarm", ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpochForTest());
+ Assert.assertEquals("recycle must open a fresh connection",
+ 2, server.handshakeCount());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("epoch-1 batch must be acked before the unplanned drop",
+ sender.awaitAckedFsn(fsn2, 5_000));
+
+ // --- Pin 1 + 2: zero catch-up frames, dictionary tiles from 0. ---
+ // Connection 2 is the FIRST connection after the recycle: its loop's
+ // sentDictCount mirror was seeded from the fresh engine's
+ // PersistedSymbolDict.recoveredSize() == 0 (nothing survived the
+ // recycle's slot wipe), so setWireBaselineWithCatchUp's
+ // `sentDictCount > 0` gate stays false for this whole connection.
+ Assert.assertEquals("first post-recycle connection must send zero catch-up "
+ + "(zero-table) frames",
+ 0, handler.zeroTableFramesFor(2));
+ Assert.assertEquals("connection 2's dictionary must tile ids from 0 with "
+ + "exactly the new epoch's symbols, none of epoch 0's a, b, x",
+ Arrays.asList("c", "d"), handler.dictFor(2));
+
+ // --- Positive control + pin 3: an UNPLANNED reconnect (server-side
+ // drop, no recycle involved) on this SAME connection DOES produce a
+ // catch-up frame, and that catch-up is bounded to exactly what this
+ // epoch has sent so far (c, d) -- proving both that the zero count
+ // above is a real property (not a handler blind spot) and that the
+ // recycle's fresh mirror does not somehow retain epoch 0's symbols.
+ handler.dropConnection(2);
+ waitFor(() -> handler.connectionsAccepted.get() >= 3, 5_000);
+ waitFor(() -> handler.dictFor(3).size() >= 2, 5_000);
+
+ Assert.assertTrue("an unplanned reconnect mid-epoch must still produce a "
+ + "catch-up frame",
+ handler.zeroTableFramesFor(3) >= 1);
+ Assert.assertEquals("the catch-up must bound itself to exactly this epoch's "
+ + "own symbols (c, d), never replaying epoch 0's a, b, x",
+ Arrays.asList("c", "d"), handler.dictFor(3));
+
+ // --- Pin 4: the plain reconnect preserved sentMaxSymbolId. A NEW
+ // symbol registered after it must ship with a delta start ABOVE 0 --
+ // resetSymbolDictStateForNewConnection only clears currentBatchMaxSymbolId,
+ // so the producer's baseline (c, d already at ids 0, 1) survives the
+ // wire boundary and e resumes at id 2. A regression that folded the
+ // recycle's sentMaxSymbolId reset into this general reconnect path
+ // would instead re-ship the whole dictionary from deltaStart 0.
+ sender.table("t").symbol("s", "e").longColumn("v", 4L).atNow();
+ long fsn3 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-reconnect row must still get acked",
+ sender.awaitAckedFsn(fsn3, 5_000));
+ Assert.assertTrue("connection 3's post-reconnect data frame carrying the new "
+ + "symbol e must ship a delta start ABOVE the surviving "
+ + "baseline (>= 1), not 0",
+ handler.sawDeltaAboveBaselineOn(3));
+ }
+
+ Assert.assertEquals("exactly 3 connections total (epoch 0, epoch 1's first "
+ + "connection, epoch 1's unplanned reconnect)",
+ 3, handler.connectionsAccepted.get());
+ }
+ });
+ }
+
+ /**
+ * Pin 5: the recycle's step 7 reconnect funnels through {@code
+ * ensureConnected()}'s {@code ASYNC} arm exactly like any other initial
+ * connect, which ends up at the same {@code swapClient} catch-up gate as
+ * the SYNC-mode scenario above. Mirrors {@code
+ * SymbolDictRecycleMemoryModeTest#testRecycleUnderAsyncInitialConnect},
+ * but in SF-disk mode (this suite's mode throughout) rather than memory
+ * mode, and asserts the zero-catch-up property instead of just the
+ * delta-start/dictionary-content pair that test already covers.
+ */
+ @Test
+ public void testRecycleUnderAsyncInitialConnectSendsZeroCatchUpFrames() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("catchup-skip-async").toString();
+ SkipCatchUpHandler handler = new SkipCatchUpHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";initial_connect_retry=async;symbol_dict_reset_threshold=2;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ // Let the I/O thread complete the deferred initial connect before
+ // driving any traffic through it (see
+ // SymbolDictRecycleMemoryModeTest.awaitWasEverConnected).
+ awaitWasEverConnected(ws);
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: batch must be acked before the recycle",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+ Assert.assertEquals(1, handler.connectionsAccepted.get());
+ Assert.assertEquals(0, ws.getSymbolDictEpochForTest());
+
+ // Recycles synchronously on the producer thread for steps 1-6; step
+ // 7's reconnect just re-arms the ASYNC path -- the actual handshake
+ // happens on the I/O thread and must be awaited via the ack below.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertFalse("recycle must disarm immediately (producer-side state)",
+ ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpochForTest());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-recycle batch must still get acked once the async "
+ + "I/O thread completes the fresh handshake",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertEquals(2, handler.connectionsAccepted.get());
+ }
+
+ Assert.assertEquals("exactly 2 connections total", 2, handler.connectionsAccepted.get());
+ Assert.assertEquals("the ASYNC path funnels through the same swapClient catch-up "
+ + "gate -- the first post-recycle connection must still send zero "
+ + "catch-up frames",
+ 0, handler.zeroTableFramesFor(2));
+ Assert.assertEquals("connection 2's dictionary must hold only the post-recycle "
+ + "symbols, not a, b",
+ Arrays.asList("c", "d"), handler.dictFor(2));
+ }
+ });
+ }
+
+ /**
+ * Spins until the I/O thread has completed the deferred ASYNC initial
+ * connect (mirrors {@code SymbolDictRecycleMemoryModeTest}'s helper of
+ * the same name).
+ */
+ private static void awaitWasEverConnected(QwpWebSocketSender ws) {
+ long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+ while (!ws.wasEverConnected()) {
+ if (System.nanoTime() > deadlineNanos) {
+ throw new AssertionError("I/O thread did not complete the async initial "
+ + "connect within 5s");
+ }
+ Compat.onSpinWait();
+ }
+ }
+
+ private static void waitFor(BoolCondition cond, long timeoutMillis) {
+ long deadline = System.currentTimeMillis() + timeoutMillis;
+ while (System.currentTimeMillis() < deadline) {
+ if (cond.test()) return;
+ try {
+ Thread.sleep(20);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ Assert.fail("interrupted");
+ }
+ }
+ Assert.fail("waitFor timed out");
+ }
+
+ @FunctionalInterface
+ private interface BoolCondition {
+ boolean test();
+ }
+
+ /**
+ * Reconstructs each connection's per-connection delta dictionary (mirrors
+ * {@code DeltaDictCatchUpTest.CatchUpHandler} / {@code
+ * SymbolDictRecycleTest.RecycleHandler}), counts zero-table (catch-up)
+ * frames per connection, tracks whether any data frame on a connection
+ * carried a delta start above 0, and -- unlike the sibling handlers --
+ * exposes {@link #dropConnection(int)} so the TEST THREAD can force an
+ * unplanned drop asynchronously, independent of the ack-driven close a
+ * handler normally does from inside {@code onBinaryMessage}.
+ */
+ private static class SkipCatchUpHandler implements TestWebSocketServer.WebSocketServerHandler {
+ final AtomicInteger connectionsAccepted = new AtomicInteger();
+ private final List
* No production change is expected to make these pass. A failure here means
* either the fresh-mirror seeding regressed (a post-recycle connection
- * started paying for catch-up again) or {@code
- * resetSymbolDictStateForNewConnection()} grew a second job (folding the
- * recycle's own state reset into itself and clobbering {@code
- * sentMaxSymbolId} on an ordinary reconnect).
+ * started paying for catch-up again) or the recycle's {@code
+ * sentMaxSymbolId} reset ({@code recycleForDictReset()}'s step 5) leaked
+ * onto the ordinary reconnect path, which today never touches that
+ * baseline.
*/
public class SymbolDictRecycleCatchUpSkipTest {
@@ -166,12 +166,15 @@ public void testRecycleSkipsCatchUpThenUnplannedReconnectBoundsCatchUpToNewEpoch
Arrays.asList("c", "d"), handler.dictFor(3));
// --- Pin 4: the plain reconnect preserved sentMaxSymbolId. A NEW
- // symbol registered after it must ship with a delta start ABOVE 0 --
- // resetSymbolDictStateForNewConnection only clears currentBatchMaxSymbolId,
- // so the producer's baseline (c, d already at ids 0, 1) survives the
- // wire boundary and e resumes at id 2. A regression that folded the
- // recycle's sentMaxSymbolId reset into this general reconnect path
- // would instead re-ship the whole dictionary from deltaStart 0.
+ // symbol registered after it must ship with a delta start ABOVE 0.
+ // Nothing on this I/O-thread reconnect path touches sentMaxSymbolId
+ // (resetSymbolDictStateForNewConnection runs only on the foreground
+ // initial-connect path, guarded by the connected flag, and never
+ // fires here), so the producer's baseline (c, d already at ids 0, 1)
+ // survives the wire boundary and e resumes at id 2. Only
+ // recycleForDictReset()'s step 5 ever zeroes that baseline; a
+ // regression that folded the reset into a path this reconnect DOES
+ // run would re-ship the whole dictionary from deltaStart 0.
sender.table("t").symbol("s", "e").longColumn("v", 4L).atNow();
long fsn3 = sender.flushAndGetSequence();
Assert.assertTrue("post-reconnect row must still get acked",
From debd45d6d8c8f831344299777fbe2fa8c222d93b Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Tue, 18 Aug 2026 10:35:02 +0100
Subject: [PATCH 17/49] Pin recycle crash-window recovery
Adds SymbolDictRecycleCrashWindowsTest, pinning what a restarted sender
recovers if the process crashes at each of four points around
QwpWebSocketSender.recycleForDictReset()'s 8-step symbol-dictionary
recycle swap:
- (a) before step 2 (the barrier that starts the swap): the pre-recycle
epoch's slot holds a fully-acked batch on disk. Recovery must find
that residue, recognize it as already acked (nothing to replay) and
resume the SAME dictionary rather than starting fresh. Constructed by
closing fast against a server that never acks (so the fully-drained
unlink never fires) and then stamping the ack watermark directly to
declare the batch acked retroactively, mirroring
DeltaDictRecoveryTest#writeAckWatermark.
- (b) between step 3 (fully-drained close of the old engine) and step 6
(rebuild): the slot is empty. Constructed by driving a real recycle to
completion and then closing immediately, before any flush touches the
freshly-rebuilt engine -- finishClose treats "nothing published yet"
as fully drained too, so this unlinks everything step 6 just created,
leaving the same empty state step 3 alone would have left.
- (c) after step 7 (reconnect), before the new epoch's first flush: the
slot holds a freshly-rebuilt engine's own state files but no data.
Constructed by snapshotting the rebuilt slot's bytes before closing
(there is no supported way to release just the slot's OS flock without
running finishClose's unlink), closing for real so nothing leaks, then
restoring the snapshot on top of the vacated directory.
- (d) an ordinary mid-operation crash one epoch into the post-recycle
steady state, to prove the epoch swap does not corrupt normal backlog
recovery. Uses the same close-fast-against-a-non-acking-server idiom
as RecoveryReplayTest, but only after the recycle's fresh connection
is established.
Arms (b) and (c) both replay nothing and both look "empty" at first
glance, but they are not the same recoverable state and a restarted
engine can tell them apart: (c)'s slot carries a manifest with collapsed
boundaries alongside a same-based, zero-frame active segment, which
SegmentRing.recover()'s chain-building accepts as a RECOVERED (if empty)
chain, while (b)'s slot carries no engine state at all and recovers as
EMPTY. wasRecoveredFromDisk() is the pinned, distinguishing observable
between the two, asserted explicitly instead of writing two
assertion-for-assertion duplicate tests.
Every arm's oracle: the recovered sender keeps ingesting; the symbols it
and its predecessor registered are exactly and correctly reconstructable
from the wire (each fresh server handler rebuilds the per-connection
delta dictionary); and no data frame is delivered more than the
at-least-once contract allows (each handler counts data frames so a
spurious re-send would show up as an unexpected count).
Co-Authored-By: Claude Fable 5
+ * Also covers the three permanent recycle-metrics getters ({@code getSymbolDictEpoch()},
+ * {@code getSymbolDictResetsPerformed()}, {@code getSymbolDictResetStarvationTimeouts()}).
+ */
+public class SymbolDictRecycleHealingTest {
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ @Test
+ public void testMetricsAfterTwoRecycles() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("metrics-sf").toString();
+ try (TestWebSocketServer server = ackingServer()) {
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=2;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+ Assert.assertEquals(0, ws.getSymbolDictResetsPerformed());
+ Assert.assertEquals(0, ws.getSymbolDictResetStarvationTimeouts());
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("armed: 2 distinct symbols crossed threshold=2", ws.isResetArmed());
+
+ // Ring drained -> this table() call recycles synchronously: epoch 1.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+ Assert.assertEquals(1, ws.getSymbolDictResetsPerformed());
+ Assert.assertEquals("no starvation wait was deliberately triggered",
+ 0, ws.getSymbolDictResetStarvationTimeouts());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertTrue("armed again: c, d cross threshold=2 in the new epoch",
+ ws.isResetArmed());
+
+ // Ring drained again -> second recycle: epoch 2.
+ sender.table("t").symbol("s", "e").longColumn("v", 4L).atNow();
+ Assert.assertEquals(2, ws.getSymbolDictEpoch());
+ Assert.assertEquals(2, ws.getSymbolDictResetsPerformed());
+ Assert.assertEquals("still no starvation wait was deliberately triggered",
+ 0, ws.getSymbolDictResetStarvationTimeouts());
+
+ long fsn3 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn3, 5_000));
+ }
+ }
+ });
+ }
+
+ /**
+ * The recovery-side sibling of {@code MmapFaultDegradesTest.testMmapAccessFaultDegradesPersistInsteadOfPropagating}:
+ * once the sender has degraded to full self-sufficient frames, the underlying fault clears,
+ * and a recycle rebuilds the engine, the fresh engine must re-derive delta-dict mode from
+ * scratch rather than staying degraded forever. Wire evidence: the first post-recycle frame
+ * (a fresh, empty dictionary) starts a delta at 0; the SECOND post-recycle frame, which
+ * introduces exactly one more symbol, starts its delta where the first one left off and
+ * carries only that one new entry -- the shape only delta mode produces. In full-dict mode
+ * every frame re-ships the whole dictionary from id 0 (see
+ * {@code QwpWebSocketSender.symbolDeltaBaseline()}: confirmedMaxId is permanently -1), so
+ * this pair of frames could not look like this if healing had not taken effect.
+ */
+ @Test
+ public void testRecycleHealsFullDictDegradeBackToDeltaMode() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("heal-sf").toString();
+ String slot = Paths.get(sfDir, "default").toString();
+ Assert.assertEquals(0, io.questdb.client.std.Files.mkdir(sfDir,
+ io.questdb.client.std.Files.DIR_MODE_DEFAULT));
+
+ CapturingAckHandler handler = new CapturingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+
+ HealableMmapFaultFacade ff = new HealableMmapFaultFacade();
+ CursorSendEngine engine = new CursorSendEngine(
+ slot, 4L * 1024 * 1024, 64L * 1024 * 1024,
+ CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, ff);
+ QwpWebSocketSender sender = buildSender(port, engine, 100_000);
+ // connect() never installs an engineRebuildFactory (only Sender.build() does),
+ // so the recycle would otherwise be a no-op. Install one that rebuilds on the
+ // SAME slot with the SAME (healable) facade -- mirroring the real factory
+ // Sender.build() installs, minus the FilesFacade seam Sender.fromConfig lacks.
+ sender.setEngineRebuildFactory(() -> new CursorSendEngine(
+ slot, 4L * 1024 * 1024, 64L * 1024 * 1024,
+ CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, ff));
+ try {
+ Assert.assertTrue("must start in delta mode", sender.isDeltaDictEnabledForTest());
+
+ // Degrade mid-life: fault the dictionary's next mmap growth.
+ ff.armed = true;
+ sender.table("m").symbol("s", "a").longColumn("v", 1L).atNow();
+ try {
+ sender.flush();
+ Assert.fail("expected the injected mmap fault to fail this flush");
+ } catch (LineSenderException expected) {
+ // same guard MmapFaultDegradesTest pins
+ }
+ Assert.assertFalse("a recognised mmap access fault must degrade the sender",
+ sender.isDeltaDictEnabledForTest());
+
+ // Heal the facade. The retry below does not itself touch mmap --
+ // persistNewSymbolsBeforePublish short-circuits once !deltaDictEnabled --
+ // so healing here matters only for what the fresh post-recycle engine sees.
+ ff.armed = false;
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("the degraded retry must still ingest the row",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertEquals(0, sender.getSymbolDictEpoch());
+
+ // Drained: arm and trigger the recycle.
+ sender.resetSymbolDictionary();
+ Assert.assertTrue(sender.isResetArmed());
+ sender.table("m").symbol("s", "b").longColumn("v", 2L).atNow();
+ Assert.assertFalse("recycle must disarm", sender.isResetArmed());
+ Assert.assertEquals(1, sender.getSymbolDictEpoch());
+ Assert.assertEquals(1, sender.getSymbolDictResetsPerformed());
+
+ // The fresh engine re-derives delta-dict mode against the healed facade.
+ Assert.assertTrue("a healed facade must let the fresh engine re-derive delta mode",
+ sender.isDeltaDictEnabledForTest());
+
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+
+ sender.table("m").symbol("s", "c").longColumn("v", 3L).atNow();
+ long fsn3 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn3, 5_000));
+
+ List
+ * Out of scope here: whether {@code symbol_dict_reset=off} actually keeps
+ * {@code armIfEligible()} from arming. That only runs from the tail of a
+ * completed {@code flush()}, which this test never performs (the fill
+ * goes through the raw dictionary test accessor, and the one
+ * {@code Sender}-routed call throws inside {@code symbol()} before a row
+ * completes) -- an {@code isResetArmed()} assertion here would pass
+ * regardless of the knob, proving nothing. That arming-vs-flush property
+ * is pinned in {@code SymbolDictRecycleArmingTest.testArmsAtThreshold}.
*/
@Test
public void testCapReachedWithResetDisabledStillThrowsAndNamesTheResetValve() throws Exception {
@@ -121,7 +130,6 @@ public void testCapReachedWithResetDisabledStillThrowsAndNamesTheResetValve() th
for (int i = 0; i < MAX_SYMBOL_DICTIONARY_SIZE; i++) {
dict.getOrAddSymbol("f" + i);
}
- Assert.assertFalse("reset disabled must never arm", ws.isResetArmed());
try {
sender.table("t").symbol("s", "one-too-many");
@@ -133,7 +141,6 @@ public void testCapReachedWithResetDisabledStillThrowsAndNamesTheResetValve() th
Assert.assertTrue("message points at the reset valve: " + message,
message.contains("symbol_dict_reset") && message.contains("resetSymbolDictionary()"));
}
- Assert.assertFalse("still not armed after the refusal", ws.isResetArmed());
}
}
});
From 6dc84e550b873cc8b46ba6bdc57bb1b55881f954 Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Tue, 18 Aug 2026 18:55:21 +0100
Subject: [PATCH 23/49] Final wave: survive a step-7 connect failure without
latching
recycleForDictReset() no longer latches the sender terminal when the
recycle's reconnect fails. Step 7 moves out of the latching try block:
steps 1-6 still latch recycleFailure (a half-swapped sender genuinely
cannot make progress), but a failed ensureConnected() logs a warning and
rethrows to the triggering caller without latching. By step 7 the swap has
committed, so the sender is coherent - connected == false, loop and client
already closed and nulled by ensureConnected's own catch, the fresh engine
attached, the step-5 epoch and swap counters correctly left incremented -
and the ordinary sendRow() -> ensureConnected() path retries the connect,
and only the connect, on the next send. Nothing re-runs a teardown step,
and nothing can fire a second swap meanwhile: the fresh dictionary sits
below the threshold, manualResetRequested was consumed at step 5, and
maybeRecycleForDictReset requires connected.
This removes a default-configuration brick. A sender built with no
reconnect_* knob resolves initialConnectMode to OFF, so step 7 is a
single-shot connect; a server restart or an LB blip across a drained,
armed sender therefore latched every later table()/flush() call forever,
including seconds later once the endpoint was back.
Deferring the connect to the next send exposed a second defect, which this
commit also fixes. ensureConnected() calls
resetSymbolDictStateForNewConnection(), which cleared
currentBatchMaxSymbolId unconditionally. That watermark is batch-scoped,
not connection-scoped - a flush ships exactly [sentMaxSymbolId+1 ..
currentBatchMaxSymbolId] - and clearing it was harmless only because
build() connects before the application can register a symbol. On the
deferred path symbol() runs first, so the clear made the next flush ship
an empty delta while its rows referenced symbol id 0: rows on the wire
pointing at ids the server never received. The reset now runs only from
the drained state the old code assumed (no pending rows, no row in
progress).
testDefaultConfigRecycleSurvivesFailedReconnect pins both. It drives a
default-config sender (no reconnect knobs), kills the listener at a
drained instant, asserts the recycle's throw reaches the caller, asserts
recycleFailure is not latched by ingesting successfully once the endpoint
returns on the same port, and asserts the recovered stream defines every
symbol its rows reference. Against the pre-fix code it fails on the latch;
with the latch fixed but the watermark clear restored it fails on the
empty dictionary.
Documentation and hygiene alongside:
- getTotalFramesReplayed/getTotalFramesSent/getTotalReconnectAttempts/
getTotalReconnectsSucceeded/getTotalServerErrors now state that they
read the live send loop and therefore restart at 0 on every recycle
("since the last recycle"), and point at the lifetime-scoped
getSymbolDictEpoch/getSymbolDictResetsPerformed for correlation.
- Sender.resetSymbolDictionary(), its QwpWebSocketSender override and
LineSenderBuilder.symbolDictReset() now say that the manual valve is a
permanent no-op while symbol_dict_reset is off, because armIfEligible
gates on that knob.
- armIfEligible's javadoc names both call sites; resetSymbolDictionary()
calls it too, and the stale single-call-site claim invited a wrong
inlining refactor.
- The builder's symbolDictReset default references
DEFAULT_SYMBOL_DICT_RESET_ENABLED instead of a hardcoded true.
- SymbolDictRecycleOutageTest joins its trigger thread in a finally so an
assert failure cannot leave a thread inside the sender, and records why
the revived server's handshake count stays >= 1 rather than == 1.
- SymbolDictRecycleCrashWindowsTest inlines an unread handler binding, and
SymbolDictRecycleHealingTest's AckAllHandler carries a warning that its
per-connection ack reset assumes every connection change is a recycle
and must not be copied into a plain-reconnect test.
Client suite: 3045 run, 0 failures, 0 errors, 2 skipped.
Co-Authored-By: Claude Fable 5
+ * Also a permanent no-op on a sender configured with
+ * {@code symbol_dict_reset=off} ({@link LineSenderBuilder#symbolDictReset(boolean)}):
+ * that knob gates the arming path this request feeds, so the request is
+ * accepted and never acted on.
*/
default void resetSymbolDictionary() {
}
@@ -1097,7 +1102,7 @@ final class LineSenderBuilder {
private int maxFrameRejections = PARAMETER_NOT_SET_EXPLICITLY;
private long poisonMinEscalationWindowMillis = PARAMETER_NOT_SET_EXPLICITLY;
private long catchUpCapGapMinEscalationWindowMillis = PARAMETER_NOT_SET_EXPLICITLY;
- private boolean symbolDictReset = true;
+ private boolean symbolDictReset = QwpWebSocketSender.DEFAULT_SYMBOL_DICT_RESET_ENABLED;
private int symbolDictResetThreshold = PARAMETER_NOT_SET_EXPLICITLY;
private long symbolDictResetMaxWaitMillis = PARAMETER_NOT_SET_EXPLICITLY;
private String httpPath;
@@ -1894,6 +1899,10 @@ public LineSenderBuilder catchUpCapGapMinEscalationWindowMillis(long millis) {
* once it reaches {@link #symbolDictResetThreshold(int)} distinct symbols,
* so a long-lived sender's dictionary does not grow without bound.
*
+ * Switching it off also disables the manual valve:
+ * {@link Sender#resetSymbolDictionary()} becomes a permanent no-op,
+ * because arming gates on this knob.
+ *
* Default {@code true} (on). WebSocket transport only.
*/
public LineSenderBuilder symbolDictReset(boolean enabled) {
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index 088b505d..0d67191c 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -446,12 +446,14 @@ public interface EngineRebuildFactory {
// its obvious reader.
private volatile long symbolDictResetStarvationTimeouts;
// Set (once) by recycleForDictReset's catch block when the recycle swap
- // itself fails -- everything was acked before the swap tore the old
- // engine down, so no data is at risk, but this sender can no longer make
- // progress (no cursor engine, no I/O loop) and refuses further use.
- // checkRecycleFailure() rethrows a fresh LineSenderException wrapping
+ // itself fails in steps 2-6 -- everything was acked before the swap tore
+ // the old engine down, so no data is at risk, but this sender can no
+ // longer make progress (no cursor engine, no I/O loop) and refuses further
+ // use. checkRecycleFailure() rethrows a fresh LineSenderException wrapping
// this cause on every later table()/flush-family call; close() still
- // works normally.
+ // works normally. A step-7 (reconnect) failure deliberately does NOT set
+ // this: the swap has already committed there, so the sender is coherent
+ // and merely disconnected, and the next send retries the connect.
private Throwable recycleFailure;
// Incremented once per completed symbol-dictionary recycle. 0 until the
// first recycle commits. volatile: this is public API (see
@@ -2150,8 +2152,11 @@ public long getFsnEpochBaseForTest() {
* Number of symbol-dictionary recycles this sender has completed. Advances
* by one at step 5 of {@link #recycleForDictReset()}, the instant the swap
* commits to the new epoch -- before the engine rebuild (step 6) or the
- * reconnect (step 7), so a later rebuild/reconnect failure that latches
- * {@link #recycleFailure} still leaves this incremented. volatile: a
+ * reconnect (step 7), so a later step-6 rebuild failure that latches
+ * {@link #recycleFailure}, and a step-7 reconnect failure that does not,
+ * both still leave this incremented. Unlike the per-send-loop
+ * {@code getTotal*} counters, it is scoped to the sender's whole lifetime
+ * and never resets. volatile: a
* concurrent read sees the latest write the producer thread completed,
* but there is no atomicity across the three symbol-dictionary-recycle
* counters -- a reader on another thread can observe this one already
@@ -2168,8 +2173,9 @@ public long getSymbolDictEpoch() {
* Incremented alongside {@link #getSymbolDictEpoch()} at step 5 of
* {@link #recycleForDictReset()} -- before the engine rebuild (step 6) or
* the reconnect (step 7), so, like the epoch counter, a later
- * rebuild/reconnect failure that latches {@link #recycleFailure} still
- * leaves this incremented. The two counts move together today -- the
+ * rebuild/reconnect failure still leaves this incremented, latched or not.
+ * Also like the epoch counter, it is scoped to the sender's whole lifetime
+ * and never resets. The two counts move together today -- the
* only way the epoch advances is through a completed recycle swap -- but
* they are defined, and incremented, independently: this one counts
* completed swaps, {@code getSymbolDictEpoch()} counts the dictionary
@@ -2295,9 +2301,15 @@ public long getTotalErrorNotificationsDelivered() {
}
/**
- * Cumulative count of frames re-sent during post-reconnect catch-up
- * windows. Zero in steady state; a sustained nonzero rate signals
- * flapping where every reconnect replays meaningful work.
+ * Count of frames re-sent during post-reconnect catch-up windows since the
+ * last symbol-dictionary recycle. Zero in steady state; a sustained nonzero
+ * rate signals flapping where every reconnect replays meaningful work.
+ *
+ * Reads the live cursor I/O loop, which a symbol-dictionary recycle
+ * rebuilds, so the count restarts at 0 on every recycle: a monitor
+ * differencing it across one sees a negative delta. Correlate with the
+ * lifetime-scoped {@link #getSymbolDictEpoch()} /
+ * {@link #getSymbolDictResetsPerformed()}, which never reset.
*/
public long getTotalFramesReplayed() {
CursorWebSocketSendLoop l = cursorSendLoop;
@@ -2305,7 +2317,14 @@ public long getTotalFramesReplayed() {
}
/**
- * Total binary frames the cursor I/O loop has issued to the wire.
+ * Binary frames the cursor I/O loop has issued to the wire since the last
+ * symbol-dictionary recycle.
+ *
+ * Reads the live cursor I/O loop, which a symbol-dictionary recycle
+ * rebuilds, so the count restarts at 0 on every recycle: a monitor
+ * differencing it across one sees a negative delta. Correlate with the
+ * lifetime-scoped {@link #getSymbolDictEpoch()} /
+ * {@link #getSymbolDictResetsPerformed()}, which never reset.
*/
public long getTotalFramesSent() {
CursorWebSocketSendLoop l = cursorSendLoop;
@@ -2313,9 +2332,16 @@ public long getTotalFramesSent() {
}
/**
- * Number of reconnect attempts the cursor I/O loop has issued —
- * succeeded plus failed. Diverges from {@link #getTotalReconnectsSucceeded}
- * when the server is flapping. Returns 0 if no I/O loop is running.
+ * Number of reconnect attempts the cursor I/O loop has issued since the
+ * last symbol-dictionary recycle -- succeeded plus failed. Diverges from
+ * {@link #getTotalReconnectsSucceeded} when the server is flapping.
+ * Returns 0 if no I/O loop is running.
+ *
+ * Reads the live cursor I/O loop, which a symbol-dictionary recycle
+ * rebuilds, so the count restarts at 0 on every recycle: a monitor
+ * differencing it across one sees a negative delta. Correlate with the
+ * lifetime-scoped {@link #getSymbolDictEpoch()} /
+ * {@link #getSymbolDictResetsPerformed()}, which never reset.
*/
public long getTotalReconnectAttempts() {
CursorWebSocketSendLoop l = cursorSendLoop;
@@ -2323,7 +2349,14 @@ public long getTotalReconnectAttempts() {
}
/**
- * Number of successful reconnects. Returns 0 if no I/O loop is running.
+ * Number of successful reconnects since the last symbol-dictionary
+ * recycle. Returns 0 if no I/O loop is running.
+ *
+ * Reads the live cursor I/O loop, which a symbol-dictionary recycle
+ * rebuilds, so the count restarts at 0 on every recycle: a monitor
+ * differencing it across one sees a negative delta. Correlate with the
+ * lifetime-scoped {@link #getSymbolDictEpoch()} /
+ * {@link #getSymbolDictResetsPerformed()}, which never reset.
*/
public long getTotalReconnectsSucceeded() {
CursorWebSocketSendLoop l = cursorSendLoop;
@@ -2331,7 +2364,14 @@ public long getTotalReconnectsSucceeded() {
}
/**
- * Total errors observed by the I/O loop (retriable and terminal combined).
+ * Errors the I/O loop has observed since the last symbol-dictionary
+ * recycle (retriable and terminal combined).
+ *
+ * Reads the live cursor I/O loop, which a symbol-dictionary recycle
+ * rebuilds, so the count restarts at 0 on every recycle: a monitor
+ * differencing it across one sees a negative delta. Correlate with the
+ * lifetime-scoped {@link #getSymbolDictEpoch()} /
+ * {@link #getSymbolDictResetsPerformed()}, which never reset.
*/
public long getTotalServerErrors() {
CursorWebSocketSendLoop l = cursorSendLoop;
@@ -2652,6 +2692,10 @@ public void reset() {
* have to wait for a later flush to observe {@code isResetArmed()}. A
* request made mid-batch is picked up by the next
* {@code resetTableBuffersAfterFlush} instead.
+ *
+ * A permanent no-op while {@code symbol_dict_reset} is off: arming gates
+ * on that knob, so a sender configured with the recycle disabled never
+ * acts on the request, however many times it is made.
*/
@Override
public void resetSymbolDictionary() {
@@ -3712,11 +3756,14 @@ private void checkNotClosed() {
}
/**
- * Terminal latch for a failed symbol-dictionary recycle swap
- * ({@link #recycleForDictReset()}). Everything was acked before the swap
- * tore the old engine down, so no data is at risk -- but the swap itself
- * left this sender without a cursor engine or I/O loop, so it refuses
- * further use. Checked by {@link #table(CharSequence)}, the flush-family
+ * Terminal latch for a symbol-dictionary recycle swap that failed in steps
+ * 2-6 ({@link #recycleForDictReset()}). Everything was acked before the
+ * swap tore the old engine down, so no data is at risk -- but the swap
+ * itself left this sender without a cursor engine or I/O loop, so it
+ * refuses further use. A step-7 reconnect failure is NOT latched (the swap
+ * has committed by then and the sender is coherent, just disconnected --
+ * see {@link #recycleForDictReset()}). Checked by
+ * {@link #table(CharSequence)}, the flush-family
* entry points ({@link #flush()}, {@link #flushAndGetSequence()},
* {@link #drain(long)}, {@link #awaitAckedFsn(long, long)}), and
* {@code sendRow()} (closing the fluent-chain corner where a caller
@@ -4180,10 +4227,11 @@ private void ensureConnected() {
ep.host, ep.port, endpoints.size());
}
// Server starts fresh on each connection, so reset the per-batch
- // symbol-dict watermark. Every frame still carries its full inline schema,
- // and the fresh server's dictionary is re-established either by a full-dict
- // frame (full-dict mode) or by an I/O-thread catch-up frame before replay
- // (delta mode), so post-reconnect replay needs no producer-side reset signal.
+ // symbol-dict watermark when nothing is staged against it. Every frame
+ // still carries its full inline schema, and the fresh server's dictionary
+ // is re-established either by a full-dict frame (full-dict mode) or by an
+ // I/O-thread catch-up frame before replay (delta mode), so post-reconnect
+ // replay needs no producer-side reset signal.
resetSymbolDictStateForNewConnection();
connectionError.set(null);
@@ -4588,11 +4636,13 @@ private void resetTableBuffersAfterFlush() {
* frames still benefits from bounding its dictionary size, and a manual
* request is honoured regardless of mode.
*
- * Called only from the tail of {@link #resetTableBuffersAfterFlush()} (a
- * safe point: no row in progress, this flush's data already handed to the
- * engine), never from the per-symbol registration path
- * ({@link #getOrAddGlobalSymbol}) -- arming mid-row or mid-encode would
- * observe a dictionary size that has not yet settled for this batch.
+ * Called from two safe points only: the tail of
+ * {@link #resetTableBuffersAfterFlush()} (no row in progress, this flush's
+ * data already handed to the engine) and {@link #resetSymbolDictionary()}
+ * when nothing is in flight ({@code pendingRowCount == 0}). Never from the
+ * per-symbol registration path ({@link #getOrAddGlobalSymbol}) -- arming
+ * mid-row or mid-encode would observe a dictionary size that has not yet
+ * settled for this batch.
*/
private void armIfEligible() {
boolean shouldArm = resetEnabled
@@ -4743,14 +4793,29 @@ private void maybeRecycleForDictReset() {
* mirroring (not calling) {@link #setCursorEngine} -- that method's
* guards refuse a second engine.
+ * Step 7 is deliberately exempt from that latch. By then the swap has
+ * committed, so a failed connect leaves a fully coherent sender that is
+ * merely disconnected: {@code connected == false}, loop and client already
+ * closed and nulled by {@link #ensureConnected()}'s own catch, the fresh
+ * engine attached, and the step-5 counters ({@link #symbolDictEpoch},
+ * {@link #symbolDictResetsPerformed}) correctly left incremented because
+ * the swap really did happen. It rethrows loudly to the triggering caller
+ * but the sender stays usable, and the ordinary
+ * {@code sendRow() -> ensureConnected()} path retries the connect -- and
+ * only the connect -- on the next send. Nothing can fire a second swap
+ * meanwhile: the fresh dictionary is below threshold,
+ * {@code manualResetRequested} was consumed at step 5, and
+ * {@link #maybeRecycleForDictReset()} requires {@code connected}.
*/
private void recycleForDictReset() {
final long lastPublishedFsn = cursorEngine.publishedFsn(); // step 1
@@ -4784,11 +4849,6 @@ private void recycleForDictReset() {
ownsCursorEngine = true;
deltaDictEnabled = cursorEngine.isDeltaDictEnabled();
cursorEngine.setSlotLockReleaseListener(this::onSlotLockReleased);
- // step 7: reconnect - rebuilds the loop with the rolled base
- connected = false;
- ensureConnected();
- LOG.info("symbol dictionary recycled [epoch={}, dictSizeAtSwap={}, pauseMicros={}]",
- symbolDictEpoch, dictSizeAtSwap, (System.nanoTime() - startNanos) / 1000L);
} catch (Throwable t) {
// step 8: terminal latch - everything was acked before step 2,
// so no data is at risk; the sender refuses further use.
@@ -4800,6 +4860,32 @@ private void recycleForDictReset() {
}
throw new LineSenderException(t).put("symbol dictionary recycle failed");
}
+ // step 7: reconnect - rebuilds the loop with the rolled base. OUTSIDE
+ // the latching try on purpose: the swap has already committed, so a
+ // connect failure here leaves a coherent fresh-epoch sender that is
+ // merely disconnected, not a half-swapped one. It throws loudly to the
+ // caller but does NOT latch -- ensureConnected's own catch has already
+ // closed and nulled the loop and the client, the epoch and swap
+ // counters incremented at step 5 stay incremented (the swap really did
+ // happen), the sender stays disarmed (a fresh dictionary is below
+ // threshold and manualResetRequested was consumed, so nothing can fire
+ // a second swap while disconnected), and the next sendRow() retries the
+ // connect - and ONLY the connect - through ensureConnected.
+ connected = false;
+ try {
+ ensureConnected();
+ } catch (Throwable t) {
+ LOG.warn("symbol dictionary swap committed but its reconnect failed; sender stays "
+ + "disconnected on the fresh epoch and retries the connect on the "
+ + "next send [epoch={}, dictSizeAtSwap={}]",
+ symbolDictEpoch, dictSizeAtSwap, t);
+ if (t instanceof LineSenderException) {
+ throw (LineSenderException) t;
+ }
+ throw new LineSenderException(t).put("symbol dictionary recycle reconnect failed");
+ }
+ LOG.info("symbol dictionary recycled [epoch={}, dictSizeAtSwap={}, pauseMicros={}]",
+ symbolDictEpoch, dictSizeAtSwap, (System.nanoTime() - startNanos) / 1000L);
}
/**
@@ -5225,14 +5311,28 @@ private void reclaimUnsentSymbolIds() {
}
private void resetSymbolDictStateForNewConnection() {
- // Runs on the foreground (initial) connect only -- NOT on the I/O thread's
- // reconnect/failover path. The per-batch watermark is drained state, so
- // clearing it here is harmless. sentMaxSymbolId is deliberately left
+ // Runs on the foreground connect only -- NOT on the I/O thread's
+ // reconnect/failover path. sentMaxSymbolId is deliberately left
// untouched: in delta mode the I/O thread re-registers the whole
// dictionary with a catch-up frame on reconnect, so the producer's
// monotonic baseline must survive the wire boundary; resetting it would
// desync the producer from the I/O thread's sent-dictionary count.
- currentBatchMaxSymbolId = -1;
+ //
+ // currentBatchMaxSymbolId is batch-scoped, not connection-scoped: a
+ // flush ships exactly [sentMaxSymbolId+1 .. currentBatchMaxSymbolId],
+ // so clearing it while a batch already references those ids ships a
+ // delta that OMITS them and puts rows on the wire pointing at symbol
+ // ids the server never received. Clearing it used to be unconditional
+ // and harmless because build() connects before the application can
+ // register anything. That no longer holds: a symbol-dictionary recycle
+ // whose step-7 connect failed defers the connect to the next
+ // sendRow(), which runs after symbol() has registered the ids of the
+ // row being built. Reset only from the drained state the old code
+ // assumed.
+ if (pendingRowCount == 0
+ && (currentTableBuffer == null || !currentTableBuffer.hasInProgressRow())) {
+ currentBatchMaxSymbolId = -1;
+ }
}
/**
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java
index d4a469c0..43a0b493 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java
@@ -378,7 +378,17 @@ private static int[] deltaStartAndCount(byte[] frame) {
return new int[]{deltaStart, deltaCount};
}
- /** ACKs every frame it receives; does not otherwise inspect the wire. */
+ /**
+ * ACKs every frame it receives; does not otherwise inspect the wire.
+ *
+ * WARNING -- recycle-only handler, do not copy into a plain-reconnect test.
+ * The per-connection sequence reset below assumes every connection change
+ * is a recycle, i.e. that a fresh engine is behind the new connection and
+ * its raw FSNs really do restart at 0. On an ordinary reconnect the SAME
+ * engine survives and keeps counting, so resetting here would ack frames
+ * the sender never published and silently advance its watermark past
+ * unsent data.
+ */
private static class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler {
private TestWebSocketServer.ClientHandler currentClient;
private final AtomicLong nextSeq = new AtomicLong(0);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleOutageTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleOutageTest.java
index 458be65a..2f0036b5 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleOutageTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleOutageTest.java
@@ -25,6 +25,7 @@
package io.questdb.client.test.cutlass.qwp.client;
import io.questdb.client.Sender;
+import io.questdb.client.cutlass.line.LineSenderException;
import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner;
import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
@@ -143,44 +144,164 @@ public void testRecycleDuringOutageReconnectsAfresh() throws Exception {
}, "recycle-trigger");
trigger.start();
- // Resolve the outage shortly after -- well inside
- // reconnect_max_duration_millis=6000 -- on the SAME port.
- Thread.sleep(150);
+ try {
+ // Resolve the outage shortly after -- well inside
+ // reconnect_max_duration_millis=6000 -- on the SAME port.
+ Thread.sleep(150);
+ OutageRecycleHandler revivedHandler = new OutageRecycleHandler();
+ try (TestWebSocketServer revived =
+ new TestWebSocketServer(revivedHandler, false, null, port)) {
+ revived.start();
+ Assert.assertTrue(revived.awaitStart(5, TimeUnit.SECONDS));
+
+ trigger.join(10_000);
+ Assert.assertFalse("recycle-trigger thread must have finished once the "
+ + "endpoint accepts again", trigger.isAlive());
+ Assert.assertNull("triggering table() must not throw once the outage "
+ + "resolves within budget: " + triggerFailure.get(),
+ triggerFailure.get());
+
+ Assert.assertFalse("recycle must disarm", ws.isResetArmed());
+ Assert.assertEquals("recycle must complete despite the outage",
+ 1, ws.getSymbolDictEpoch());
+ // Deliberately >= 1, not == 1: the pre-recycle I/O thread is
+ // banging on the refused port when the revive binds, and
+ // nothing orders step 2's close+join against that bind, so a
+ // stray handshake from the outgoing loop is legal here.
+ Assert.assertTrue("revived server must observe a fresh handshake",
+ revived.handshakeCount() >= 1);
+
+ // The "c" row was built (atNow()) inside the triggering
+ // call but not yet flushed -- flush now and prove it
+ // lands on the fresh connection.
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-recycle row must land once reconnected",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertTrue("post-recycle FSN must exceed pre-recycle FSN",
+ fsn2 > fsn1);
+
+ Assert.assertEquals("the fresh connection's first frame must carry a "
+ + "fresh (empty) dictionary, not a, b",
+ 0, revivedHandler.firstFrameDeltaStart);
+ Assert.assertEquals("post-recycle dictionary must hold only the new "
+ + "epoch's symbol, nothing lost or duplicated from "
+ + "before the outage",
+ Collections.singletonList("c"), revivedHandler.dict());
+ }
+ } finally {
+ // Never leave the trigger thread running past this test:
+ // a thread still inside the sender on an assert-failure
+ // path muddies assertMemoryLeak's diagnostics.
+ trigger.join(10_000);
+ }
+ }
+ }
+ });
+ }
+
+ /**
+ * Default configuration: no {@code reconnect_*} knob and no
+ * {@code initial_connect_retry}, so the builder resolves
+ * {@code initialConnectMode} to OFF and step 7's {@code ensureConnected()}
+ * is a single-shot connect that fails outright while the endpoint refuses.
+ * Steps 1-6 have already committed by then, so latching {@code
+ * recycleFailure} here would brick the sender permanently over an ordinary
+ * transient outage -- the shipped default for every sender that crosses
+ * the threshold.
+ *
+ * Proves the step-7 failure reaches the caller WITHOUT latching, that the
+ * swap committed exactly one epoch, and that the very next send recovers
+ * through the existing {@code sendRow() -> ensureConnected()} path once
+ * the endpoint is back -- reconnecting only, never re-running a teardown
+ * step and never swapping a second time.
+ */
+ @Test
+ public void testDefaultConfigRecycleSurvivesFailedReconnect() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("default-config-outage").toString();
+ AckAllHandler firstHandler = new AckAllHandler();
+ int port;
+ try (TestWebSocketServer server = new TestWebSocketServer(firstHandler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=2;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ Assert.assertTrue("the recycle must be on under a default configuration",
+ ws.isSymbolDictResetEnabled());
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: the arming batch must be acked before the outage",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ // Kill the listener AND the live connection. The ring is
+ // drained, so the sender-level connected flag is still true
+ // and the next table() call fires the recycle into a wire
+ // that is already down.
+ server.close();
+
+ LineSenderException triggering = null;
+ try {
+ sender.table("t");
+ Assert.fail("step 7's single-shot connect must throw while the endpoint "
+ + "refuses connections");
+ } catch (LineSenderException e) {
+ triggering = e;
+ }
+ Assert.assertNotNull(triggering);
+
+ Assert.assertEquals("the swap committed exactly one epoch before the connect "
+ + "failed", 1, ws.getSymbolDictEpoch());
+ Assert.assertEquals(1, ws.getSymbolDictResetsPerformed());
+ Assert.assertFalse("a committed swap disarms even when its reconnect fails",
+ ws.isResetArmed());
+
+ // Endpoint back on the SAME port. The sender must not be
+ // terminal: the next send reconnects on its own.
OutageRecycleHandler revivedHandler = new OutageRecycleHandler();
try (TestWebSocketServer revived =
new TestWebSocketServer(revivedHandler, false, null, port)) {
revived.start();
Assert.assertTrue(revived.awaitStart(5, TimeUnit.SECONDS));
- trigger.join(10_000);
- Assert.assertFalse("recycle-trigger thread must have finished once the "
- + "endpoint accepts again", trigger.isAlive());
- Assert.assertNull("triggering table() must not throw once the outage "
- + "resolves within budget: " + triggerFailure.get(),
- triggerFailure.get());
-
- Assert.assertFalse("recycle must disarm", ws.isResetArmed());
- Assert.assertEquals("recycle must complete despite the outage",
- 1, ws.getSymbolDictEpoch());
- Assert.assertTrue("revived server must observe a fresh handshake",
- revived.handshakeCount() >= 1);
-
- // The "c" row was built (atNow()) inside the triggering
- // call but not yet flushed -- flush now and prove it
- // lands on the fresh connection.
+ // "d" registers BEFORE the deferred connect -- symbol()
+ // runs ahead of sendRow(), which is what finally
+ // performs it -- so this first batch is the one that
+ // proves the connect left the batch's symbol watermark
+ // alone on its way through.
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
long fsn2 = sender.flushAndGetSequence();
- Assert.assertTrue("post-recycle row must land once reconnected",
+ Assert.assertTrue("a sender whose step-7 connect failed must still ingest "
+ + "once the endpoint returns",
sender.awaitAckedFsn(fsn2, 5_000));
Assert.assertTrue("post-recycle FSN must exceed pre-recycle FSN",
fsn2 > fsn1);
+ Assert.assertEquals("the recovery reconnects only -- it must not run a "
+ + "second swap", 1, ws.getSymbolDictEpoch());
+ Assert.assertEquals(1, ws.getSymbolDictResetsPerformed());
- Assert.assertEquals("the fresh connection's first frame must carry a "
+ Assert.assertEquals("the recovered connection's first frame must carry a "
+ "fresh (empty) dictionary, not a, b",
0, revivedHandler.firstFrameDeltaStart);
- Assert.assertEquals("post-recycle dictionary must hold only the new "
- + "epoch's symbol, nothing lost or duplicated from "
- + "before the outage",
- Collections.singletonList("c"), revivedHandler.dict());
+ Assert.assertEquals("the recovered stream must define every symbol its "
+ + "rows reference: a deferred connect that cleared the "
+ + "batch watermark would ship a row pointing at an id "
+ + "the server never received",
+ Collections.singletonList("d"), revivedHandler.dict());
+
+ // And the epoch keeps extending normally from there.
+ sender.table("t").symbol("s", "e").longColumn("v", 4L).atNow();
+ long fsn3 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn3, 5_000));
+ Assert.assertEquals("later batches must extend the same fresh dictionary",
+ Arrays.asList("d", "e"), revivedHandler.dict());
}
}
}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SymbolDictRecycleCrashWindowsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SymbolDictRecycleCrashWindowsTest.java
index acd98450..0f4a5e97 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SymbolDictRecycleCrashWindowsTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SymbolDictRecycleCrashWindowsTest.java
@@ -191,8 +191,7 @@ public void testCrashBeforeRecycleStartsRecoversAckedResidueOnly() throws Except
String sfDir = temporaryFolder.getRoot().toPath().resolve("crash-a-pre-swap").toString();
String slot = Paths.get(sfDir, "default").toString();
long fsn;
- SilentHandler crashedHandler = new SilentHandler();
- try (TestWebSocketServer crashed = startedServer(crashedHandler)) {
+ try (TestWebSocketServer crashed = startedServer(new SilentHandler())) {
String cfg = "ws::addr=localhost:" + crashed.getPort() + ";sf_dir=" + sfDir
+ ";symbol_dict_reset_threshold=2;close_flush_timeout_millis=0;";
try (Sender sender = Sender.fromConfig(cfg)) {
From 06dcb6dbcd59888ae73e27e1226155496274d16e Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Wed, 19 Aug 2026 13:46:03 +0100
Subject: [PATCH 24/49] Fix round 2: recycle awaits a deferred engine close
Review C1: recycleForDictReset step 3 re-acquired the slot flock without
waiting for a deferred close to release it. When the SF worker is wedged
in a syscall past SegmentManager's bounded join, CursorSendEngine.close()
returns with the flock retained and isCloseCompleted() false, releasing
both from the worker's exit path; the step-6 rebuild then threw
SlotLockContentionException on the retained flock and latched the sender
permanently terminal -- converting exactly the transient disk stall the
deferred-close machinery exists to survive into a hard sender death on
the default (SF + recycle-on) path.
recycleForDictReset now mirrors close()'s deferred-close discipline:
awaitDeferredEngineClose parks (awaitAckedFsn-shaped) until the deferred
cleanup confirms the flock release, re-arming the shared flock-release
retry driver each pass like isSlotLockReleased() does. Only exhausting
the 30 s budget -- a genuinely dead worker -- latches terminal, and that
path hands the still-locked engine to retainedEngine so a pool re-probe
recovers the slot's capacity if the worker ever exits; close() no longer
clobbers slotLockReleased to true while such an engine is pending.
SymbolDictRecycleDeferredCloseTest pins both branches with a wedged-
worker harness (red-proofed: without the await, the survival test dies
with the exact SlotLockContentionException the review traced).
Review Mo1: getAckedFsn and awaitAckedFsn snapshot cursorEngine into a
local (the recycle transitions it non-null -> null -> non-null), and
while it is null they report the durable watermark the recycle barrier
proved (new lastRecycleDurableFsn) instead of collapsing to -1.
Review Mo2: PooledSender forwards resetSymbolDictionary() to the live
delegate instead of inheriting the interface's no-op default; pinned by
a pooled-path test.
Review Mo5 (verified: released 9.4.x servers cap the dictionary at 1M):
document the pre-10.0.0 compatibility constraint on the 2M client cap at
QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE and symbolDictReset(boolean).
Review mi7/mi8: step 6 refuses a rebuilt engine that recovered from disk
(the empties-the-slot contract was breached) and resets slotLockReleased
after a successful swap. Review mi3: recycle javadoc now says seven
steps / steps 2-6, the resetArmed and resetSymbolDictionary docs match
the code, and EngineRebuildFactory moves out of the field block.
Review Mo4: SymbolDictRecycleFsnContinuityTest wraps every test in
assertMemoryLeak, and LineSenderBuilderWebSocketTest pins the config
boundaries: threshold == 2M accepted on both config paths, the
symbol_dict_reset=on parse branch, the invalid-value message, and the
three fluent-setter transport guards.
Co-Authored-By: Claude Fable 5
+ * Switching it off removes the only bound on dictionary growth below
+ * the hard cap ({@link io.questdb.client.cutlass.qwp.protocol.QwpConstants#MAX_SYMBOL_DICTIONARY_SIZE},
+ * 2,000,000). Servers released before QuestDB 10.0.0 cap the
+ * dictionary at 1,000,000 and reject anything beyond it as a terminal
+ * parse error, so with the recycle off against a pre-10.0.0 server
+ * keep symbol cardinality below 1M.
+ *
* Default {@code true} (on). WebSocket transport only.
*/
public LineSenderBuilder symbolDictReset(boolean enabled) {
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index 0d67191c..2f2bf08e 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -159,6 +159,13 @@ public class QwpWebSocketSender implements Sender {
// sf-client.md section 4.4 floor: drop-oldest under bursts needs a wide
// enough window to preserve the trailing category distribution.
private static final int MIN_ERROR_INBOX_CAPACITY = 16;
+ // Upper bound on how long recycleForDictReset step 3 waits for a DEFERRED
+ // engine close (SF worker wedged in a syscall past SegmentManager's
+ // bounded join) to release the slot flock before giving up and latching
+ // the sender terminal. Sized well past any transient disk/NFS stall the
+ // deferred-close machinery exists to survive; a worker still wedged after
+ // this long is treated as a genuinely dead disk.
+ private static final long RECYCLE_DEFERRED_CLOSE_MAX_WAIT_MILLIS = 30_000L;
private static final String WRITE_PATH = "/write/v4";
private final String authorizationHeader;
private final int autoFlushBytes;
@@ -301,15 +308,6 @@ public class QwpWebSocketSender implements Sender {
// while the producer thread reads it from sendRow without
// holding the sender monitor.
private volatile int effectiveAutoFlushBytes;
- /**
- * Rebuilds a fresh {@link CursorSendEngine} on this sender's own slot, going
- * through the identical construct/quarantine code path
- * {@link Sender.LineSenderBuilder#build} uses.
- */
- public interface EngineRebuildFactory {
- CursorSendEngine rebuild();
- }
-
// Installed by build() once connect() succeeds; null for a sender that
// has never connected. See setEngineRebuildFactory.
private EngineRebuildFactory engineRebuildFactory;
@@ -432,8 +430,10 @@ public interface EngineRebuildFactory {
// runs and consumes the request.
private boolean manualResetRequested;
// True once armIfEligible has determined a recycle should happen. Consumed
- // by the (later-task) recycle trigger; set only at the tail of
- // resetTableBuffersAfterFlush, never on the per-symbol registration path.
+ // by the recycle trigger; set only from armIfEligible's two safe call
+ // points (the tail of resetTableBuffersAfterFlush, and
+ // resetSymbolDictionary() when no flush is in flight), never on the
+ // per-symbol registration path.
private boolean resetArmed;
// Cleared on the false -> true armed transition; a later task's
// opportunistic-wait step sets it once it has waited out its window for
@@ -445,6 +445,19 @@ public interface EngineRebuildFactory {
// getSymbolDictResetStarvationTimeouts()), and a monitoring thread is
// its obvious reader.
private volatile long symbolDictResetStarvationTimeouts;
+ // External-scale FSN of the last frame proven durably acked by a recycle's
+ // barrier, recorded at recycleForDictReset step 1 before any teardown.
+ // -1 until the first recycle that had published anything. Lets the
+ // monitoring accessors (getAckedFsn, awaitAckedFsn's null-engine branch)
+ // keep reporting the durable watermark instead of collapsing to -1 while
+ // cursorEngine is transiently null mid-swap or permanently null after a
+ // failed recycle -- all pre-swap data really is acked, so the watermark
+ // stays truthful.
+ private long lastRecycleDurableFsn = -1L;
+ // Budget for recycleForDictReset's deferred-close await (see
+ // RECYCLE_DEFERRED_CLOSE_MAX_WAIT_MILLIS); non-final only so tests can
+ // shrink it to drive the timeout branch.
+ private long recycleDeferredCloseMaxWaitMillis = RECYCLE_DEFERRED_CLOSE_MAX_WAIT_MILLIS;
// Set (once) by recycleForDictReset's catch block when the recycle swap
// itself fails in steps 2-6 -- everything was acked before the swap tore
// the old engine down, so no data is at risk, but this sender can no
@@ -1050,10 +1063,16 @@ public void atNow() {
public boolean awaitAckedFsn(long targetFsn, long timeoutMillis) {
checkNotClosed();
checkRecycleFailure();
- if (cursorEngine == null) {
- return targetFsn < 0L;
+ // Snapshot: the recycle transitions cursorEngine non-null -> null ->
+ // non-null on the producer thread; reading the field once keeps this
+ // method from dereferencing a half-swapped null. While it is null,
+ // anything at or below the watermark the recycle barrier proved
+ // durable is truthfully "acked".
+ CursorSendEngine engine = cursorEngine;
+ if (engine == null) {
+ return targetFsn < 0L || targetFsn <= lastRecycleDurableFsn;
}
- cursorEngine.checkDurability();
+ engine.checkDurability();
// Surface latched errors before any early-return path, so a caller
// polling with timeoutMillis <= 0 to drive their own loop sees the
// throw instead of an indefinite "not yet". The durability latch
@@ -1071,15 +1090,15 @@ public boolean awaitAckedFsn(long targetFsn, long timeoutMillis) {
}
targetFsn = internalTarget;
}
- if (cursorEngine.ackedFsn() >= targetFsn) {
+ if (engine.ackedFsn() >= targetFsn) {
return true;
}
if (timeoutMillis <= 0L) {
return false;
}
long deadlineNanos = System.nanoTime() + timeoutMillis * 1_000_000L;
- while (cursorEngine.ackedFsn() < targetFsn) {
- cursorEngine.checkDurability();
+ while (engine.ackedFsn() < targetFsn) {
+ engine.checkDurability();
if (cursorSendLoop != null) {
cursorSendLoop.checkError();
}
@@ -1899,7 +1918,13 @@ public QwpWebSocketSender geoHashColumn(CharSequence columnName, CharSequence va
*/
@Override
public long getAckedFsn() {
- return cursorEngine != null ? fsnEpochBase + cursorEngine.ackedFsn() : -1L;
+ // Snapshot: the recycle transitions cursorEngine non-null -> null ->
+ // non-null on the producer thread, so read the field once. While it
+ // is null (mid-swap, or for good after a failed swap) report the last
+ // watermark a recycle barrier proved durable instead of collapsing
+ // to -1 -- all pre-swap data really is acked.
+ CursorSendEngine engine = cursorEngine;
+ return engine != null ? fsnEpochBase + engine.ackedFsn() : lastRecycleDurableFsn;
}
/**
@@ -2686,11 +2711,13 @@ public void reset() {
/**
* Advisory request to start a fresh symbol-dictionary epoch. Sets
- * {@link #manualResetRequested}; if no row is currently in progress and no
- * flush is in flight ({@code pendingRowCount == 0}), re-evaluates arming
- * immediately so a caller that requests a reset between batches does not
- * have to wait for a later flush to observe {@code isResetArmed()}. A
- * request made mid-batch is picked up by the next
+ * {@link #manualResetRequested}; if no flush is in flight
+ * ({@code pendingRowCount == 0} -- a first row may still be under
+ * construction; arming is harmless there because the recycle trigger
+ * itself refuses to run mid-row), re-evaluates arming immediately so a
+ * caller that requests a reset between batches does not have to wait for
+ * a later flush to observe {@code isResetArmed()}. A request made
+ * mid-batch is picked up by the next
* {@code resetTableBuffersAfterFlush} instead.
*
* A permanent no-op while {@code symbol_dict_reset} is off: arming gates
@@ -2870,6 +2897,11 @@ public void setErrorInboxCapacity(int capacity) {
this.errorInboxCapacity = capacity;
}
+ @TestOnly
+ public void setRecycleDeferredCloseMaxWaitMillisForTesting(long millis) {
+ this.recycleDeferredCloseMaxWaitMillis = millis;
+ }
+
public void setTransactional(boolean transactional) {
this.transactional = transactional;
}
@@ -3832,7 +3864,11 @@ private synchronized Throwable closeRemainingResources(Throwable terminalError)
slotLockReleased = false;
retainedEngine = engine;
}
- } else {
+ } else if (retainedEngine == null) {
+ // No engine and nothing retained: no flock left to report. A
+ // non-null retainedEngine (a recycle's deferred-close await
+ // timed out) still holds the slot flock, so leave the flag
+ // false and let isSlotLockReleased() re-probe it.
slotLockReleased = true;
}
if (errorDispatcher != null) {
@@ -4654,6 +4690,48 @@ private void armIfEligible() {
resetArmed = shouldArm;
}
+ /**
+ * Recycle step 3's deferred-close await. A fully-drained engine close
+ * normally completes inline ({@code isCloseCompleted()} true on return),
+ * making this a single volatile read. When the SF worker was wedged in a
+ * syscall past {@code SegmentManager}'s bounded join, the close instead
+ * returned with the slot flock retained and its release deferred to the
+ * worker's exit path -- exactly the transient disk stall the
+ * deferred-close machinery exists to survive. Park (the same
+ * {@code awaitAckedFsn}-shaped wait the starvation policy uses) until the
+ * deferred cleanup confirms the release; each pass also re-arms the
+ * shared flock-release retry driver for the close-ran-but-release-failed
+ * case, mirroring {@link #isSlotLockReleased()}'s re-probe.
+ *
+ * Exhausting {@link #recycleDeferredCloseMaxWaitMillis} means the worker
+ * is genuinely dead (not stalled), so throw -- the recycle's catch then
+ * latches the sender terminal. Before throwing, hand the still-locked
+ * engine to {@link #retainedEngine} so a pool re-probe
+ * ({@link #isSlotLockReleased()}) can still recover the slot's capacity
+ * if the worker ever exits.
+ */
+ private void awaitDeferredEngineClose(CursorSendEngine outgoing) {
+ if (outgoing.isCloseCompleted()) {
+ return;
+ }
+ LOG.warn("symbol dictionary recycle waiting for a deferred engine close: the SF worker "
+ + "did not quiesce, so the slot lock is still held [maxWaitMillis={}]",
+ recycleDeferredCloseMaxWaitMillis);
+ long deadlineNanos = System.nanoTime() + recycleDeferredCloseMaxWaitMillis * 1_000_000L;
+ while (!outgoing.isCloseCompleted()) {
+ if (System.nanoTime() >= deadlineNanos) {
+ retainedEngine = outgoing;
+ slotLockReleased = false;
+ throw new LineSenderException("symbol dictionary recycle could not reclaim its "
+ + "slot: the outgoing engine's deferred close did not release the slot "
+ + "lock within " + recycleDeferredCloseMaxWaitMillis
+ + " ms (SF worker wedged)");
+ }
+ outgoing.ensureFlockReleaseRetryScheduled();
+ java.util.concurrent.locks.LockSupport.parkNanos(50_000L);
+ }
+ }
+
/**
* True once every FSN this engine has published has also been
* server-acknowledged (or nothing has been published yet). The barrier
@@ -4765,7 +4843,7 @@ private void maybeRecycleForDictReset() {
* The symbol-dictionary recycle swap. Runs synchronously on the producer
* thread from the {@link #table(CharSequence)} barrier, once
* {@link #maybeRecycleForDictReset()} has proven the ring is drained.
- * Eight steps, strictly ordered:
+ * Seven steps, strictly ordered:
*
+ * Step 3 mirrors {@code close()}'s deferred-close discipline: when the
+ * outgoing engine's close could not confirm SF-worker quiescence it
+ * returns with the slot flock retained and {@code isCloseCompleted()}
+ * false, releasing both from the worker's exit path. Step 3 then awaits
+ * that deferred release (bounded by
+ * {@link #RECYCLE_DEFERRED_CLOSE_MAX_WAIT_MILLIS}) before step 6 rebuilds
+ * on the slot -- rebuilding against the retained flock would throw
+ * {@code SlotLockContentionException} and needlessly latch the sender
+ * terminal for what is usually a transient disk stall. Only exhausting
+ * the await budget (a genuinely dead worker) latches terminal.
*
* Step 7 is deliberately exempt from that latch. By then the swap has
* committed, so a failed connect leaves a fully coherent sender that is
@@ -4821,6 +4911,12 @@ private void recycleForDictReset() {
final long lastPublishedFsn = cursorEngine.publishedFsn(); // step 1
final int dictSizeAtSwap = globalSymbolDictionary.size();
final long startNanos = System.nanoTime();
+ if (lastPublishedFsn >= 0) {
+ // The barrier proved every published frame acked, so this is the
+ // durable watermark the monitoring accessors keep reporting while
+ // cursorEngine is null (mid-swap, or for good after a failed swap).
+ lastRecycleDurableFsn = fsnEpochBase + lastPublishedFsn;
+ }
try {
// step 2: close the loop - joins the I/O thread, closes the client
if (cursorSendLoop != null) {
@@ -4830,9 +4926,14 @@ private void recycleForDictReset() {
client = null;
// step 3: fully-drained close of the engine - empties the slot.
// Holds NO logical slot lock here: close(true) unlinks the logical
- // lock file (CursorSendEngine close javadoc).
- cursorEngine.close();
+ // lock file (CursorSendEngine close javadoc). When the SF worker
+ // is wedged in a syscall the close defers flock release to the
+ // worker's exit path -- await it (bounded) rather than let step 6
+ // throw SlotLockContentionException on the retained flock.
+ CursorSendEngine outgoing = cursorEngine;
+ outgoing.close();
cursorEngine = null;
+ awaitDeferredEngineClose(outgoing);
// step 4: roll the external FSN base (-1 no-publish case adds 0)
rollFsnEpochBase(lastPublishedFsn);
// step 5: producer state swap - replace, don't clear()
@@ -4847,10 +4948,24 @@ private void recycleForDictReset() {
// step 6: rebuild the engine on the now-empty slot
cursorEngine = engineRebuildFactory.rebuild();
ownsCursorEngine = true;
+ if (cursorEngine.wasRecoveredFromDisk()) {
+ // close(true) above emptied the slot, so a rebuild that found
+ // a persisted dictionary to recover from means the outgoing
+ // close's empties-the-slot contract was breached -- the fresh
+ // producer dictionary and the slot's on-disk state have
+ // diverged, so refuse to run on it.
+ throw new LineSenderException(
+ "symbol dictionary recycle rebuilt on a non-empty slot: "
+ + "the outgoing engine's fully-drained close did not empty it");
+ }
deltaDictEnabled = cursorEngine.isDeltaDictEnabled();
cursorEngine.setSlotLockReleaseListener(this::onSlotLockReleased);
+ // The fresh engine holds the slot flock again; step 3's release
+ // flipped slotLockReleased true via the outgoing engine's
+ // listener, which no longer describes this sender's state.
+ slotLockReleased = false;
} catch (Throwable t) {
- // step 8: terminal latch - everything was acked before step 2,
+ // terminal latch - everything was acked before step 2,
// so no data is at risk; the sender refuses further use.
recycleFailure = t;
LOG.error("symbol dictionary recycle failed; sender is now terminal "
@@ -5674,6 +5789,15 @@ public Endpoint(String host, int port) {
}
}
+ /**
+ * Rebuilds a fresh {@link CursorSendEngine} on this sender's own slot, going
+ * through the identical construct/quarantine code path
+ * {@link Sender.LineSenderBuilder#build} uses.
+ */
+ public interface EngineRebuildFactory {
+ CursorSendEngine rebuild();
+ }
+
private final class ReconnectSupplier implements CursorWebSocketSendLoop.ReconnectFactory {
/**
* Optional caller-owned liveness gate. {@code null} means this factory
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
index 5d1b2f5c..ec51437c 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
@@ -92,6 +92,14 @@ public final class QwpConstants {
*
* NOT the result-direction cap: {@code QwpResultBatchDecoder.MAX_CONN_DICT_SIZE}
* (8,388,608) governs server-to-client result batches and is unrelated.
+ *
+ * Compatibility: servers released before QuestDB 10.0.0 cap their
+ * dictionary at 1,000,000, and QWP has no wire-level negotiation of the
+ * limit -- a dictionary this client lets grow past 1M is rejected by
+ * those servers as a terminal parse error. Unreachable on defaults
+ * ({@code symbol_dict_reset} recycles at 100k), but a sender configured
+ * with the recycle off (or a threshold above 1M) against a pre-10.0.0
+ * server must keep its symbol cardinality below the old 1M cap.
*/
public static final int MAX_SYMBOL_DICTIONARY_SIZE = 2_000_000;
/**
diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java
index 7b4e5f80..095f64ab 100644
--- a/core/src/main/java/io/questdb/client/impl/PooledSender.java
+++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java
@@ -335,6 +335,11 @@ public void reset() {
slot.live(generation).reset();
}
+ @Override
+ public void resetSymbolDictionary() {
+ slot.live(generation).resetSymbolDictionary();
+ }
+
@Override
public Sender shortColumn(CharSequence name, short value) {
slot.live(generation).shortColumn(name, value);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
index 0bb792e3..75540eef 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
@@ -334,6 +334,64 @@ public void testSymbolDictResetRejectedForNonWebSocketTransport() {
() -> Sender.builder("http::addr=" + LOCALHOST + ":9000;symbol_dict_reset_max_wait_millis=0;"));
}
+ /**
+ * The three fluent setters carry the same transport guard as the
+ * connect-string keys, but sit on a separate code path -- pin them
+ * directly so a guard dropped from the setters alone cannot ship green.
+ */
+ @Test
+ public void testSymbolDictResetFluentSettersRejectNonWebSocketTransport() {
+ assertThrows("symbol_dict_reset is only supported for WebSocket transport",
+ () -> Sender.builder(Sender.Transport.HTTP).symbolDictReset(true));
+ assertThrows("symbol_dict_reset_threshold is only supported for WebSocket transport",
+ () -> Sender.builder(Sender.Transport.HTTP).symbolDictResetThreshold(500));
+ assertThrows("symbol_dict_reset_max_wait_millis is only supported for WebSocket transport",
+ () -> Sender.builder(Sender.Transport.HTTP).symbolDictResetMaxWaitMillis(0));
+ }
+
+ /**
+ * {@code symbol_dict_reset=on} must survive the parse as {@code true} --
+ * distinct from the default-true path, which passes with the parse branch
+ * deleted. Contrast against {@code off} on an otherwise identical builder.
+ */
+ @Test
+ public void testSymbolDictResetOnParsesTrue() {
+ Assert.assertEquals(Boolean.TRUE,
+ Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset=on;")
+ .wsConfigSnapshotForTest()
+ .get("symbol_dict_reset"));
+ Assert.assertEquals(Boolean.FALSE,
+ Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset=off;")
+ .wsConfigSnapshotForTest()
+ .get("symbol_dict_reset"));
+ }
+
+ @Test
+ public void testSymbolDictResetRejectsInvalidValue() {
+ assertThrows("invalid symbol_dict_reset [value=banana, allowed-values=[on, off]]",
+ () -> Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset=banana;"));
+ }
+
+ /**
+ * The accepted upper edge: exactly {@code MAX_SYMBOL_DICTIONARY_SIZE}
+ * (2M) must pass validation -- a {@code >} -> {@code >=} regression at
+ * the bound would reject it. Both the connect-string and the fluent
+ * setter paths.
+ */
+ @Test
+ public void testSymbolDictResetThresholdAcceptsHardCapBoundary() {
+ Assert.assertEquals(QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE,
+ Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset_threshold="
+ + QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE + ";")
+ .wsConfigSnapshotForTest()
+ .get("symbol_dict_reset_threshold"));
+ Assert.assertEquals(QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE,
+ Sender.builder(Sender.Transport.WEBSOCKET)
+ .symbolDictResetThreshold(QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE)
+ .wsConfigSnapshotForTest()
+ .get("symbol_dict_reset_threshold"));
+ }
+
@Test
public void testConnectionRefused() throws Exception {
assertMemoryLeak(() -> {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleDeferredCloseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleDeferredCloseTest.java
new file mode 100644
index 00000000..4b3d2962
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleDeferredCloseTest.java
@@ -0,0 +1,296 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * The recycle's deferred-close discipline ({@code recycleForDictReset} step 3,
+ * {@code awaitDeferredEngineClose}): when the SF worker is wedged in a syscall
+ * past {@code SegmentManager}'s bounded join, the outgoing engine's close
+ * returns with the slot flock retained and its release deferred to the
+ * worker's exit path. The recycle must await that release before rebuilding on
+ * the slot -- rebuilding against the retained flock throws
+ * {@code SlotLockContentionException} and would latch the sender permanently
+ * terminal for what is usually a transient disk stall. Only exhausting the
+ * await budget (a genuinely dead worker) may latch terminal.
+ *
+ * The wedge: {@code SegmentManager}'s trim-sync hook runs unconditionally once
+ * per service pass, so parking the worker there leaves it un-joinable exactly
+ * the way a stalled disk/NFS syscall does, and a shrunken
+ * {@code workerJoinTimeoutMillis} lets the close's bounded join give up
+ * promptly (the same recipe as {@code SlotLockReleasedContractTest}).
+ */
+public class SymbolDictRecycleDeferredCloseTest {
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ /**
+ * A transient wedge: the worker un-wedges while the recycle is parked in
+ * its deferred-close await. The recycle must ride the stall out and
+ * complete -- fresh epoch, rebuilt engine, sender fully usable -- instead
+ * of latching terminal on the retained flock.
+ */
+ @Test(timeout = 60_000L)
+ public void testRecycleSurvivesDeferredEngineClose() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("recycle-deferred-survive").toString();
+ try (TestWebSocketServer server = ackingServer()) {
+ String cfg = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir + ";";
+ CountDownLatch workerBlocked = new CountDownLatch(1);
+ CountDownLatch releaseWorker = new CountDownLatch(1);
+ AtomicBoolean wedgeFired = new AtomicBoolean();
+ AtomicReference
+ * Call on the producing thread only: like every other {@code Sender}
+ * method, this mutates producer-side state and is not thread-safe.
*/
default void resetSymbolDictionary() {
}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index 2f2bf08e..cb4d8a7b 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -269,8 +269,11 @@ public class QwpWebSocketSender implements Sender {
private String currentTableName;
// Cursor SF engine: the producer (user thread) writes encoded QWP frames
// into the engine's mmap'd ring; the cursorSendLoop is the I/O thread
- // that walks the ring and sends frames.
- private CursorSendEngine cursorEngine;
+ // that walks the ring and sends frames. Volatile since the recycle
+ // started reassigning it (non-null -> null -> non-null on the producer
+ // thread): the monitoring accessors (getAckedFsn, awaitAckedFsn) read it
+ // from a monitor thread, same reasoning as symbolDictEpoch.
+ private volatile CursorSendEngine cursorEngine;
private CursorWebSocketSendLoop cursorSendLoop;
private boolean deferCommit;
// True when the sender emits incremental (delta) symbol dictionaries: each
@@ -327,8 +330,11 @@ public class QwpWebSocketSender implements Sender {
// recycle path) advance it past every FSN already handed out, so the
// external sequence stays strictly monotone across the internal reset.
// Rule everywhere it is applied: external = fsnEpochBase + raw: raw
- // -1 (no-data) sentinels are never translated.
- private long fsnEpochBase = 0;
+ // -1 (no-data) sentinels are never translated. Volatile because the
+ // recycle rolls it while a monitor thread may be inside getAckedFsn /
+ // awaitAckedFsn: a stale base paired with a fresh engine would report
+ // an FSN dip to -1 (same reasoning as symbolDictEpoch).
+ private volatile long fsnEpochBase = 0;
private boolean hasDeferredMessages;
// FSN of the last commit-bearing (non-FLAG_DEFER_COMMIT) frame this session
// published, or -1 when none. Frames above it are deferred and uncommitted:
@@ -452,8 +458,9 @@ public class QwpWebSocketSender implements Sender {
// keep reporting the durable watermark instead of collapsing to -1 while
// cursorEngine is transiently null mid-swap or permanently null after a
// failed recycle -- all pre-swap data really is acked, so the watermark
- // stays truthful.
- private long lastRecycleDurableFsn = -1L;
+ // stays truthful. Volatile: those accessors are exactly the surface a
+ // monitoring thread reads mid-swap, same reasoning as symbolDictEpoch.
+ private volatile long lastRecycleDurableFsn = -1L;
// Budget for recycleForDictReset's deferred-close await (see
// RECYCLE_DEFERRED_CLOSE_MAX_WAIT_MILLIS); non-final only so tests can
// shrink it to drive the timeout branch.
From 6793928a840e950f9b671b19a0ce1d8dddcd8834 Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Thu, 20 Aug 2026 16:14:11 +0100
Subject: [PATCH 26/49] Pin buffer-through-outage contract for the recycle
Red first: the outage tests now assert the store-and-forward contract
(producer never sees a transport error nor a reconnect budget after the
initial connect) instead of pinning the step-7 foreground connect they
used to. The production change lands in the next commit.
---
.../client/SymbolDictRecycleOutageTest.java | 204 +++++++-----------
1 file changed, 80 insertions(+), 124 deletions(-)
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleOutageTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleOutageTest.java
index 2f0036b5..624804f4 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleOutageTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleOutageTest.java
@@ -25,7 +25,6 @@
package io.questdb.client.test.cutlass.qwp.client;
import io.questdb.client.Sender;
-import io.questdb.client.cutlass.line.LineSenderException;
import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner;
import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
@@ -43,7 +42,6 @@
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE;
import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
@@ -57,8 +55,9 @@
*
* (a) proves the recycle's step 2 ({@code cursorSendLoop.close()}) correctly
* joins an I/O thread that is itself mid-reconnect (not idle, not yet given
- * up), and that step 7's fresh {@code ensureConnected()} recovers once the
- * endpoint accepts connections again -- exercising
+ * up), and that step 7 no longer recovers the connection on the calling
+ * thread -- it defers to the I/O loop, so the swap returns promptly and the
+ * producer never observes the outage -- exercising
* {@code CursorWebSocketSendLoop.close()}'s "handles both states" contract
* under a real outage rather than a synthetic one.
*
@@ -78,16 +77,19 @@ public class SymbolDictRecycleOutageTest {
/**
* Kills the server out from under an armed, fully-drained sender, waits
* for the pre-recycle I/O thread to actually enter its own reconnect
- * loop (not just assumed via a fixed sleep), then triggers the recycle
- * from a background thread -- because step 7's fresh
- * {@code ensureConnected()} blocks the calling {@code table()} call (sync
- * initial-connect mode) until the endpoint accepts again or
- * {@code reconnect_max_duration_millis} elapses. The main thread revives
- * a fresh server on the same port shortly after, well inside that
- * budget, mirroring {@code ReconnectTest}'s down-then-up realism.
+ * loop (not just assumed via a fixed sleep) -- so the recycle's step 2
+ * ({@code cursorSendLoop.close()}) provably joins a MID-reconnect
+ * thread -- then triggers the recycle inline, on the calling thread.
+ * {@code reconnect_max_duration_millis} bounds only the sender's initial
+ * connect; under the store-and-forward contract step 7 no longer
+ * re-enters {@code connectWithRetry} on the producer thread, so the
+ * triggering {@code table()} call must return well within that budget
+ * even though the endpoint is still down when it fires. The main thread
+ * revives a fresh server on the same port after asserting the bound,
+ * mirroring {@code ReconnectTest}'s down-then-up realism.
*/
@Test
- public void testRecycleDuringOutageReconnectsAfresh() throws Exception {
+ public void testSyncModeRecycleDoesNotBlockProducerDuringOutage() throws Exception {
assertMemoryLeak(() -> {
String sfDir = temporaryFolder.getRoot().toPath().resolve("outage-recycle").toString();
AckAllHandler firstHandler = new AckAllHandler();
@@ -131,68 +133,31 @@ public void testRecycleDuringOutageReconnectsAfresh() throws Exception {
+ "the triggering table() call",
ws.getTotalReconnectAttempts() > 0);
- // Trigger the recycle off-thread: step 7's fresh
- // ensureConnected() blocks the caller (sync initial-connect
- // mode) until the endpoint accepts again.
- AtomicReference
- * Proves the step-7 failure reaches the caller WITHOUT latching, that the
- * swap committed exactly one epoch, and that the very next send recovers
- * through the existing {@code sendRow() -> ensureConnected()} path once
- * the endpoint is back -- reconnecting only, never re-running a teardown
- * step and never swapping a second time.
+ * Proves the swap commits exactly one epoch and disarms without the
+ * caller ever observing a transport failure, that the flush right after
+ * publishes into the fresh epoch's SF slot, and that once the endpoint
+ * returns on the same port the I/O loop's own reconnect replays every
+ * row sent during the outage with zero loss -- reconnecting only, never
+ * re-running a teardown step and never swapping a second time.
*/
@Test
- public void testDefaultConfigRecycleSurvivesFailedReconnect() throws Exception {
+ public void testDefaultConfigRecycleBuffersThroughOutage() throws Exception {
assertMemoryLeak(() -> {
String sfDir = temporaryFolder.getRoot().toPath().resolve("default-config-outage").toString();
AckAllHandler firstHandler = new AckAllHandler();
@@ -247,61 +212,52 @@ public void testDefaultConfigRecycleSurvivesFailedReconnect() throws Exception {
// that is already down.
server.close();
- LineSenderException triggering = null;
- try {
- sender.table("t");
- Assert.fail("step 7's single-shot connect must throw while the endpoint "
- + "refuses connections");
- } catch (LineSenderException e) {
- triggering = e;
- }
- Assert.assertNotNull(triggering);
-
- Assert.assertEquals("the swap committed exactly one epoch before the connect "
- + "failed", 1, ws.getSymbolDictEpoch());
+ // The ring is drained, so the next table() fires the recycle
+ // into a wire that is already down. The swap must complete AND
+ // return normally -- the reconnect is the I/O loop's job, so
+ // no transport failure may reach the producer. "c" registers
+ // into the fresh dictionary after the swap's
+ // resetSymbolDictStateForNewConnection but before the wire is
+ // up, which keeps pinning the drained-guard: a deferred
+ // connect that cleared the batch watermark would ship a row
+ // pointing at an id the server never received.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertEquals("the swap must commit exactly one epoch",
+ 1, ws.getSymbolDictEpoch());
Assert.assertEquals(1, ws.getSymbolDictResetsPerformed());
- Assert.assertFalse("a committed swap disarms even when its reconnect fails",
- ws.isResetArmed());
+ Assert.assertFalse("a committed swap disarms", ws.isResetArmed());
- // Endpoint back on the SAME port. The sender must not be
- // terminal: the next send reconnects on its own.
+ // Producer keeps working against the dead endpoint: the
+ // flush publishes into the fresh epoch's SF slot.
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-recycle FSN must exceed pre-recycle FSN",
+ fsn2 > fsn1);
+
+ // Endpoint back on the SAME port: the I/O loop's own
+ // reconnect must land the buffered rows -- zero loss.
OutageRecycleHandler revivedHandler = new OutageRecycleHandler();
try (TestWebSocketServer revived =
new TestWebSocketServer(revivedHandler, false, null, port)) {
revived.start();
Assert.assertTrue(revived.awaitStart(5, TimeUnit.SECONDS));
- // "d" registers BEFORE the deferred connect -- symbol()
- // runs ahead of sendRow(), which is what finally
- // performs it -- so this first batch is the one that
- // proves the connect left the batch's symbol watermark
- // alone on its way through.
- sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
- long fsn2 = sender.flushAndGetSequence();
- Assert.assertTrue("a sender whose step-7 connect failed must still ingest "
- + "once the endpoint returns",
- sender.awaitAckedFsn(fsn2, 5_000));
- Assert.assertTrue("post-recycle FSN must exceed pre-recycle FSN",
- fsn2 > fsn1);
- Assert.assertEquals("the recovery reconnects only -- it must not run a "
- + "second swap", 1, ws.getSymbolDictEpoch());
+ Assert.assertTrue("rows sent during the outage must replay once "
+ + "the endpoint returns",
+ sender.awaitAckedFsn(fsn2, 10_000));
+ Assert.assertEquals("the recovery reconnects only -- no second swap",
+ 1, ws.getSymbolDictEpoch());
Assert.assertEquals(1, ws.getSymbolDictResetsPerformed());
-
- Assert.assertEquals("the recovered connection's first frame must carry a "
+ Assert.assertEquals("the fresh connection's first frame must carry a "
+ "fresh (empty) dictionary, not a, b",
0, revivedHandler.firstFrameDeltaStart);
- Assert.assertEquals("the recovered stream must define every symbol its "
- + "rows reference: a deferred connect that cleared the "
- + "batch watermark would ship a row pointing at an id "
- + "the server never received",
- Collections.singletonList("d"), revivedHandler.dict());
+ Assert.assertEquals(Collections.singletonList("c"), revivedHandler.dict());
// And the epoch keeps extending normally from there.
sender.table("t").symbol("s", "e").longColumn("v", 4L).atNow();
long fsn3 = sender.flushAndGetSequence();
Assert.assertTrue(sender.awaitAckedFsn(fsn3, 5_000));
Assert.assertEquals("later batches must extend the same fresh dictionary",
- Arrays.asList("d", "e"), revivedHandler.dict());
+ Arrays.asList("c", "e"), revivedHandler.dict());
}
}
}
From abed2a8d3bf633dfc7f8d7136948ee0bfeb0b9ba Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Thu, 20 Aug 2026 16:22:26 +0100
Subject: [PATCH 27/49] Defer the recycle reconnect to the I/O loop
Step 7 of the symbol-dict recycle re-ran the initial-connect policy on
the producer thread: OFF senders got a single-shot connect whose failure
made every subsequent send throw before buffering until the endpoint
returned, and SYNC senders blocked the producer up to
reconnect_max_duration_millis. Both violate the store-and-forward
contract (post-init, the client never exposes transport problems and
never imposes a reconnect budget on the producer).
ensureConnected() now latches hasConnectedOnce on its first completion
and routes every later entry through the existing ASYNC (deferred)
branch: the loop is built and started synchronously, the socket connect
happens on the I/O thread with indefinite retry, and the producer keeps
buffering into the fresh epoch's slot. This is the same path ASYNC-mode
senders already took at step 7. The server clears its per-connection
dictionary on disconnect, so the buffered fresh-epoch frames
(deltaStart=0) replay correctly on reconnect.
---
.../qwp/client/QwpWebSocketSender.java | 51 +++++++++++++------
1 file changed, 35 insertions(+), 16 deletions(-)
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index cb4d8a7b..dae800e3 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -335,6 +335,14 @@ public class QwpWebSocketSender implements Sender {
// awaitAckedFsn: a stale base paired with a fresh engine would report
// an FSN dip to -1 (same reasoning as symbolDictEpoch).
private volatile long fsnEpochBase = 0;
+ // Latched true the first time ensureConnected() completes. Once set,
+ // every later ensureConnected() -- today only the recycle's step 7 and
+ // its retry-on-next-send path -- takes the deferred (ASYNC-style)
+ // branch regardless of initialConnectMode: the store-and-forward
+ // contract scopes foreground connectivity errors to initialization
+ // only, so post-initial (re)connects belong to the I/O loop's
+ // indefinite retry (Invariant B), never to the producer thread.
+ private boolean hasConnectedOnce;
private boolean hasDeferredMessages;
// FSN of the last commit-bearing (non-FLAG_DEFER_COMMIT) frame this session
// published, or -1 when none. Frames above it are deferred and uncommitted:
@@ -4150,7 +4158,14 @@ private void ensureConnected() {
connectionListener, connectionListenerInboxCapacity);
}
CursorWebSocketSendLoop.ReconnectFactory reconnectFactory = newReconnectFactory();
- switch (initialConnectMode) {
+ // initialConnectMode is an *initialization* policy. After the first
+ // successful connect the SF contract forbids foreground connects on
+ // the producer thread, so re-entries (the recycle's step 7, or its
+ // retry after a failed loop start) always defer to the I/O thread.
+ Sender.InitialConnectMode effectiveMode = hasConnectedOnce
+ ? Sender.InitialConnectMode.ASYNC
+ : initialConnectMode;
+ switch (effectiveMode) {
case SYNC:
client = CursorWebSocketSendLoop.connectWithRetry(
reconnectFactory,
@@ -4168,9 +4183,10 @@ private void ensureConnected() {
// a future version bump must account for that. Transport
// failures retry indefinitely on the I/O thread (Invariant B).
// But a terminal auth, upgrade or capability rejection on this
- // initial connect -- before the wire is ever up -- is surfaced
- // to the async SenderErrorHandler and latched for a close()
- // rethrow, not retried.
+ // deferred connect (the initial one, or a later re-entry such
+ // as the recycle's step 7) -- before the wire is ever up -- is
+ // surfaced to the async SenderErrorHandler and latched for a
+ // close() rethrow, not retried.
client = null;
break;
case OFF:
@@ -4279,6 +4295,7 @@ private void ensureConnected() {
connectionError.set(null);
connected = true;
+ hasConnectedOnce = true;
}
private void ensureNoInProgressRow() {
@@ -4878,7 +4895,8 @@ private void maybeRecycleForDictReset() {
* mirroring (not calling) {@link #setCursorEngine} -- that method's
* guards refuse a second engine.
*
* Step 7 is deliberately exempt from that latch. By then the swap has
- * committed, so a failed connect leaves a fully coherent sender that is
+ * committed, so a failed step-7 setup (dispatcher construction, loop
+ * build/start -- environmental, since the socket connect itself is
+ * deferred to the I/O thread) leaves a fully coherent sender that is
* merely disconnected: {@code connected == false}, loop and client already
* closed and nulled by {@link #ensureConnected()}'s own catch, the fresh
* engine attached, and the step-5 counters ({@link #symbolDictEpoch},
* {@link #symbolDictResetsPerformed}) correctly left incremented because
* the swap really did happen. It rethrows loudly to the triggering caller
* but the sender stays usable, and the ordinary
- * {@code sendRow() -> ensureConnected()} path retries the connect -- and
- * only the connect -- on the next send. Nothing can fire a second swap
+ * {@code sendRow() -> ensureConnected()} path retries the deferred setup
+ * -- and only that -- on the next send. Nothing can fire a second swap
* meanwhile: the fresh dictionary is below threshold,
* {@code manualResetRequested} was consumed at step 5, and
* {@link #maybeRecycleForDictReset()} requires {@code connected}.
@@ -5060,9 +5067,9 @@ private void recycleForDictReset() {
try {
ensureConnected();
} catch (Throwable t) {
- LOG.warn("symbol dictionary swap committed but its reconnect failed; sender stays "
- + "disconnected on the fresh epoch and retries the connect on the "
- + "next send [epoch={}, dictSizeAtSwap={}]",
+ LOG.warn("symbol dictionary swap committed but starting its deferred reconnect "
+ + "failed; sender stays disconnected on the fresh epoch and retries "
+ + "the setup on the next send [epoch={}, dictSizeAtSwap={}]",
symbolDictEpoch, dictSizeAtSwap, t);
if (t instanceof LineSenderException) {
throw (LineSenderException) t;
From d00acf9c0cdda94bf82d3343afc4d8e5e37a8261 Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Mon, 24 Aug 2026 12:32:15 +0100
Subject: [PATCH 31/49] Pin step-7 non-latching with a fault-injection test
Adds a @TestOnly loop-start fault hook and a regression test proving a
failed recycle reconnect leaves the sender usable (review r3, C5). The
test the PR body credited for this was deleted by 6793928a; this one is
red-proofed against re-latching step 7.
---
.../qwp/client/QwpWebSocketSender.java | 14 ++
.../SymbolDictRecycleStep7FaultTest.java | 131 ++++++++++++++++++
2 files changed, 145 insertions(+)
create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index 086ca657..dabe7a35 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -498,6 +498,12 @@ public class QwpWebSocketSender implements Sender {
// this: the swap has already committed there, so the sender is coherent
// and merely disconnected, and the next send retries the connect.
private Throwable recycleFailure;
+ // Test seam: recycle step-7 fault injection. When set, runs (and is
+ // expected to throw) inside ensureConnected()'s loop-construction try,
+ // after cursorSendLoop is assigned but before start() -- exercising the
+ // catch that closes and nulls the fresh loop, i.e. the failed-reconnect
+ // state the C4/C5 regression tests pin.
+ private Runnable loopStartFault;
// Incremented once per completed symbol-dictionary recycle. 0 until the
// first recycle commits. volatile: this is public API (see
// getSymbolDictEpoch()), and a monitoring thread is its obvious reader.
@@ -2932,6 +2938,11 @@ public void setErrorInboxCapacity(int capacity) {
this.errorInboxCapacity = capacity;
}
+ @TestOnly
+ public void setLoopStartFaultForTesting(Runnable fault) {
+ this.loopStartFault = fault;
+ }
+
@TestOnly
public void setRecycleDeferredCloseMaxWaitMillisForTesting(long millis) {
this.recycleDeferredCloseMaxWaitMillis = millis;
@@ -4268,6 +4279,9 @@ private void ensureConnected() {
if (hasLoopEverConnected) {
cursorSendLoop.markEverConnected();
}
+ if (loopStartFault != null) {
+ loopStartFault.run();
+ }
cursorSendLoop.start();
} catch (Throwable t) {
// start() (or dispatcher construction) failed after cursorSendLoop was
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java
new file mode 100644
index 00000000..92db63ac
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java
@@ -0,0 +1,131 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * Fault-injects a failure into recycle step 7 (the deferred reconnect) via
+ * {@link QwpWebSocketSender#setLoopStartFaultForTesting(Runnable)} and pins
+ * that the failure does NOT latch the sender terminal.
+ */
+public class SymbolDictRecycleStep7FaultTest {
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ /**
+ * Pins that a step-7 (reconnect) failure does NOT latch the sender
+ * terminal: the swap has committed, the sender is coherent and merely
+ * disconnected, and the next send retries the deferred setup. Review
+ * round 3, finding C5 (the test the PR body credited was deleted by
+ * commit 6793928a).
+ */
+ @Test
+ public void testStep7FailureDoesNotLatchAndRecovers() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.newFolder("step7-c5").getAbsolutePath();
+ try (TestWebSocketServer server = ackingServer()) {
+ try (Sender sender = Sender.fromConfig(cfg(server, sfDir))) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long f1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(f1, 5_000));
+ Assert.assertTrue(ws.isResetArmed());
+
+ RuntimeException fault = new RuntimeException("injected step-7 fault");
+ ws.setLoopStartFaultForTesting(() -> {
+ throw fault;
+ });
+ try {
+ sender.table("t");
+ Assert.fail("the triggering table() must surface the step-7 failure");
+ } catch (LineSenderException e) {
+ Assert.assertSame(fault, findRootCause(e));
+ }
+ // The swap committed; the sender must NOT be terminal.
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+ ws.setLoopStartFaultForTesting(null);
+ // Next send retries the deferred setup and data flows again.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ long f2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-recovery batch must be acked",
+ sender.awaitAckedFsn(f2, 5_000));
+ }
+ }
+ });
+ }
+
+ private static TestWebSocketServer ackingServer() throws Exception {
+ TestWebSocketServer server = new TestWebSocketServer(new AckAllHandler());
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ return server;
+ }
+
+ private static String cfg(TestWebSocketServer server, String sfDir) {
+ return "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=2"
+ + ";reconnect_initial_backoff_millis=20"
+ + ";reconnect_max_backoff_millis=80;";
+ }
+
+ /** Follows {@code getCause()} to the deepest non-null cause. */
+ private static Throwable findRootCause(Throwable t) {
+ Throwable cause = t;
+ while (cause.getCause() != null) {
+ cause = cause.getCause();
+ }
+ return cause;
+ }
+
+ /** ACKs every frame it receives; does not otherwise inspect the wire. */
+ private static class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ try {
+ client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}
From 895cc22a8db102b53b231bb521a4a36cd21fcf0e Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Mon, 24 Aug 2026 12:44:05 +0100
Subject: [PATCH 32/49] Add regression test for the batch-watermark reset guard
Drives a real failed step-7 reconnect and asserts the next flushed
delta still carries ids registered while the row was in progress
(review r3, C4). Red-proofed: reverting the guard to the unconditional
clear fails the test.
---
.../SymbolDictRecycleStep7FaultTest.java | 94 +++++++++++++++++++
1 file changed, 94 insertions(+)
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java
index 92db63ac..66921574 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java
@@ -34,6 +34,9 @@
import org.junit.rules.TemporaryFolder;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@@ -92,6 +95,64 @@ public void testStep7FailureDoesNotLatchAndRecovers() throws Exception {
});
}
+ /**
+ * Pins the conditional in resetSymbolDictStateForNewConnection(): ids a
+ * row registered before the deferred reconnect completes must still ship
+ * in the next delta. Review round 3, finding C4 (empirically untested:
+ * suite was green with the guard reverted).
+ */
+ @Test
+ public void testPostFailedReconnectDeltaCoversStagedSymbolIds() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.newFolder("step7-c4").getAbsolutePath();
+ CapturingAckHandler handler = new CapturingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ try (Sender sender = Sender.fromConfig(cfg(server, sfDir))) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long f1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(f1, 5_000));
+
+ ws.setLoopStartFaultForTesting(() -> {
+ throw new RuntimeException("injected step-7 fault");
+ });
+ try {
+ sender.table("t");
+ Assert.fail("expected the step-7 failure to surface");
+ } catch (LineSenderException ignore) {
+ }
+ // Everything before this point belongs to the old epoch's
+ // dictionary; the swap already committed (steps 1-6), only
+ // the reconnect (step 7) failed, so no new frame reached
+ // the wire during the failed table() call above.
+ int firstPostRecycle = handler.framesSnapshot().size();
+ ws.setLoopStartFaultForTesting(null);
+
+ // This row registers "c" in the FRESH dictionary during
+ // symbol(); its sendRow() then completes the deferred
+ // reconnect, which runs resetSymbolDictStateForNewConnection
+ // with the row in progress -- the C4 window.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ long f2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(f2, 5_000));
+
+ // Replay every captured frame's delta sections; a delta that
+ // omits "c" while rows reference it throws DictionaryGapException.
+ List
* Step 3 mirrors {@code close()}'s deferred-close discipline: when the
* outgoing engine's close could not confirm SF-worker quiescence it
* returns with the slot flock retained and {@code isCloseCompleted()}
* false, releasing both from the worker's exit path. Step 3 then awaits
* that deferred release (bounded by
- * {@link #RECYCLE_DEFERRED_CLOSE_MAX_WAIT_MILLIS}) before step 6 rebuilds
+ * {@link #RECYCLE_DEFERRED_CLOSE_MAX_WAIT_MILLIS}) before step 5 rebuilds
* on the slot -- rebuilding against the retained flock would throw
* {@code SlotLockContentionException} and needlessly latch the sender
* terminal for what is usually a transient disk stall. Only exhausting
* the await budget (a genuinely dead worker) latches terminal.
*
* Step 7 is deliberately exempt from that latch. By then the swap has
- * committed, so a failed step-7 setup (dispatcher construction, loop
- * build/start -- environmental, since the socket connect itself is
+ * committed (step 6), so a failed step-7 setup (dispatcher construction,
+ * loop build/start -- environmental, since the socket connect itself is
* deferred to the I/O thread) leaves a fully coherent sender that is
* merely disconnected: {@code connected == false}, loop and client already
* closed and nulled by {@link #ensureConnected()}'s own catch, the fresh
- * engine attached, and the step-5 counters ({@link #symbolDictEpoch},
+ * engine attached, and the step-6 counters ({@link #symbolDictEpoch},
* {@link #symbolDictResetsPerformed}) correctly left incremented because
* the swap really did happen. It rethrows loudly to the triggering caller
* but the sender stays usable, and the ordinary
* {@code sendRow() -> ensureConnected()} path retries the deferred setup
* -- and only that -- on the next send. Nothing can fire a second swap
* meanwhile: the fresh dictionary is below threshold,
- * {@code manualResetRequested} was consumed at step 5, and
+ * {@code manualResetRequested} was consumed at step 6, and
* {@link #maybeRecycleForDictReset()} requires {@code connected}.
*/
private void recycleForDictReset() {
@@ -5012,49 +5030,32 @@ private void recycleForDictReset() {
final int dictSizeAtSwap = globalSymbolDictionary.size();
final long startNanos = System.nanoTime();
if (lastPublishedFsn >= 0) {
- // The barrier proved every published frame acked, so this is the
- // durable watermark the monitoring accessors keep reporting while
- // cursorEngine is null (mid-swap, or for good after a failed swap).
lastRecycleDurableFsn = fsnEpochBase + lastPublishedFsn;
}
try {
// step 2: close the loop - joins the I/O thread, closes the client.
- // Capture the outgoing loop's own ever-connected state into the
- // sender-lifetime sticky OR BEFORE closing it: this is the only
- // place an ASYNC-initial sender's connect (which happens only on
- // the I/O thread, never observed by ensureConnected's client !=
- // null branch) reaches hasLoopEverConnected.
if (cursorSendLoop != null) {
- hasLoopEverConnected |= cursorSendLoop.hasEverConnected();
cursorSendLoop.close();
+ // Read the sticky AFTER close(): close joins the I/O thread,
+ // so a connect that landed mid-window is final here. This is
+ // the only place an ASYNC-initial sender's connect (observed
+ // only by the I/O thread) reaches hasLoopEverConnected.
+ hasLoopEverConnected |= cursorSendLoop.hasEverConnected();
cursorSendLoop = null;
}
client = null;
- // step 3: fully-drained close of the engine - empties the slot.
- // Holds NO logical slot lock here: close(true) unlinks the logical
- // lock file (CursorSendEngine close javadoc). When the SF worker
- // is wedged in a syscall the close defers flock release to the
- // worker's exit path -- await it (bounded) rather than let step 6
- // throw SlotLockContentionException on the retained flock.
+ // step 3: fully-drained close of the engine - empties the slot
+ // and unlinks the parent-anchored logical slot lock. When the SF
+ // worker is wedged the close defers flock release to the worker's
+ // exit path; await it (bounded) before rebuilding on the slot.
CursorSendEngine outgoing = cursorEngine;
- outgoing.close();
cursorEngine = null;
+ connected = false;
+ outgoing.close();
awaitDeferredEngineClose(outgoing);
// step 4: roll the external FSN base (-1 no-publish case adds 0)
rollFsnEpochBase(lastPublishedFsn);
- // step 5: producer state swap - replace, don't clear()
- globalSymbolDictionary = new GlobalSymbolDictionary();
- sentMaxSymbolId = -1;
- currentBatchMaxSymbolId = -1;
- lastCommitBoundaryFsn = -1L;
- symbolDictEpoch++;
- symbolDictResetsPerformed++;
- // C1 anti-thrash floor: see resetFloorSymbols.
- resetFloorSymbols = Math.min(dictSizeAtSwap * 2,
- QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE / 2);
- resetArmed = false;
- manualResetRequested = false;
- // step 6: rebuild the engine on the now-empty slot
+ // step 5: rebuild the engine on the now-empty slot.
cursorEngine = engineRebuildFactory.rebuild();
ownsCursorEngine = true;
if (cursorEngine.wasRecoveredFromDisk()) {
@@ -5067,12 +5068,34 @@ private void recycleForDictReset() {
"symbol dictionary recycle rebuilt on a non-empty slot: "
+ "the outgoing engine's fully-drained close did not empty it");
}
+ // step 6: producer-side swap COMMIT - replace, don't clear().
+ globalSymbolDictionary = new GlobalSymbolDictionary();
+ sentMaxSymbolId = -1;
+ currentBatchMaxSymbolId = -1;
+ lastCommitBoundaryFsn = -1L;
+ symbolDictEpoch++;
+ symbolDictResetsPerformed++;
+ resetArmed = false;
+ manualResetRequested = false;
+ // C1 anti-thrash floor: see resetFloorSymbols.
+ resetFloorSymbols = Math.min(dictSizeAtSwap * 2,
+ QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE / 2);
+ // Deliberately re-derived (not carried over): the healing half
+ // of the recycle contract -- a sender that degraded to full
+ // frames heals back into delta mode once the underlying fault
+ // clears; a persistent fault just degrades the fresh engine
+ // again on its first append (SymbolDictRecycleHealingTest).
deltaDictEnabled = cursorEngine.isDeltaDictEnabled();
cursorEngine.setSlotLockReleaseListener(this::onSlotLockReleased);
// The fresh engine holds the slot flock again; step 3's release
// flipped slotLockReleased true via the outgoing engine's
// listener, which no longer describes this sender's state.
slotLockReleased = false;
+ } catch (Error e) {
+ // OOM/SOE/linkage must reach the caller untouched (same
+ // convention as rethrowTerminal) and must not masquerade as a
+ // recycle verdict.
+ throw e;
} catch (Throwable t) {
// terminal latch - everything was acked before step 2,
// so no data is at risk; the sender refuses further use.
@@ -5084,24 +5107,14 @@ private void recycleForDictReset() {
}
throw new LineSenderException(t).put("symbol dictionary recycle failed");
}
- // step 7: reconnect - rebuilds the loop with the rolled base, deferring
- // the socket connect to the I/O thread (hasInitialConnectRun forces
- // ensureConnected's ASYNC branch, and hasLoopEverConnected -- if this
- // sender ever reached the server -- seeds the fresh loop's own
- // hasEverConnected so Invariant B's classification survives the
- // rebuild): the loop retries indefinitely with
- // backoff while the producer keeps buffering into the fresh slot, so a
- // transient outage in this window never surfaces to the producer and
- // never imposes a reconnect budget on it (the store-and-forward
- // contract). OUTSIDE the latching try on purpose: the swap has already
- // committed, so the residual failures here (dispatcher construction,
- // loop build/start -- environmental, not transport) leave a coherent
- // fresh-epoch sender that is merely disconnected. They throw loudly
- // but do NOT latch; the next sendRow() retries the deferred setup --
- // and ONLY that -- through ensureConnected.
- connected = false;
+ // step 7: reconnect -- unchanged from head (outside the latch; the
+ // loop retries indefinitely on the I/O thread; a failed setup here
+ // leaves a coherent, merely-disconnected sender and is retried by
+ // the next sendRow()'s ensureConnected()).
try {
ensureConnected();
+ } catch (Error e) {
+ throw e;
} catch (Throwable t) {
LOG.warn("symbol dictionary swap committed but starting its deferred reconnect "
+ "failed; sender stays disconnected on the fresh epoch and retries "
@@ -5176,8 +5189,10 @@ private void advanceSentMaxSymbolId() {
}
/**
- * Stops emitting delta dictionaries for the rest of this sender's life, after the
- * per-slot {@code .symbol-dict} has proved unwritable.
+ * Stops emitting delta dictionaries for the rest of this epoch, after the
+ * per-slot {@code .symbol-dict} has proved unwritable -- a symbol-dictionary
+ * recycle re-derives {@code deltaDictEnabled} from the fresh engine (the
+ * healing contract; see {@link #recycleForDictReset()}).
*
* The side-file can stop accepting appends mid-run -- a full disk or an exhausted
* quota, where SF's own segments stay writable because they are pre-allocated mmap
@@ -5200,8 +5215,9 @@ private void disableDeltaDict(Throwable cause) {
}
deltaDictEnabled = false;
LOG.warn("symbol dictionary persistence failed; this sender has switched to full "
- + "self-sufficient frames for the rest of its life (bandwidth cost only -- "
- + "no data is at risk, and recovery replays such frames without a side file)",
+ + "self-sufficient frames for the rest of this epoch (bandwidth cost only -- "
+ + "no data is at risk, and recovery replays such frames without a side file; "
+ + "a symbol dictionary recycle re-derives delta mode from the fresh engine)",
cause);
}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java
index 43a0b493..779c2443 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java
@@ -99,7 +99,12 @@ public void testMetricsAfterTwoRecycles() throws Exception {
sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
long fsn2 = sender.flushAndGetSequence();
Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
- Assert.assertTrue("armed again: c, d cross threshold=2 in the new epoch",
+ // The C1 anti-thrash floor (resetFloorSymbols = 2x the first swap's
+ // dictSizeAtSwap = 4) keeps c,d (2 symbols, == threshold but < floor)
+ // from re-arming on their own; a manual request bypasses the floor by
+ // design, so drive the second recycle through resetSymbolDictionary().
+ sender.resetSymbolDictionary();
+ Assert.assertTrue("manual reset request bypasses the C1 floor",
ws.isResetArmed());
// Ring drained again -> second recycle: epoch 2.
From b0a01269bf5a39570d7de93c3084cc45c180c5c1 Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Mon, 24 Aug 2026 13:33:38 +0100
Subject: [PATCH 36/49] Fix stale step-number docs left by the recycle reorder
Task 5's reorder made the engine rebuild step 5 and the producer-side
swap commit step 6 (after a successful rebuild), but several comments
and javadocs still described the old order where the swap committed
before the rebuild. Fixes getSymbolDictEpoch()/getSymbolDictResetsPerformed()
javadocs (a rebuild failure now leaves the counters un-bumped, not
incremented), a field comment, maybeRecycleForDictReset()'s NPE note,
and SymbolDictRecycleHealingTest's class javadoc. Also restores two
load-bearing notes the reorder's shorter step comments had dropped
(lastRecycleDurableFsn is written before teardown for the monitoring
accessors; hasInitialConnectRun forces the step-7 reconnect's ASYNC
branch) and fixes a missing comma.
---
.../qwp/client/QwpWebSocketSender.java | 38 +++++++++++--------
.../client/SymbolDictRecycleHealingTest.java | 2 +-
2 files changed, 23 insertions(+), 17 deletions(-)
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index 61fed5bc..2150e204 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -518,7 +518,7 @@ public class QwpWebSocketSender implements Sender {
// getSymbolDictEpoch()), and a monitoring thread is its obvious reader.
private volatile long symbolDictEpoch;
// Incremented once per completed symbol-dictionary recycle swap, beside
- // symbolDictEpoch (recycleForDictReset step 5). The two move together
+ // symbolDictEpoch (recycleForDictReset step 6). The two move together
// today -- the only way symbolDictEpoch advances is through a committed
// recycle swap -- but they count different things (dictionary generation
// vs. completed swaps) and are incremented independently in case a
@@ -2234,11 +2234,12 @@ public long getFsnEpochBaseForTest() {
/**
* Number of symbol-dictionary recycles this sender has completed. Advances
- * by one at step 5 of {@link #recycleForDictReset()}, the instant the swap
- * commits to the new epoch -- before the engine rebuild (step 6) or the
- * reconnect (step 7), so a later step-6 rebuild failure that latches
- * {@link #recycleFailure}, and a step-7 reconnect failure that does not,
- * both still leave this incremented. Unlike the per-send-loop
+ * by one at step 6 of {@link #recycleForDictReset()}, the instant the swap
+ * commits to the new epoch -- after the engine rebuild (step 5) has
+ * already succeeded, so a step-5 rebuild failure that latches
+ * {@link #recycleFailure} leaves this counter un-bumped, while a later
+ * step-7 reconnect failure (which cannot latch: the swap already
+ * committed by then) still leaves this incremented. Unlike the per-send-loop
* {@code getTotal*} counters, it is scoped to the sender's whole lifetime
* and never resets. volatile: a
* concurrent read sees the latest write the producer thread completed,
@@ -2254,10 +2255,11 @@ public long getSymbolDictEpoch() {
/**
* Number of symbol-dictionary recycle swaps this sender has completed.
- * Incremented alongside {@link #getSymbolDictEpoch()} at step 5 of
- * {@link #recycleForDictReset()} -- before the engine rebuild (step 6) or
- * the reconnect (step 7), so, like the epoch counter, a later
- * rebuild/reconnect failure still leaves this incremented, latched or not.
+ * Incremented alongside {@link #getSymbolDictEpoch()} at step 6 of
+ * {@link #recycleForDictReset()} -- after the engine rebuild (step 5) has
+ * already succeeded, so, like the epoch counter, a step-5 rebuild failure
+ * leaves this un-bumped while a step-7 reconnect failure still leaves it
+ * incremented (the swap has already committed by then).
* Also like the epoch counter, it is scoped to the sender's whole lifetime
* and never resets. The two counts move together today -- the
* only way the epoch advances is through a completed recycle swap -- but
@@ -4907,7 +4909,7 @@ private void maybeBlockForStarvedReset() {
* {@code QwpWebSocketSender.connect(...)} overload leaves it null --
* only {@code Sender.build()} installs one -- and the recycle feature is
* default-on, so a connect()-built sender must simply stay unarmed rather
- * than NPE at step 6 and latch terminal), or a cursor engine this sender
+ * than NPE at step 5 and latch terminal), or a cursor engine this sender
* does not own ({@code setCursorEngine(engine, false)}'s contract: the
* caller retains ownership, so closing it out from under them at step 3
* would be a use-after-free from the caller's point of view).
@@ -4987,7 +4989,7 @@ private void maybeRecycleForDictReset() {
*
* The producer-visible swap (dictionary, counters, epoch) commits only
* once a fresh engine stands on the emptied slot: a throw in steps 2-6
- * is caught before that point leaves the counters un-bumped, latches
+ * is caught before that point, leaves the counters un-bumped, latches
* {@link #recycleFailure}, and rethrows -- every frame that existed
* before this call was already proven acked, so no data is at risk, but
* the sender that made the throw observe a half-swapped engine/loop
@@ -5030,6 +5032,8 @@ private void recycleForDictReset() {
final int dictSizeAtSwap = globalSymbolDictionary.size();
final long startNanos = System.nanoTime();
if (lastPublishedFsn >= 0) {
+ // Written before teardown: the monitoring accessors keep
+ // reporting this durable watermark while cursorEngine is null.
lastRecycleDurableFsn = fsnEpochBase + lastPublishedFsn;
}
try {
@@ -5107,10 +5111,12 @@ private void recycleForDictReset() {
}
throw new LineSenderException(t).put("symbol dictionary recycle failed");
}
- // step 7: reconnect -- unchanged from head (outside the latch; the
- // loop retries indefinitely on the I/O thread; a failed setup here
- // leaves a coherent, merely-disconnected sender and is retried by
- // the next sendRow()'s ensureConnected()).
+ // step 7: reconnect (outside the latch; the swap has already
+ // committed). hasInitialConnectRun forces ensureConnected's ASYNC
+ // branch here, so the deferred socket connect never parks the
+ // producer thread; the loop retries indefinitely on the I/O thread,
+ // and a failed setup here leaves a coherent, merely-disconnected
+ // sender that is retried by the next sendRow()'s ensureConnected().
try {
ensureConnected();
} catch (Error e) {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java
index 779c2443..0c264ec2 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java
@@ -54,7 +54,7 @@
* full self-sufficient frames ({@code QwpWebSocketSender.disableDeltaDict}, e.g. after a
* recognised mmap access fault on the persisted dictionary -- see {@link MmapFaultDegradesTest})
* is not stuck there forever. {@code recycleForDictReset()} rebuilds the cursor engine from
- * scratch (step 6), and a fresh engine re-derives {@code deltaDictEnabled} independently of
+ * scratch (step 5), and a fresh engine re-derives {@code deltaDictEnabled} independently of
* whatever the outgoing epoch's engine decided -- so once the underlying fault clears, the
* next recycle heals the sender back into delta mode. If the fault has not cleared, the fresh
* engine simply degrades again on its own first append: a normal, catchable
From dc990429523c316dfd0720279d1d5380c407b349 Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Mon, 24 Aug 2026 14:06:16 +0100
Subject: [PATCH 37/49] Make the recycle resumable instead of latching terminal
Review r3, C2: transient failures in the recycle (a wedged SF worker,
an interrupted producer thread, a momentary EMFILE at rebuild, a
post-cleanup fsync warning) latched a healthy sender terminal although
zero data was at risk. The recycle now records a two-state resume point
(CLOSE_LOOP / REBUILD) and the next send finishes the swap; the commit
refuses while a row is in progress so old-dictionary symbol ids can
never cross the swap. The terminal latch survives for exactly one case:
a rebuilt engine that recovered UNACKED frames, which proves the
outgoing close's fully-drained contract was breached. Benign recovered
leftovers (a transiently failed segment unlink) heal by closing the
recovered engine (which retries the unlink by design) and rebuilding.
---
.../qwp/client/QwpWebSocketSender.java | 436 +++++++++++++-----
.../SymbolDictRecycleDeferredCloseTest.java | 87 ++++
.../qwp/client/SymbolDictRecycleTest.java | 121 +++--
3 files changed, 475 insertions(+), 169 deletions(-)
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index 2150e204..a4d90542 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -497,16 +497,29 @@ public class QwpWebSocketSender implements Sender {
// RECYCLE_DEFERRED_CLOSE_MAX_WAIT_MILLIS); non-final only so tests can
// shrink it to drive the timeout branch.
private long recycleDeferredCloseMaxWaitMillis = RECYCLE_DEFERRED_CLOSE_MAX_WAIT_MILLIS;
- // Set (once) by recycleForDictReset's catch block when the recycle swap
- // itself fails in steps 2-6 -- everything was acked before the swap tore
- // the old engine down, so no data is at risk, but this sender can no
- // longer make progress (no cursor engine, no I/O loop) and refuses further
- // use. checkRecycleFailure() rethrows a fresh LineSenderException wrapping
- // this cause on every later table()/flush-family call; close() still
- // works normally. A step-7 (reconnect) failure deliberately does NOT set
- // this: the swap has already committed there, so the sender is coherent
- // and merely disconnected, and the next send retries the connect.
+ // Set (once) by completeRecycleRebuild when a rebuilt engine recovered
+ // UNACKED frames from the slot the outgoing close was supposed to have
+ // emptied -- the one failure that proves the fully-drained close contract
+ // was breached, so the producer's fresh dictionary and the slot's on-disk
+ // state have provably diverged and this sender refuses further use.
+ // checkRecycleFailure() rethrows a fresh LineSenderException wrapping this
+ // cause on every later table()/flush-family call; close() still works
+ // normally. Every other recycle failure is transient and resumable (see
+ // recycleResume), never latched here.
private Throwable recycleFailure;
+ // Resumable recycle (review r3, C2): a transient failure mid-recycle no
+ // longer latches the sender terminal. CLOSE_LOOP = step 2 failed, the
+ // old loop is still dying and the old engine/dictionary are intact.
+ // REBUILD = the old engine is closed (possibly still releasing its slot
+ // flock); await/rebuild/commit are pending. resumeRecycleIfPending()
+ // advances the state from the table() barrier and ensureConnected().
+ private RecycleResume recycleResume = RecycleResume.NONE;
+ // The closed-but-not-yet-released outgoing engine a REBUILD resume still
+ // awaits; null once its deferred close completes.
+ private CursorSendEngine recyclePendingOutgoing;
+ // Raw last-published FSN of the outgoing epoch (step-1 snapshot),
+ // consumed by the commit when a REBUILD resume completes.
+ private long recyclePendingLastPublishedFsn = -1L;
// Test seam: recycle step-7 fault injection. When set, runs (and is
// expected to throw) inside ensureConnected()'s loop-construction try,
// after cursorSendLoop is assigned but before start() -- exercising the
@@ -547,7 +560,10 @@ public class QwpWebSocketSender implements Sender {
// Lifetime-monotonic in delta mode -- it is NOT reset on reconnect, because
// the I/O thread re-registers the full dictionary via a catch-up frame before
// replaying, so the producer's delta baseline stays valid across the wire
- // boundary. Used only when deltaDictEnabled; ignored in full-dict mode.
+ // boundary. It drops back to -1 only where the LOOP ITSELF is replaced and
+ // its catch-up mirror dies with it: the recycle's swap commit, and the
+ // CLOSE_LOOP resume (see resumeRecycleIfPending). Used only when
+ // deltaDictEnabled; ignored in full-dict mode.
private int sentMaxSymbolId = -1;
// When true, auto-flush sends messages with FLAG_DEFER_COMMIT and only
// explicit flush() triggers the server-side commit. Enables accumulating
@@ -2081,6 +2097,16 @@ public int getEffectiveAutoFlushBytes() {
return effectiveAutoFlushBytes;
}
+ /**
+ * The installed engine-rebuild factory, so a test can wrap the real one
+ * (e.g. fault-inject the first rebuild and delegate afterwards) instead of
+ * replacing it outright. {@code null} for a {@code connect()}-built sender.
+ */
+ @TestOnly
+ public EngineRebuildFactory getEngineRebuildFactoryForTesting() {
+ return engineRebuildFactory;
+ }
+
/**
* Snapshot of the typed payload for the latched terminal server-rejection error,
* or {@code null} if the I/O loop has not latched a server-rejection terminal
@@ -3148,7 +3174,9 @@ public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) {
public QwpWebSocketSender table(CharSequence tableName) {
checkNotClosed();
checkRecycleFailure();
- if (resetArmed) {
+ if (recycleResume != RecycleResume.NONE) {
+ resumeRecycleIfPending();
+ } else if (resetArmed) {
maybeRecycleForDictReset();
}
// Fast path: if table name matches current, skip hashmap lookup
@@ -3854,13 +3882,18 @@ private void checkNotClosed() {
}
/**
- * Terminal latch for a symbol-dictionary recycle swap that failed in steps
- * 2-6 ({@link #recycleForDictReset()}). Everything was acked before the
- * swap tore the old engine down, so no data is at risk -- but the swap
- * itself left this sender without a cursor engine or I/O loop, so it
- * refuses further use. A step-7 reconnect failure is NOT latched (the swap
- * has committed by then and the sender is coherent, just disconnected --
- * see {@link #recycleForDictReset()}). Checked by
+ * Terminal latch for the one symbol-dictionary recycle failure that is not
+ * resumable: a rebuilt engine that recovered UNACKED frames from the slot
+ * the outgoing engine's fully-drained close was supposed to have emptied
+ * (see {@link #completeRecycleRebuild}). That proves the everything-acked
+ * barrier the swap rests on was breached, so the producer's fresh
+ * dictionary and the slot's on-disk state have diverged and this sender
+ * refuses further use. Every OTHER recycle failure -- a wedged SF worker,
+ * an interrupted producer thread, a momentary rebuild fault, a
+ * post-cleanup fsync warning, a failed step-7 reconnect -- is transient:
+ * it throws to the triggering caller, leaves the counters un-bumped and
+ * the recycle pending ({@link #recycleResume}), and the next send
+ * finishes the swap. Checked by
* {@link #table(CharSequence)}, the flush-family
* entry points ({@link #flush()}, {@link #flushAndGetSequence()},
* {@link #drain(long)}, {@link #awaitAckedFsn(long, long)}), and
@@ -4193,6 +4226,7 @@ private void ensureActiveBufferReady() {
private void ensureConnected() {
checkNotClosed();
+ resumeRecycleIfPending();
if (connected) {
return;
}
@@ -4802,10 +4836,10 @@ private void armIfEligible() {
* shared flock-release retry driver for the close-ran-but-release-failed
* case, mirroring {@link #isSlotLockReleased()}'s re-probe.
*
- * Exhausting {@link #recycleDeferredCloseMaxWaitMillis} means the worker
- * is genuinely dead (not stalled), so throw -- the recycle's catch then
- * latches the sender terminal. Before throwing, hand the still-locked
- * engine to {@link #retainedEngine} so a pool re-probe
+ * Exhausting {@link #recycleDeferredCloseMaxWaitMillis} throws; the
+ * recycle stays pending ({@link RecycleResume#REBUILD}) and the next send
+ * retries the await. Before throwing, hand the still-locked engine to
+ * {@link #retainedEngine} so a pool re-probe
* ({@link #isSlotLockReleased()}) can still recover the slot's capacity
* if the worker ever exits.
*/
@@ -4821,16 +4855,133 @@ private void awaitDeferredEngineClose(CursorSendEngine outgoing) {
if (System.nanoTime() >= deadlineNanos) {
retainedEngine = outgoing;
slotLockReleased = false;
- throw new LineSenderException("symbol dictionary recycle could not reclaim its "
- + "slot: the outgoing engine's deferred close did not release the slot "
- + "lock within " + recycleDeferredCloseMaxWaitMillis
- + " ms (SF worker wedged)");
+ throw new LineSenderException("symbol dictionary recycle could not yet reclaim "
+ + "its slot: the outgoing engine's deferred close did not release the "
+ + "slot lock within " + recycleDeferredCloseMaxWaitMillis
+ + " ms (SF worker stalled); the recycle stays pending and is retried "
+ + "on the next send");
}
outgoing.ensureFlockReleaseRetryScheduled();
java.util.concurrent.locks.LockSupport.parkNanos(50_000L);
}
}
+ private static void closeQuietly(CursorSendEngine engine) {
+ try {
+ engine.close();
+ } catch (Throwable ignored) {
+ // best-effort; the retained files are the next rebuild's problem
+ }
+ }
+
+ /**
+ * The recycle's tail: await the outgoing engine's (possibly deferred)
+ * close, rebuild a fresh engine on the emptied slot, and only then
+ * commit the swap -- roll the FSN base, install the fresh dictionary,
+ * advance the counters, wire the engine, reconnect. Every phase before
+ * the commit is idempotent, so both {@link #recycleForDictReset()} and a
+ * REBUILD resume run this; a transient throw leaves
+ * {@code recycleResume == REBUILD} for the next attempt. Only a rebuild
+ * that recovered UNACKED frames -- a genuine breach of the barrier's
+ * everything-acked proof -- latches {@link #recycleFailure}.
+ */
+ private void completeRecycleRebuild(int dictSizeAtSwap, long startNanos) {
+ CursorSendEngine outgoing = recyclePendingOutgoing;
+ if (outgoing != null) {
+ awaitDeferredEngineClose(outgoing); // throws transient while wedged
+ recyclePendingOutgoing = null;
+ retainedEngine = null;
+ }
+ // step 5: rebuild the engine on the now-empty slot.
+ CursorSendEngine rebuilt = rebuildEngineOrAbandon(
+ "symbol dictionary recycle could not rebuild its engine; retried on the next send");
+ if (rebuilt.wasRecoveredFromDisk()) {
+ // The outgoing close's empties-the-slot contract can miss
+ // benignly: a transiently failed segment unlink retains the ack
+ // watermark, and the SF design is that the NEXT engine on the
+ // slot recovers those segments as fully acked and retries the
+ // unlink on its own close. Heal by doing exactly that. Only a
+ // recovery holding UNACKED frames is a genuine breach: latch.
+ if (rebuilt.publishedFsn() > rebuilt.ackedFsn()) {
+ LineSenderException breach = new LineSenderException(
+ "symbol dictionary recycle rebuilt on a slot holding unacknowledged "
+ + "frames: the outgoing engine's fully-drained close contract "
+ + "was breached");
+ recycleFailure = breach;
+ recycleResume = RecycleResume.NONE;
+ closeQuietly(rebuilt);
+ LOG.error("symbol dictionary recycle failed; sender is now terminal "
+ + "[epoch={}, dictSizeAtSwap={}]", symbolDictEpoch, dictSizeAtSwap, breach);
+ throw breach;
+ }
+ closeQuietly(rebuilt); // fully drained: retries the segment unlink
+ rebuilt = rebuildEngineOrAbandon(
+ "symbol dictionary recycle could not rebuild its engine after healing "
+ + "leftover acked segments; retried on the next send");
+ if (rebuilt.wasRecoveredFromDisk()) {
+ closeQuietly(rebuilt);
+ throw new LineSenderException(
+ "symbol dictionary recycle keeps recovering leftover acked segments "
+ + "(slot cleanup not durable yet); retried on the next send");
+ }
+ }
+ // COMMIT (steps 4 + 6): pure producer-side state, and nothing below
+ // can throw. Step 4 rolls the external FSN base past every FSN the
+ // outgoing epoch handed out (the -1 no-publish case adds 0); it must
+ // run with cursorSendLoop == null, which step 2 guarantees and the
+ // step-7 reconnect below only undoes afterwards.
+ rollFsnEpochBase(recyclePendingLastPublishedFsn);
+ // Replace the dictionary, don't clear().
+ globalSymbolDictionary = new GlobalSymbolDictionary();
+ sentMaxSymbolId = -1;
+ currentBatchMaxSymbolId = -1;
+ lastCommitBoundaryFsn = -1L;
+ symbolDictEpoch++;
+ symbolDictResetsPerformed++;
+ resetArmed = false;
+ manualResetRequested = false;
+ // C1 anti-thrash floor: see resetFloorSymbols.
+ resetFloorSymbols = Math.min(dictSizeAtSwap * 2,
+ QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE / 2);
+ // Deliberately re-derived (not carried over): the healing half
+ // of the recycle contract -- a sender that degraded to full
+ // frames heals back into delta mode once the underlying fault
+ // clears; a persistent fault just degrades the fresh engine
+ // again on its first append (SymbolDictRecycleHealingTest).
+ deltaDictEnabled = rebuilt.isDeltaDictEnabled();
+ cursorEngine = rebuilt;
+ ownsCursorEngine = true;
+ cursorEngine.setSlotLockReleaseListener(this::onSlotLockReleased);
+ // The fresh engine holds the slot flock again; step 3's release
+ // flipped slotLockReleased true via the outgoing engine's
+ // listener, which no longer describes this sender's state.
+ slotLockReleased = false;
+ recycleResume = RecycleResume.NONE;
+ recyclePendingLastPublishedFsn = -1L;
+ // step 7: reconnect (the swap has already committed).
+ // hasInitialConnectRun forces ensureConnected's ASYNC branch here, so
+ // the deferred socket connect never parks the producer thread; the
+ // loop retries indefinitely on the I/O thread, and a failed setup
+ // here leaves a coherent, merely-disconnected sender that is retried
+ // by the next sendRow()'s ensureConnected().
+ try {
+ ensureConnected();
+ } catch (Error e) {
+ throw e;
+ } catch (Throwable t) {
+ LOG.warn("symbol dictionary swap committed but starting its deferred reconnect "
+ + "failed; sender stays disconnected on the fresh epoch and retries "
+ + "the setup on the next send [epoch={}, dictSizeAtSwap={}]",
+ symbolDictEpoch, dictSizeAtSwap, t);
+ if (t instanceof LineSenderException) {
+ throw (LineSenderException) t;
+ }
+ throw new LineSenderException(t).put("symbol dictionary recycle reconnect failed");
+ }
+ LOG.info("symbol dictionary recycled [epoch={}, dictSizeAtSwap={}, pauseMicros={}]",
+ symbolDictEpoch, dictSizeAtSwap, (System.nanoTime() - startNanos) / 1000L);
+ }
+
/**
* True once every FSN this engine has published has also been
* server-acknowledged (or nothing has been published yet). The barrier
@@ -4938,6 +5089,16 @@ private void maybeRecycleForDictReset() {
}
}
+ private CursorSendEngine rebuildEngineOrAbandon(String message) {
+ try {
+ return engineRebuildFactory.rebuild();
+ } catch (Error e) {
+ throw e;
+ } catch (Throwable t) {
+ throw new LineSenderException(t).put(message);
+ }
+ }
+
/**
* The symbol-dictionary recycle swap. Runs synchronously on the producer
* thread from the {@link #table(CharSequence)} barrier, once
@@ -4984,36 +5145,33 @@ private void maybeRecycleForDictReset() {
* engine.
*
* Step 3 mirrors {@code close()}'s deferred-close discipline: when the
* outgoing engine's close could not confirm SF-worker quiescence it
* returns with the slot flock retained and {@code isCloseCompleted()}
- * false, releasing both from the worker's exit path. Step 3 then awaits
+ * false, releasing both from the worker's exit path. The tail then awaits
* that deferred release (bounded by
* {@link #RECYCLE_DEFERRED_CLOSE_MAX_WAIT_MILLIS}) before step 5 rebuilds
* on the slot -- rebuilding against the retained flock would throw
- * {@code SlotLockContentionException} and needlessly latch the sender
- * terminal for what is usually a transient disk stall. Only exhausting
- * the await budget (a genuinely dead worker) latches terminal.
+ * {@code SlotLockContentionException} for what is usually a transient disk
+ * stall.
*
- * Step 7 is deliberately exempt from that latch. By then the swap has
- * committed (step 6), so a failed step-7 setup (dispatcher construction,
- * loop build/start -- environmental, since the socket connect itself is
+ * Step 7's failure mode is unchanged: by then the swap has committed
+ * (step 6), so a failed step-7 setup (dispatcher construction, loop
+ * build/start -- environmental, since the socket connect itself is
* deferred to the I/O thread) leaves a fully coherent sender that is
* merely disconnected: {@code connected == false}, loop and client already
* closed and nulled by {@link #ensureConnected()}'s own catch, the fresh
@@ -5036,8 +5194,8 @@ private void recycleForDictReset() {
// reporting this durable watermark while cursorEngine is null.
lastRecycleDurableFsn = fsnEpochBase + lastPublishedFsn;
}
+ // step 2: close the loop - joins the I/O thread, closes the client.
try {
- // step 2: close the loop - joins the I/O thread, closes the client.
if (cursorSendLoop != null) {
cursorSendLoop.close();
// Read the sticky AFTER close(): close joins the I/O thread,
@@ -5048,91 +5206,112 @@ private void recycleForDictReset() {
cursorSendLoop = null;
}
client = null;
- // step 3: fully-drained close of the engine - empties the slot
- // and unlinks the parent-anchored logical slot lock. When the SF
- // worker is wedged the close defers flock release to the worker's
- // exit path; await it (bounded) before rebuilding on the slot.
- CursorSendEngine outgoing = cursorEngine;
- cursorEngine = null;
- connected = false;
- outgoing.close();
- awaitDeferredEngineClose(outgoing);
- // step 4: roll the external FSN base (-1 no-publish case adds 0)
- rollFsnEpochBase(lastPublishedFsn);
- // step 5: rebuild the engine on the now-empty slot.
- cursorEngine = engineRebuildFactory.rebuild();
- ownsCursorEngine = true;
- if (cursorEngine.wasRecoveredFromDisk()) {
- // close(true) above emptied the slot, so a rebuild that found
- // a persisted dictionary to recover from means the outgoing
- // close's empties-the-slot contract was breached -- the fresh
- // producer dictionary and the slot's on-disk state have
- // diverged, so refuse to run on it.
- throw new LineSenderException(
- "symbol dictionary recycle rebuilt on a non-empty slot: "
- + "the outgoing engine's fully-drained close did not empty it");
- }
- // step 6: producer-side swap COMMIT - replace, don't clear().
- globalSymbolDictionary = new GlobalSymbolDictionary();
- sentMaxSymbolId = -1;
- currentBatchMaxSymbolId = -1;
- lastCommitBoundaryFsn = -1L;
- symbolDictEpoch++;
- symbolDictResetsPerformed++;
- resetArmed = false;
- manualResetRequested = false;
- // C1 anti-thrash floor: see resetFloorSymbols.
- resetFloorSymbols = Math.min(dictSizeAtSwap * 2,
- QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE / 2);
- // Deliberately re-derived (not carried over): the healing half
- // of the recycle contract -- a sender that degraded to full
- // frames heals back into delta mode once the underlying fault
- // clears; a persistent fault just degrades the fresh engine
- // again on its first append (SymbolDictRecycleHealingTest).
- deltaDictEnabled = cursorEngine.isDeltaDictEnabled();
- cursorEngine.setSlotLockReleaseListener(this::onSlotLockReleased);
- // The fresh engine holds the slot flock again; step 3's release
- // flipped slotLockReleased true via the outgoing engine's
- // listener, which no longer describes this sender's state.
- slotLockReleased = false;
} catch (Error e) {
- // OOM/SOE/linkage must reach the caller untouched (same
- // convention as rethrowTerminal) and must not masquerade as a
- // recycle verdict.
throw e;
} catch (Throwable t) {
- // terminal latch - everything was acked before step 2,
- // so no data is at risk; the sender refuses further use.
- recycleFailure = t;
- LOG.error("symbol dictionary recycle failed; sender is now terminal "
- + "[epoch={}, dictSizeAtSwap={}]", symbolDictEpoch, dictSizeAtSwap, t);
- if (t instanceof LineSenderException) {
- throw (LineSenderException) t;
- }
- throw new LineSenderException(t).put("symbol dictionary recycle failed");
- }
- // step 7: reconnect (outside the latch; the swap has already
- // committed). hasInitialConnectRun forces ensureConnected's ASYNC
- // branch here, so the deferred socket connect never parks the
- // producer thread; the loop retries indefinitely on the I/O thread,
- // and a failed setup here leaves a coherent, merely-disconnected
- // sender that is retried by the next sendRow()'s ensureConnected().
+ // close() set the loop's stop flag before throwing, so the loop
+ // is irreversibly dying but its I/O thread may still own the
+ // engine. Neither proceed with the swap nor claim connectivity;
+ // abandon, and let resumeRecycleIfPending() finish the close
+ // (a repeated close() converges once the I/O thread exits).
+ connected = false;
+ recycleResume = RecycleResume.CLOSE_LOOP;
+ LOG.warn("symbol dictionary recycle abandoned: closing the outgoing I/O loop "
+ + "failed; the close is finished on the next send [epoch={}]",
+ symbolDictEpoch, t);
+ rethrowRecycleAbandoned(t, "symbol dictionary recycle abandoned while closing "
+ + "the outgoing I/O loop; retried on the next send");
+ }
+ // step 3: fully-drained close of the engine - empties the slot and
+ // unlinks the parent-anchored logical slot lock. From here the old
+ // engine cannot come back, so record the REBUILD resume point BEFORE
+ // anything that can throw.
+ CursorSendEngine outgoing = cursorEngine;
+ cursorEngine = null;
+ connected = false;
+ recycleResume = RecycleResume.REBUILD;
+ recyclePendingOutgoing = outgoing;
+ recyclePendingLastPublishedFsn = lastPublishedFsn;
try {
- ensureConnected();
+ outgoing.close();
} catch (Error e) {
throw e;
} catch (Throwable t) {
- LOG.warn("symbol dictionary swap committed but starting its deferred reconnect "
- + "failed; sender stays disconnected on the fresh epoch and retries "
- + "the setup on the next send [epoch={}, dictSizeAtSwap={}]",
- symbolDictEpoch, dictSizeAtSwap, t);
- if (t instanceof LineSenderException) {
- throw (LineSenderException) t;
+ // A throw with the terminal cleanup nevertheless completed (the
+ // post-cleanup fsyncDir durability warning, C2(b)) is not a swap
+ // failure -- finishClose's finally released the slot regardless.
+ // awaitDeferredEngineClose() below tells the two apart: it
+ // returns immediately when isCloseCompleted(), parks while the
+ // deferred close is in flight, and throws only on a genuinely
+ // dead worker.
+ LOG.warn("outgoing engine close reported a failure during the symbol dictionary "
+ + "recycle; deferring to the close-completion probe", t);
+ }
+ completeRecycleRebuild(dictSizeAtSwap, startNanos);
+ }
+
+ /**
+ * Advances an abandoned recycle. CLOSE_LOOP finishes killing the old
+ * loop (no swap -- the old engine and dictionary are intact and the
+ * armed recycle re-fires from a later barrier, once the reconnect the
+ * next send drives has restored {@code connected}); it drops the
+ * producer's delta baseline with the dead loop's catch-up mirror, see
+ * below. REBUILD completes the await/rebuild/commit tail; because the
+ * commit swaps the dictionary, it refuses while producer state could
+ * carry old-dictionary symbol ids (staged rows or a row in progress) --
+ * the caller's row fails, rolls back, and the next table() resumes
+ * cleanly.
+ */
+ private void resumeRecycleIfPending() {
+ if (recycleResume == RecycleResume.NONE) {
+ return;
+ }
+ if (recycleResume == RecycleResume.CLOSE_LOOP) {
+ try {
+ cursorSendLoop.close(); // re-signals; converges once the I/O thread exits
+ } catch (Error e) {
+ throw e;
+ } catch (Throwable t) {
+ rethrowRecycleAbandoned(t, "the outgoing I/O loop is still stopping; "
+ + "retried on the next send");
}
- throw new LineSenderException(t).put("symbol dictionary recycle reconnect failed");
+ hasLoopEverConnected |= cursorSendLoop.hasEverConnected();
+ cursorSendLoop = null;
+ client = null;
+ // The dead loop took its catch-up mirror with it, and the fresh
+ // loop ensureConnected() builds next seeds that mirror only from a
+ // RECOVERED persisted dictionary -- this engine is live, so the
+ // fresh loop starts at sentDictCount == 0. sentMaxSymbolId is the
+ // producer's model of the same number (it normally survives a
+ // reconnect precisely because the SAME loop re-registers from its
+ // mirror), so it has to drop with the mirror; otherwise the next
+ // frame's delta starts above the new loop's coverage and trips its
+ // torn-dictionary guard. Nothing is invalidated by the drop: the
+ // barrier proved the ring drained before step 2, and every publish
+ // path runs ensureConnected() -- hence this resume -- first, so no
+ // frame referencing those ids can be waiting to replay.
+ sentMaxSymbolId = -1;
+ recycleResume = RecycleResume.NONE;
+ return;
}
- LOG.info("symbol dictionary recycled [epoch={}, dictSizeAtSwap={}, pauseMicros={}]",
- symbolDictEpoch, dictSizeAtSwap, (System.nanoTime() - startNanos) / 1000L);
+ // REBUILD
+ if (pendingRowCount != 0
+ || (currentTableBuffer != null && currentTableBuffer.hasInProgressRow())) {
+ throw new LineSenderException(
+ "a symbol dictionary recycle is completing; finish or cancel the "
+ + "in-progress row and retry");
+ }
+ completeRecycleRebuild(globalSymbolDictionary.size(), System.nanoTime());
+ }
+
+ private void rethrowRecycleAbandoned(Throwable t, String message) {
+ if (t instanceof Error) {
+ throw (Error) t;
+ }
+ if (t instanceof LineSenderException) {
+ throw (LineSenderException) t;
+ }
+ throw new LineSenderException(t).put(message);
}
/**
@@ -5933,6 +6112,15 @@ public interface EngineRebuildFactory {
CursorSendEngine rebuild();
}
+ /**
+ * How far an abandoned symbol-dictionary recycle got, and therefore what
+ * {@link #resumeRecycleIfPending()} must still do. See
+ * {@link #recycleResume}.
+ */
+ private enum RecycleResume {
+ NONE, CLOSE_LOOP, REBUILD
+ }
+
private final class ReconnectSupplier implements CursorWebSocketSendLoop.ReconnectFactory {
/**
* Optional caller-owned liveness gate. {@code null} means this factory
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleDeferredCloseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleDeferredCloseTest.java
index 4b3d2962..34736ba5 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleDeferredCloseTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleDeferredCloseTest.java
@@ -273,6 +273,93 @@ public void testRecycleLatchesTerminalWhenDeferredCloseNeverReleases() throws Ex
});
}
+ /**
+ * The await budget runs out while the worker is still wedged, but the
+ * wedge is transient after all. Exhausting the budget must NOT latch the
+ * sender terminal (review r3, C2): the recycle stays pending in its
+ * REBUILD resume state, and once the worker exits and the deferred close
+ * releases the flock, the next send finishes the await, rebuilds and
+ * commits the swap.
+ */
+ @Test(timeout = 60_000L)
+ public void testExhaustedDeferredCloseAwaitResumesOnNextSend() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("recycle-deferred-resume").toString();
+ try (TestWebSocketServer server = ackingServer()) {
+ String cfg = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir + ";";
+ CountDownLatch workerBlocked = new CountDownLatch(1);
+ CountDownLatch releaseWorker = new CountDownLatch(1);
+ AtomicBoolean wedgeFired = new AtomicBoolean();
+ AtomicReference
* The wedge: {@code SegmentManager}'s trim-sync hook runs unconditionally once
* per service pass, so parking the worker there leaves it un-joinable exactly
@@ -117,13 +120,22 @@ public void testRecycleSurvivesDeferredEngineClose() throws Exception {
sender.resetSymbolDictionary();
Assert.assertTrue(ws.isResetArmed());
+ // Positive witness that the await really parked (M15).
+ // A sleep could not tell "the await is parked" from
+ // "the close completed inline and the test skipped the
+ // code under test"; this fires from inside the await's
+ // own deferred-close branch.
+ CountDownLatch parked = new CountDownLatch(1);
+ ws.setDeferredCloseParkWitnessForTesting(parked::countDown);
+
// Un-wedges the worker while the recycle is parked in its
// deferred-close await. The pre-release assert can never
// race: the flock release needs the worker's exit, which
// needs this very countDown.
releaser = new Thread(() -> {
try {
- Thread.sleep(1_500L);
+ Assert.assertTrue("the await must actually park",
+ parked.await(10, TimeUnit.SECONDS));
Assert.assertFalse(
"deferred close cannot complete while the worker is wedged",
outgoing.isCloseCompleted());
@@ -175,15 +187,17 @@ public void testRecycleSurvivesDeferredEngineClose() throws Exception {
}
/**
- * A permanent wedge: the await budget (shrunk via the test seam) runs out
- * with the flock still held -- a genuinely dead worker. The recycle must
- * latch terminal BEFORE committing any of the swap (epoch stays 0), and
- * the still-locked engine must stay reachable through
- * {@code isSlotLockReleased()}'s re-probe so the slot's capacity is
- * recoverable if the worker ever exits.
+ * A wedge that outlives the await budget (shrunk via the test seam): every
+ * send while the worker is stuck runs the await afresh and throws again,
+ * and none of them may commit any part of the swap (epoch stays 0). The
+ * repeated throw is NOT a latch, which the tail proves: once the worker
+ * exits, the still-locked engine's late release becomes visible through
+ * {@code isSlotLockReleased()}'s retained-engine re-probe -- so the slot's
+ * capacity is recoverable -- and the very next send resumes the pending
+ * recycle and completes it. A latched sender could do neither.
*/
@Test(timeout = 60_000L)
- public void testRecycleLatchesTerminalWhenDeferredCloseNeverReleases() throws Exception {
+ public void testExhaustedDeferredCloseAwaitKeepsThrowingWhileWedged() throws Exception {
assertMemoryLeak(() -> {
String sfDir = temporaryFolder.getRoot().toPath().resolve("recycle-deferred-timeout").toString();
try (TestWebSocketServer server = ackingServer()) {
@@ -228,8 +242,8 @@ public void testRecycleLatchesTerminalWhenDeferredCloseNeverReleases() throws Ex
try {
sender.table("t").symbol("s", "b").longColumn("v", 2L).atNow();
- Assert.fail("expected the recycle to latch terminal once the "
- + "deferred-close await budget ran out");
+ Assert.fail("expected the exhausted deferred-close await to throw "
+ + "while the worker stays wedged");
} catch (LineSenderException e) {
TestUtils.assertContains(e.getMessage(),
"deferred close did not release the slot lock");
@@ -237,13 +251,16 @@ public void testRecycleLatchesTerminalWhenDeferredCloseNeverReleases() throws Ex
// The await runs at step 3, before the step-5 swap: no
// epoch may have committed, and every later entry point
- // must rethrow the latched failure.
+ // re-runs the await and throws the same transient
+ // verdict for as long as the worker stays wedged.
Assert.assertEquals("the swap must not have committed",
0, ws.getSymbolDictEpoch());
try {
sender.table("t");
- Assert.fail("expected a latched terminal sender to rethrow");
- } catch (LineSenderException expected) {
+ Assert.fail("expected the retried await to throw again");
+ } catch (LineSenderException e) {
+ TestUtils.assertContains(e.getMessage(),
+ "deferred close did not release the slot lock");
}
Assert.assertFalse("the slot flock is still held by the wedged engine",
ws.isSlotLockReleased());
@@ -261,6 +278,17 @@ public void testRecycleLatchesTerminalWhenDeferredCloseNeverReleases() throws Ex
outgoing.isCloseCompleted());
Assert.assertTrue("sender must expose the late flock release",
ws.isSlotLockReleased());
+
+ // The proof that none of the throws above latched: the
+ // next send resumes the pending recycle and commits it.
+ sender.table("t").symbol("s", "c").longColumn("v", 3L).atNow();
+ Assert.assertEquals("the pending recycle must complete once the wedge "
+ + "clears", 1, ws.getSymbolDictEpoch());
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-resume batch must still get acked",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertTrue("post-recycle FSN must exceed pre-recycle FSN "
+ + "[fsn1=" + fsn1 + ", fsn2=" + fsn2 + ']', fsn2 > fsn1);
} finally {
manager.setBeforeTrimSyncHook(null);
releaseWorker.countDown();
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleSlotHealTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleSlotHealTest.java
new file mode 100644
index 00000000..c459340b
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleSlotHealTest.java
@@ -0,0 +1,354 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager;
+import io.questdb.client.std.Files;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * The two verdicts {@code QwpWebSocketSender.completeRecycleRebuild} reaches
+ * when the recycle's step-5 rebuild comes back
+ * {@code wasRecoveredFromDisk()} -- i.e. when the outgoing engine's
+ * fully-drained close did NOT leave the slot empty.
+ *
+ * The acked-leftover recipe injects the unlink failure the way
+ * {@code CursorSendEngineCloseUnlinkFailureTest} does: it drops write
+ * permission on the slot directory (POSIX unlink needs a writable parent), so
+ * that test skips on Windows and wherever permissions are not enforced (root).
+ */
+public class SymbolDictRecycleSlotHealTest {
+
+ /**
+ * Big enough for the real QWP frames the sender appends to the rebuilt
+ * engine after the heal, and identical in the prep helpers so recovery
+ * reads the doctored segments back at the size they were written with.
+ */
+ private static final long SEGMENT_BYTES = 1024L * 1024L;
+ private static final int PAYLOAD_BYTES = 32;
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ /**
+ * Review r3, C2(a): a benign fully-drained close verdict (segment unlink
+ * transiently failed; watermark retained by design) must not brick the
+ * recycle. The rebuild recovers fully-acked leftovers; the sender heals by
+ * closing the recovered engine (which retries the unlink) and rebuilding
+ * once more.
+ */
+ @Test(timeout = 60_000L)
+ public void testRecoveredFullyAckedLeftoversHealAndRecycleCompletes() throws Exception {
+ assertMemoryLeak(() -> {
+ // Phase 1: doctor a slot -- fully-acked frames whose close-time
+ // unlink failed (CursorSendEngineCloseUnlinkFailureTest's recipe).
+ String doctoredSlot = temporaryFolder.getRoot().toPath()
+ .resolve("doctored-slot").toString();
+ prepareFullyAckedLeftoverSlot(doctoredSlot);
+
+ // Phase 2: a live sender whose rebuild factory lands on that slot.
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("heal-sf").toString();
+ try (TestWebSocketServer server = ackingServer()) {
+ try (Sender sender = Sender.fromConfig(config(server, sfDir))) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: batch must be acked before the recycle",
+ sender.awaitAckedFsn(fsn1, 5_000));
+
+ AtomicInteger rebuilds = new AtomicInteger();
+ ws.setEngineRebuildFactory(() -> {
+ rebuilds.incrementAndGet();
+ return new CursorSendEngine(doctoredSlot, SEGMENT_BYTES);
+ });
+
+ sender.resetSymbolDictionary();
+ Assert.assertTrue(ws.isResetArmed());
+ // Recycle: rebuild #1 recovers the acked leftovers -> heal
+ // -> rebuild #2 stands on a genuinely empty slot.
+ sender.table("t").symbol("s", "b").longColumn("v", 2L).atNow();
+
+ Assert.assertEquals("heal must close the recovered engine and rebuild again",
+ 2, rebuilds.get());
+ Assert.assertEquals("the recycle must have committed",
+ 1, ws.getSymbolDictEpoch());
+ Assert.assertFalse("recycle must disarm", ws.isResetArmed());
+ // The heal's close retried the unlink the outgoing close
+ // could not do, so the engine the swap committed on stands
+ // on a genuinely emptied slot -- not on the leftovers.
+ Assert.assertFalse("the recycle must commit on a non-recovered engine",
+ ws.getCursorEngineForTesting().wasRecoveredFromDisk());
+
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-heal batch must still get acked",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertTrue("post-recycle FSN must exceed pre-recycle FSN "
+ + "[fsn1=" + fsn1 + ", fsn2=" + fsn2 + ']', fsn2 > fsn1);
+ }
+ }
+ });
+ }
+
+ /**
+ * Review r3: the terminal latch's one surviving case. A rebuild that
+ * recovers UNACKED frames proves the fully-drained-close contract was
+ * breached -- the fresh producer dictionary and the slot's state have
+ * genuinely diverged, so the sender must refuse further use ({@code close()}
+ * still works).
+ */
+ @Test(timeout = 60_000L)
+ public void testRecoveredUnackedFramesLatchTerminal() throws Exception {
+ assertMemoryLeak(() -> {
+ String doctoredSlot = temporaryFolder.getRoot().toPath()
+ .resolve("breach-slot").toString();
+ prepareUnackedLeftoverSlot(doctoredSlot);
+
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("breach-sf").toString();
+ try (TestWebSocketServer server = ackingServer()) {
+ try (Sender sender = Sender.fromConfig(config(server, sfDir))) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: batch must be acked before the recycle",
+ sender.awaitAckedFsn(fsn1, 5_000));
+
+ ws.setEngineRebuildFactory(() -> new CursorSendEngine(doctoredSlot, SEGMENT_BYTES));
+ sender.resetSymbolDictionary();
+ try {
+ sender.table("t").symbol("s", "b").longColumn("v", 2L).atNow();
+ Assert.fail("a breached slot must latch terminal");
+ } catch (LineSenderException expected) {
+ TestUtils.assertContains(expected.getMessage(), "unacknowledged");
+ }
+ Assert.assertEquals("the swap must not have committed",
+ 0, ws.getSymbolDictEpoch());
+ try {
+ sender.flush();
+ Assert.fail("the latch must gate every later call");
+ } catch (LineSenderException expected) {
+ TestUtils.assertContains(expected.getMessage(),
+ "sender is terminal: symbol dictionary recycle failed");
+ }
+ try {
+ sender.table("t");
+ Assert.fail("the latch must gate every later call");
+ } catch (LineSenderException expected) {
+ TestUtils.assertContains(expected.getMessage(),
+ "sender is terminal: symbol dictionary recycle failed");
+ }
+ sender.close(); // must still work
+ }
+ }
+ });
+ }
+
+ private static TestWebSocketServer ackingServer() throws Exception {
+ TestWebSocketServer server = new TestWebSocketServer(new AckAllHandler());
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ return server;
+ }
+
+ private static String config(TestWebSocketServer server, String sfDir) {
+ return "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir + ";";
+ }
+
+ private static void fill(long address, int len, byte value) {
+ for (int i = 0; i < len; i++) {
+ Unsafe.getUnsafe().putByte(address + i, value);
+ }
+ }
+
+ /**
+ * Leaves {@code slot} holding one segment whose only frame the server
+ * already acknowledged, plus the ack watermark that covers it -- the exact
+ * residue a fully-drained close produces when its segment unlink fails
+ * transiently. Follows {@code CursorSendEngineCloseUnlinkFailureTest}: a
+ * shared {@link SegmentManager} that is deliberately never started (no
+ * worker thread can persist the watermark or trim behind the test's back,
+ * and the close-path quiescence barrier is trivially satisfied), and the
+ * unlink failure injected by dropping write permission on the slot dir.
+ * Restores the permissions before returning, so the successor engine the
+ * sender's rebuild factory constructs can heal the slot.
+ */
+ private static void prepareFullyAckedLeftoverSlot(String slot) throws Exception {
+ Path slotPath = Paths.get(slot);
+ long payload = Unsafe.malloc(PAYLOAD_BYTES, MemoryTag.NATIVE_DEFAULT);
+ SegmentManager manager = new SegmentManager(SEGMENT_BYTES, TimeUnit.SECONDS.toNanos(60));
+ CursorSendEngine pred = null;
+ boolean slotDirReadOnly = false;
+ try {
+ fill(payload, PAYLOAD_BYTES, (byte) 0x33);
+ pred = new CursorSendEngine(slot, SEGMENT_BYTES, manager);
+ Assert.assertEquals(0L, pred.appendBlocking(payload, PAYLOAD_BYTES));
+ Assert.assertEquals(0L, pred.publishedFsn());
+ // The server durably acknowledged FSN 0 in this session.
+ Assert.assertTrue(pred.acknowledge(0L));
+ Assert.assertEquals(0L, pred.ackedFsn());
+
+ // Inject the close-time unlink failure, and prove the injection
+ // works with a probe file -- root (and some filesystems) ignore
+ // directory permissions.
+ String probePath = slot + "/probe";
+ Assert.assertTrue(java.nio.file.Files.exists(
+ java.nio.file.Files.createFile(Paths.get(probePath))));
+ try {
+ setPermissions(slotPath, "r-xr-xr-x");
+ } catch (UnsupportedOperationException e) {
+ Assume.assumeNoException("POSIX permissions unavailable on this platform", e);
+ }
+ slotDirReadOnly = true;
+ boolean probeRemoved = Files.remove(probePath);
+ if (probeRemoved) {
+ setPermissions(slotPath, "rwxr-xr-x");
+ slotDirReadOnly = false;
+ }
+ Assume.assumeFalse("directory permissions not enforced (running as root?)",
+ probeRemoved);
+
+ // Fully-drained close: tries to unlink the acknowledged segment
+ // file and fails, so the watermark stays behind to cover it.
+ pred.close();
+ Assert.assertTrue("flock release needs no dir write; close must complete",
+ pred.isCloseCompleted());
+ pred = null;
+
+ // The transient failure clears before the sender's rebuild arrives.
+ setPermissions(slotPath, "rwxr-xr-x");
+ slotDirReadOnly = false;
+ Files.remove(probePath);
+
+ Assert.assertTrue("prep: the injected unlink failure must leave the acked segment",
+ Files.exists(slot + "/sf-initial.sfa"));
+ Assert.assertTrue("prep: the watermark must be retained to cover the residue",
+ Files.exists(slot + "/" + AckWatermark.FILE_NAME));
+ } finally {
+ if (slotDirReadOnly) {
+ try {
+ setPermissions(slotPath, "rwxr-xr-x");
+ } catch (Throwable ignored) {
+ }
+ }
+ if (pred != null) {
+ pred.close();
+ }
+ manager.close();
+ Unsafe.free(payload, PAYLOAD_BYTES, MemoryTag.NATIVE_DEFAULT);
+ }
+ }
+
+ /**
+ * Leaves {@code slot} holding one published-but-never-acknowledged frame.
+ * A close that is not fully drained retains the segment files by design
+ * (they still have to reach the server), so the successor recovers them
+ * with {@code publishedFsn > ackedFsn} -- the breach signature. Same
+ * never-started shared {@link SegmentManager} as the acked variant.
+ */
+ private static void prepareUnackedLeftoverSlot(String slot) {
+ long payload = Unsafe.malloc(PAYLOAD_BYTES, MemoryTag.NATIVE_DEFAULT);
+ SegmentManager manager = new SegmentManager(SEGMENT_BYTES, TimeUnit.SECONDS.toNanos(60));
+ CursorSendEngine pred = null;
+ try {
+ fill(payload, PAYLOAD_BYTES, (byte) 0x44);
+ pred = new CursorSendEngine(slot, SEGMENT_BYTES, manager);
+ Assert.assertEquals(0L, pred.appendBlocking(payload, PAYLOAD_BYTES));
+ Assert.assertEquals(0L, pred.publishedFsn());
+ Assert.assertTrue("prep: the frame must stay unacknowledged",
+ pred.ackedFsn() < pred.publishedFsn());
+ pred.close();
+ Assert.assertTrue(pred.isCloseCompleted());
+ pred = null;
+ Assert.assertTrue("prep: an unacknowledged frame must survive the close",
+ Files.exists(slot + "/sf-initial.sfa"));
+ } finally {
+ if (pred != null) {
+ pred.close();
+ }
+ manager.close();
+ Unsafe.free(payload, PAYLOAD_BYTES, MemoryTag.NATIVE_DEFAULT);
+ }
+ }
+
+ private static void setPermissions(Path path, String posix) throws Exception {
+ Set
+ * A sender that cannot rebuild -- no {@link #engineRebuildFactory} (every
+ * public {@code QwpWebSocketSender.connect(...)} overload leaves it null
+ * -- only {@code Sender.build()} installs one), or a cursor engine this
+ * sender does not own ({@code setCursorEngine(engine, false)}'s contract:
+ * the caller retains ownership, so closing it out from under them would
+ * be a use-after-free from the caller's point of view) -- must never arm.
+ * Since the recycle feature is default-on and {@code
+ * resetSymbolDictionary()} is a public advisory API, arming a sender with
+ * no way to ever act on the request would leave {@code isResetArmed()}
+ * reading true forever alongside a permanently-0 resets counter,
+ * misleading monitoring.
*
* Called from two safe points only: the tail of
* {@link #resetTableBuffersAfterFlush()} (no row in progress, this flush's
@@ -4829,6 +4843,8 @@ private void resetTableBuffersAfterFlush() {
*/
private void armIfEligible() {
boolean shouldArm = resetEnabled
+ && engineRebuildFactory != null
+ && ownsCursorEngine
&& (globalSymbolDictionary.size() >= Math.max(resetThresholdSymbols, resetFloorSymbols)
|| manualResetRequested);
if (shouldArm && !resetArmed) {
@@ -5091,20 +5107,12 @@ private void maybeBlockForStarvedReset() {
/**
* Evaluates whether the barrier in {@link #table(CharSequence)} may run
* the symbol-dictionary recycle right now. Only ever called with
- * {@link #resetArmed} true.
- *
- * Refuses before any teardown when the rebuild itself is impossible or
- * unsafe: no {@link #engineRebuildFactory} (every public
- * {@code QwpWebSocketSender.connect(...)} overload leaves it null --
- * only {@code Sender.build()} installs one -- and the recycle feature is
- * default-on, so a connect()-built sender must simply stay unarmed rather
- * than NPE at step 5 and latch terminal), or a cursor engine this sender
- * does not own ({@code setCursorEngine(engine, false)}'s contract: the
- * caller retains ownership, so closing it out from under them at step 3
- * would be a use-after-free from the caller's point of view).
+ * {@link #resetArmed} true -- {@link #armIfEligible()} already refused to
+ * arm a sender that cannot rebuild, so this method only has to weigh
+ * producer-side state.
*
- * Also refuses when there is producer-side state the swap cannot safely
- * tear down: no connection yet (a V4 sender that has never sent is never
+ * Refuses when there is producer-side state the swap cannot safely tear
+ * down: no connection yet (a V4 sender that has never sent is never
* pre-connected here), a flush in flight ({@code pendingRowCount != 0}),
* or a row under construction. Otherwise proceeds to the ring-drained
* check: if the backlog is empty, recycle immediately; if not, defer to
@@ -5112,9 +5120,6 @@ private void maybeBlockForStarvedReset() {
* of blocking the caller indefinitely here.
*/
private void maybeRecycleForDictReset() {
- if (engineRebuildFactory == null || !ownsCursorEngine) {
- return;
- }
if (!connected
|| pendingRowCount != 0
|| (currentTableBuffer != null && currentTableBuffer.hasInProgressRow())) {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleArmingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleArmingTest.java
index e83dfa45..b3de45e2 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleArmingTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleArmingTest.java
@@ -80,7 +80,7 @@ public void testArmsAtThreshold() throws Exception {
/**
* Decision 5: arming ignores {@code deltaDictEnabled} -- threshold-based
- * arming must still fire once the sender has degraded to full self-sufficient
+ * evaluation must still run once the sender has degraded to full self-sufficient
* frames. Reaching a custom low {@code symbol_dict_reset_threshold} on a
* sender that also carries the fault-injecting {@code FilesFacade} needs
* {@code QwpWebSocketSender}'s widest {@code connect(List
+ * This {@code connect(...)} overload installs no {@link
+ * io.questdb.client.cutlass.qwp.client.QwpWebSocketSender.EngineRebuildFactory
+ * EngineRebuildFactory} (only {@code Sender.build()} does), so per review
+ * r3 M3 crossing the threshold must never actually arm -- {@code
+ * armIfEligible()} folds the capability check in ahead of the threshold
+ * comparison. Decision 5 is instead pinned negatively here: full-dict
+ * degradation does not change that verdict either way.
*/
@Test
public void testArmsInFullDictMode() throws Exception {
@@ -160,11 +168,14 @@ public void testArmsInFullDictMode() throws Exception {
sender.isResetArmed());
// No manual resetSymbolDictionary() call anywhere in this test: crossing
- // the threshold alone must arm the recycle, even while degraded.
+ // the threshold, even while degraded, still must not arm -- this
+ // connect(...) overload installs no engineRebuildFactory (review r3,
+ // M3), and that capability check now runs ahead of the threshold
+ // comparison in armIfEligible().
sender.table("m").symbol("s", "c").longColumn("v", 3L).atNow();
sender.flush();
- Assert.assertTrue("threshold-based arming must fire even in full-dict mode "
- + "(Decision 5: arming ignores deltaDictEnabled)",
+ Assert.assertFalse("a sender with no rebuild factory must never arm, even once "
+ + "the threshold is crossed in full-dict mode (review r3, M3)",
sender.isResetArmed());
} finally {
sender.close();
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java
index ad6d8924..6360654a 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java
@@ -126,13 +126,16 @@ public void testRecycleAtEmptyBacklog() throws Exception {
/**
* {@code engineRebuildFactory} is only installed by {@code Sender.build()}
- * ({@code Sender.java:1752}) -- every public {@code QwpWebSocketSender.connect(...)}
+ * ({@code Sender.java:1760}) -- every public {@code QwpWebSocketSender.connect(...)}
* overload leaves it null. Since the recycle feature is default-on and
* {@code resetSymbolDictionary()} is a public advisory API, a connect()-built
- * sender can become "armed" with no way to ever act on it.
- * {@code maybeRecycleForDictReset()} must refuse before any teardown in that
- * case, not attempt step 6 and NPE into a latched terminal state -- covers
- * both ways a sender can arm: the manual request and threshold crossing.
+ * sender could previously become "armed" with no way to ever act on it --
+ * {@code isResetArmed()} reading true forever alongside a permanently-0
+ * resets counter misled monitoring (review r3, M3). {@code armIfEligible()}
+ * now folds the same capability check ({@code engineRebuildFactory != null
+ * && ownsCursorEngine}) into the arming decision itself, so a sender that
+ * cannot rebuild never arms in the first place -- covers both ways a
+ * sender can otherwise arm: the manual request and threshold crossing.
*/
@Test
public void testConnectBuiltSenderNeverRecyclesWithoutFactory() throws Exception {
@@ -143,7 +146,8 @@ public void testConnectBuiltSenderNeverRecyclesWithoutFactory() throws Exception
// Manual reset request on the simplest connect() overload.
try (QwpWebSocketSender sender = QwpWebSocketSender.connect("localhost", port)) {
sender.resetSymbolDictionary();
- Assert.assertTrue("a manual request arms immediately (no row/flush in flight)",
+ Assert.assertFalse("a sender with no rebuild factory must never arm, not even "
+ + "for a manual request (review r3, M3)",
sender.isResetArmed());
// Drained instant (nothing published yet, no row in progress): with a
@@ -155,7 +159,7 @@ public void testConnectBuiltSenderNeverRecyclesWithoutFactory() throws Exception
sender.awaitAckedFsn(fsn, 5_000));
Assert.assertEquals("no factory -> the recycle can never actually run",
0, sender.getSymbolDictEpoch());
- Assert.assertTrue("stays armed forever -- nothing ever consumes the request",
+ Assert.assertFalse("still never armed -- nothing changed that would flip it",
sender.isResetArmed());
}
@@ -195,7 +199,9 @@ public void testConnectBuiltSenderNeverRecyclesWithoutFactory() throws Exception
sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
long fsn1 = sender.flushAndGetSequence();
Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
- Assert.assertTrue("threshold=2 crossed by a, b", sender.isResetArmed());
+ Assert.assertFalse("a sender with no rebuild factory must never arm: "
+ + "isResetArmed()==true with a permanently-0 resets counter "
+ + "misleads monitoring", sender.isResetArmed());
// Drained instant again: must not recycle, must not throw.
sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
@@ -204,7 +210,8 @@ public void testConnectBuiltSenderNeverRecyclesWithoutFactory() throws Exception
sender.awaitAckedFsn(fsn2, 5_000));
Assert.assertEquals("no factory -> the recycle can never actually run",
0, sender.getSymbolDictEpoch());
- Assert.assertTrue("stays armed -- nothing ever consumes the threshold arming",
+ Assert.assertFalse("still never armed -- crossing the threshold again changes "
+ + "nothing",
sender.isResetArmed());
} finally {
sender.close();
From 4bb08fb7a28551d0f1261fa117fcb26905cbd173 Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Mon, 24 Aug 2026 15:13:12 +0100
Subject: [PATCH 40/49] Fix accessor truthfulness across the recycle window
Review r3 M1/M4/M9: getAckedFsn reads fsnEpochBase before the engine
so a torn pair can never fabricate an FSN above anything published;
wasEverConnected honors its documented stickiness through the loop-null
window via the sender-lifetime flag; the getAckedFsn javadocs now state
the real post-recycle sentinel behavior (the value was already
truthful).
---
.../main/java/io/questdb/client/Sender.java | 6 ++-
.../qwp/client/QwpWebSocketSender.java | 37 ++++++++++++-------
.../SymbolDictRecycleStep7FaultTest.java | 2 +
3 files changed, 29 insertions(+), 16 deletions(-)
diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java
index 7252ce03..6ce3c684 100644
--- a/core/src/main/java/io/questdb/client/Sender.java
+++ b/core/src/main/java/io/questdb/client/Sender.java
@@ -611,8 +611,10 @@ default Sender geoHashColumn(CharSequence name, CharSequence value) {
/**
* Highest frame sequence number (FSN) the server has acknowledged.
- * Returns {@code -1} when no batch has been published yet, and on transports that
- * do not track FSNs (HTTP, TCP, UDP).
+ * Returns {@code -1} only while nothing has ever been published in this
+ * sender's lifetime. After a symbol-dictionary recycle the accessor keeps
+ * reporting the last pre-swap durable watermark until the fresh epoch
+ * publishes -- it never collapses back to {@code -1}.
*
* Snapshot accessor — for a bounded wait, use
* {@link #awaitAckedFsn(long, long)}.
*/
@Override
public long getAckedFsn() {
- // Snapshot: the recycle transitions cursorEngine non-null -> null ->
- // non-null on the producer thread, so read the field once. While it
- // is null (mid-swap, or for good after a failed swap) report the last
- // watermark a recycle barrier proved durable instead of collapsing
- // to -1 -- all pre-swap data really is acked.
+ // Read fsnEpochBase FIRST, then cursorEngine: the recycle writes
+ // engine=null -> base+=L+1 -> engine=fresh, so a reader that saw the
+ // NEW base is ordered after the null write and can only observe null
+ // or the fresh engine -- never (new base, stale engine), which would
+ // fabricate an FSN above anything ever published. Sender is
+ // documented single-threaded; this ordering just keeps best-effort
+ // monitor reads truthful rather than promising thread safety.
+ long base = fsnEpochBase;
CursorSendEngine engine = cursorEngine;
- return engine != null ? fsnEpochBase + engine.ackedFsn() : lastRecycleDurableFsn;
+ return engine != null ? base + engine.ackedFsn() : lastRecycleDurableFsn;
}
/**
@@ -3291,15 +3296,19 @@ public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi)
/**
* True iff this sender has at least once installed a live (connected
* + upgraded) WebSocket. Sticky — once true, stays true even after a
- * subsequent disconnect. Lets a {@link SenderErrorHandler}
- * disambiguate a "never reached the server" terminal failure (likely
- * a config typo or firewall block) from a "lost connection after we
- * were up" failure (likely transient). Returns {@code false} if no
- * I/O loop is running.
+ * subsequent disconnect, including through a symbol-dict recycle's
+ * loop-null window (mid-swap, or after a failed reconnect setup).
+ * Lets a {@link SenderErrorHandler} disambiguate a "never reached the
+ * server" terminal failure (likely a config typo or firewall block)
+ * from a "lost connection after we were up" failure (likely
+ * transient). Returns {@code false} only if no loop has ever
+ * connected in this sender's lifetime.
*/
public boolean wasEverConnected() {
+ // Sticky by contract: fall back to the sender-lifetime flag while no
+ // loop is installed (mid-recycle, or after a failed reconnect setup).
CursorWebSocketSendLoop l = cursorSendLoop;
- return l != null && l.hasEverConnected();
+ return hasLoopEverConnected || (l != null && l.hasEverConnected());
}
private static Throwable captureCloseError(Throwable terminalError, Throwable t) {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java
index 66921574..9a40130c 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java
@@ -84,6 +84,8 @@ public void testStep7FailureDoesNotLatchAndRecovers() throws Exception {
}
// The swap committed; the sender must NOT be terminal.
Assert.assertEquals(1, ws.getSymbolDictEpoch());
+ Assert.assertTrue("wasEverConnected() is documented sticky and must survive "
+ + "the recycle's loop-null window", ws.wasEverConnected());
ws.setLoopStartFaultForTesting(null);
// Next send retries the deferred setup and data flows again.
sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
From dc6526bccaf0281871d9521a9d4bce0c3368e7fe Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Mon, 24 Aug 2026 15:23:46 +0100
Subject: [PATCH 41/49] Fix up M9 javadoc overclaim and make
hasLoopEverConnected volatile
Review of task 9: scope the getAckedFsn 'never collapses to -1' claim to
a live sender (close() nulls cursorEngine and lastRecycleDurableFsn is
only ever set by a recycle, so a published+acked, never-recycled,
closed sender does return -1 again -- the prior wording overclaimed).
hasLoopEverConnected is now load-bearing for wasEverConnected(), which
is documented for cross-thread use from a SenderErrorHandler callback;
mark it volatile like every other monitoring-accessor field in this
class.
---
core/src/main/java/io/questdb/client/Sender.java | 5 +++--
.../cutlass/qwp/client/QwpWebSocketSender.java | 13 ++++++++-----
2 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java
index 6ce3c684..4e1834f8 100644
--- a/core/src/main/java/io/questdb/client/Sender.java
+++ b/core/src/main/java/io/questdb/client/Sender.java
@@ -612,9 +612,10 @@ default Sender geoHashColumn(CharSequence name, CharSequence value) {
/**
* Highest frame sequence number (FSN) the server has acknowledged.
* Returns {@code -1} only while nothing has ever been published in this
- * sender's lifetime. After a symbol-dictionary recycle the accessor keeps
+ * sender's lifetime. On a live sender the value never collapses back to
+ * {@code -1}: after a symbol-dictionary recycle the accessor keeps
* reporting the last pre-swap durable watermark until the fresh epoch
- * publishes -- it never collapses back to {@code -1}.
+ * publishes. (After {@code close()} the reading is unspecified.)
*
* Snapshot accessor — for a bounded wait, use
* {@link #awaitAckedFsn(long, long)}.
From 4b3894d5a248756efcebce632aca20e0fa7eccd1 Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Mon, 24 Aug 2026 16:05:02 +0100
Subject: [PATCH 42/49] Fix stale doc claims left by the review wave
- Sender.getAckedFsn: restore the transport caveat -- HTTP/TCP/UDP
senders inherit the -1 default and never track FSNs.
- QwpWebSocketSender.getSymbolDictEpoch: a step-5 rebuild failure no
longer latches recycleFailure; the resume machinery abandons the
recycle instead, to be resumed by a later send.
- QwpWebSocketSender.resetSymbolDictionary: note that the request is
also a permanent no-op on senders that cannot recycle (no engine
rebuild factory, or an engine they do not own).
- QwpWebSocketSender.isDeltaDictEnabledForTest: reword the one-way
"permanently" claim to epoch-scoped -- a symbol-dictionary recycle
re-derives delta mode from the fresh engine.
- QwpWebSocketSender.sealAndSwapBuffer: guard the append-failure
catch's cursorSendLoop.checkError() call against a null loop so it
cannot NPE and mask the real append failure.
---
.../main/java/io/questdb/client/Sender.java | 5 ++--
.../qwp/client/QwpWebSocketSender.java | 26 ++++++++++++-------
2 files changed, 20 insertions(+), 11 deletions(-)
diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java
index 4e1834f8..f97ae399 100644
--- a/core/src/main/java/io/questdb/client/Sender.java
+++ b/core/src/main/java/io/questdb/client/Sender.java
@@ -611,8 +611,9 @@ default Sender geoHashColumn(CharSequence name, CharSequence value) {
/**
* Highest frame sequence number (FSN) the server has acknowledged.
- * Returns {@code -1} only while nothing has ever been published in this
- * sender's lifetime. On a live sender the value never collapses back to
+ * Returns {@code -1} while nothing has ever been published in this
+ * sender's lifetime, and always on transports that do not track FSNs
+ * (HTTP, TCP, UDP). On a live sender the value never collapses back to
* {@code -1}: after a symbol-dictionary recycle the accessor keeps
* reporting the last pre-swap durable watermark until the fresh epoch
* publishes. (After {@code close()} the reading is unspecified.)
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index e7d66473..60a37e90 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -2239,9 +2239,11 @@ public QwpTableBuffer getTableBuffer(String tableName) {
/**
* Whether this sender is still in delta-encoded mode. Flips to {@code false}
- * permanently once {@link #disableDeltaDict} fires (a persisted-dictionary
- * write failure, including a recognised mmap access fault) -- every later
- * flush then ships full self-sufficient frames instead.
+ * for the rest of this epoch once {@link #disableDeltaDict} fires (a
+ * persisted-dictionary write failure, including a recognised mmap access
+ * fault) -- every later flush this epoch then ships full self-sufficient
+ * frames instead. A symbol-dictionary recycle re-derives this from the
+ * fresh engine.
*/
@TestOnly
public boolean isDeltaDictEnabledForTest() {
@@ -2274,10 +2276,11 @@ public long getFsnEpochBaseForTest() {
* Number of symbol-dictionary recycles this sender has completed. Advances
* by one at step 6 of {@link #recycleForDictReset()}, the instant the swap
* commits to the new epoch -- after the engine rebuild (step 5) has
- * already succeeded, so a step-5 rebuild failure that latches
- * {@link #recycleFailure} leaves this counter un-bumped, while a later
- * step-7 reconnect failure (which cannot latch: the swap already
- * committed by then) still leaves this incremented. Unlike the per-send-loop
+ * already succeeded, so a step-5 rebuild failure -- which abandons the
+ * recycle to be resumed by a later send -- leaves this counter
+ * un-bumped, while a later step-7 reconnect failure (which cannot
+ * latch: the swap already committed by then) still leaves this
+ * incremented. Unlike the per-send-loop
* {@code getTotal*} counters, it is scoped to the sender's whole lifetime
* and never resets. volatile: a
* concurrent read sees the latest write the producer thread completed,
@@ -2821,7 +2824,10 @@ public void reset() {
*
* A permanent no-op while {@code symbol_dict_reset} is off: arming gates
* on that knob, so a sender configured with the recycle disabled never
- * acts on the request, however many times it is made.
+ * acts on the request, however many times it is made. The request is
+ * likewise a permanent no-op on senders that cannot recycle -- ones
+ * without an engine rebuild factory (every {@code connect()}-built
+ * sender) or running on an engine they do not own -- which never arm.
*/
@Override
public void resetSymbolDictionary() {
@@ -6026,7 +6032,9 @@ private void sealAndSwapBuffer() {
// Surface any I/O thread error first — appendBlocking itself only
// throws on PAYLOAD_TOO_LARGE / backpressure deadline, but the
// I/O loop can have failed independently.
- cursorSendLoop.checkError();
+ if (cursorSendLoop != null) {
+ cursorSendLoop.checkError();
+ }
throw new LineSenderException("cursor SF append failed", t);
}
}
From 1d869cc579e3e9a720303599525c29a5028d0a6e Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Thu, 27 Aug 2026 01:51:31 +0100
Subject: [PATCH 43/49] Pin the recycle re-arm floor arithmetic
The floor (2x the dictionary size at the last swap, capped at half the
protocol cap) had no test reading it, which is how two integration
suites with bounded symbol sets went red when the floor landed: they
still expected organic re-arms that the floor makes impossible. Pin
the doubling, the organic-re-arm refusal below the floor, and the
advisory-request bypass. Rename the arming test whose every assertion
is assertFalse(isResetArmed()) to say what it proves.
---
.../qwp/client/QwpWebSocketSender.java | 4 ++
.../client/SymbolDictRecycleArmingTest.java | 49 ++++++++++++++++++-
2 files changed, 52 insertions(+), 1 deletion(-)
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index 60a37e90..d3d273c9 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -2219,6 +2219,10 @@ public int getSymbolDictResetThreshold() {
return resetThresholdSymbols;
}
+ /**
+ * The current re-arm floor: 0 before the first swap, then
+ * {@code min(2 x dictSizeAtSwap, MAX_SYMBOL_DICTIONARY_SIZE / 2)}.
+ */
@TestOnly
public int getResetFloorSymbolsForTesting() {
return resetFloorSymbols;
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleArmingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleArmingTest.java
index b3de45e2..33d659f9 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleArmingTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleArmingTest.java
@@ -100,7 +100,7 @@ public void testArmsAtThreshold() throws Exception {
* degradation does not change that verdict either way.
*/
@Test
- public void testArmsInFullDictMode() throws Exception {
+ public void testDoesNotArmWithoutRebuildFactory() throws Exception {
assertMemoryLeak(() -> {
String sfDir = temporaryFolder.getRoot().toPath().resolve("arm-full-dict-sf").toString();
String slot = Paths.get(sfDir, "default").toString();
@@ -231,6 +231,53 @@ public void testManualResetRequestArms() throws Exception {
});
}
+ @Test
+ public void testReArmFloorDoublesPerSwapAndBlocksOrganicReArm() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("floor-sf").toString();
+ try (TestWebSocketServer server = ackingServer()) {
+ String config = cfg(server) + "sf_dir=" + sfDir + ";symbol_dict_reset_threshold=2;";
+ try (Sender sender = Sender.fromConfig(config)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ Assert.assertEquals("no swap yet: floor is 0", 0, ws.getResetFloorSymbolsForTesting());
+
+ // epoch 0: two symbols == threshold -> arms
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000));
+ Assert.assertTrue(ws.isResetArmed());
+
+ // swap #1 runs inside this table() with dictSizeAtSwap == 2
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+ Assert.assertEquals("floor = 2 x size-at-swap", 4, ws.getResetFloorSymbolsForTesting());
+
+ // epoch 1: c,d,e,f == floor -> arms again
+ sender.table("t").symbol("s", "d").longColumn("v", 2L).atNow();
+ sender.table("t").symbol("s", "e").longColumn("v", 2L).atNow();
+ sender.table("t").symbol("s", "f").longColumn("v", 2L).atNow();
+ Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000));
+ Assert.assertTrue("size 4 >= max(threshold 2, floor 4) must arm", ws.isResetArmed());
+
+ // swap #2 with dictSizeAtSwap == 4
+ sender.table("t").symbol("s", "g").longColumn("v", 3L).atNow();
+ Assert.assertEquals(2, ws.getSymbolDictEpoch());
+ Assert.assertEquals("floor doubles again", 8, ws.getResetFloorSymbolsForTesting());
+
+ // epoch 2: four symbols is above the threshold but below the floor
+ sender.table("t").symbol("s", "h").longColumn("v", 4L).atNow();
+ sender.table("t").symbol("s", "i").longColumn("v", 4L).atNow();
+ sender.table("t").symbol("s", "j").longColumn("v", 4L).atNow();
+ Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000));
+ Assert.assertFalse("size 4 < floor 8 must not re-arm organically", ws.isResetArmed());
+
+ sender.resetSymbolDictionary();
+ Assert.assertTrue("the advisory request bypasses the floor", ws.isResetArmed());
+ }
+ }
+ });
+ }
+
@Test
public void testResetSymbolDictionaryOnNonWsSenderIsNoOp() throws Exception {
assertMemoryLeak(() -> {
From 269aa8a5b2e617eac37890e17f57ac65e6f4857d Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Thu, 27 Aug 2026 01:54:07 +0100
Subject: [PATCH 44/49] Point comments at the renamed arming test
SymbolDictRecycleArmingTest.testArmsInFullDictMode was renamed to
testDoesNotArmWithoutRebuildFactory; update the two comment references
that still named the old method.
---
.../test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java | 2 +-
.../client/test/cutlass/qwp/client/SymbolDictRecycleTest.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java
index 0c264ec2..9749d7ac 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java
@@ -339,7 +339,7 @@ private static TestWebSocketServer ackingServer() throws Exception {
/**
* Widest {@code QwpWebSocketSender.connect(...)} overload with everything but the
* fault-injecting engine and the reset threshold pinned to defaults -- mirrors
- * {@code SymbolDictRecycleArmingTest.testArmsInFullDictMode}. {@code Sender.fromConfig} has
+ * {@code SymbolDictRecycleArmingTest.testDoesNotArmWithoutRebuildFactory}. {@code Sender.fromConfig} has
* no {@code FilesFacade} seam, so this is the only way to combine a custom facade with a
* custom low threshold; it leaves {@code engineRebuildFactory} null, same as every other
* {@code connect(...)} overload, so callers that need a working recycle must install one
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java
index 6360654a..413242b0 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java
@@ -165,7 +165,7 @@ public void testConnectBuiltSenderNeverRecyclesWithoutFactory() throws Exception
// Threshold-based arming needs a custom low threshold, only reachable (without
// routing through Sender.build(), which WOULD install a factory) via the
- // widest connect() overload -- mirrors SymbolDictRecycleArmingTest.testArmsInFullDictMode.
+ // widest connect() overload -- mirrors SymbolDictRecycleArmingTest.testDoesNotArmWithoutRebuildFactory.
CursorSendEngine engine = new CursorSendEngine(
null, 4L * 1024 * 1024, 128L * 1024 * 1024,
CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS);
From 5591741edcbc3781a4f8179dbbb2e24590a59898 Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Thu, 27 Aug 2026 02:05:30 +0100
Subject: [PATCH 45/49] Hand the live error handler to the rebuild factory
The builder's rebuild factory read the builder's own errorHandler
field at rebuild time, so a slot quarantined during a recycle rebuild
never reached a handler installed on the sender after build() via
setErrorHandler(). The factory gains a rebuild(SenderErrorHandler)
overload the sender calls with its current user handler (null when
only the default is installed, matching build-time behaviour); the
builder's factory forwards it and snapshots sfDir/senderId instead of
reading builder fields live.
---
.../main/java/io/questdb/client/Sender.java | 23 +++++++---
.../qwp/client/QwpWebSocketSender.java | 16 ++++++-
.../qwp/client/SymbolDictRecycleTest.java | 44 +++++++++++++++++++
3 files changed, 77 insertions(+), 6 deletions(-)
diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java
index f97ae399..317ae232 100644
--- a/core/src/main/java/io/questdb/client/Sender.java
+++ b/core/src/main/java/io/questdb/client/Sender.java
@@ -1761,11 +1761,24 @@ public Sender build() {
// dispatcher daemon, drainer pool, microbatch buffers and
// WebSocketClient inside the abandoned `connected`.
connected.setTransactional(transactional);
- connected.setEngineRebuildFactory(() -> LineSenderBuilder.constructEngineOnSlot(
- sfDir, senderId, slotPath,
- actualSfMaxSegmentBytes, actualSfMaxTotalBytes,
- actualSfAppendDeadlineNanos, actualSfSyncIntervalNanos,
- errorHandler));
+ final String rebuildSfDir = sfDir;
+ final String rebuildSenderId = senderId;
+ final SenderErrorHandler buildTimeHandler = errorHandler;
+ connected.setEngineRebuildFactory(new QwpWebSocketSender.EngineRebuildFactory() {
+ @Override
+ public CursorSendEngine rebuild() {
+ return rebuild(buildTimeHandler);
+ }
+
+ @Override
+ public CursorSendEngine rebuild(SenderErrorHandler liveHandler) {
+ return LineSenderBuilder.constructEngineOnSlot(
+ rebuildSfDir, rebuildSenderId, slotPath,
+ actualSfMaxSegmentBytes, actualSfMaxTotalBytes,
+ actualSfAppendDeadlineNanos, actualSfSyncIntervalNanos,
+ liveHandler);
+ }
+ });
try {
// Install the drainer listener BEFORE startOrphanDrainers
// below: drainers must see the listener at submit time so
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index d3d273c9..99dce59c 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -5156,7 +5156,7 @@ private void maybeRecycleForDictReset() {
private CursorSendEngine rebuildEngineOrAbandon(String message) {
try {
- return engineRebuildFactory.rebuild();
+ return engineRebuildFactory.rebuild(userErrorHandler());
} catch (Error e) {
throw e;
} catch (Throwable t) {
@@ -5431,6 +5431,11 @@ private void sendCommitMessage() {
lastCommitBoundaryFsn = cursorEngine.publishedFsn();
}
+ private SenderErrorHandler userErrorHandler() {
+ SenderErrorHandler h = errorHandler;
+ return h == DefaultSenderErrorHandler.INSTANCE ? null : h;
+ }
+
/**
* Advances the delta baseline once a frame carrying the current batch's
* symbols has been queued onto the ring. No-op in full-dict mode. Only ever
@@ -6182,6 +6187,15 @@ public Endpoint(String host, int port) {
*/
public interface EngineRebuildFactory {
CursorSendEngine rebuild();
+
+ /**
+ * Rebuild with the sender's current user-supplied error handler ({@code null}
+ * when only the default handler is installed), so a quarantine during the
+ * rebuild reaches a handler installed after {@code build()}.
+ */
+ default CursorSendEngine rebuild(SenderErrorHandler liveHandler) {
+ return rebuild();
+ }
}
/**
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java
index 413242b0..6dca288b 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java
@@ -25,6 +25,7 @@
package io.questdb.client.test.cutlass.qwp.client;
import io.questdb.client.Sender;
+import io.questdb.client.SenderErrorHandler;
import io.questdb.client.cutlass.line.LineSenderException;
import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
@@ -49,8 +50,10 @@
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE;
import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
@@ -407,6 +410,47 @@ public void testFailedRebuildAbandonsAndRecovers() throws Exception {
});
}
+ @Test
+ public void testRebuildFactoryReceivesTheLiveErrorHandler() throws Exception {
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = ackingServer()) {
+ try (Sender sender = Sender.fromConfig(cfg(server))) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ SenderErrorHandler installedAfterBuild = error -> { };
+ ws.setErrorHandler(installedAfterBuild);
+
+ QwpWebSocketSender.EngineRebuildFactory real = ws.getEngineRebuildFactoryForTesting();
+ AtomicReference
@@ -4923,12 +4925,19 @@ private void awaitDeferredEngineClose(CursorSendEngine outgoing) {
}
}
- private static void closeQuietly(CursorSendEngine engine) {
+ private void closeRecoveredEngine(CursorSendEngine recovered) {
+ recyclePendingOutgoing = recovered;
try {
- engine.close();
- } catch (Throwable ignored) {
- // best-effort; the retained files are the next rebuild's problem
+ recovered.close();
+ } catch (Error e) {
+ throw e;
+ } catch (Throwable t) {
+ LOG.warn("recovered engine close reported a failure during the symbol dictionary "
+ + "recycle; deferring to the close-completion probe", t);
}
+ awaitDeferredEngineClose(recovered);
+ recyclePendingOutgoing = null;
+ retainedEngine = null;
}
/**
@@ -4962,7 +4971,7 @@ private void completeRecycleRebuild(int dictSizeAtSwap, long startNanos) {
if (rebuilt.publishedFsn() > rebuilt.ackedFsn()) {
throw latchRecycleBreach(rebuilt, dictSizeAtSwap);
}
- closeQuietly(rebuilt); // fully drained: retries the segment unlink
+ closeRecoveredEngine(rebuilt); // fully drained: retries the segment unlink
rebuilt = rebuildEngineOrAbandon(
"symbol dictionary recycle could not rebuild its engine after healing "
+ "leftover acked segments; retried on the next send");
@@ -4973,7 +4982,7 @@ private void completeRecycleRebuild(int dictSizeAtSwap, long startNanos) {
if (rebuilt.publishedFsn() > rebuilt.ackedFsn()) {
throw latchRecycleBreach(rebuilt, dictSizeAtSwap);
}
- closeQuietly(rebuilt);
+ closeRecoveredEngine(rebuilt);
throw new LineSenderException(
"symbol dictionary recycle keeps recovering leftover acked segments "
+ "(slot cleanup not durable yet); retried on the next send");
@@ -5006,9 +5015,9 @@ private void completeRecycleRebuild(int dictSizeAtSwap, long startNanos) {
cursorEngine = rebuilt;
ownsCursorEngine = true;
cursorEngine.setSlotLockReleaseListener(this::onSlotLockReleased);
- // The fresh engine holds the slot flock again; step 3's release
- // flipped slotLockReleased true via the outgoing engine's
- // listener, which no longer describes this sender's state.
+ // The fresh engine holds the slot flock again; a stale true (an
+ // isSlotLockReleased() re-probe of the outgoing engine while the
+ // recycle stayed pending) no longer describes this sender's state.
slotLockReleased = false;
recycleResume = RecycleResume.NONE;
recyclePendingLastPublishedFsn = -1L;
@@ -5069,7 +5078,17 @@ private RuntimeException latchRecycleBreach(CursorSendEngine rebuilt, int dictSi
+ "was breached");
recycleFailure = breach;
recycleResume = RecycleResume.NONE;
- closeQuietly(rebuilt);
+ try {
+ rebuilt.close();
+ } catch (Error e) {
+ throw e;
+ } catch (Throwable ignored) {
+ // terminal either way; the retained-engine probe below covers a deferred close
+ }
+ if (!rebuilt.isCloseCompleted()) {
+ retainedEngine = rebuilt;
+ slotLockReleased = false;
+ }
LOG.error("symbol dictionary recycle failed; sender is now terminal "
+ "[epoch={}, dictSizeAtSwap={}]", symbolDictEpoch, dictSizeAtSwap, breach);
throw breach;
@@ -5298,6 +5317,7 @@ private void recycleForDictReset() {
recyclePendingOutgoing = outgoing;
recyclePendingLastPublishedFsn = lastPublishedFsn;
try {
+ outgoing.setSlotLockReleaseListener(null);
outgoing.close();
} catch (Error e) {
throw e;
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
index 66a0635e..b06d3f39 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
@@ -1349,6 +1349,11 @@ public SlotLock getSlotLockForTesting() {
return slotLock;
}
+ @TestOnly
+ public Runnable getSlotLockReleaseListenerForTesting() {
+ return slotLockReleaseListener;
+ }
+
@TestOnly
public long getSyncIntervalNanosForTesting() {
return syncIntervalNanos;
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleDeferredCloseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleDeferredCloseTest.java
index 169f5927..687ddb7b 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleDeferredCloseTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleDeferredCloseTest.java
@@ -301,6 +301,41 @@ public void testExhaustedDeferredCloseAwaitKeepsThrowingWhileWedged() throws Exc
});
}
+ /**
+ * The outgoing engine's slot-lock listener is the sender's own
+ * onSlotLockReleased. Left attached, a release that completes after step 6
+ * (a preempted retry thread between closeCompleted = true and listener.run())
+ * would mark the REBUILT engine's flock released. Step 3 detaches it.
+ */
+ @Test(timeout = 60_000L)
+ public void testRecycleDetachesTheOutgoingEngineListener() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("recycle-detach-listener").toString();
+ try (TestWebSocketServer server = ackingServer()) {
+ String cfg = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir + ";";
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000));
+
+ CursorSendEngine outgoing = ws.getCursorEngineForTesting();
+ Assert.assertNotNull("the live engine carries the sender's listener",
+ outgoing.getSlotLockReleaseListenerForTesting());
+
+ sender.resetSymbolDictionary();
+ sender.table("t").symbol("s", "b").longColumn("v", 2L).atNow();
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+
+ Assert.assertNull("step 3 must detach the outgoing engine's listener",
+ outgoing.getSlotLockReleaseListenerForTesting());
+ Assert.assertNotNull("the rebuilt engine carries the listener instead",
+ ws.getCursorEngineForTesting().getSlotLockReleaseListenerForTesting());
+ Assert.assertFalse("the rebuilt engine holds the flock", ws.isSlotLockReleased());
+ }
+ }
+ });
+ }
+
/**
* The await budget runs out while the worker is still wedged, but the
* wedge is transient after all. Exhausting the budget must NOT latch the
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleSlotHealTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleSlotHealTest.java
index c459340b..de2629cf 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleSlotHealTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleSlotHealTest.java
@@ -47,9 +47,11 @@
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.Set;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
@@ -152,6 +154,164 @@ public void testRecoveredFullyAckedLeftoversHealAndRecycleCompletes() throws Exc
});
}
+ /**
+ * The heal closes the engine that recovered the leftovers. When that engine's
+ * SF worker is wedged, its close returns with the slot flock retained, exactly
+ * like the outgoing engine's close in step 3 -- and must be awaited the same
+ * way, or rebuild #2 collides with the retained flock.
+ */
+ @Test(timeout = 60_000L)
+ public void testHealRidesOutADeferredCloseOfTheRecoveredEngine() throws Exception {
+ assertMemoryLeak(() -> {
+ String doctoredSlot = temporaryFolder.getRoot().toPath().resolve("doctored-deferred").toString();
+ prepareFullyAckedLeftoverSlot(doctoredSlot);
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("heal-deferred-sf").toString();
+ CountDownLatch workerBlocked = new CountDownLatch(1);
+ CountDownLatch releaseWorker = new CountDownLatch(1);
+ AtomicReference
* Default {@code 100_000}. WebSocket transport only.
@@ -3259,9 +3245,9 @@ static ConstructedEngine constructEngineOnSlotLocked(
// frames that were never delivered. All three types below are load
// bearing; narrowing this catch to UnreplayableSlotException would
// restore the permanent build() brick for the segment-skip case. That verdict gets
- // the exact same quarantine-and-continue treatment as the connect()-time
- // verdict below -- constructing cursorEngine is not inside the loop below,
- // so a throw here would otherwise escape build() entirely, uncaught.
+ // the same quarantine-and-continue treatment as build()'s connect()-time
+ // verdict; for build() the construction runs outside its retry loop, and
+ // for a recycle rebuild there is no loop at all.
// quarantineTornSlot(null, ...) renames the WHOLE slot directory aside
// (not just the unreadable segment file) before building the replacement
// at the original slotPath, so the replacement starts on a genuinely empty
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index e29bc09c..7cc3ac98 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -454,7 +454,7 @@ public class QwpWebSocketSender implements Sender {
// Distinct-symbol count that triggers a recycle once resetEnabled is on
// (connect-string key symbol_dict_reset_threshold).
private int resetThresholdSymbols = DEFAULT_SYMBOL_DICT_RESET_THRESHOLD_SYMBOLS;
- // Anti-thrash floor for the automatic reset (review r3, C1). 0 until the
+ // Anti-thrash floor for the automatic reset. 0 until the
// first swap; the effective re-arm bar is max(resetThresholdSymbols,
// resetFloorSymbols). Each swap raises it to twice the dictionary size
// at that swap, so a live symbol set larger than the threshold stops
@@ -513,15 +513,15 @@ public class QwpWebSocketSender implements Sender {
// normally. Every other recycle failure is transient and resumable (see
// recycleResume), never latched here.
private Throwable recycleFailure;
- // Resumable recycle (review r3, C2): a transient failure mid-recycle no
+ // Resumable recycle: a transient failure mid-recycle no
// longer latches the sender terminal. CLOSE_LOOP = step 2 failed, the
// old loop is still dying and the old engine/dictionary are intact.
// REBUILD = the old engine is closed (possibly still releasing its slot
// flock); await/rebuild/commit are pending. resumeRecycleIfPending()
// advances the state from the table() barrier and ensureConnected().
private RecycleResume recycleResume = RecycleResume.NONE;
- // The closed-but-not-yet-released outgoing engine a REBUILD resume still
- // awaits; null once its deferred close completes.
+ // The closed-but-not-yet-released outgoing or recovered engine a REBUILD
+ // resume still awaits; null once its deferred close completes.
private CursorSendEngine recyclePendingOutgoing;
// Raw last-published FSN of the outgoing epoch (step-1 snapshot),
// consumed by the commit when a REBUILD resume completes.
@@ -530,7 +530,7 @@ public class QwpWebSocketSender implements Sender {
// expected to throw) inside ensureConnected()'s loop-construction try,
// after cursorSendLoop is assigned but before start() -- exercising the
// catch that closes and nulls the fresh loop, i.e. the failed-reconnect
- // state the C4/C5 regression tests pin.
+ // state SymbolDictRecycleStep7FaultTest pins.
private Runnable loopStartFault;
// Incremented once per completed symbol-dictionary recycle. 0 until the
// first recycle commits. volatile: this is public API (see
@@ -2003,12 +2003,14 @@ public long getAckedFsn() {
// engine=null -> base+=L+1 -> engine=fresh, so a reader that saw the
// NEW base is ordered after the null write and can only observe null
// or the fresh engine -- never (new base, stale engine), which would
- // fabricate an FSN above anything ever published. Sender is
- // documented single-threaded; this ordering just keeps best-effort
- // monitor reads truthful rather than promising thread safety.
+ // fabricate an FSN above anything ever published. The clamp against
+ // lastRecycleDurableFsn keeps the other torn pair (old base, fresh
+ // engine) from reading below a value already returned. Sender is
+ // documented single-threaded; this keeps best-effort monitor reads
+ // truthful rather than promising thread safety.
long base = fsnEpochBase;
CursorSendEngine engine = cursorEngine;
- return engine != null ? base + engine.ackedFsn() : lastRecycleDurableFsn;
+ return engine != null ? Math.max(lastRecycleDurableFsn, base + engine.ackedFsn()) : lastRecycleDurableFsn;
}
/**
@@ -2281,8 +2283,8 @@ public long getFsnEpochBaseForTest() {
/**
* Number of symbol-dictionary recycles this sender has completed. Advances
* by one at step 6 of {@link #recycleForDictReset()}, the instant the swap
- * commits to the new epoch -- after the engine rebuild (step 5) has
- * already succeeded, so a step-5 rebuild failure -- which abandons the
+ * commits to the new epoch -- after the engine rebuild (step 4) has
+ * already succeeded, so a step-4 rebuild failure -- which abandons the
* recycle to be resumed by a later send -- leaves this counter
* un-bumped, while a later step-7 reconnect failure (which cannot
* latch: the swap already committed by then) still leaves this
@@ -2303,8 +2305,8 @@ public long getSymbolDictEpoch() {
/**
* Number of symbol-dictionary recycle swaps this sender has completed.
* Incremented alongside {@link #getSymbolDictEpoch()} at step 6 of
- * {@link #recycleForDictReset()} -- after the engine rebuild (step 5) has
- * already succeeded, so, like the epoch counter, a step-5 rebuild failure
+ * {@link #recycleForDictReset()} -- after the engine rebuild (step 4) has
+ * already succeeded, so, like the epoch counter, a step-4 rebuild failure
* leaves this un-bumped while a step-7 reconnect failure still leaves it
* incremented (the swap has already committed by then).
* Also like the epoch counter, it is scoped to the sender's whole lifetime
@@ -4306,11 +4308,13 @@ private void ensureConnected() {
// connect commit to V1 because cursor segments are immutable;
// a future version bump must account for that. Transport
// failures retry indefinitely on the I/O thread (Invariant B).
- // But a terminal auth, upgrade or capability rejection on this
- // deferred connect (the initial one, or a later re-entry such
- // as the recycle's step 7) -- before the wire is ever up -- is
+ // But a terminal auth, upgrade or capability rejection on the
+ // INITIAL deferred connect -- before the wire is ever up -- is
// surfaced to the async SenderErrorHandler and latched for a
- // close() rethrow, not retried.
+ // close() rethrow, not retried. A re-entry after a prior
+ // connect (the recycle's step 7) seeds the fresh loop with
+ // markEverConnected(), so the same rejection there is retried
+ // like any post-connect failure.
client = null;
break;
case OFF:
@@ -4915,7 +4919,7 @@ private void awaitDeferredEngineClose(CursorSendEngine outgoing) {
retainedEngine = outgoing;
slotLockReleased = false;
throw new LineSenderException("symbol dictionary recycle could not yet reclaim "
- + "its slot: the outgoing engine's deferred close did not release the "
+ + "its slot: the engine's deferred close did not release the "
+ "slot lock within " + recycleDeferredCloseMaxWaitMillis
+ " ms (SF worker stalled); the recycle stays pending and is retried "
+ "on the next send");
@@ -4958,7 +4962,7 @@ private void completeRecycleRebuild(int dictSizeAtSwap, long startNanos) {
recyclePendingOutgoing = null;
retainedEngine = null;
}
- // step 5: rebuild the engine on the now-empty slot.
+ // step 4: rebuild the engine on the now-empty slot.
CursorSendEngine rebuilt = rebuildEngineOrAbandon(
"symbol dictionary recycle could not rebuild its engine; retried on the next send");
if (rebuilt.wasRecoveredFromDisk()) {
@@ -4988,14 +4992,14 @@ private void completeRecycleRebuild(int dictSizeAtSwap, long startNanos) {
+ "(slot cleanup not durable yet); retried on the next send");
}
}
- // COMMIT (steps 4 + 6): pure producer-side state, and nothing below
- // can throw. Step 4 rolls the external FSN base past every FSN the
+ // COMMIT (steps 5 + 6): pure producer-side state, and nothing below
+ // can throw. Step 5 rolls the external FSN base past every FSN the
// outgoing epoch handed out (the -1 no-publish case adds 0); it must
// run with cursorSendLoop == null, which step 2 guarantees and the
// step-7 reconnect below only undoes afterwards.
rollFsnEpochBase(recyclePendingLastPublishedFsn);
// Replace the dictionary, don't clear().
- globalSymbolDictionary = new GlobalSymbolDictionary();
+ globalSymbolDictionary = new GlobalSymbolDictionary(Math.max(dictSizeAtSwap, 64));
sentMaxSymbolId = -1;
currentBatchMaxSymbolId = -1;
lastCommitBoundaryFsn = -1L;
@@ -5003,7 +5007,7 @@ private void completeRecycleRebuild(int dictSizeAtSwap, long startNanos) {
symbolDictResetsPerformed++;
resetArmed = false;
manualResetRequested = false;
- // C1 anti-thrash floor: see resetFloorSymbols.
+ // Anti-thrash floor: see resetFloorSymbols.
resetFloorSymbols = Math.min(dictSizeAtSwap * 2,
QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE / 2);
// Deliberately re-derived (not carried over): the healing half
@@ -5205,22 +5209,24 @@ private CursorSendEngine rebuildEngineOrAbandon(String message) {
* javadoc) -- this sender holds no other lock on the slot at this
* point, so releasing it here is safe. Awaits the (possibly
* deferred) release -- see below.
+ *
* This {@code connect(...)} overload installs no {@link
* io.questdb.client.cutlass.qwp.client.QwpWebSocketSender.EngineRebuildFactory
- * EngineRebuildFactory} (only {@code Sender.build()} does), so per review
- * r3 M3 crossing the threshold must never actually arm -- {@code
+ * EngineRebuildFactory} (only {@code Sender.build()} does), so
+ * crossing the threshold must never actually arm -- {@code
* armIfEligible()} folds the capability check in ahead of the threshold
* comparison. Decision 5 is instead pinned negatively here: full-dict
* degradation does not change that verdict either way.
@@ -169,13 +169,13 @@ public void testDoesNotArmWithoutRebuildFactory() throws Exception {
// No manual resetSymbolDictionary() call anywhere in this test: crossing
// the threshold, even while degraded, still must not arm -- this
- // connect(...) overload installs no engineRebuildFactory (review r3,
- // M3), and that capability check now runs ahead of the threshold
+ // connect(...) overload installs no engineRebuildFactory,
+ // and that capability check now runs ahead of the threshold
// comparison in armIfEligible().
sender.table("m").symbol("s", "c").longColumn("v", 3L).atNow();
sender.flush();
Assert.assertFalse("a sender with no rebuild factory must never arm, even once "
- + "the threshold is crossed in full-dict mode (review r3, M3)",
+ + "the threshold is crossed in full-dict mode",
sender.isResetArmed());
} finally {
sender.close();
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleDeferredCloseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleDeferredCloseTest.java
index 687ddb7b..0908e0ff 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleDeferredCloseTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleDeferredCloseTest.java
@@ -54,7 +54,7 @@
* the slot -- rebuilding against the retained flock throws
* {@code SlotLockContentionException} and would fail the recycle for what is
* usually a transient disk stall. Exhausting the await budget does not latch
- * the sender terminal either (review r3, C2): it throws to the triggering
+ * the sender terminal either: it throws to the triggering
* caller and leaves the recycle pending in its {@code RecycleResume.REBUILD}
* state, so each later send retries the await, and one of them finishes the
* swap once the worker finally exits. Nothing on this path is terminal.
@@ -120,7 +120,7 @@ public void testRecycleSurvivesDeferredEngineClose() throws Exception {
sender.resetSymbolDictionary();
Assert.assertTrue(ws.isResetArmed());
- // Positive witness that the await really parked (M15).
+ // Positive witness that the await really parked.
// A sleep could not tell "the await is parked" from
// "the close completed inline and the test skipped the
// code under test"; this fires from inside the await's
@@ -339,7 +339,7 @@ public void testRecycleDetachesTheOutgoingEngineListener() throws Exception {
/**
* The await budget runs out while the worker is still wedged, but the
* wedge is transient after all. Exhausting the budget must NOT latch the
- * sender terminal (review r3, C2): the recycle stays pending in its
+ * sender terminal: the recycle stays pending in its
* REBUILD resume state, and once the worker exits and the deferred close
* releases the flock, the next send finishes the await, rebuilds and
* commits the swap.
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java
index 9749d7ac..01816540 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleHealingTest.java
@@ -99,12 +99,12 @@ public void testMetricsAfterTwoRecycles() throws Exception {
sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
long fsn2 = sender.flushAndGetSequence();
Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
- // The C1 anti-thrash floor (resetFloorSymbols = 2x the first swap's
+ // The anti-thrash floor (resetFloorSymbols = 2x the first swap's
// dictSizeAtSwap = 4) keeps c,d (2 symbols, == threshold but < floor)
// from re-arming on their own; a manual request bypasses the floor by
// design, so drive the second recycle through resetSymbolDictionary().
sender.resetSymbolDictionary();
- Assert.assertTrue("manual reset request bypasses the C1 floor",
+ Assert.assertTrue("manual reset request bypasses the re-arm floor",
ws.isResetArmed());
// Ring drained again -> second recycle: epoch 2.
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleSlotHealTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleSlotHealTest.java
index de2629cf..af0b243e 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleSlotHealTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleSlotHealTest.java
@@ -95,7 +95,7 @@ public class SymbolDictRecycleSlotHealTest {
public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
/**
- * Review r3, C2(a): a benign fully-drained close verdict (segment unlink
+ * A benign fully-drained close verdict (segment unlink
* transiently failed; watermark retained by design) must not brick the
* recycle. The rebuild recovers fully-acked leftovers; the sender heals by
* closing the recovered engine (which retries the unlink) and rebuilding
@@ -313,7 +313,7 @@ public void testHealDeferredCloseExhaustionRetainsTheRecoveredEngine() throws Ex
}
/**
- * Review r3: the terminal latch's one surviving case. A rebuild that
+ * The terminal latch's one surviving case. A rebuild that
* recovers UNACKED frames proves the fully-drained-close contract was
* breached -- the fresh producer dictionary and the slot's state have
* genuinely diverged, so the sender must refuse further use ({@code close()}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java
index 9a40130c..76919f48 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java
@@ -55,9 +55,8 @@ public class SymbolDictRecycleStep7FaultTest {
/**
* Pins that a step-7 (reconnect) failure does NOT latch the sender
* terminal: the swap has committed, the sender is coherent and merely
- * disconnected, and the next send retries the deferred setup. Review
- * round 3, finding C5 (the test the PR body credited was deleted by
- * commit 6793928a).
+ * disconnected, and the next send retries the deferred setup. The test
+ * the PR body credited was deleted by commit 6793928a.
*/
@Test
public void testStep7FailureDoesNotLatchAndRecovers() throws Exception {
@@ -100,8 +99,8 @@ public void testStep7FailureDoesNotLatchAndRecovers() throws Exception {
/**
* Pins the conditional in resetSymbolDictStateForNewConnection(): ids a
* row registered before the deferred reconnect completes must still ship
- * in the next delta. Review round 3, finding C4 (empirically untested:
- * suite was green with the guard reverted).
+ * in the next delta. Empirically untested: suite was green with the
+ * guard reverted.
*/
@Test
public void testPostFailedReconnectDeltaCoversStagedSymbolIds() throws Exception {
@@ -136,7 +135,7 @@ public void testPostFailedReconnectDeltaCoversStagedSymbolIds() throws Exception
// This row registers "c" in the FRESH dictionary during
// symbol(); its sendRow() then completes the deferred
// reconnect, which runs resetSymbolDictStateForNewConnection
- // with the row in progress -- the C4 window.
+ // with the row in progress -- this race window.
sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
long f2 = sender.flushAndGetSequence();
Assert.assertTrue(sender.awaitAckedFsn(f2, 5_000));
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java
index 6dca288b..630a778d 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java
@@ -134,7 +134,7 @@ public void testRecycleAtEmptyBacklog() throws Exception {
* {@code resetSymbolDictionary()} is a public advisory API, a connect()-built
* sender could previously become "armed" with no way to ever act on it --
* {@code isResetArmed()} reading true forever alongside a permanently-0
- * resets counter misled monitoring (review r3, M3). {@code armIfEligible()}
+ * resets counter misled monitoring. {@code armIfEligible()}
* now folds the same capability check ({@code engineRebuildFactory != null
* && ownsCursorEngine}) into the arming decision itself, so a sender that
* cannot rebuild never arms in the first place -- covers both ways a
@@ -150,7 +150,7 @@ public void testConnectBuiltSenderNeverRecyclesWithoutFactory() throws Exception
try (QwpWebSocketSender sender = QwpWebSocketSender.connect("localhost", port)) {
sender.resetSymbolDictionary();
Assert.assertFalse("a sender with no rebuild factory must never arm, not even "
- + "for a manual request (review r3, M3)",
+ + "for a manual request",
sender.isResetArmed());
// Drained instant (nothing published yet, no row in progress): with a
@@ -367,8 +367,8 @@ public void testFactoryRebuildsOnEmptySlot() throws Exception {
/**
* A transient engine-rebuild failure must NOT latch the sender terminal:
* the recycle is abandoned before the swap commits and resumes on the
- * next send. Replaces testFailedRebuildLatchesTerminal (review r3, C2(d):
- * build() has a retry-and-quarantine loop for exactly these operational
+ * next send. Replaces testFailedRebuildLatchesTerminal (build() has a
+ * retry-and-quarantine loop for exactly these operational
* failures; killing a healthy sender on a provably empty slot mid-life
* was strictly worse than the build()-time behavior).
*/
@@ -404,7 +404,6 @@ public void testFailedRebuildAbandonsAndRecovers() throws Exception {
Assert.assertEquals(1, ws.getSymbolDictEpoch());
long f = sender.flushAndGetSequence();
Assert.assertTrue(sender.awaitAckedFsn(f, 5_000));
- sender.close();
}
}
});
@@ -456,7 +455,6 @@ public CursorSendEngine rebuild(SenderErrorHandler liveHandler) {
* loop close throw deterministically (CountDownLatch.await throws on
* entry). That must abandon the recycle non-terminally; once the flag is
* cleared the next call finishes the loop close and the sender recovers.
- * Review r3, C2(c).
*/
@Test
public void testInterruptedRecycleAbandonsAndRecovers() throws Exception {
@@ -506,7 +504,7 @@ public void testInterruptedRecycleAbandonsAndRecovers() throws Exception {
/**
* A live symbol set larger than the threshold must not thrash the
* recycle: after a swap, re-arming requires the dictionary to reach
- * max(threshold, 2 * size-at-swap). Review round 3, finding C1.
+ * max(threshold, 2 * size-at-swap).
*/
@Test
public void testLiveSetAboveThresholdDoesNotThrash() throws Exception {
From 3ba9fbd7aaaefd43cb55644cdef5eefb235adf5f Mon Sep 17 00:00:00 2001
From: Sergei Minaev <5072859+jovfer@users.noreply.github.com>
Date: Thu, 27 Aug 2026 04:27:07 +0100
Subject: [PATCH 49/49] Pin the rebuild-time quarantine notification
The builder's rebuild factory is the half that forwards the live error
handler into constructEngineOnSlot -> quarantineTornSlot, and that
hand-off is only observable when a rebuild really has a slot to set
aside. The new test plants SegmentSkipQuarantineTest's tainted slot
(several never-acked segments, the oldest one's magic overwritten) into
the empty slot the recycle's step-3 close just left behind, then lets the
real factory run: the recycle still commits, and the handler installed
after build() receives the DATA_LOSS SenderError naming where the bytes
were set aside. The sender is built without an error handler at all, so a
regression that forwarded the build-time handler instead of the live one
leaves the notification nowhere -- verified by mutating the delegation to
the handlerless overload, which fails the new assertion.
Also align the test javadocs with the recycle step numbering and drop the
last review-era names.
Co-Authored-By: Claude Fable 5
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index 8c2b3608..1f3bf438 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -402,6 +402,24 @@ public interface EngineRebuildFactory {
// Distinct-symbol count that triggers a recycle once resetEnabled is on
// (connect-string key symbol_dict_reset_threshold).
private int resetThresholdSymbols = DEFAULT_SYMBOL_DICT_RESET_THRESHOLD_SYMBOLS;
+ // Wall-clock time (System.nanoTime()) at which resetArmed last flipped
+ // false -> true. Recorded by armIfEligible so a later task's opportunistic
+ // wait can measure how long the recycle has been armed against
+ // resetMaxWaitMillis.
+ private long armedSinceNanos;
+ // Set by resetSymbolDictionary() (the public advisory API) and never
+ // cleared by armIfEligible itself -- once a caller asks for a fresh epoch,
+ // every later armIfEligible call keeps arming until the recycle actually
+ // runs and consumes the request.
+ private boolean manualResetRequested;
+ // True once armIfEligible has determined a recycle should happen. Consumed
+ // by the (later-task) recycle trigger; set only at the tail of
+ // resetTableBuffersAfterFlush, never on the per-symbol registration path.
+ private boolean resetArmed;
+ // Cleared on the false -> true armed transition; a later task's
+ // opportunistic-wait step sets it once it has waited out its window for
+ // THIS arm cycle, so a subsequent forced-wait check does not re-wait.
+ private boolean starvationWaitDoneThisArm;
private long reconnectInitialBackoffMillis =
CursorWebSocketSendLoop.DEFAULT_RECONNECT_INITIAL_BACKOFF_MILLIS;
private long reconnectMaxBackoffMillis =
@@ -2048,6 +2066,16 @@ public boolean isDeltaDictEnabledForTest() {
return deltaDictEnabled;
}
+ /**
+ * Whether the symbol-dictionary recycle is currently armed. Set by
+ * {@link #armIfEligible()} at the tail of every flush (and immediately by
+ * {@link #resetSymbolDictionary()} when no row or flush is in progress).
+ */
+ @TestOnly
+ public boolean isResetArmed() {
+ return resetArmed;
+ }
+
/** Resolved value of {@code symbol_dict_reset}. */
@TestOnly
public boolean isSymbolDictResetEnabled() {
@@ -2466,6 +2494,24 @@ public void reset() {
cachedTimestampNanosColumn = null;
}
+ /**
+ * Advisory request to start a fresh symbol-dictionary epoch. Sets
+ * {@link #manualResetRequested}; if no row is currently in progress and no
+ * flush is in flight ({@code pendingRowCount == 0}), re-evaluates arming
+ * immediately so a caller that requests a reset between batches does not
+ * have to wait for a later flush to observe {@code isResetArmed()}. A
+ * request made mid-batch is picked up by the next
+ * {@code resetTableBuffersAfterFlush} instead.
+ */
+ @Override
+ public void resetSymbolDictionary() {
+ checkNotClosed();
+ manualResetRequested = true;
+ if (pendingRowCount == 0) {
+ armIfEligible();
+ }
+ }
+
/**
* Register an async listener for connection-state transitions: initial
* connect, primary failover, endpoint attempt failures, the full address
@@ -4354,6 +4400,32 @@ private void resetTableBuffersAfterFlush() {
currentTableBufferSnapshotBytes = 0;
pendingRowCount = 0;
firstPendingRowTimeNanos = 0;
+ armIfEligible();
+ }
+
+ /**
+ * Re-evaluates whether the symbol-dictionary recycle should be armed:
+ * {@code resetEnabled} is on AND either the global dictionary has reached
+ * {@code resetThresholdSymbols} distinct entries or a caller requested a
+ * reset via {@link #resetSymbolDictionary()}. Deliberately ignores
+ * {@code deltaDictEnabled} -- a producer degraded to full self-sufficient
+ * frames still benefits from bounding its dictionary size, and a manual
+ * request is honoured regardless of mode.
+ *
+ *
+ * A throw at any step is caught, latches {@link #recycleFailure} (step 8),
+ * and rethrows: every frame that existed before this call was already
+ * proven acked, so no data is at risk, but the sender that made the throw
+ * observe a torn-down engine/loop refuses further use from here on --
+ * {@link #checkRecycleFailure()} enforces that at every later
+ * {@link #table(CharSequence)} and flush-family call.
+ */
+ private void recycleForDictReset() {
+ final long lastPublishedFsn = cursorEngine.publishedFsn(); // step 1
+ final int dictSizeAtSwap = globalSymbolDictionary.size();
+ final long startNanos = System.nanoTime();
+ try {
+ // step 2: close the loop - joins the I/O thread, closes the client
+ if (cursorSendLoop != null) {
+ cursorSendLoop.close();
+ cursorSendLoop = null;
+ }
+ client = null;
+ // step 3: fully-drained close of the engine - empties the slot.
+ // Holds NO logical slot lock here: close(true) unlinks the logical
+ // lock file (CursorSendEngine close javadoc).
+ cursorEngine.close();
+ cursorEngine = null;
+ // step 4: roll the external FSN base (-1 no-publish case adds 0)
+ rollFsnEpochBase(lastPublishedFsn);
+ // step 5: producer state swap - replace, don't clear()
+ globalSymbolDictionary = new GlobalSymbolDictionary();
+ sentMaxSymbolId = -1;
+ currentBatchMaxSymbolId = -1;
+ symbolDictEpoch++;
+ resetArmed = false;
+ manualResetRequested = false;
+ // step 6: rebuild the engine on the now-empty slot
+ cursorEngine = engineRebuildFactory.rebuild();
+ ownsCursorEngine = true;
+ deltaDictEnabled = cursorEngine.isDeltaDictEnabled();
+ // step 7: reconnect - rebuilds the loop with the rolled base
+ connected = false;
+ ensureConnected();
+ LOG.info("symbol dictionary recycled [epoch={}, dictSizeAtSwap={}, pauseMicros={}]",
+ symbolDictEpoch, dictSizeAtSwap, (System.nanoTime() - startNanos) / 1000L);
+ } catch (Throwable t) {
+ // step 8: terminal latch - everything was acked before step 2,
+ // so no data is at risk; the sender refuses further use.
+ recycleFailure = t;
+ LOG.error("symbol dictionary recycle failed; sender is now terminal "
+ + "[epoch={}, dictSizeAtSwap={}]", symbolDictEpoch, dictSizeAtSwap, t);
+ if (t instanceof LineSenderException) {
+ throw (LineSenderException) t;
+ }
+ throw new LineSenderException(t).put("symbol dictionary recycle failed");
+ }
+ }
+
/**
* Sends an empty QWP message without FLAG_DEFER_COMMIT to trigger
* the server-side commit of all previously deferred rows.
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java
new file mode 100644
index 00000000..9c08192b
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java
@@ -0,0 +1,448 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.std.Files;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE;
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * The symbol-dictionary recycle swap ({@code QwpWebSocketSender.table()}'s
+ * barrier hook + {@code recycleForDictReset()}): tears the cursor engine and
+ * I/O loop down once the ring is proven drained, replaces the producer's
+ * global symbol dictionary, and rebuilds the engine on the same (now-empty)
+ * slot before reconnecting -- all synchronously inside a single
+ * {@code table()} call.
+ */
+public class SymbolDictRecycleTest {
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ @Test
+ public void testRecycleAtEmptyBacklog() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("recycle-empty-backlog").toString();
+ RecycleHandler handler = new RecycleHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=2;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: batch must be acked before the recycle",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+ Assert.assertEquals(1, handler.connectionsAccepted.get());
+ Assert.assertEquals(0, ws.getSymbolDictEpochForTest());
+
+ // The ring is drained (everything acked) and no row is in
+ // progress, so this table() call must recycle synchronously.
+ // ensureConnected() performs the fresh WebSocket handshake
+ // synchronously as part of the swap, before any data is sent --
+ // handshakeCount (not the handler's own onBinaryMessage-driven
+ // counter, which only advances once a frame actually arrives)
+ // observes that handshake immediately.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertFalse("recycle must disarm", ws.isResetArmed());
+ Assert.assertEquals("recycle must open a fresh connection",
+ 2, server.handshakeCount());
+ Assert.assertEquals(1, ws.getSymbolDictEpochForTest());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-recycle batch must still get acked",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertTrue("post-recycle FSN must exceed pre-recycle FSN "
+ + "[fsn1=" + fsn1 + ", fsn2=" + fsn2 + ']',
+ fsn2 > fsn1);
+ }
+
+ Assert.assertEquals("exactly 2 connections total", 2, handler.connectionsAccepted.get());
+ Assert.assertEquals("connection 2's first data frame must carry deltaStart == 0 "
+ + "(a fresh, empty dictionary)",
+ 0, handler.conn2FirstFrameDeltaStart);
+ Assert.assertEquals("connection 2's dictionary must hold only the post-recycle "
+ + "symbols, not a, b",
+ Arrays.asList("c", "d"), handler.dictFor(2));
+ }
+ });
+ }
+
+ @Test
+ public void testPostRecycleSlotContents() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("recycle-slot-contents").toString();
+ String slot = Paths.get(sfDir, "default").toString();
+ try (TestWebSocketServer server = ackingServer()) {
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=2;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue(ws.isResetArmed());
+
+ CursorSendEngine before = ws.getCursorEngineForTesting();
+
+ // Synchronous swap: by the time table() returns, the old engine
+ // is gone and a fresh one is rebuilt and reconnected. Asserting
+ // right here needs no polling -- there is no window to race.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+
+ CursorSendEngine after = ws.getCursorEngineForTesting();
+ Assert.assertNotSame("recycle must swap in a fresh engine instance",
+ before, after);
+ Assert.assertTrue("the rebuilt engine must create its own initial segment",
+ Files.exists(slot + "/sf-initial.sfa"));
+ Assert.assertEquals("post-recycle dictionary must start empty, not continue "
+ + "the outgoing epoch's 2 entries",
+ 0, after.getPersistedSymbolDict().size());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+
+ // The new epoch's persisted dictionary holds exactly c, d --
+ // proof it is a genuinely fresh dictionary, not a, b continued.
+ Assert.assertEquals("post-recycle dictionary must hold only the new epoch's "
+ + "symbols",
+ 2, after.getPersistedSymbolDict().size());
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testRecycleUnderDurableAck() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("recycle-durable-ack").toString();
+ DurableAckHandler handler = new DurableAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler, true)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=2;request_durable_ack=on;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: the batch must be DURABLY acked before the recycle "
+ + "-- isRingDrained() reads the durable-ack-gated watermark",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue(ws.isResetArmed());
+ Assert.assertEquals(1, handler.connectionsAccepted.get());
+
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertFalse("recycle must disarm", ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpochForTest());
+ Assert.assertEquals("recycle must open a fresh connection",
+ 2, server.handshakeCount());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-recycle batch must still get durably acked on the "
+ + "fresh connection",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertTrue(fsn2 > fsn1);
+ }
+ }
+ });
+ }
+
+ /**
+ * Exercises the same code path {@code engineRebuildFactory.rebuild()} calls
+ * in production ({@code LineSenderBuilder.constructEngineOnSlot}) -- that
+ * method is package-private to {@code io.questdb.client} and unreachable
+ * directly from this package, so this observes its result through the
+ * sender's own {@code @TestOnly} engine accessor instead of calling it in
+ * isolation.
+ */
+ @Test
+ public void testFactoryRebuildsOnEmptySlot() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("recycle-factory-rebuild").toString();
+ try (TestWebSocketServer server = ackingServer()) {
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=2;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue(ws.isResetArmed());
+
+ sender.table("t");
+
+ CursorSendEngine rebuilt = ws.getCursorEngineForTesting();
+ Assert.assertTrue("a freshly rebuilt slot must support delta encoding",
+ rebuilt.isDeltaDictEnabled());
+ Assert.assertFalse("a freshly emptied slot has nothing to recover",
+ rebuilt.wasRecoveredFromDisk());
+ Assert.assertEquals(-1L, rebuilt.recoveredMaxSymbolId());
+ Assert.assertEquals("a freshly rebuilt engine has published nothing yet",
+ -1L, rebuilt.publishedFsn());
+ }
+ }
+ });
+ }
+
+ /**
+ * A step-6 rebuild failure (the {@link QwpWebSocketSender.EngineRebuildFactory}
+ * itself throwing) must latch the sender terminal rather than leaving it in
+ * the torn-down state the swap's earlier steps produced. Uses
+ * {@code setEngineRebuildFactory} directly to inject the fault -- the real
+ * factory has no seam for a custom {@code FilesFacade} (it always goes
+ * through {@code LineSenderBuilder.constructEngineOnSlot} against the real
+ * filesystem), and this isolates the exception-path behaviour under test
+ * from any particular failure cause.
+ */
+ @Test
+ public void testFailedRebuildLatchesTerminal() throws Exception {
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = ackingServer()) {
+ try (Sender sender = Sender.fromConfig(cfg(server))) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ RuntimeException fault = new RuntimeException("injected engine rebuild fault");
+ ws.setEngineRebuildFactory(() -> {
+ throw fault;
+ });
+
+ // pendingRowCount == 0 -> resetSymbolDictionary() arms immediately.
+ sender.resetSymbolDictionary();
+ Assert.assertTrue(ws.isResetArmed());
+
+ LineSenderException triggering = null;
+ try {
+ sender.table("t");
+ Assert.fail("expected the triggering table() call to throw");
+ } catch (LineSenderException e) {
+ triggering = e;
+ }
+ Assert.assertNotNull(triggering);
+ Assert.assertSame("the latched failure must be the exact rebuild fault",
+ fault, triggering.getCause());
+
+ // Every subsequent table()/flush-family call must rethrow --
+ // never touch the torn-down (null cursorEngine/cursorSendLoop) state.
+ assertRethrowsWithCause(fault, () -> sender.table("t"));
+ assertRethrowsWithCause(fault, sender::flush);
+ assertRethrowsWithCause(fault, sender::flushAndGetSequence);
+ assertRethrowsWithCause(fault, () -> sender.drain(0));
+ assertRethrowsWithCause(fault, () -> sender.awaitAckedFsn(0, 0));
+
+ // close() must still work despite the latched terminal failure.
+ sender.close();
+ }
+ }
+ });
+ }
+
+ private static TestWebSocketServer ackingServer() throws Exception {
+ TestWebSocketServer server = new TestWebSocketServer(new AckAllHandler());
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ return server;
+ }
+
+ private static void assertRethrowsWithCause(Throwable expectedCause, ThrowingRunnable action)
+ throws Exception {
+ try {
+ action.run();
+ Assert.fail("expected a latched LineSenderException to rethrow");
+ } catch (LineSenderException e) {
+ Assert.assertSame("a latched terminal sender must keep rethrowing the same cause",
+ expectedCause, e.getCause());
+ }
+ }
+
+ private static String cfg(TestWebSocketServer server) {
+ return "ws::addr=localhost:" + server.getPort() + ";";
+ }
+
+ private static byte[] buildDurableAckFrame(String tableName, long seqTxn) {
+ byte[] name = tableName.getBytes(StandardCharsets.UTF_8);
+ ByteBuffer bb = ByteBuffer.allocate(1 + 2 + 2 + name.length + 8).order(ByteOrder.LITTLE_ENDIAN);
+ bb.put((byte) 0x02); // STATUS_DURABLE_ACK
+ bb.putShort((short) 1); // tableCount
+ bb.putShort((short) name.length);
+ bb.put(name);
+ bb.putLong(seqTxn);
+ return bb.array();
+ }
+
+ private static byte[] buildOkFrame(String tableName, long wireSeq, long seqTxn) {
+ byte[] name = tableName.getBytes(StandardCharsets.UTF_8);
+ ByteBuffer bb = ByteBuffer.allocate(1 + 8 + 2 + 2 + name.length + 8).order(ByteOrder.LITTLE_ENDIAN);
+ bb.put((byte) 0x00); // STATUS_OK
+ bb.putLong(wireSeq);
+ bb.putShort((short) 1); // tableCount
+ bb.putShort((short) name.length);
+ bb.put(name);
+ bb.putLong(seqTxn);
+ return bb.array();
+ }
+
+ @FunctionalInterface
+ private interface ThrowingRunnable {
+ void run() throws Exception;
+ }
+
+ /** ACKs every frame it receives; does not otherwise inspect the wire. */
+ private static class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ try {
+ client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Immediately follows every OK ack with a durable ack for the same
+ * transaction, so a durable-ack-mode sender's {@code ackedFsn} advances
+ * without a separate release phase. Counters reset per connection, since
+ * the recycle's fresh loop restarts its own wire sequence at 0.
+ */
+ private static class DurableAckHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private static final String TABLE_NAME = "t";
+ final AtomicInteger connectionsAccepted = new AtomicInteger();
+ private TestWebSocketServer.ClientHandler currentClient;
+ private long nextSeqTxn;
+ private long nextWireSeq;
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ if (currentClient != client) {
+ currentClient = client;
+ connectionsAccepted.incrementAndGet();
+ nextWireSeq = 0;
+ nextSeqTxn = 0;
+ }
+ try {
+ long wireSeq = nextWireSeq++;
+ long seqTxn = nextSeqTxn++;
+ client.sendBinary(buildOkFrame(TABLE_NAME, wireSeq, seqTxn));
+ client.sendBinary(buildDurableAckFrame(TABLE_NAME, seqTxn));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Reconstructs each connection's per-connection delta dictionary (mirrors
+ * {@code DeltaDictCatchUpTest.CatchUpHandler}) and records the delta-start
+ * id of connection 2's first non-empty data frame.
+ */
+ private static class RecycleHandler implements TestWebSocketServer.WebSocketServerHandler {
+ final AtomicInteger connectionsAccepted = new AtomicInteger();
+ volatile int conn2FirstFrameDeltaStart = -1;
+ private boolean conn2SeenFirstDataFrame;
+ private TestWebSocketServer.ClientHandler currentClient;
+ private final List
+ * 1. lastPublishedFsn = cursorEngine.publishedFsn()
+ * 2. close cursorSendLoop (I/O thread + client)
+ * 3. cursorEngine.close() -- FULLY DRAINED (the barrier only fires the swap once
+ * isRingDrained() is true), so this unlinks every *.sfa, the ack watermark,
+ * the persisted dictionary and the logical slot lock, leaving the slot empty.
+ * 4. rollFsnEpochBase(lastPublishedFsn)
+ * 5. producer state swap: fresh GlobalSymbolDictionary, sentMaxSymbolId=-1,
+ * symbolDictEpoch++, resetArmed=false
+ * 6. cursorEngine = engineRebuildFactory.rebuild() -- a brand-new CursorSendEngine
+ * on the now-empty slot (fresh .lock/.ack-watermark/.symbol-dict/segments)
+ * 7. reconnect (ensureConnected())
+ * 8. (catch) recycleFailure latch on any throw
+ *
+ * This suite pins what a restarted sender recovers if the process dies at each
+ * of four points around that sequence, per the phase-11 brief:
+ *
+ *
+ *
+ * Why these are simulated, not paused mid-sequence
+ * {@code recycleForDictReset()} runs synchronously inside one {@code table()}
+ * call with no external hook between its steps, so a test cannot literally
+ * suspend a live sender between step 3 and step 6. And unlike
+ * {@code CursorSendEngineCrashConsistencyTest}'s bare {@code CursorSendEngine} +
+ * fault-injecting {@code FilesFacade}, a {@code Sender} built through the public
+ * API (as production always does) has no seam for a custom {@code FilesFacade}
+ * -- {@code LineSenderBuilder.constructEngineOnSlot} always goes through the real
+ * filesystem. Each arm below instead constructs the exact on-disk image a crash
+ * at that point would leave, using only real production code paths plus
+ * filesystem-level fixtures already established elsewhere in this test suite
+ * ({@code DeltaDictRecoveryTest}'s {@code writeAckWatermark}, {@code
+ * RecoveryReplayTest}'s close-fast-with-a-silent-server idiom):
+ *
+ *
+ *
+ * (b) and (c) are NOT the same recoverable state
+ * Both look empty of data and both replay nothing, but they are not
+ * byte-identical on disk, and a restarted engine can tell them apart. Arm (b)'s
+ * directory holds nothing this engine ever created (no manifest, no segment).
+ * Arm (c)'s directory holds the fresh rebuild's own {@code sf-manifest.bin}
+ * (boundaries collapsed at 0) and its zero-frame {@code sf-initial.sfa} /
+ * {@code sf-...0000.sfa} pair. {@code SegmentRing.recover()}'s manifest branch
+ * (the {@code chain.size() == 0} check) accepts a manifest whose {@code
+ * headBase == activeBase} alongside a same-based, zero-frame active segment as
+ * a RECOVERED (if empty) chain -- it does not collapse that case to EMPTY the
+ * way a manifest with NO segment files at all does. So {@code
+ * wasRecoveredFromDisk()} comes back {@code false} for (b) and {@code true} for
+ * (c): the pinned, distinguishing observable between the two, asserted
+ * explicitly below instead of writing two assertion-for-assertion duplicate
+ * tests.
+ *
+ * Oracle
+ * Every arm asserts the same three things about the RECOVERED sender: it keeps
+ * ingesting after recovery; the symbols it and its predecessor registered are
+ * exactly and correctly reconstructable from the wire (each fresh server
+ * handler rebuilds the per-connection delta dictionary via {@link
+ * QwpWireTestUtils#accumulateDeltaDictionary}); and no data (table-carrying)
+ * frame is delivered more than the at-least-once contract allows (each handler
+ * also counts data frames, so a spurious re-send shows up as an unexpected
+ * count).
+ */
+public class SymbolDictRecycleCrashWindowsTest {
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ /**
+ * Arm (a): before step 2. The pre-recycle epoch has flushed a fully-acked
+ * batch but the barrier that starts {@code recycleForDictReset()} has not
+ * fired yet -- the crash lands with an intact, acked epoch-0 slot on disk.
+ * Recovery must find that residue, recognize it as already acked (nothing
+ * to replay) and resume the SAME (epoch-0) dictionary rather than starting
+ * fresh.
+ */
+ @Test
+ public void testCrashBeforeRecycleStartsRecoversAckedResidueOnly() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("crash-a-pre-swap").toString();
+ String slot = Paths.get(sfDir, "default").toString();
+ long fsn;
+ SilentHandler crashedHandler = new SilentHandler();
+ try (TestWebSocketServer crashed = startedServer(crashedHandler)) {
+ String cfg = "ws::addr=localhost:" + crashed.getPort() + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=2;close_flush_timeout_millis=0;";
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ fsn = sender.flushAndGetSequence();
+ Assert.assertTrue("threshold=2 crossed by a, b must arm the recycle "
+ + "immediately (arming does not wait for an ack)",
+ ws.isResetArmed());
+ // close() below hits close_flush_timeout_millis=0 against a server
+ // that never acks: not fully drained, so finishClose does NOT unlink
+ // the segment/manifest/dictionary. It still releases the slot flock
+ // unconditionally (CursorSendEngine.finishClose's retryFlockReleaseIfReady
+ // runs in the outer finally regardless of drain state), so the
+ // successor below can acquire the slot cleanly. No recycle step ever
+ // ran -- the crash is strictly "before step 2".
+ }
+ }
+ // Retroactively declare the flush's only fsn acked: simulates the ack
+ // having actually landed moments before the process died, mirroring
+ // DeltaDictRecoveryTest#writeAckWatermark.
+ writeAckWatermark(slot, fsn);
+
+ AckAllHandler freshHandler = new AckAllHandler();
+ try (TestWebSocketServer fresh = startedServer(freshHandler)) {
+ String cfg2 = "ws::addr=localhost:" + fresh.getPort() + ";sf_dir=" + sfDir + ";";
+ try (Sender successor = Sender.fromConfig(cfg2)) {
+ QwpWebSocketSender ws2 = (QwpWebSocketSender) successor;
+ CursorSendEngine recovered = ws2.getCursorEngineForTesting();
+ Assert.assertTrue("acked residue on disk must be recognized as recovered",
+ recovered.wasRecoveredFromDisk());
+ Assert.assertTrue("the watermark stamp must seed ackedFsn at least up to "
+ + "the only published fsn -- nothing left to replay",
+ recovered.ackedFsn() >= fsn);
+ Assert.assertTrue("the recovered producer must resume epoch 0's a, b "
+ + "dictionary, not restart at -1",
+ recovered.recoveredMaxSymbolId() >= 1);
+
+ successor.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ long fsn2 = successor.flushAndGetSequence();
+ Assert.assertTrue("recovered sender must keep ingesting",
+ successor.awaitAckedFsn(fsn2, 5_000));
+ }
+ Assert.assertEquals("no replay of the already-acked a, b frame -- exactly "
+ + "the one new post-recovery frame reaches the server",
+ 1, freshHandler.dataFrameCount());
+ Assert.assertEquals("the reconstructed dictionary is epoch 0's a, b plus "
+ + "the new c, correct and in order",
+ Arrays.asList("a", "b", "c"), freshHandler.dict());
+ }
+ });
+ }
+
+ /**
+ * Arm (b): between steps 3 and 6. See the class javadoc for how this is
+ * constructed (drive a real recycle to completion, then close before any
+ * flush touches the rebuilt engine -- the "never published" fully-drained
+ * branch empties it exactly like step 3 alone would have).
+ */
+ @Test
+ public void testCrashBetweenEngineCloseAndRebuildRecoversAsFreshStart() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("crash-b-mid-swap").toString();
+ String slot = Paths.get(sfDir, "default").toString();
+ AckAllHandler crashedHandler = new AckAllHandler();
+ try (TestWebSocketServer crashed = startedServer(crashedHandler)) {
+ String cfg = "ws::addr=localhost:" + crashed.getPort() + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=2;";
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue(ws.isResetArmed());
+
+ // A bare table() call fires the barrier hook before any row is
+ // constructed (mirrors SymbolDictRecycleTest#testFactoryRebuildsOnEmptySlot),
+ // so steps 1-7 run to completion with nothing left pending to flush.
+ sender.table("t");
+ Assert.assertFalse("recycle must disarm", ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpochForTest());
+ // close() below: the fresh engine has published nothing, so
+ // finishClose's "never published" branch is fully-drained too --
+ // it unlinks everything step 6 just created, leaving the slot as
+ // empty as it was right after step 3 alone emptied the OLD engine.
+ }
+ }
+
+ AckAllHandler freshHandler = new AckAllHandler();
+ try (TestWebSocketServer fresh = startedServer(freshHandler)) {
+ String cfg2 = "ws::addr=localhost:" + fresh.getPort() + ";sf_dir=" + sfDir + ";";
+ try (Sender successor = Sender.fromConfig(cfg2)) {
+ QwpWebSocketSender ws2 = (QwpWebSocketSender) successor;
+ CursorSendEngine recovered = ws2.getCursorEngineForTesting();
+ Assert.assertFalse("an empty slot has nothing to recover -- see the "
+ + "class javadoc for why this differs from arm (c)",
+ recovered.wasRecoveredFromDisk());
+ Assert.assertEquals(-1L, recovered.recoveredMaxSymbolId());
+ Assert.assertEquals(-1L, recovered.publishedFsn());
+
+ successor.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn = successor.flushAndGetSequence();
+ Assert.assertTrue(successor.awaitAckedFsn(fsn, 5_000));
+ }
+ Assert.assertEquals("exactly the one post-crash frame reaches the server",
+ 1, freshHandler.dataFrameCount());
+ Assert.assertEquals("the new epoch's dictionary tiles from id 0 -- none of "
+ + "the pre-crash a, b survive",
+ Arrays.asList("d"), freshHandler.dict());
+ }
+ Assert.assertFalse("the slot dir must hold nothing from the emptied-and-abandoned "
+ + "rebuild once the successor's own recovery has cleaned up any "
+ + "stale (collapsed-boundary) manifest",
+ Files.exists(slot + "/sf-manifest.bin"));
+ });
+ }
+
+ /**
+ * Arm (c): after step 7, pre-first-flush. See the class javadoc for how
+ * this is constructed (snapshot the freshly-rebuilt slot before closing,
+ * close for real so nothing leaks, then restore the snapshot on top of the
+ * vacated directory) and for why this recovers differently from arm (b)
+ * despite carrying no data either.
+ */
+ @Test
+ public void testCrashAfterRebuildBeforeFirstFlushRecoversAsRecoveredButEmpty() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("crash-c-post-swap").toString();
+ String slot = Paths.get(sfDir, "default").toString();
+ Map(b) and (c) are NOT the same recoverable state
* Both look empty of data and both replay nothing, but they are not
* byte-identical on disk, and a restarted engine can tell them apart. Arm (b)'s
- * directory holds nothing this engine ever created (no manifest, no segment).
- * Arm (c)'s directory holds the fresh rebuild's own {@code sf-manifest.bin}
- * (boundaries collapsed at 0) and its zero-frame {@code sf-initial.sfa} /
- * {@code sf-...0000.sfa} pair. {@code SegmentRing.recover()}'s manifest branch
- * (the {@code chain.size() == 0} check) accepts a manifest whose {@code
- * headBase == activeBase} alongside a same-based, zero-frame active segment as
- * a RECOVERED (if empty) chain -- it does not collapse that case to EMPTY the
- * way a manifest with NO segment files at all does. So {@code
+ * directory holds nothing this engine ever created -- no manifest, no segment.
+ * (The crashed sender's own fully-drained close already removed {@code
+ * sf-manifest.bin} along with the last segment, so recovery finds NO {@code
+ * .sfa} files and NO manifest, and falls straight through to {@code
+ * Recovery.empty()}.) Arm (c)'s directory holds the fresh rebuild's own
+ * {@code sf-manifest.bin} (boundaries collapsed at 0) and its zero-frame
+ * {@code sf-initial.sfa} / {@code sf-...0000.sfa} pair. {@code
+ * SegmentRing.recover()}'s manifest branch (the {@code chain.size() == 0}
+ * check) accepts a manifest whose {@code headBase == activeBase} alongside a
+ * same-based, zero-frame active segment as a RECOVERED (if empty) chain -- a
+ * different branch entirely from the one arm (b) falls through to. So {@code
* wasRecoveredFromDisk()} comes back {@code false} for (b) and {@code true} for
* (c): the pinned, distinguishing observable between the two, asserted
* explicitly below instead of writing two assertion-for-assertion duplicate
@@ -152,6 +164,16 @@
*/
public class SymbolDictRecycleCrashWindowsTest {
+ /**
+ * The exact file set a freshly-rebuilt (never-flushed) engine's own slot
+ * holds -- matches {@code SymbolDictRecycleTest#testPostRecycleSlotContents}'s
+ * {@code freshSlotFiles}. Shared by arm (c)'s pre-snapshot wait and its
+ * post-restore assertion so the two can never drift apart.
+ */
+ private static final List
*
- * A throw in steps 1-6 is caught, latches {@link #recycleFailure} (step 8),
+ * A throw in steps 2-6 is caught, latches {@link #recycleFailure},
* and rethrows: every frame that existed before this call was already
* proven acked, so no data is at risk, but the sender that made the throw
* observe a half-swapped engine/loop refuses further use from here on --
* {@link #checkRecycleFailure()} enforces that at every later
- * {@link #table(CharSequence)} and flush-family call.
+ * {@link #table(CharSequence)} and flush-family call. (Step 1 is a pair
+ * of plain reads and runs before the latching try.)
+ *
*
- * A throw in steps 2-6 is caught, latches {@link #recycleFailure},
- * and rethrows: every frame that existed before this call was already
- * proven acked, so no data is at risk, but the sender that made the throw
- * observe a half-swapped engine/loop refuses further use from here on --
- * {@link #checkRecycleFailure()} enforces that at every later
- * {@link #table(CharSequence)} and flush-family call. (Step 1 is a pair
- * of plain reads and runs before the latching try.)
+ * The producer-visible swap (dictionary, counters, epoch) commits only
+ * once a fresh engine stands on the emptied slot: a throw in steps 2-6
+ * is caught before that point leaves the counters un-bumped, latches
+ * {@link #recycleFailure}, and rethrows -- every frame that existed
+ * before this call was already proven acked, so no data is at risk, but
+ * the sender that made the throw observe a half-swapped engine/loop
+ * refuses further use from here on -- {@link #checkRecycleFailure()}
+ * enforces that at every later {@link #table(CharSequence)} and
+ * flush-family call. An {@link Error} (OOM/SOE/linkage) passes through
+ * this catch untouched, neither latched nor wrapped -- it is not a
+ * recycle verdict. (Step 1 is a pair of plain reads and runs before the
+ * latching try.)
*
+ *
+ * Both tests doctor a slot directly at the engine level and then point a live
+ * sender's rebuild factory at it, so the verdict is driven by real on-disk
+ * state rather than by a mocked engine.
+ *
* Snapshot accessor: for a bounded blocking wait, use
* {@link #awaitAckedFsn(long, long)}.
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index 1b0aca84..67fee551 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -1984,21 +1984,26 @@ public QwpWebSocketSender geoHashColumn(CharSequence columnName, CharSequence va
/**
* Highest FSN that has been server-acknowledged. Rejections never advance
- * the watermark. {@code -1} if
- * the I/O loop has not yet started or no batch has been published.
+ * the watermark. Returns {@code -1} only while nothing has ever been
+ * published in this sender's lifetime. After a symbol-dictionary recycle
+ * the accessor keeps reporting the last pre-swap durable watermark until
+ * the fresh epoch publishes -- it never collapses back to {@code -1}.
*
* Snapshot accessor: for a bounded blocking wait, use
* {@link #awaitAckedFsn(long, long)}.
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index 67fee551..e7d66473 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -361,8 +361,9 @@ public class QwpWebSocketSender implements Sender {
// does not reset CursorWebSocketSendLoop's own hasEverConnected back to
// false -- which would wrongly re-arm its startup-terminal
// classification (endpointPolicyFailureIsTerminal()) and misclassify
- // wasEverConnected() for the whole post-recycle outage window.
- private boolean hasLoopEverConnected;
+ // wasEverConnected() for the whole post-recycle outage window. Volatile:
+ // wasEverConnected() is consulted from the error-dispatcher daemon.
+ private volatile boolean hasLoopEverConnected;
// FSN of the last commit-bearing (non-FLAG_DEFER_COMMIT) frame this session
// published, or -1 when none. Frames above it are deferred and uncommitted:
// the server withholds their acks by design (their rows are rolled back on
@@ -1985,9 +1986,11 @@ public QwpWebSocketSender geoHashColumn(CharSequence columnName, CharSequence va
/**
* Highest FSN that has been server-acknowledged. Rejections never advance
* the watermark. Returns {@code -1} only while nothing has ever been
- * published in this sender's lifetime. After a symbol-dictionary recycle
- * the accessor keeps reporting the last pre-swap durable watermark until
- * the fresh epoch publishes -- it never collapses back to {@code -1}.
+ * published in this sender's lifetime. On a live sender the value never
+ * collapses back to {@code -1}: after a symbol-dictionary recycle the
+ * accessor keeps reporting the last pre-swap durable watermark until the
+ * fresh epoch publishes. (After {@code close()} the reading is
+ * unspecified.)
*
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java
index 76919f48..d95e0872 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleStep7FaultTest.java
@@ -55,13 +55,12 @@ public class SymbolDictRecycleStep7FaultTest {
/**
* Pins that a step-7 (reconnect) failure does NOT latch the sender
* terminal: the swap has committed, the sender is coherent and merely
- * disconnected, and the next send retries the deferred setup. The test
- * the PR body credited was deleted by commit 6793928a.
+ * disconnected, and the next send retries the deferred setup.
*/
@Test
public void testStep7FailureDoesNotLatchAndRecovers() throws Exception {
assertMemoryLeak(() -> {
- String sfDir = temporaryFolder.newFolder("step7-c5").getAbsolutePath();
+ String sfDir = temporaryFolder.newFolder("step7-reconnect").getAbsolutePath();
try (TestWebSocketServer server = ackingServer()) {
try (Sender sender = Sender.fromConfig(cfg(server, sfDir))) {
QwpWebSocketSender ws = (QwpWebSocketSender) sender;
@@ -105,7 +104,7 @@ public void testStep7FailureDoesNotLatchAndRecovers() throws Exception {
@Test
public void testPostFailedReconnectDeltaCoversStagedSymbolIds() throws Exception {
assertMemoryLeak(() -> {
- String sfDir = temporaryFolder.newFolder("step7-c4").getAbsolutePath();
+ String sfDir = temporaryFolder.newFolder("step7-staged-ids").getAbsolutePath();
CapturingAckHandler handler = new CapturingAckHandler();
try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
server.start();
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java
index 630a778d..e7aeea5d 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleTest.java
@@ -25,6 +25,7 @@
package io.questdb.client.test.cutlass.qwp.client;
import io.questdb.client.Sender;
+import io.questdb.client.SenderError;
import io.questdb.client.SenderErrorHandler;
import io.questdb.client.cutlass.line.LineSenderException;
import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
@@ -33,7 +34,10 @@
import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher;
import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher;
import io.questdb.client.std.Files;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Unsafe;
import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.TestUtils;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -450,6 +454,83 @@ public CursorSendEngine rebuild(SenderErrorHandler liveHandler) {
});
}
+ /**
+ * The builder's rebuild factory ({@code Sender.build()}) is the half that
+ * actually forwards the live handler into {@code constructEngineOnSlot} ->
+ * {@code quarantineTornSlot}, and that hand-off is only observable when a
+ * rebuild really has a slot to set aside: the notification is dispatched
+ * synchronously from inside the rebuild, to whatever handler it was handed.
+ * Step 3's fully-drained close leaves the slot empty, so the fixture plants
+ * {@code SegmentSkipQuarantineTest}'s tainted slot -- several never-acked
+ * segments with the oldest one's magic overwritten, so recovery must skip
+ * it -- into that empty slot just before the real factory runs. The sender
+ * is built with no error handler at all, so a regression that forwarded the
+ * BUILD-time handler instead of the live one would leave the recycle green
+ * and the data-loss notification nowhere.
+ */
+ @Test
+ public void testRebuildTimeQuarantineReachesTheHandlerInstalledAfterBuild() throws Exception {
+ assertMemoryLeak(() -> {
+ String taintedSfDir = temporaryFolder.newFolder("rebuild-quarantine-tainted").getAbsolutePath();
+ writeSlotWithCorruptedOldestSegment(taintedSfDir);
+ final String taintedSlot = taintedSfDir + "/default";
+
+ String sfDir = temporaryFolder.newFolder("rebuild-quarantine-live").getAbsolutePath();
+ final String slotPath = sfDir + "/default";
+ final List