+ * Also a permanent no-op on a sender configured with + * {@code symbol_dict_reset=off} ({@link LineSenderBuilder#symbolDictReset(boolean)}): + * that knob gates the arming path this request feeds, so the request is + * accepted and never acted on. + *
+ * Call on the producing thread only: like every other {@code Sender}
+ * method, this mutates producer-side state and is not thread-safe.
+ */
+ default void resetSymbolDictionary() {
+ }
+
/**
* Clear the internal buffers, discarding any unsent data.
*
@@ -1087,6 +1109,9 @@ final class LineSenderBuilder {
private int maxFrameRejections = PARAMETER_NOT_SET_EXPLICITLY;
private long poisonMinEscalationWindowMillis = PARAMETER_NOT_SET_EXPLICITLY;
private long catchUpCapGapMinEscalationWindowMillis = PARAMETER_NOT_SET_EXPLICITLY;
+ private boolean symbolDictReset = QwpWebSocketSender.DEFAULT_SYMBOL_DICT_RESET_ENABLED;
+ private int symbolDictResetThreshold = PARAMETER_NOT_SET_EXPLICITLY;
+ private long symbolDictResetMaxWaitMillis = PARAMETER_NOT_SET_EXPLICITLY;
private String httpPath;
private String httpSettingsPath;
private int httpTimeout = PARAMETER_NOT_SET_EXPLICITLY;
@@ -1545,6 +1570,12 @@ public Sender build() {
catchUpCapGapMinEscalationWindowMillis != PARAMETER_NOT_SET_EXPLICITLY
? catchUpCapGapMinEscalationWindowMillis
: CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS;
+ int actualSymbolDictResetThreshold = symbolDictResetThreshold != PARAMETER_NOT_SET_EXPLICITLY
+ ? symbolDictResetThreshold
+ : QwpWebSocketSender.DEFAULT_SYMBOL_DICT_RESET_THRESHOLD_SYMBOLS;
+ long actualSymbolDictResetMaxWaitMillis = symbolDictResetMaxWaitMillis != PARAMETER_NOT_SET_EXPLICITLY
+ ? symbolDictResetMaxWaitMillis
+ : QwpWebSocketSender.DEFAULT_SYMBOL_DICT_RESET_MAX_WAIT_MILLIS;
// sfDir is the parent (group root); the actual slot lives
// under sfDir/senderId. This is what the engine sees — the
@@ -1599,77 +1630,21 @@ public Sender build() {
try (SlotLock logicalSlotLock = slotPath == null
? null
: SlotLock.acquireLogical(slotPath)) {
- // The constructor's own recovery seed can also fail terminally, and
- // not only as UnreplayableSlotException: when SegmentRing.openExisting
- // had to skip an unreadable segment it throws SfRecoveryException (it
- // constructs UnreplayableSlotException nowhere), and where it cannot
- // even prove the chain's identity -- no manifest -- it quarantines the
- // corrupt files and returns an EMPTY recovery rather than refusing.
- // Either way the frame range cannot be shown already-acked, so recovery
- // sets the slot aside rather than risk seeding the ack cursor past
- // frames that were never delivered. All three types below are load
- // bearing; narrowing this catch to UnreplayableSlotException would
- // restore the permanent build() brick for the segment-skip case. That verdict gets
- // the exact same quarantine-and-continue treatment as the connect()-time
- // verdict below -- constructing cursorEngine is not inside the loop below,
- // so a throw here would otherwise escape build() entirely, uncaught.
- // quarantineTornSlot(null, ...) renames the WHOLE slot directory aside
- // (not just the unreadable segment file) before building the replacement
- // at the original slotPath, so the replacement starts on a genuinely empty
- // directory with nothing left to skip -- it cannot throw the same way
- // twice, which is what makes looping unnecessary here.
- boolean quarantined = false;
- CursorSendEngine cursorEngine;
- try {
- try {
- cursorEngine = new CursorSendEngine(
- slotPath, actualSfMaxSegmentBytes,
- actualSfMaxTotalBytes, actualSfAppendDeadlineNanos,
- actualSfSyncIntervalNanos);
- } catch (SfSanitizedResidueException first) {
- // NOT terminal, and it must be intercepted ahead of its
- // SfRecoveryException parent below. Recovery durably zeroed
- // proven-dead sealed residue BEFORE failing closed, so the
- // chain on disk is already healed: quarantining here would
- // set aside a slot whose backlog replays perfectly. Retry
- // once over the healed chain; a repeat is genuine and takes
- // the terminal arm.
- LOG.info("sf slot {}: sealed residue sanitized during recovery ({}); "
- + "retrying over the healed chain",
- slotPath, first.getMessage());
- cursorEngine = new CursorSendEngine(
- slotPath, actualSfMaxSegmentBytes,
- actualSfMaxTotalBytes, actualSfAppendDeadlineNanos,
- actualSfSyncIntervalNanos);
- }
- } catch (UnreplayableSlotException | SfRecoveryException
- | MmapSegmentCorruptionException e) {
- // The terminal recovery verdicts, and the only ones build()
- // sets a slot aside for. UnreplayableSlotException says the
- // symbol dictionary cannot be rebuilt from any source;
- // SfRecoveryException and MmapSegmentCorruptionException say
- // the durable chain itself is proven corrupt or incomplete.
- // None of the three clears on a retry, and senderId is stable
- // with a not-fully-drained slot retained on close -- so
- // without this arm every restart re-recovers the same slot and
- // throws again, and the application cannot construct a Sender
- // at all, not even to BUFFER new rows.
- //
- // Deliberately NOT catching plain MmapSegmentException or
- // SfOperationalException: those are operational (EMFILE,
- // ENOMEM, an unreadable-but-possibly-intact file). Aborting
- // startup on them is correct; quarantining on them would
- // convert a transient into the permanent loss of a healthy
- // slot's durable frames.
- if (slotPath == null) {
- throw e;
- }
- quarantined = true;
- cursorEngine = quarantineTornSlot(
- null, e, sfDir, senderId, slotPath, actualSfMaxSegmentBytes,
- actualSfMaxTotalBytes, actualSfAppendDeadlineNanos,
- actualSfSyncIntervalNanos, errorHandler);
- }
+ // Recovery-verdict handling lives in constructEngineOnSlotLocked.
+ ConstructedEngine constructed = constructEngineOnSlotLocked(
+ sfDir, senderId, slotPath,
+ actualSfMaxSegmentBytes, actualSfMaxTotalBytes,
+ actualSfAppendDeadlineNanos, actualSfSyncIntervalNanos,
+ errorHandler);
+ // Seeded from constructEngineOnSlotLocked's own verdict, not
+ // hardcoded false: if construction already quarantined this
+ // slot, the connect loop below must count that as the one
+ // quarantine build() allows per attempt (see its "quarantined
+ // || slotPath == null" guard) rather than starting blind and
+ // risking a second quarantineTornSlot pass on what should be
+ // an immediate close-and-rethrow.
+ boolean quarantined = constructed.quarantined;
+ CursorSendEngine cursorEngine = constructed.engine;
int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY
? errorInboxCapacity
: io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY;
@@ -1712,7 +1687,10 @@ public Sender build() {
actualConnectionListenerInboxCapacity,
actualMaxFrameRejections,
actualPoisonMinEscalationWindowMillis,
- actualCatchUpCapGapMinEscalationWindowMillis
+ actualCatchUpCapGapMinEscalationWindowMillis,
+ symbolDictReset,
+ actualSymbolDictResetThreshold,
+ actualSymbolDictResetMaxWaitMillis
);
} catch (UnreplayableSlotException e) {
// The one failure build() recovers from. The slot's frames reference ids
@@ -1765,6 +1743,24 @@ public Sender build() {
// dispatcher daemon, drainer pool, microbatch buffers and
// WebSocketClient inside the abandoned `connected`.
connected.setTransactional(transactional);
+ final String rebuildSfDir = sfDir;
+ final String rebuildSenderId = senderId;
+ final SenderErrorHandler buildTimeHandler = errorHandler;
+ connected.setEngineRebuildFactory(new QwpWebSocketSender.EngineRebuildFactory() {
+ @Override
+ public CursorSendEngine rebuild() {
+ return rebuild(buildTimeHandler);
+ }
+
+ @Override
+ public CursorSendEngine rebuild(SenderErrorHandler liveHandler) {
+ return LineSenderBuilder.constructEngineOnSlot(
+ rebuildSfDir, rebuildSenderId, slotPath,
+ actualSfMaxSegmentBytes, actualSfMaxTotalBytes,
+ actualSfAppendDeadlineNanos, actualSfSyncIntervalNanos,
+ liveHandler);
+ }
+ });
try {
// Install the drainer listener BEFORE startOrphanDrainers
// below: drainers must see the listener at submit time so
@@ -1900,6 +1896,83 @@ public LineSenderBuilder catchUpCapGapMinEscalationWindowMillis(long millis) {
return this;
}
+ /**
+ * Enables periodic recycling (rebuilding) of the sender's symbol dictionary
+ * once it reaches {@link #symbolDictResetThreshold(int)} distinct symbols,
+ * so a long-lived sender's dictionary does not grow without bound.
+ *
+ * Switching it off also disables the manual valve: + * {@link Sender#resetSymbolDictionary()} becomes a permanent no-op, + * because arming gates on this knob. + *
+ * Switching it off removes the only bound on dictionary growth below + * the hard cap ({@link io.questdb.client.cutlass.qwp.protocol.QwpConstants#MAX_SYMBOL_DICTIONARY_SIZE}, + * 2,000,000). Servers released before QuestDB 10.0.0 cap the + * dictionary at 1,000,000 and reject anything beyond it as a terminal + * parse error, so with the recycle off against a pre-10.0.0 server + * keep symbol cardinality below 1M. + *
+ * Default {@code true} (on). WebSocket transport only. + */ + public LineSenderBuilder symbolDictReset(boolean enabled) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("symbol_dict_reset is only supported for WebSocket transport"); + } + this.symbolDictReset = enabled; + return this; + } + + /** + * Number of distinct symbols the sender's dictionary may accumulate before + * {@link #symbolDictReset(boolean)} triggers a recycle. Each recycle raises + * the effective bar to {@code max(threshold, 2 x dictionary size at the swap)}, + * capped at half of {@link QwpConstants#MAX_SYMBOL_DICTIONARY_SIZE}, so a + * bounded live set larger than the threshold recycles once and settles + * instead of recycling on every refill. Must be greater than + * {@code 0} and no larger than {@link QwpConstants#MAX_SYMBOL_DICTIONARY_SIZE}. + *
+ * Default {@code 100_000}. WebSocket transport only. + */ + public LineSenderBuilder symbolDictResetThreshold(int threshold) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("symbol_dict_reset_threshold is only supported for WebSocket transport"); + } + if (threshold <= 0 || threshold > QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE) { + throw new LineSenderException("symbol_dict_reset_threshold must be > 0 and <= ") + .put(QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE).put(": ").put(threshold); + } + this.symbolDictResetThreshold = threshold; + return this; + } + + /** + * Upper bound, in milliseconds, on how long a triggered symbol-dictionary + * recycle stays armed before it may block the calling thread to force + * progress. Once a recycle has been armed for longer than this window + * without an opportunistic (idle) drain, the NEXT row-start call + * ({@code table(...)}) blocks the calling thread for up to this many + * millis waiting for the outstanding backlog to drain, then recycles + * before returning. If the backlog still has not drained by the + * deadline, that call gives up (logging a warning) and returns without + * blocking further -- the recycle stays armed and is retried + * opportunistically on a later {@code table(...)} call that happens to + * find the backlog already drained. {@code 0} disables blocking + * entirely (opportunistic-only): the recycle then only ever runs when a + * {@code table(...)} call finds the backlog already drained on its own. + *
+ * Default {@code 30_000} (30 s). WebSocket transport only.
+ */
+ public LineSenderBuilder symbolDictResetMaxWaitMillis(long maxWaitMillis) {
+ if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) {
+ throw new LineSenderException("symbol_dict_reset_max_wait_millis is only supported for WebSocket transport");
+ }
+ if (maxWaitMillis < 0) {
+ throw new LineSenderException("symbol_dict_reset_max_wait_millis must be >= 0: ").put(maxWaitMillis);
+ }
+ this.symbolDictResetMaxWaitMillis = maxWaitMillis;
+ return this;
+ }
+
/**
* close() drain timeout in milliseconds. The sender's {@code close()}
* method blocks up to this many millis waiting for the server to ACK
@@ -3132,6 +3205,135 @@ private static long parseSizeValue(@NotNull StringSink value, @NotNull String na
}
}
+ /**
+ * Result of {@link #constructEngineOnSlotLocked}: the constructed engine, plus
+ * whether construction itself had to quarantine a torn slot to produce it.
+ * {@link #build} folds {@code quarantined} into its own connect-loop retry
+ * guard, so a construction-time quarantine still counts toward the one
+ * quarantine build() allows per attempt -- the invariant a single shared
+ * {@code quarantined} local enforced before this method existed.
+ */
+ static final class ConstructedEngine {
+ final CursorSendEngine engine;
+ final boolean quarantined;
+
+ ConstructedEngine(CursorSendEngine engine, boolean quarantined) {
+ this.engine = engine;
+ this.quarantined = quarantined;
+ }
+ }
+
+ /**
+ * Constructs a {@code CursorSendEngine} on {@code slotPath}, quarantining a torn
+ * slot exactly as {@link #build}'s connect loop does when the constructor itself
+ * hits a terminal recovery verdict. Assumes the caller already holds
+ * {@code slotPath}'s logical lock (or {@code slotPath == null}, memory mode).
+ */
+ static ConstructedEngine constructEngineOnSlotLocked(
+ String sfDir, String senderId, String slotPath,
+ long maxSegmentBytes, long maxTotalBytes,
+ long appendDeadlineNanos, long syncIntervalNanos,
+ SenderErrorHandler errorHandler) {
+ // The constructor's own recovery seed can also fail terminally, and
+ // not only as UnreplayableSlotException: when SegmentRing.openExisting
+ // had to skip an unreadable segment it throws SfRecoveryException (it
+ // constructs UnreplayableSlotException nowhere), and where it cannot
+ // even prove the chain's identity -- no manifest -- it quarantines the
+ // corrupt files and returns an EMPTY recovery rather than refusing.
+ // Either way the frame range cannot be shown already-acked, so recovery
+ // sets the slot aside rather than risk seeding the ack cursor past
+ // frames that were never delivered. All three types below are load
+ // bearing; narrowing this catch to UnreplayableSlotException would
+ // restore the permanent build() brick for the segment-skip case. That verdict gets
+ // the same quarantine-and-continue treatment as build()'s connect()-time
+ // verdict; for build() the construction runs outside its retry loop, and
+ // for a recycle rebuild there is no loop at all.
+ // quarantineTornSlot(null, ...) renames the WHOLE slot directory aside
+ // (not just the unreadable segment file) before building the replacement
+ // at the original slotPath, so the replacement starts on a genuinely empty
+ // directory with nothing left to skip -- it cannot throw the same way
+ // twice, which is what makes looping unnecessary here.
+ boolean quarantined = false;
+ CursorSendEngine cursorEngine;
+ try {
+ try {
+ cursorEngine = new CursorSendEngine(
+ slotPath, maxSegmentBytes,
+ maxTotalBytes, appendDeadlineNanos,
+ syncIntervalNanos);
+ } catch (SfSanitizedResidueException first) {
+ // NOT terminal, and it must be intercepted ahead of its
+ // SfRecoveryException parent below. Recovery durably zeroed
+ // proven-dead sealed residue BEFORE failing closed, so the
+ // chain on disk is already healed: quarantining here would
+ // set aside a slot whose backlog replays perfectly. Retry
+ // once over the healed chain; a repeat is genuine and takes
+ // the terminal arm.
+ LOG.info("sf slot {}: sealed residue sanitized during recovery ({}); "
+ + "retrying over the healed chain",
+ slotPath, first.getMessage());
+ cursorEngine = new CursorSendEngine(
+ slotPath, maxSegmentBytes,
+ maxTotalBytes, appendDeadlineNanos,
+ syncIntervalNanos);
+ }
+ } catch (UnreplayableSlotException | SfRecoveryException
+ | MmapSegmentCorruptionException e) {
+ // The terminal recovery verdicts, and the only ones build()
+ // sets a slot aside for. UnreplayableSlotException says the
+ // symbol dictionary cannot be rebuilt from any source;
+ // SfRecoveryException and MmapSegmentCorruptionException say
+ // the durable chain itself is proven corrupt or incomplete.
+ // None of the three clears on a retry, and senderId is stable
+ // with a not-fully-drained slot retained on close -- so
+ // without this arm every restart re-recovers the same slot and
+ // throws again, and the application cannot construct a Sender
+ // at all, not even to BUFFER new rows.
+ //
+ // Deliberately NOT catching plain MmapSegmentException or
+ // SfOperationalException: those are operational (EMFILE,
+ // ENOMEM, an unreadable-but-possibly-intact file). Aborting
+ // startup on them is correct; quarantining on them would
+ // convert a transient into the permanent loss of a healthy
+ // slot's durable frames.
+ if (slotPath == null) {
+ throw e;
+ }
+ quarantined = true;
+ cursorEngine = quarantineTornSlot(
+ null, e, sfDir, senderId, slotPath, maxSegmentBytes,
+ maxTotalBytes, appendDeadlineNanos,
+ syncIntervalNanos, errorHandler);
+ }
+ return new ConstructedEngine(cursorEngine, quarantined);
+ }
+
+ /**
+ * {@link #constructEngineOnSlotLocked} wrapped in its own narrow acquisition of
+ * {@code slotPath}'s logical lock. {@link #build} itself does not call this --
+ * its own lock spans the connect loop too, see the comment at its call site --
+ * this entry point is for callers that only need a freshly (re)built engine on
+ * an already-owned slot, such as a symbol-dictionary epoch rebuild. Recovery
+ * verdicts still quarantine here exactly as they do under {@link #build} --
+ * that happens inside {@link #constructEngineOnSlotLocked}. Only the
+ * quarantined FLAG is discarded: it exists to seed {@code build}'s connect-loop
+ * retry guard, and a recycle rebuild has no such loop. A rebuild that fails
+ * outright does not latch the sender terminal either; the recycle abandons and
+ * retries on the next send.
+ */
+ static CursorSendEngine constructEngineOnSlot(
+ String sfDir, String senderId, String slotPath,
+ long maxSegmentBytes, long maxTotalBytes,
+ long appendDeadlineNanos, long syncIntervalNanos,
+ SenderErrorHandler errorHandler) {
+ try (SlotLock logicalSlotLock = slotPath == null
+ ? null : SlotLock.acquireLogical(slotPath)) {
+ return constructEngineOnSlotLocked(sfDir, senderId, slotPath,
+ maxSegmentBytes, maxTotalBytes, appendDeadlineNanos,
+ syncIntervalNanos, errorHandler).engine;
+ }
+ }
+
/**
* Sets a slot aside that either connect() (a symbol dictionary that cannot cover its
* surviving frames, {@code UnreplayableSlotException}) or the
@@ -3778,6 +3980,30 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) {
}
pos = getValue(configurationString, pos, sink, "catch_up_cap_gap_min_escalation_window_millis");
catchUpCapGapMinEscalationWindowMillis(parseLongValue(sink, "catch_up_cap_gap_min_escalation_window_millis"));
+ } else if (Chars.equals("symbol_dict_reset", sink)) {
+ if (protocol != PROTOCOL_WEBSOCKET) {
+ throw new LineSenderException("symbol_dict_reset is only supported for WebSocket transport");
+ }
+ pos = getValue(configurationString, pos, sink, "symbol_dict_reset");
+ if (Chars.equalsIgnoreCase("on", sink)) {
+ symbolDictReset(true);
+ } else if (Chars.equalsIgnoreCase("off", sink)) {
+ symbolDictReset(false);
+ } else {
+ throw new LineSenderException("invalid symbol_dict_reset [value=").put(sink).put(", allowed-values=[on, off]]");
+ }
+ } else if (Chars.equals("symbol_dict_reset_threshold", sink)) {
+ if (protocol != PROTOCOL_WEBSOCKET) {
+ throw new LineSenderException("symbol_dict_reset_threshold is only supported for WebSocket transport");
+ }
+ pos = getValue(configurationString, pos, sink, "symbol_dict_reset_threshold");
+ symbolDictResetThreshold(parseIntValue(sink, "symbol_dict_reset_threshold"));
+ } else if (Chars.equals("symbol_dict_reset_max_wait_millis", sink)) {
+ if (protocol != PROTOCOL_WEBSOCKET) {
+ throw new LineSenderException("symbol_dict_reset_max_wait_millis is only supported for WebSocket transport");
+ }
+ pos = getValue(configurationString, pos, sink, "symbol_dict_reset_max_wait_millis");
+ symbolDictResetMaxWaitMillis(parseLongValue(sink, "symbol_dict_reset_max_wait_millis"));
} else if (Chars.equals("initial_connect_retry", sink)) {
if (protocol != PROTOCOL_WEBSOCKET) {
throw new LineSenderException("initial_connect_retry is only supported for WebSocket transport");
@@ -4053,6 +4279,12 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString)
if (view.has("catch_up_cap_gap_min_escalation_window_millis")) {
catchUpCapGapMinEscalationWindowMillis(wsLong(view, v, "catch_up_cap_gap_min_escalation_window_millis"));
}
+ if (view.has("symbol_dict_reset_threshold")) {
+ symbolDictResetThreshold(wsInt(view, v, "symbol_dict_reset_threshold"));
+ }
+ if (view.has("symbol_dict_reset_max_wait_millis")) {
+ symbolDictResetMaxWaitMillis(wsLong(view, v, "symbol_dict_reset_max_wait_millis"));
+ }
if (view.has("sf_append_deadline_millis")) {
sfAppendDeadlineMillis(wsLong(view, v, "sf_append_deadline_millis"));
}
@@ -4122,6 +4354,16 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString)
throw new LineSenderException("invalid initial_connect_retry [value=").put(s).put(", allowed-values=[on, off, true, false, sync, async]]");
}
}
+ s = view.getStr("symbol_dict_reset");
+ if (s != null) {
+ if (s.equalsIgnoreCase("on")) {
+ symbolDictReset(true);
+ } else if (s.equalsIgnoreCase("off")) {
+ symbolDictReset(false);
+ } else {
+ throw new LineSenderException("invalid symbol_dict_reset [value=").put(s).put(", allowed-values=[on, off]]");
+ }
+ }
return this;
} catch (IllegalArgumentException e) {
throw new LineSenderException(e.getMessage());
@@ -4237,6 +4479,9 @@ public java.util.Map
@@ -1609,6 +1825,7 @@ public QwpWebSocketSender floatColumn(CharSequence columnName, float value) {
*/
@Override
public void flush() {
+ checkRecycleFailure();
flushAndGetSequence();
}
@@ -1627,6 +1844,7 @@ public void flush() {
@Override
public long flushAndGetSequence() {
checkNotClosed();
+ checkRecycleFailure();
if (cursorEngine != null) {
cursorEngine.checkDurability();
}
@@ -1658,7 +1876,7 @@ public long flushAndGetSequence() {
checkConnectionError();
long afterFsn = cursorEngine != null ? cursorEngine.publishedFsn() : -1L;
- return afterFsn > beforeFsn ? afterFsn : -1L;
+ return afterFsn > beforeFsn ? fsnEpochBase + afterFsn : -1L;
}
/**
@@ -1687,8 +1905,10 @@ public long flushAndGetSequence() {
*/
@Override
public boolean drain(long timeoutMillis) {
+ checkRecycleFailure();
flush();
- long targetFsn = cursorEngine != null ? cursorEngine.publishedFsn() : -1L;
+ long targetRaw = cursorEngine != null ? cursorEngine.publishedFsn() : -1L;
+ long targetFsn = targetRaw < 0 ? targetRaw : fsnEpochBase + targetRaw;
return awaitAckedFsn(targetFsn, timeoutMillis);
}
@@ -1767,15 +1987,30 @@ public QwpWebSocketSender geoHashColumn(CharSequence columnName, CharSequence va
/**
* Highest FSN that has been server-acknowledged. Rejections never advance
- * the watermark. {@code -1} if
- * the I/O loop has not yet started or no batch has been published.
+ * the watermark. Returns {@code -1} only while nothing has ever been
+ * published in this sender's lifetime. On a live sender the value never
+ * collapses back to {@code -1}: after a symbol-dictionary recycle the
+ * accessor keeps reporting the last pre-swap durable watermark until the
+ * fresh epoch publishes. (After {@code close()} the reading is
+ * unspecified.)
*
* Snapshot accessor — for a bounded wait, use
* {@link #awaitAckedFsn(long, long)}.
*/
@Override
public long getAckedFsn() {
- return cursorEngine != null ? cursorEngine.ackedFsn() : -1L;
+ // Read fsnEpochBase FIRST, then cursorEngine: the recycle writes
+ // engine=null -> base+=L+1 -> engine=fresh, so a reader that saw the
+ // NEW base is ordered after the null write and can only observe null
+ // or the fresh engine -- never (new base, stale engine), which would
+ // fabricate an FSN above anything ever published. The clamp against
+ // lastRecycleDurableFsn keeps the other torn pair (old base, fresh
+ // engine) from reading below a value already returned. Sender is
+ // documented single-threaded; this keeps best-effort monitor reads
+ // truthful rather than promising thread safety.
+ long base = fsnEpochBase;
+ CursorSendEngine engine = cursorEngine;
+ return engine != null ? Math.max(lastRecycleDurableFsn, base + engine.ackedFsn()) : lastRecycleDurableFsn;
}
/**
@@ -1878,6 +2113,16 @@ public int getEffectiveAutoFlushBytes() {
return effectiveAutoFlushBytes;
}
+ /**
+ * The installed engine-rebuild factory, so a test can wrap the real one
+ * (e.g. fault-inject the first rebuild and delegate afterwards) instead of
+ * replacing it outright. {@code null} for a {@code connect()}-built sender.
+ */
+ @TestOnly
+ public EngineRebuildFactory getEngineRebuildFactoryForTesting() {
+ return engineRebuildFactory;
+ }
+
/**
* Snapshot of the typed payload for the latched terminal server-rejection error,
* or {@code null} if the I/O loop has not latched a server-rejection terminal
@@ -1966,6 +2211,27 @@ public int getServerMaxBatchSize() {
return serverMaxBatchSize;
}
+ /** Resolved value of {@code symbol_dict_reset_max_wait_millis}. */
+ @TestOnly
+ public long getSymbolDictResetMaxWaitMillis() {
+ return resetMaxWaitMillis;
+ }
+
+ /** Resolved value of {@code symbol_dict_reset_threshold}. */
+ @TestOnly
+ public int getSymbolDictResetThreshold() {
+ return resetThresholdSymbols;
+ }
+
+ /**
+ * The current re-arm floor: 0 before the first swap, then
+ * {@code min(2 x dictSizeAtSwap, MAX_SYMBOL_DICTIONARY_SIZE / 2)}.
+ */
+ @TestOnly
+ public int getResetFloorSymbolsForTesting() {
+ return resetFloorSymbols;
+ }
+
@TestOnly
public QwpTableBuffer getTableBuffer(String tableName) {
QwpTableBuffer buffer = tableBuffers.get(tableName);
@@ -1981,15 +2247,132 @@ public QwpTableBuffer getTableBuffer(String tableName) {
/**
* Whether this sender is still in delta-encoded mode. Flips to {@code false}
- * permanently once {@link #disableDeltaDict} fires (a persisted-dictionary
- * write failure, including a recognised mmap access fault) -- every later
- * flush then ships full self-sufficient frames instead.
+ * for the rest of this epoch once {@link #disableDeltaDict} fires (a
+ * persisted-dictionary write failure, including a recognised mmap access
+ * fault) -- every later flush this epoch then ships full self-sufficient
+ * frames instead. A symbol-dictionary recycle re-derives this from the
+ * fresh engine.
*/
@TestOnly
public boolean isDeltaDictEnabledForTest() {
return deltaDictEnabled;
}
+ /**
+ * Whether the symbol-dictionary recycle is currently armed. Set by
+ * {@link #armIfEligible()} at the tail of every flush (and immediately by
+ * {@link #resetSymbolDictionary()} when no row or flush is in progress).
+ */
+ @TestOnly
+ public boolean isResetArmed() {
+ return resetArmed;
+ }
+
+ /** Resolved value of {@code symbol_dict_reset}. */
+ @TestOnly
+ public boolean isSymbolDictResetEnabled() {
+ return resetEnabled;
+ }
+
+ /** Current value of {@link #fsnEpochBase}. */
+ @TestOnly
+ public long getFsnEpochBaseForTest() {
+ return fsnEpochBase;
+ }
+
+ /**
+ * Number of symbol-dictionary recycles this sender has completed. Advances
+ * by one at step 6 of {@link #recycleForDictReset()}, the instant the swap
+ * commits to the new epoch -- after the engine rebuild (step 4) has
+ * already succeeded, so a step-4 rebuild failure -- which abandons the
+ * recycle to be resumed by a later send -- leaves this counter
+ * un-bumped, while a later step-7 reconnect failure (which cannot
+ * latch: the swap already committed by then) still leaves this
+ * incremented. Unlike the per-send-loop
+ * {@code getTotal*} counters, it is scoped to the sender's whole lifetime
+ * and never resets. volatile: a
+ * concurrent read sees the latest write the producer thread completed,
+ * but there is no atomicity across the three symbol-dictionary-recycle
+ * counters -- a reader on another thread can observe this one already
+ * advanced while {@link #getSymbolDictResetsPerformed()} still reflects
+ * the prior value, even though the producer thread writes them on
+ * adjacent lines.
+ */
+ public long getSymbolDictEpoch() {
+ return symbolDictEpoch;
+ }
+
+ /**
+ * Number of symbol-dictionary recycle swaps this sender has completed.
+ * Incremented alongside {@link #getSymbolDictEpoch()} at step 6 of
+ * {@link #recycleForDictReset()} -- after the engine rebuild (step 4) has
+ * already succeeded, so, like the epoch counter, a step-4 rebuild failure
+ * leaves this un-bumped while a step-7 reconnect failure still leaves it
+ * incremented (the swap has already committed by then).
+ * Also like the epoch counter, it is scoped to the sender's whole lifetime
+ * and never resets. The two counts move together today -- the
+ * only way the epoch advances is through a completed recycle swap -- but
+ * they are defined, and incremented, independently: this one counts
+ * completed swaps, {@code getSymbolDictEpoch()} counts the dictionary
+ * generation. They would diverge if a future change ever rolled the
+ * epoch by some path other than a recycle swap. Same thread-safety
+ * caveat as {@link #getSymbolDictEpoch()}.
+ */
+ public long getSymbolDictResetsPerformed() {
+ return symbolDictResetsPerformed;
+ }
+
+ /**
+ * Number of times {@link #maybeBlockForStarvedReset()} has timed out
+ * without the backlog draining. 0 until the first such timeout. volatile,
+ * written only from the producer thread inside
+ * {@link #maybeBlockForStarvedReset()}: same thread-safety caveat as
+ * {@link #getSymbolDictEpoch()} -- a concurrent read sees the latest
+ * completed write, with no atomicity across the three counters.
+ */
+ public long getSymbolDictResetStarvationTimeouts() {
+ return symbolDictResetStarvationTimeouts;
+ }
+
+ /**
+ * Test-only entry point for {@link #rollFsnEpochBase}, the same private
+ * roll the symbol-dict recycle swap calls in production once the engine
+ * rebuild has committed. See that method's precondition: {@code cursorSendLoop}
+ * must be {@code null} -- roll before the sender's first connect (e.g. via
+ * {@link #createForTesting}), never on an already-connected sender.
+ */
+ @TestOnly
+ public void rollFsnEpochBaseForTest(long lastPublishedFsn) {
+ rollFsnEpochBase(lastPublishedFsn);
+ }
+
+ /**
+ * Advances {@link #fsnEpochBase} past every FSN handed out under the
+ * epoch that just ended. {@code lastPublishedFsn} is the highest raw FSN
+ * the outgoing cursor engine ever published ({@code -1} if it published
+ * nothing), so the next raw FSN the fresh engine hands out --
+ * {@code 0} -- maps to external {@code lastPublishedFsn + 1 + 0}, one
+ * past the last external FSN this sender ever reported.
+ *
+ * Precondition: {@code cursorSendLoop} must be {@code null}. A live loop's
+ * {@code externalFsnBase} is a construction-time snapshot -- it is never updated
+ * on an already-built loop -- so rolling while one is attached would silently
+ * desynchronize the two: {@link #getAckedFsn()} / {@link #flushAndGetSequence()}
+ * would report post-roll values while every {@code SenderProgressHandler} advance
+ * and {@link SenderError} span the loop emits would stay pinned at pre-roll
+ * values. The recycle swap must call this strictly between tearing the old loop
+ * down and constructing the new one.
+ */
+ private void rollFsnEpochBase(long lastPublishedFsn) {
+ if (cursorSendLoop != null) {
+ throw new IllegalStateException("rollFsnEpochBase must run while cursorSendLoop"
+ + " is null -- the loop's externalFsnBase is a construction-time snapshot,"
+ + " never updated on a live loop; roll strictly between tearing the old"
+ + " loop down and building the new one");
+ }
+ fsnEpochBase += lastPublishedFsn + 1L;
+ }
+
/**
* Total binary frames whose ACKs have been received and applied.
*/
@@ -2053,9 +2436,15 @@ public long getTotalErrorNotificationsDelivered() {
}
/**
- * Cumulative count of frames re-sent during post-reconnect catch-up
- * windows. Zero in steady state; a sustained nonzero rate signals
- * flapping where every reconnect replays meaningful work.
+ * Count of frames re-sent during post-reconnect catch-up windows since the
+ * last symbol-dictionary recycle. Zero in steady state; a sustained nonzero
+ * rate signals flapping where every reconnect replays meaningful work.
+ *
+ * Reads the live cursor I/O loop, which a symbol-dictionary recycle
+ * rebuilds, so the count restarts at 0 on every recycle: a monitor
+ * differencing it across one sees a negative delta. Correlate with the
+ * lifetime-scoped {@link #getSymbolDictEpoch()} /
+ * {@link #getSymbolDictResetsPerformed()}, which never reset.
*/
public long getTotalFramesReplayed() {
CursorWebSocketSendLoop l = cursorSendLoop;
@@ -2063,7 +2452,14 @@ public long getTotalFramesReplayed() {
}
/**
- * Total binary frames the cursor I/O loop has issued to the wire.
+ * Binary frames the cursor I/O loop has issued to the wire since the last
+ * symbol-dictionary recycle.
+ *
+ * Reads the live cursor I/O loop, which a symbol-dictionary recycle
+ * rebuilds, so the count restarts at 0 on every recycle: a monitor
+ * differencing it across one sees a negative delta. Correlate with the
+ * lifetime-scoped {@link #getSymbolDictEpoch()} /
+ * {@link #getSymbolDictResetsPerformed()}, which never reset.
*/
public long getTotalFramesSent() {
CursorWebSocketSendLoop l = cursorSendLoop;
@@ -2071,9 +2467,16 @@ public long getTotalFramesSent() {
}
/**
- * Number of reconnect attempts the cursor I/O loop has issued —
- * succeeded plus failed. Diverges from {@link #getTotalReconnectsSucceeded}
- * when the server is flapping. Returns 0 if no I/O loop is running.
+ * Number of reconnect attempts the cursor I/O loop has issued since the
+ * last symbol-dictionary recycle -- succeeded plus failed. Diverges from
+ * {@link #getTotalReconnectsSucceeded} when the server is flapping.
+ * Returns 0 if no I/O loop is running.
+ *
+ * Reads the live cursor I/O loop, which a symbol-dictionary recycle
+ * rebuilds, so the count restarts at 0 on every recycle: a monitor
+ * differencing it across one sees a negative delta. Correlate with the
+ * lifetime-scoped {@link #getSymbolDictEpoch()} /
+ * {@link #getSymbolDictResetsPerformed()}, which never reset.
*/
public long getTotalReconnectAttempts() {
CursorWebSocketSendLoop l = cursorSendLoop;
@@ -2081,7 +2484,14 @@ public long getTotalReconnectAttempts() {
}
/**
- * Number of successful reconnects. Returns 0 if no I/O loop is running.
+ * Number of successful reconnects since the last symbol-dictionary
+ * recycle. Returns 0 if no I/O loop is running.
+ *
+ * Reads the live cursor I/O loop, which a symbol-dictionary recycle
+ * rebuilds, so the count restarts at 0 on every recycle: a monitor
+ * differencing it across one sees a negative delta. Correlate with the
+ * lifetime-scoped {@link #getSymbolDictEpoch()} /
+ * {@link #getSymbolDictResetsPerformed()}, which never reset.
*/
public long getTotalReconnectsSucceeded() {
CursorWebSocketSendLoop l = cursorSendLoop;
@@ -2089,7 +2499,14 @@ public long getTotalReconnectsSucceeded() {
}
/**
- * Total errors observed by the I/O loop (retriable and terminal combined).
+ * Errors the I/O loop has observed since the last symbol-dictionary
+ * recycle (retriable and terminal combined).
+ *
+ * Reads the live cursor I/O loop, which a symbol-dictionary recycle
+ * rebuilds, so the count restarts at 0 on every recycle: a monitor
+ * differencing it across one sees a negative delta. Correlate with the
+ * lifetime-scoped {@link #getSymbolDictEpoch()} /
+ * {@link #getSymbolDictResetsPerformed()}, which never reset.
*/
public long getTotalServerErrors() {
CursorWebSocketSendLoop l = cursorSendLoop;
@@ -2402,6 +2819,33 @@ public void reset() {
cachedTimestampNanosColumn = null;
}
+ /**
+ * Advisory request to start a fresh symbol-dictionary epoch. Sets
+ * {@link #manualResetRequested}; if no flush is in flight
+ * ({@code pendingRowCount == 0} -- a first row may still be under
+ * construction; arming is harmless there because the recycle trigger
+ * itself refuses to run mid-row), re-evaluates arming immediately so a
+ * caller that requests a reset between batches does not have to wait for
+ * a later flush to observe {@code isResetArmed()}. A request made
+ * mid-batch is picked up by the next
+ * {@code resetTableBuffersAfterFlush} instead.
+ *
+ * A permanent no-op while {@code symbol_dict_reset} is off: arming gates
+ * on that knob, so a sender configured with the recycle disabled never
+ * acts on the request, however many times it is made. The request is
+ * likewise a permanent no-op on senders that cannot recycle -- ones
+ * without an engine rebuild factory (every {@code connect()}-built
+ * sender) or running on an engine they do not own -- which never arm.
+ */
+ @Override
+ public void resetSymbolDictionary() {
+ checkNotClosed();
+ manualResetRequested = true;
+ if (pendingRowCount == 0) {
+ armIfEligible();
+ }
+ }
+
/**
* Register an async listener for connection-state transitions: initial
* connect, primary failover, endpoint attempt failures, the full address
@@ -2533,6 +2977,21 @@ public synchronized void setDrainerListener(BackgroundDrainerListener listener)
}
}
+ /**
+ * Installs the positive witness {@link #awaitDeferredEngineClose} runs
+ * once it actually begins parking, so a test can prove the await engaged
+ * instead of completing inline -- see
+ * {@code SymbolDictRecycleDeferredCloseTest}.
+ */
+ @TestOnly
+ public void setDeferredCloseParkWitnessForTesting(Runnable witness) {
+ this.deferredCloseParkWitness = witness;
+ }
+
+ public void setEngineRebuildFactory(EngineRebuildFactory factory) {
+ this.engineRebuildFactory = factory;
+ }
+
/**
* Configure the user-supplied error handler. May be called either before
* or after {@code connect()} — when called after, the change propagates
@@ -2562,6 +3021,16 @@ public void setErrorInboxCapacity(int capacity) {
this.errorInboxCapacity = capacity;
}
+ @TestOnly
+ public void setLoopStartFaultForTesting(Runnable fault) {
+ this.loopStartFault = fault;
+ }
+
+ @TestOnly
+ public void setRecycleDeferredCloseMaxWaitMillisForTesting(long millis) {
+ this.recycleDeferredCloseMaxWaitMillis = millis;
+ }
+
public void setTransactional(boolean transactional) {
this.transactional = transactional;
}
@@ -2741,6 +3210,12 @@ public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) {
@Override
public QwpWebSocketSender table(CharSequence tableName) {
checkNotClosed();
+ checkRecycleFailure();
+ if (recycleResume != RecycleResume.NONE) {
+ resumeRecycleIfPending();
+ } else if (resetArmed) {
+ maybeRecycleForDictReset();
+ }
// Fast path: if table name matches current, skip hashmap lookup
if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) {
return this;
@@ -2838,15 +3313,19 @@ public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi)
/**
* True iff this sender has at least once installed a live (connected
* + upgraded) WebSocket. Sticky — once true, stays true even after a
- * subsequent disconnect. Lets a {@link SenderErrorHandler}
- * disambiguate a "never reached the server" terminal failure (likely
- * a config typo or firewall block) from a "lost connection after we
- * were up" failure (likely transient). Returns {@code false} if no
- * I/O loop is running.
+ * subsequent disconnect, including through a symbol-dict recycle's
+ * loop-null window (mid-swap, or after a failed reconnect setup).
+ * Lets a {@link SenderErrorHandler} disambiguate a "never reached the
+ * server" terminal failure (likely a config typo or firewall block)
+ * from a "lost connection after we were up" failure (likely
+ * transient). Returns {@code false} only if no loop has ever
+ * connected in this sender's lifetime.
*/
public boolean wasEverConnected() {
+ // Sticky by contract: fall back to the sender-lifetime flag while no
+ // loop is installed (mid-recycle, or after a failed reconnect setup).
CursorWebSocketSendLoop l = cursorSendLoop;
- return l != null && l.hasEverConnected();
+ return hasLoopEverConnected || (l != null && l.hasEverConnected());
}
private static Throwable captureCloseError(Throwable terminalError, Throwable t) {
@@ -3443,6 +3922,35 @@ private void checkNotClosed() {
checkConnectionError();
}
+ /**
+ * Terminal latch for the one symbol-dictionary recycle failure that is not
+ * resumable: a rebuilt engine that recovered UNACKED frames from the slot
+ * the outgoing engine's fully-drained close was supposed to have emptied
+ * (see {@link #completeRecycleRebuild}). That proves the everything-acked
+ * barrier the swap rests on was breached, so the producer's fresh
+ * dictionary and the slot's on-disk state have diverged and this sender
+ * refuses further use. Every OTHER recycle failure -- a wedged SF worker,
+ * an interrupted producer thread, a momentary rebuild fault, a
+ * post-cleanup fsync warning, a failed step-7 reconnect -- is transient:
+ * it throws to the triggering caller, leaves the counters un-bumped and
+ * the recycle pending ({@link #recycleResume}), and the next send
+ * finishes the swap. Checked by
+ * {@link #table(CharSequence)}, the flush-family
+ * entry points ({@link #flush()}, {@link #flushAndGetSequence()},
+ * {@link #drain(long)}, {@link #awaitAckedFsn(long, long)}), and
+ * {@code sendRow()} (closing the fluent-chain corner where a caller
+ * continues {@code .symbol(...).atNow()} against a {@code currentTableBuffer}
+ * selected before the latch, without an intervening {@code table()} call)
+ * -- deliberately NOT by {@link #close()}, which must still be able to
+ * tear down a latched sender.
+ */
+ private void checkRecycleFailure() {
+ if (recycleFailure != null) {
+ throw new LineSenderException(recycleFailure)
+ .put("sender is terminal: symbol dictionary recycle failed");
+ }
+ }
+
private void checkTableSelected() {
if (currentTableBuffer == null) {
throw new LineSenderException("table() must be called before adding columns");
@@ -3496,7 +4004,11 @@ private synchronized Throwable closeRemainingResources(Throwable terminalError)
slotLockReleased = false;
retainedEngine = engine;
}
- } else {
+ } else if (retainedEngine == null) {
+ // No engine and nothing retained: no flock left to report. A
+ // non-null retainedEngine (a recycle's deferred-close await
+ // timed out) still holds the slot flock, so leave the flag
+ // false and let isSlotLockReleased() re-probe it.
slotLockReleased = true;
}
if (errorDispatcher != null) {
@@ -3755,6 +4267,7 @@ private void ensureActiveBufferReady() {
private void ensureConnected() {
checkNotClosed();
+ resumeRecycleIfPending();
if (connected) {
return;
}
@@ -3771,7 +4284,14 @@ private void ensureConnected() {
connectionListener, connectionListenerInboxCapacity);
}
CursorWebSocketSendLoop.ReconnectFactory reconnectFactory = newReconnectFactory();
- switch (initialConnectMode) {
+ // initialConnectMode is an *initialization* policy. After the first
+ // successful connect the SF contract forbids foreground connects on
+ // the producer thread, so re-entries (the recycle's step 7, or its
+ // retry after a failed loop start) always defer to the I/O thread.
+ Sender.InitialConnectMode effectiveMode = hasInitialConnectRun
+ ? Sender.InitialConnectMode.ASYNC
+ : initialConnectMode;
+ switch (effectiveMode) {
case SYNC:
client = CursorWebSocketSendLoop.connectWithRetry(
reconnectFactory,
@@ -3788,10 +4308,13 @@ private void ensureConnected() {
// connect commit to V1 because cursor segments are immutable;
// a future version bump must account for that. Transport
// failures retry indefinitely on the I/O thread (Invariant B).
- // But a terminal auth, upgrade or capability rejection on this
- // initial connect -- before the wire is ever up -- is surfaced
- // to the async SenderErrorHandler and latched for a close()
- // rethrow, not retried.
+ // But a terminal auth, upgrade or capability rejection on the
+ // INITIAL deferred connect -- before the wire is ever up -- is
+ // surfaced to the async SenderErrorHandler and latched for a
+ // close() rethrow, not retried. A re-entry after a prior
+ // connect (the recycle's step 7) seeds the fresh loop with
+ // markEverConnected(), so the same rejection there is retried
+ // like any post-connect failure.
client = null;
break;
case OFF:
@@ -3818,7 +4341,8 @@ private void ensureConnected() {
maxFrameRejections,
poisonMinEscalationWindowMillis,
catchUpCapGapMinEscalationWindowMillis,
- CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND);
+ CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND,
+ fsnEpochBase);
// Plug the async-delivery sink before start() so the I/O thread
// never observes a null dispatcher between recordFatal and
// notification — the test for null in dispatchError handles
@@ -3842,6 +4366,19 @@ private void ensureConnected() {
// the loop no longer fires a terminal budget-exhaustion event -- it
// retries indefinitely.)
cursorSendLoop.setConnectionDispatcher(connectionDispatcher);
+ // Seed the fresh loop's own hasEverConnected before it can observe
+ // any endpoint-policy failure: without this, a symbol-dict
+ // recycle's rebuilt loop starts believing it has never connected
+ // (ASYNC startup always hands the constructor a null client),
+ // which would wrongly re-arm endpointPolicyFailureIsTerminal()'s
+ // startup-terminal branch for a FOREGROUND sender that already
+ // reached the server in a prior loop instance.
+ if (hasLoopEverConnected) {
+ cursorSendLoop.markEverConnected();
+ }
+ if (loopStartFault != null) {
+ loopStartFault.run();
+ }
cursorSendLoop.start();
} catch (Throwable t) {
// start() (or dispatcher construction) failed after cursorSendLoop was
@@ -3880,24 +4417,36 @@ private void ensureConnected() {
// client; same path runs on every reconnect.
LOG.info("Connected to WebSocket [host={}, port={}, qwpVersion={}, serverMaxBatchSize={}, effectiveAutoFlushBytes={}]",
host, port, client.getServerQwpVersion(), serverMaxBatchSize, effectiveAutoFlushBytes);
+ hasLoopEverConnected = true;
} else {
- // Async mode: I/O thread will drive the connect. Encoder uses
- // its default version (V1). The per-batch symbol-dict watermark still
- // gets reset for consistency with the sync path; the post-connect
- // replay path needs no producer-side reset signal (see below).
+ // Deferred connect: the I/O thread will drive it, on the sender's
+ // true initial connect (hasInitialConnectRun still false here) or
+ // on a post-initial re-entry such as the recycle's step 7. Either
+ // way the encoder keeps whatever version was already negotiated
+ // (V1 -- the only supported wire version today); a re-entry never
+ // resets it. The per-batch symbol-dict watermark still gets reset
+ // for consistency with the sync path; the post-connect replay
+ // path needs no producer-side reset signal (see below).
Endpoint ep = endpoints.get(0);
- LOG.info("Async initial connect deferred to I/O thread [firstHost={}, firstPort={}, endpointCount={}]",
- ep.host, ep.port, endpoints.size());
+ if (hasInitialConnectRun) {
+ LOG.info("Reconnect deferred to I/O thread [firstHost={}, firstPort={}, endpointCount={}]",
+ ep.host, ep.port, endpoints.size());
+ } else {
+ LOG.info("Initial connect deferred to I/O thread [firstHost={}, firstPort={}, endpointCount={}]",
+ ep.host, ep.port, endpoints.size());
+ }
}
// Server starts fresh on each connection, so reset the per-batch
- // symbol-dict watermark. Every frame still carries its full inline schema,
- // and the fresh server's dictionary is re-established either by a full-dict
- // frame (full-dict mode) or by an I/O-thread catch-up frame before replay
- // (delta mode), so post-reconnect replay needs no producer-side reset signal.
+ // symbol-dict watermark when nothing is staged against it. Every frame
+ // still carries its full inline schema, and the fresh server's dictionary
+ // is re-established either by a full-dict frame (full-dict mode) or by an
+ // I/O-thread catch-up frame before replay (delta mode), so post-reconnect
+ // replay needs no producer-side reset signal.
resetSymbolDictStateForNewConnection();
connectionError.set(null);
connected = true;
+ hasInitialConnectRun = true;
}
private void ensureNoInProgressRow() {
@@ -4286,6 +4835,579 @@ private void resetTableBuffersAfterFlush() {
currentTableBufferSnapshotBytes = 0;
pendingRowCount = 0;
firstPendingRowTimeNanos = 0;
+ armIfEligible();
+ }
+
+ /**
+ * Re-evaluates whether the symbol-dictionary recycle should be armed:
+ * {@code resetEnabled} is on, the sender can actually rebuild ({@link
+ * #engineRebuildFactory} is set and {@link #ownsCursorEngine}), AND
+ * either the global dictionary has reached the effective bar
+ * {@code max(resetThresholdSymbols, resetFloorSymbols)} distinct entries
+ * or a caller requested a reset via {@link #resetSymbolDictionary()}.
+ * Deliberately ignores {@code deltaDictEnabled} -- a producer degraded to
+ * full self-sufficient frames still benefits from bounding its dictionary
+ * size, and a manual request is honoured regardless of mode.
+ *
+ * A sender that cannot rebuild -- no {@link #engineRebuildFactory} (every
+ * public {@code QwpWebSocketSender.connect(...)} overload leaves it null
+ * -- only {@code Sender.build()} installs one), or a cursor engine this
+ * sender does not own ({@code setCursorEngine(engine, false)}'s contract:
+ * the caller retains ownership, so closing it out from under them would
+ * be a use-after-free from the caller's point of view) -- must never arm.
+ * Since the recycle feature is default-on and {@code
+ * resetSymbolDictionary()} is a public advisory API, arming a sender with
+ * no way to ever act on the request would leave {@code isResetArmed()}
+ * reading true forever alongside a permanently-0 resets counter,
+ * misleading monitoring.
+ *
+ * Called from two safe points only: the tail of
+ * {@link #resetTableBuffersAfterFlush()} (no row in progress, this flush's
+ * data already handed to the engine) and {@link #resetSymbolDictionary()}
+ * when nothing is in flight ({@code pendingRowCount == 0}). Never from the
+ * per-symbol registration path ({@link #getOrAddGlobalSymbol}) -- arming
+ * mid-row or mid-encode would observe a dictionary size that has not yet
+ * settled for this batch.
+ */
+ private void armIfEligible() {
+ boolean shouldArm = resetEnabled
+ && engineRebuildFactory != null
+ && ownsCursorEngine
+ && (globalSymbolDictionary.size() >= Math.max(resetThresholdSymbols, resetFloorSymbols)
+ || manualResetRequested);
+ if (shouldArm && !resetArmed) {
+ armedSinceNanos = System.nanoTime();
+ starvationWaitDoneThisArm = false;
+ }
+ resetArmed = shouldArm;
+ }
+
+ /**
+ * Recycle step 3's deferred-close await. A fully-drained engine close
+ * normally completes inline ({@code isCloseCompleted()} true on return),
+ * making this a single volatile read. When the SF worker was wedged in a
+ * syscall past {@code SegmentManager}'s bounded join, the close instead
+ * returned with the slot flock retained and its release deferred to the
+ * worker's exit path -- exactly the transient disk stall the
+ * deferred-close machinery exists to survive. Park (the same
+ * {@code awaitAckedFsn}-shaped wait the starvation policy uses) until the
+ * deferred cleanup confirms the release; each pass also re-arms the
+ * shared flock-release retry driver for the close-ran-but-release-failed
+ * case, mirroring {@link #isSlotLockReleased()}'s re-probe.
+ *
+ * Exhausting {@link #recycleDeferredCloseMaxWaitMillis} throws; the
+ * recycle stays pending ({@link RecycleResume#REBUILD}) and the next send
+ * retries the await. Before throwing, hand the still-locked engine to
+ * {@link #retainedEngine} so a pool re-probe
+ * ({@link #isSlotLockReleased()}) can still recover the slot's capacity
+ * if the worker ever exits.
+ */
+ private void awaitDeferredEngineClose(CursorSendEngine outgoing) {
+ if (outgoing.isCloseCompleted()) {
+ return;
+ }
+ LOG.warn("symbol dictionary recycle waiting for a deferred engine close: the SF worker "
+ + "did not quiesce, so the slot lock is still held [maxWaitMillis={}]",
+ recycleDeferredCloseMaxWaitMillis);
+ Runnable witness = deferredCloseParkWitness;
+ if (witness != null) {
+ witness.run();
+ }
+ long deadlineNanos = System.nanoTime() + recycleDeferredCloseMaxWaitMillis * 1_000_000L;
+ while (!outgoing.isCloseCompleted()) {
+ if (System.nanoTime() >= deadlineNanos) {
+ retainedEngine = outgoing;
+ slotLockReleased = false;
+ throw new LineSenderException("symbol dictionary recycle could not yet reclaim "
+ + "its slot: the engine's deferred close did not release the "
+ + "slot lock within " + recycleDeferredCloseMaxWaitMillis
+ + " ms (SF worker stalled); the recycle stays pending and is retried "
+ + "on the next send");
+ }
+ outgoing.ensureFlockReleaseRetryScheduled();
+ java.util.concurrent.locks.LockSupport.parkNanos(50_000L);
+ }
+ }
+
+ private void closeRecoveredEngine(CursorSendEngine recovered) {
+ recyclePendingOutgoing = recovered;
+ try {
+ recovered.close();
+ } catch (Error e) {
+ throw e;
+ } catch (Throwable t) {
+ LOG.warn("recovered engine close reported a failure during the symbol dictionary "
+ + "recycle; deferring to the close-completion probe", t);
+ }
+ awaitDeferredEngineClose(recovered);
+ recyclePendingOutgoing = null;
+ retainedEngine = null;
+ }
+
+ /**
+ * The recycle's tail: await the outgoing engine's (possibly deferred)
+ * close, rebuild a fresh engine on the emptied slot, and only then
+ * commit the swap -- roll the FSN base, install the fresh dictionary,
+ * advance the counters, wire the engine, reconnect. Every phase before
+ * the commit is idempotent, so both {@link #recycleForDictReset()} and a
+ * REBUILD resume run this; a transient throw leaves
+ * {@code recycleResume == REBUILD} for the next attempt. Only a rebuild
+ * that recovered UNACKED frames -- a genuine breach of the barrier's
+ * everything-acked proof -- latches {@link #recycleFailure}.
+ */
+ private void completeRecycleRebuild(int dictSizeAtSwap, long startNanos) {
+ CursorSendEngine outgoing = recyclePendingOutgoing;
+ if (outgoing != null) {
+ awaitDeferredEngineClose(outgoing); // throws transient while wedged
+ recyclePendingOutgoing = null;
+ retainedEngine = null;
+ }
+ // step 4: rebuild the engine on the now-empty slot.
+ CursorSendEngine rebuilt = rebuildEngineOrAbandon(
+ "symbol dictionary recycle could not rebuild its engine; retried on the next send");
+ if (rebuilt.wasRecoveredFromDisk()) {
+ // The outgoing close's empties-the-slot contract can miss
+ // benignly: a transiently failed segment unlink retains the ack
+ // watermark, and the SF design is that the NEXT engine on the
+ // slot recovers those segments as fully acked and retries the
+ // unlink on its own close. Heal by doing exactly that. Only a
+ // recovery holding UNACKED frames is a genuine breach: latch.
+ if (rebuilt.publishedFsn() > rebuilt.ackedFsn()) {
+ throw latchRecycleBreach(rebuilt, dictSizeAtSwap);
+ }
+ closeRecoveredEngine(rebuilt); // fully drained: retries the segment unlink
+ rebuilt = rebuildEngineOrAbandon(
+ "symbol dictionary recycle could not rebuild its engine after healing "
+ + "leftover acked segments; retried on the next send");
+ if (rebuilt.wasRecoveredFromDisk()) {
+ // Re-check: a breach the first pass could not see (the heal's
+ // close reshaped what recovery finds) must latch here too,
+ // otherwise it loops forever behind a resumable "acked" message.
+ if (rebuilt.publishedFsn() > rebuilt.ackedFsn()) {
+ throw latchRecycleBreach(rebuilt, dictSizeAtSwap);
+ }
+ closeRecoveredEngine(rebuilt);
+ throw new LineSenderException(
+ "symbol dictionary recycle keeps recovering leftover acked segments "
+ + "(slot cleanup not durable yet); retried on the next send");
+ }
+ }
+ // COMMIT (steps 5 + 6): pure producer-side state, and nothing below
+ // can throw. Step 5 rolls the external FSN base past every FSN the
+ // outgoing epoch handed out (the -1 no-publish case adds 0); it must
+ // run with cursorSendLoop == null, which step 2 guarantees and the
+ // step-7 reconnect below only undoes afterwards.
+ rollFsnEpochBase(recyclePendingLastPublishedFsn);
+ // Replace the dictionary, don't clear().
+ globalSymbolDictionary = new GlobalSymbolDictionary(Math.max(dictSizeAtSwap, 64));
+ sentMaxSymbolId = -1;
+ currentBatchMaxSymbolId = -1;
+ lastCommitBoundaryFsn = -1L;
+ symbolDictEpoch++;
+ symbolDictResetsPerformed++;
+ resetArmed = false;
+ manualResetRequested = false;
+ // Anti-thrash floor: see resetFloorSymbols.
+ resetFloorSymbols = Math.min(dictSizeAtSwap * 2,
+ QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE / 2);
+ // Deliberately re-derived (not carried over): the healing half
+ // of the recycle contract -- a sender that degraded to full
+ // frames heals back into delta mode once the underlying fault
+ // clears; a persistent fault just degrades the fresh engine
+ // again on its first append (SymbolDictRecycleHealingTest).
+ deltaDictEnabled = rebuilt.isDeltaDictEnabled();
+ cursorEngine = rebuilt;
+ ownsCursorEngine = true;
+ cursorEngine.setSlotLockReleaseListener(this::onSlotLockReleased);
+ // The fresh engine holds the slot flock again; a stale true (an
+ // isSlotLockReleased() re-probe of the outgoing engine while the
+ // recycle stayed pending) no longer describes this sender's state.
+ slotLockReleased = false;
+ recycleResume = RecycleResume.NONE;
+ recyclePendingLastPublishedFsn = -1L;
+ // step 7: reconnect (the swap has already committed).
+ // hasInitialConnectRun forces ensureConnected's ASYNC branch here, so
+ // the deferred socket connect never parks the producer thread; the
+ // loop retries indefinitely on the I/O thread, and a failed setup
+ // here leaves a coherent, merely-disconnected sender that is retried
+ // by the next sendRow()'s ensureConnected().
+ try {
+ ensureConnected();
+ } catch (Error e) {
+ throw e;
+ } catch (Throwable t) {
+ LOG.warn("symbol dictionary swap committed but starting its deferred reconnect "
+ + "failed; sender stays disconnected on the fresh epoch and retries "
+ + "the setup on the next send [epoch={}, dictSizeAtSwap={}]",
+ symbolDictEpoch, dictSizeAtSwap, t);
+ if (t instanceof LineSenderException) {
+ throw (LineSenderException) t;
+ }
+ throw new LineSenderException(t).put("symbol dictionary recycle reconnect failed");
+ }
+ LOG.info("symbol dictionary recycled [epoch={}, dictSizeAtSwap={}, pauseMicros={}]",
+ symbolDictEpoch, dictSizeAtSwap, (System.nanoTime() - startNanos) / 1000L);
+ }
+
+ /**
+ * True once every FSN this engine has published has also been
+ * server-acknowledged (or nothing has been published yet). The barrier
+ * {@link #recycleForDictReset()} waits for: the swap tears the cursor
+ * engine down, so it must never run while a frame is still in flight.
+ *
+ * Read order matters: {@code publishedFsn} (producer-written, cannot move
+ * during this call -- we ARE the producer) first, then {@code ackedFsn}
+ * (monotone, I/O-thread-written) second. Reading them in the other order
+ * could observe a published advance without its matching ack and falsely
+ * report drained.
+ */
+ private boolean isRingDrained() {
+ long published = cursorEngine.publishedFsn();
+ return published < 0 || cursorEngine.ackedFsn() >= published;
+ }
+
+ /**
+ * The recycle's one non-resumable verdict: a rebuild recovered UNACKED
+ * frames from the slot the outgoing engine's fully-drained close was
+ * supposed to have emptied, so the producer's fresh dictionary and the
+ * slot's on-disk state have genuinely diverged. Latches
+ * {@link #recycleFailure}, disposes the rebuilt engine, and always throws
+ * -- the declared return type only lets callers write
+ * {@code throw latchRecycleBreach(...)}.
+ */
+ private RuntimeException latchRecycleBreach(CursorSendEngine rebuilt, int dictSizeAtSwap) {
+ LineSenderException breach = new LineSenderException(
+ "symbol dictionary recycle rebuilt on a slot holding unacknowledged "
+ + "frames: the outgoing engine's fully-drained close contract "
+ + "was breached");
+ recycleFailure = breach;
+ recycleResume = RecycleResume.NONE;
+ try {
+ rebuilt.close();
+ } catch (Error e) {
+ throw e;
+ } catch (Throwable ignored) {
+ // terminal either way; the retained-engine probe below covers a deferred close
+ }
+ if (!rebuilt.isCloseCompleted()) {
+ retainedEngine = rebuilt;
+ slotLockReleased = false;
+ }
+ LOG.error("symbol dictionary recycle failed; sender is now terminal "
+ + "[epoch={}, dictSizeAtSwap={}]", symbolDictEpoch, dictSizeAtSwap, breach);
+ throw breach;
+ }
+
+ /**
+ * Starvation policy: when the ring is NOT drained at arming time, waits
+ * out an opportunistic window before giving up for this armed window.
+ * Refuses (returns immediately) in three cases: {@code resetMaxWaitMillis
+ * <= 0} (blocking disabled), a wait already ran for this arm cycle
+ * ({@link #starvationWaitDoneThisArm} -- at most one blocking wait per
+ * armed window), or a deferred-commit group is open
+ * ({@link #hasDeferredMessages}). That last guard is a data-safety
+ * requirement, not an optimisation: the server withholds acks for
+ * {@code FLAG_DEFER_COMMIT} frames by design until the closing commit
+ * lands, and this producer thread is the only one that could ever send
+ * that commit -- blocking here would just run out the clock every time,
+ * while starving the caller of the thread it needs to actually close the
+ * group.
+ *
+ * Otherwise waits (parked, {@code awaitAckedFsn}-shaped) until either the
+ * ring drains -- in which case the recycle runs synchronously before
+ * returning -- or {@code resetMaxWaitMillis} elapses from THIS call, in
+ * which case it gives up, counts the timeout, and leaves
+ * {@link #resetArmed} set so a later drained {@link #table(CharSequence)}
+ * call can still recycle opportunistically.
+ */
+ private void maybeBlockForStarvedReset() {
+ if (resetMaxWaitMillis <= 0 || starvationWaitDoneThisArm) {
+ return;
+ }
+ if (hasDeferredMessages) {
+ return;
+ }
+ if (System.nanoTime() - armedSinceNanos < resetMaxWaitMillis * 1_000_000L) {
+ return;
+ }
+ starvationWaitDoneThisArm = true;
+ long deadlineNanos = System.nanoTime() + resetMaxWaitMillis * 1_000_000L;
+ while (!isRingDrained()) {
+ cursorEngine.checkDurability();
+ if (cursorSendLoop != null) {
+ cursorSendLoop.checkError();
+ }
+ checkConnectionError();
+ if (System.nanoTime() >= deadlineNanos) {
+ symbolDictResetStarvationTimeouts++;
+ LOG.warn("symbol dictionary reset starved: backlog not drained within {} ms; "
+ + "staying armed", resetMaxWaitMillis);
+ return;
+ }
+ java.util.concurrent.locks.LockSupport.parkNanos(50_000L);
+ }
+ recycleForDictReset();
+ }
+
+ /**
+ * Evaluates whether the barrier in {@link #table(CharSequence)} may run
+ * the symbol-dictionary recycle right now. Only ever called with
+ * {@link #resetArmed} true -- {@link #armIfEligible()} already refused to
+ * arm a sender that cannot rebuild, so this method only has to weigh
+ * producer-side state.
+ *
+ * Refuses when there is producer-side state the swap cannot safely tear
+ * down: no connection yet (a V4 sender that has never sent is never
+ * pre-connected here), a flush in flight ({@code pendingRowCount != 0}),
+ * or a row under construction. Otherwise proceeds to the ring-drained
+ * check: if the backlog is empty, recycle immediately; if not, defer to
+ * {@link #maybeBlockForStarvedReset()}'s starvation-wait policy instead
+ * of blocking the caller indefinitely here.
+ */
+ private void maybeRecycleForDictReset() {
+ if (!connected
+ || pendingRowCount != 0
+ || (currentTableBuffer != null && currentTableBuffer.hasInProgressRow())) {
+ return;
+ }
+ if (isRingDrained()) {
+ recycleForDictReset();
+ } else {
+ maybeBlockForStarvedReset();
+ }
+ }
+
+ private CursorSendEngine rebuildEngineOrAbandon(String message) {
+ try {
+ return engineRebuildFactory.rebuild(userErrorHandler());
+ } catch (Error e) {
+ throw e;
+ } catch (Throwable t) {
+ throw new LineSenderException(t).put(message);
+ }
+ }
+
+ /**
+ * The symbol-dictionary recycle swap. Runs synchronously on the producer
+ * thread from the {@link #table(CharSequence)} barrier, once
+ * {@link #maybeRecycleForDictReset()} has proven the ring is drained.
+ * Seven steps, strictly ordered:
+ *
+ * Step 3 mirrors {@code close()}'s deferred-close discipline: when the
+ * outgoing engine's close could not confirm SF-worker quiescence it
+ * returns with the slot flock retained and {@code isCloseCompleted()}
+ * false, releasing both from the worker's exit path. The tail then awaits
+ * that deferred release (bounded by
+ * {@link #RECYCLE_DEFERRED_CLOSE_MAX_WAIT_MILLIS}) before step 4 rebuilds
+ * on the slot -- rebuilding against the retained flock would throw
+ * {@code SlotLockContentionException} for what is usually a transient disk
+ * stall.
+ *
+ * Step 7's failure mode is unchanged: by then the swap has committed
+ * (step 6), so a failed step-7 setup (dispatcher construction, loop
+ * build/start -- environmental, since the socket connect itself is
+ * deferred to the I/O thread) leaves a fully coherent sender that is
+ * merely disconnected: {@code connected == false}, loop and client already
+ * closed and nulled by {@link #ensureConnected()}'s own catch, the fresh
+ * engine attached, and the step-6 counters ({@link #symbolDictEpoch},
+ * {@link #symbolDictResetsPerformed}) correctly left incremented because
+ * the swap really did happen. It rethrows loudly to the triggering caller
+ * but the sender stays usable, and the ordinary
+ * {@code sendRow() -> ensureConnected()} path retries the deferred setup
+ * -- and only that -- on the next send. Nothing can fire a second swap
+ * meanwhile: the fresh dictionary is below threshold,
+ * {@code manualResetRequested} was consumed at step 6, and
+ * {@link #maybeRecycleForDictReset()} requires {@code connected}.
+ */
+ private void recycleForDictReset() {
+ final long lastPublishedFsn = cursorEngine.publishedFsn(); // step 1
+ final int dictSizeAtSwap = globalSymbolDictionary.size();
+ final long startNanos = System.nanoTime();
+ if (lastPublishedFsn >= 0) {
+ // Written before teardown: the monitoring accessors keep
+ // reporting this durable watermark while cursorEngine is null.
+ lastRecycleDurableFsn = fsnEpochBase + lastPublishedFsn;
+ }
+ // step 2: close the loop - joins the I/O thread, closes the client.
+ try {
+ if (cursorSendLoop != null) {
+ cursorSendLoop.close();
+ // Read the sticky AFTER close(): close joins the I/O thread,
+ // so a connect that landed mid-window is final here. This is
+ // the only place an ASYNC-initial sender's connect (observed
+ // only by the I/O thread) reaches hasLoopEverConnected.
+ hasLoopEverConnected |= cursorSendLoop.hasEverConnected();
+ cursorSendLoop = null;
+ }
+ client = null;
+ } catch (Error e) {
+ throw e;
+ } catch (Throwable t) {
+ // close() set the loop's stop flag before throwing, so the loop
+ // is irreversibly dying but its I/O thread may still own the
+ // engine. Neither proceed with the swap nor claim connectivity;
+ // abandon, and let resumeRecycleIfPending() finish the close
+ // (a repeated close() converges once the I/O thread exits).
+ connected = false;
+ recycleResume = RecycleResume.CLOSE_LOOP;
+ LOG.warn("symbol dictionary recycle abandoned: closing the outgoing I/O loop "
+ + "failed; the close is finished on the next send [epoch={}]",
+ symbolDictEpoch, t);
+ throw rethrowRecycleAbandoned(t, "symbol dictionary recycle abandoned while closing "
+ + "the outgoing I/O loop; retried on the next send");
+ }
+ // step 3: fully-drained close of the engine - empties the slot and
+ // unlinks the parent-anchored logical slot lock. From here the old
+ // engine cannot come back, so record the REBUILD resume point BEFORE
+ // anything that can throw.
+ CursorSendEngine outgoing = cursorEngine;
+ cursorEngine = null;
+ connected = false;
+ recycleResume = RecycleResume.REBUILD;
+ recyclePendingOutgoing = outgoing;
+ recyclePendingLastPublishedFsn = lastPublishedFsn;
+ try {
+ outgoing.setSlotLockReleaseListener(null);
+ outgoing.close();
+ } catch (Error e) {
+ throw e;
+ } catch (Throwable t) {
+ // A throw with the terminal cleanup nevertheless completed (the
+ // post-cleanup fsyncDir durability warning) is not a swap
+ // failure -- finishClose's finally released the slot regardless.
+ // awaitDeferredEngineClose() below tells the two apart: it
+ // returns immediately when isCloseCompleted(), parks while the
+ // deferred close is in flight, and throws only on a genuinely
+ // dead worker.
+ LOG.warn("outgoing engine close reported a failure during the symbol dictionary "
+ + "recycle; deferring to the close-completion probe", t);
+ }
+ completeRecycleRebuild(dictSizeAtSwap, startNanos);
+ }
+
+ /**
+ * Advances an abandoned recycle. CLOSE_LOOP finishes killing the old
+ * loop (no swap -- the old engine and dictionary are intact and the
+ * armed recycle re-fires from a later barrier, once the reconnect the
+ * next send drives has restored {@code connected}); it drops the
+ * producer's delta baseline with the dead loop's catch-up mirror, see
+ * below. REBUILD completes the await/rebuild/commit tail; because the
+ * commit swaps the dictionary, it refuses while producer state could
+ * carry old-dictionary symbol ids (staged rows or a row in progress) --
+ * the caller's row fails, rolls back, and the next table() resumes
+ * cleanly.
+ */
+ private void resumeRecycleIfPending() {
+ if (recycleResume == RecycleResume.NONE) {
+ return;
+ }
+ if (recycleResume == RecycleResume.CLOSE_LOOP) {
+ try {
+ cursorSendLoop.close(); // re-signals; converges once the I/O thread exits
+ } catch (Error e) {
+ throw e;
+ } catch (Throwable t) {
+ throw rethrowRecycleAbandoned(t, "the outgoing I/O loop is still stopping; "
+ + "retried on the next send");
+ }
+ hasLoopEverConnected |= cursorSendLoop.hasEverConnected();
+ cursorSendLoop = null;
+ client = null;
+ // The dead loop took its catch-up mirror with it, and the fresh
+ // loop ensureConnected() builds next seeds that mirror only from a
+ // RECOVERED persisted dictionary -- this engine is live, so the
+ // fresh loop starts at sentDictCount == 0. sentMaxSymbolId is the
+ // producer's model of the same number (it normally survives a
+ // reconnect precisely because the SAME loop re-registers from its
+ // mirror), so it has to drop with the mirror; otherwise the next
+ // frame's delta starts above the new loop's coverage and trips its
+ // torn-dictionary guard. Nothing is invalidated by the drop: the
+ // barrier proved the ring drained before step 2, and every publish
+ // path runs ensureConnected() -- hence this resume -- first, so no
+ // frame referencing those ids can be waiting to replay.
+ sentMaxSymbolId = -1;
+ recycleResume = RecycleResume.NONE;
+ return;
+ }
+ // REBUILD
+ if (pendingRowCount != 0
+ || (currentTableBuffer != null && currentTableBuffer.hasInProgressRow())) {
+ throw new LineSenderException(
+ "a symbol dictionary recycle is completing; finish or cancel the "
+ + "in-progress row and retry");
+ }
+ completeRecycleRebuild(globalSymbolDictionary.size(), System.nanoTime());
+ }
+
+ /**
+ * Always throws; the declared return type exists purely so every caller
+ * can write {@code throw rethrowRecycleAbandoned(...)} and make an
+ * accidental fall-through past an abandoned recycle unrepresentable.
+ */
+ private RuntimeException rethrowRecycleAbandoned(Throwable t, String message) {
+ if (t instanceof Error) {
+ throw (Error) t;
+ }
+ if (t instanceof LineSenderException) {
+ throw (LineSenderException) t;
+ }
+ throw new LineSenderException(t).put(message);
}
/**
@@ -4335,6 +5457,11 @@ private void sendCommitMessage() {
lastCommitBoundaryFsn = cursorEngine.publishedFsn();
}
+ private SenderErrorHandler userErrorHandler() {
+ SenderErrorHandler h = errorHandler;
+ return h == DefaultSenderErrorHandler.INSTANCE ? null : h;
+ }
+
/**
* Advances the delta baseline once a frame carrying the current batch's
* symbols has been queued onto the ring. No-op in full-dict mode. Only ever
@@ -4348,8 +5475,10 @@ private void advanceSentMaxSymbolId() {
}
/**
- * Stops emitting delta dictionaries for the rest of this sender's life, after the
- * per-slot {@code .symbol-dict} has proved unwritable.
+ * Stops emitting delta dictionaries for the rest of this epoch, after the
+ * per-slot {@code .symbol-dict} has proved unwritable -- a symbol-dictionary
+ * recycle re-derives {@code deltaDictEnabled} from the fresh engine (the
+ * healing contract; see {@link #recycleForDictReset()}).
*
* The side-file can stop accepting appends mid-run -- a full disk or an exhausted
* quota, where SF's own segments stay writable because they are pre-allocated mmap
@@ -4372,8 +5501,9 @@ private void disableDeltaDict(Throwable cause) {
}
deltaDictEnabled = false;
LOG.warn("symbol dictionary persistence failed; this sender has switched to full "
- + "self-sufficient frames for the rest of its life (bandwidth cost only -- "
- + "no data is at risk, and recovery replays such frames without a side file)",
+ + "self-sufficient frames for the rest of this epoch (bandwidth cost only -- "
+ + "no data is at risk, and recovery replays such frames without a side file; "
+ + "a symbol dictionary recycle re-derives delta mode from the fresh engine)",
cause);
}
@@ -4711,14 +5841,28 @@ private void reclaimUnsentSymbolIds() {
}
private void resetSymbolDictStateForNewConnection() {
- // Runs on the foreground (initial) connect only -- NOT on the I/O thread's
- // reconnect/failover path. The per-batch watermark is drained state, so
- // clearing it here is harmless. sentMaxSymbolId is deliberately left
+ // Runs on the foreground connect only -- NOT on the I/O thread's
+ // reconnect/failover path. sentMaxSymbolId is deliberately left
// untouched: in delta mode the I/O thread re-registers the whole
// dictionary with a catch-up frame on reconnect, so the producer's
// monotonic baseline must survive the wire boundary; resetting it would
// desync the producer from the I/O thread's sent-dictionary count.
- currentBatchMaxSymbolId = -1;
+ //
+ // currentBatchMaxSymbolId is batch-scoped, not connection-scoped: a
+ // flush ships exactly [sentMaxSymbolId+1 .. currentBatchMaxSymbolId],
+ // so clearing it while a batch already references those ids ships a
+ // delta that OMITS them and puts rows on the wire pointing at symbol
+ // ids the server never received. Clearing it used to be unconditional
+ // and harmless because build() connects before the application can
+ // register anything. That no longer holds: a symbol-dictionary recycle
+ // whose step-7 connect failed defers the connect to the next
+ // sendRow(), which runs after symbol() has registered the ids of the
+ // row being built. Reset only from the drained state the old code
+ // assumed.
+ if (pendingRowCount == 0
+ && (currentTableBuffer == null || !currentTableBuffer.hasInProgressRow())) {
+ currentBatchMaxSymbolId = -1;
+ }
}
/**
@@ -4923,7 +6067,9 @@ private void sealAndSwapBuffer() {
// Surface any I/O thread error first — appendBlocking itself only
// throws on PAYLOAD_TOO_LARGE / backpressure deadline, but the
// I/O loop can have failed independently.
- cursorSendLoop.checkError();
+ if (cursorSendLoop != null) {
+ cursorSendLoop.checkError();
+ }
throw new LineSenderException("cursor SF append failed", t);
}
}
@@ -4933,6 +6079,7 @@ private void sealAndSwapBuffer() {
* Rows buffer until flush (explicit or auto-flush).
*/
private void sendRow() {
+ checkRecycleFailure();
ensureConnected();
// Hard guard: a single row whose bytes exceed the server's wire cap
@@ -5059,6 +6206,33 @@ public Endpoint(String host, int port) {
}
}
+ /**
+ * Rebuilds a fresh {@link CursorSendEngine} on this sender's own slot, going
+ * through the identical construct/quarantine code path
+ * {@link Sender.LineSenderBuilder#build} uses.
+ */
+ public interface EngineRebuildFactory {
+ CursorSendEngine rebuild();
+
+ /**
+ * Rebuild with the sender's current user-supplied error handler ({@code null}
+ * when only the default handler is installed), so a quarantine during the
+ * rebuild reaches a handler installed after {@code build()}.
+ */
+ default CursorSendEngine rebuild(SenderErrorHandler liveHandler) {
+ return rebuild();
+ }
+ }
+
+ /**
+ * How far an abandoned symbol-dictionary recycle got, and therefore what
+ * {@link #resumeRecycleIfPending()} must still do. See
+ * {@link #recycleResume}.
+ */
+ private enum RecycleResume {
+ NONE, CLOSE_LOOP, REBUILD
+ }
+
private final class ReconnectSupplier implements CursorWebSocketSendLoop.ReconnectFactory {
/**
* Optional caller-owned liveness gate. {@code null} means this factory
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
index a9b5545e..13e80da3 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
@@ -789,7 +789,8 @@ public void run() {
maxHeadFrameRejections,
poisonMinEscalationWindowMillis,
catchUpCapGapMinEscalationWindowMillis,
- CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN);
+ CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN,
+ 0L);
loop.start();
while (!stopRequestedOrInterrupted()) {
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
index 66a0635e..b06d3f39 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
@@ -1349,6 +1349,11 @@ public SlotLock getSlotLockForTesting() {
return slotLock;
}
+ @TestOnly
+ public Runnable getSlotLockReleaseListenerForTesting() {
+ return slotLockReleaseListener;
+ }
+
@TestOnly
public long getSyncIntervalNanosForTesting() {
return syncIntervalNanos;
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
index 5a66c302..4a100143 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
@@ -415,6 +415,14 @@ public final class CursorWebSocketSendLoop implements QuietCloseable {
// it is engine.ackedFsn() + 1, so the first replayed frame on the new
// connection is wireSeq=0 and server-side cumulative ACKs still line up.
private long fsnAtZero;
+ // Third coordinate: additive offset applied on top of the engine FSN
+ // (fsnAtZero already folded in) to produce the FSN this loop hands to a
+ // user-visible surface -- the progress dispatcher and every SenderError
+ // [fromFsn,toFsn] span. Fixed for the lifetime of one loop instance: 0
+ // for a loop built directly against a live engine, or the sender's
+ // fsnEpochBase snapshot when a symbol-dict recycle rebuilt the engine and
+ // restarted its internal FSNs at 0. Rule: external = externalFsnBase + raw.
+ private final long externalFsnBase;
// Bounded-await backstop budget for close() (see
// DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS). Overridable via
// setShutdownAwaitTimeoutMillis so tests can exercise the timeout branch
@@ -712,7 +720,7 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
reconnectMaxBackoffMillis, durableAckMode,
durableAckKeepaliveIntervalMillis, maxHeadFrameRejections,
poisonMinEscalationWindowMillis, catchUpCapGapMinEscalationWindowMillis,
- CatchUpCapGapPolicy.RETRY_FOREVER);
+ CatchUpCapGapPolicy.RETRY_FOREVER, 0L);
}
/**
@@ -730,7 +738,8 @@ private CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
int maxHeadFrameRejections,
long poisonMinEscalationWindowMillis,
long catchUpCapGapMinEscalationWindowMillis,
- CatchUpCapGapPolicy catchUpCapGapPolicy) {
+ CatchUpCapGapPolicy catchUpCapGapPolicy,
+ long externalFsnBase) {
if (maxHeadFrameRejections < 1) {
throw new IllegalArgumentException(
"maxHeadFrameRejections must be >= 1: " + maxHeadFrameRejections);
@@ -882,6 +891,7 @@ private CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
// always outlive their borrower. Any growth copy-on-writes into loop-owned memory
// (ensureSentDictCapacity), and releaseSentDictBytes frees only what the loop owns.
this.fsnAtZero = fsnAtZero;
+ this.externalFsnBase = externalFsnBase;
this.parkNanos = parkNanos;
this.reconnectFactory = reconnectFactory;
this.reconnectInitialBackoffMillis = reconnectInitialBackoffMillis;
@@ -923,6 +933,11 @@ private CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
* establishing its first connection, then retries endpoint-policy failures
* indefinitely after it has been live. An orphan drainer returns such failures
* to its owner so the slot can follow its settle/quarantine policy.
+ *
+ * {@code externalFsnBase} is the additive offset this loop folds into every
+ * user-visible FSN it produces (progress-dispatcher advances and
+ * {@link SenderError} spans) -- see {@link #externalFsnBase}. Pass {@code 0L}
+ * unless the caller is replacing an engine a symbol-dict recycle rebuilt.
*/
public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
long fsnAtZero, long parkNanos,
@@ -934,13 +949,14 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
int maxHeadFrameRejections,
long poisonMinEscalationWindowMillis,
long catchUpCapGapMinEscalationWindowMillis,
- ReconnectPolicy reconnectPolicy) {
+ ReconnectPolicy reconnectPolicy,
+ long externalFsnBase) {
this(client, engine, fsnAtZero, parkNanos, reconnectFactory,
reconnectInitialBackoffMillis,
reconnectMaxBackoffMillis, durableAckMode,
durableAckKeepaliveIntervalMillis, maxHeadFrameRejections,
poisonMinEscalationWindowMillis, catchUpCapGapMinEscalationWindowMillis,
- catchUpPolicyFor(reconnectPolicy));
+ catchUpPolicyFor(reconnectPolicy), externalFsnBase);
}
private static CatchUpCapGapPolicy catchUpPolicyFor(ReconnectPolicy reconnectPolicy) {
@@ -1497,6 +1513,25 @@ public boolean isRunning() {
return running;
}
+ /**
+ * Called by the sender before {@link #start()} when a prior loop of the
+ * same sender already reached the server: restores Invariant B's
+ * past-initialization classification (see {@link
+ * #endpointPolicyFailureIsTerminal()}) across a symbol-dict recycle's
+ * loop rebuild, where the constructor would otherwise seed a fresh
+ * {@code hasEverConnected = false} for the new loop instance (ASYNC
+ * startup always hands the constructor a null client). Public rather
+ * than package-private only because the owning sender lives in a
+ * different package; it is not part of the public {@code Sender} API.
+ * {@code hasEverConnected} is volatile, so this write needs no extra
+ * synchronization to be visible to the I/O thread -- callers still call
+ * it before {@code start()} so the invariant is established before the
+ * loop can observe any endpoint-policy failure.
+ */
+ public void markEverConnected() {
+ hasEverConnected = true;
+ }
+
/**
* Plug an async-delivery sink for {@link SenderConnectionEvent}
* notifications. Connection events fire from
@@ -1786,8 +1821,8 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM
// role rejects are classified into the transient branch below.
LOG.error("terminal upgrade error during {} -- won't retry: {}",
phase, e.getMessage());
- long fromFsn = engine.ackedFsn() + 1L;
- long toFsn = Math.max(fromFsn, engine.publishedFsn());
+ long fromFsn = externalFsnBase + engine.ackedFsn() + 1L;
+ long toFsn = Math.max(fromFsn, externalFsnBase + engine.publishedFsn());
SenderError err = new SenderError(
SenderError.Category.SECURITY_ERROR,
SenderError.Policy.TERMINAL,
@@ -1826,8 +1861,8 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM
// volatile first-writer-wins latch observed by the owner.
capabilityGapTerminal = e;
}
- long fromFsn = engine.ackedFsn() + 1L;
- long toFsn = Math.max(fromFsn, engine.publishedFsn());
+ long fromFsn = externalFsnBase + engine.ackedFsn() + 1L;
+ long toFsn = Math.max(fromFsn, externalFsnBase + engine.publishedFsn());
SenderError err = new SenderError(
SenderError.Category.PROTOCOL_VIOLATION,
SenderError.Policy.TERMINAL,
@@ -1958,7 +1993,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM
* producer stays alive and no data is at risk.
*/
private void dispatchRetriedEndpointPolicyFailure(SenderError.Category category, String message) {
- long fromFsn = engine.ackedFsn() + 1L;
+ long fromFsn = externalFsnBase + engine.ackedFsn() + 1L;
dispatchError(new SenderError(
category,
SenderError.Policy.RETRIABLE,
@@ -1966,7 +2001,7 @@ private void dispatchRetriedEndpointPolicyFailure(SenderError.Category category,
message,
SenderError.NO_MESSAGE_SEQUENCE,
fromFsn,
- Math.max(fromFsn, engine.publishedFsn()),
+ Math.max(fromFsn, externalFsnBase + engine.publishedFsn()),
null,
System.nanoTime()
));
@@ -2072,9 +2107,11 @@ private void haltOnPoisonedFrame(String lastRejection, long toFsnHint) {
// the operator at those bytes would misattribute the poison. The
// caller supplies the span end: a NACK names the exact frame, so the
// span is that single frame; a non-orderly close cannot single one
- // out, so it spans to publishedFsn.
- long fromFsn = poisonFsn;
- long toFsn = Math.max(fromFsn, toFsnHint);
+ // out, so it spans to publishedFsn. poisonFsn and toFsnHint are both
+ // raw internal FSNs (fsnAtZero already folded in by the caller where
+ // relevant); rebase both by externalFsnBase here.
+ long fromFsn = externalFsnBase + poisonFsn;
+ long toFsn = Math.max(fromFsn, externalFsnBase + toFsnHint);
String msg = "frame at fsn=" + fromFsn + " rejected " + poisonStrikes
+ " consecutive times with no acceptance at or beyond it -- poisoned frame, replay cannot succeed (last: "
+ lastRejection + ')';
@@ -2098,12 +2135,14 @@ private void haltOnPoisonedFrame(String lastRejection, long toFsnHint) {
* Notify the progress dispatcher that the ack watermark advanced to
* {@code ackedFsn}. Caller must already have observed the advance via
* {@link CursorSendEngine#acknowledge}'s boolean return; this method
- * does no further filtering.
+ * does no further filtering. {@code ackedFsn} is the engine-relative FSN
+ * (fsnAtZero already folded in by the caller); this rebases it by
+ * {@link #externalFsnBase} before it reaches the user-visible dispatcher.
*/
private void dispatchProgress(long ackedFsn) {
SenderProgressDispatcher d = progressDispatcher;
if (d != null) {
- d.offer(ackedFsn);
+ d.offer(externalFsnBase + ackedFsn);
}
}
@@ -3842,8 +3881,8 @@ private void handlePreSendRejection(long wireSeq, byte status,
// protocol-violation close path uses (see onClose above): there
// is no FSN we can attribute the rejection to, so we report
// the unacked range the producer can correlate against.
- long fromFsn = engine.ackedFsn() + 1L;
- long toFsn = Math.max(fromFsn, engine.publishedFsn());
+ long fromFsn = externalFsnBase + engine.ackedFsn() + 1L;
+ long toFsn = Math.max(fromFsn, externalFsnBase + engine.publishedFsn());
String tableName = response.getTableEntryCount() == 1
? response.getTableName(0)
: null;
@@ -3966,8 +4005,8 @@ private void handleServerRejection(long wireSeq) {
status & 0xFF,
response.getErrorMessage(),
wireSeq,
- fsn,
- fsn,
+ externalFsnBase + fsn,
+ externalFsnBase + fsn,
tableName,
System.nanoTime()
);
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
index 074cf0e3..0fa7a645 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
@@ -92,8 +92,17 @@ public final class QwpConstants {
*
* NOT the result-direction cap: {@code QwpResultBatchDecoder.MAX_CONN_DICT_SIZE}
* (8,388,608) governs server-to-client result batches and is unrelated.
- */
- public static final int MAX_SYMBOL_DICTIONARY_SIZE = 1_000_000;
+ *
+ * Compatibility: servers released before QuestDB 10.0.0 cap their
+ * dictionary at 1,000,000, and QWP has no wire-level negotiation of the
+ * limit -- a dictionary this client lets grow past 1M is rejected by
+ * those servers as a terminal parse error. Reachable on defaults: each
+ * recycle raises the re-arm bar to twice the dictionary size at the
+ * swap, capped at half of this constant, so an unbounded-cardinality
+ * producer's dictionary grows to 1M entries per epoch. Only 10.0.0+
+ * servers are supported.
+ */
+ public static final int MAX_SYMBOL_DICTIONARY_SIZE = 2_000_000;
/**
* Maximum table name length in bytes. Mirrors the server's same-named
* constant; used by the decoder to reject malformed wire bytes.
diff --git a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java
index c9529a13..b9725608 100644
--- a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java
+++ b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java
@@ -90,6 +90,9 @@ public final class ConfigSchema {
str("sf_max_segment_bytes", Side.INGRESS);
str("sf_max_total_bytes", Side.INGRESS);
str("sf_sync_interval_millis", Side.INGRESS);
+ str("symbol_dict_reset", Side.INGRESS);
+ str("symbol_dict_reset_max_wait_millis", Side.INGRESS);
+ str("symbol_dict_reset_threshold", Side.INGRESS);
str("transaction", Side.INGRESS);
// EGRESS -- the QwpQueryClient applies. Typed where there is a range or
diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java
index 7b4e5f80..095f64ab 100644
--- a/core/src/main/java/io/questdb/client/impl/PooledSender.java
+++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java
@@ -335,6 +335,11 @@ public void reset() {
slot.live(generation).reset();
}
+ @Override
+ public void resetSymbolDictionary() {
+ slot.live(generation).resetSymbolDictionary();
+ }
+
@Override
public Sender shortColumn(CharSequence name, short value) {
slot.live(generation).shortColumn(name, value);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCeilingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCeilingTest.java
index d9ab6026..846b2e8a 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCeilingTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCeilingTest.java
@@ -45,7 +45,7 @@
/**
* The producer-side dictionary cap ({@code MAX_SYMBOL_DICTIONARY_SIZE}) as the
* application sees it: {@code symbol()} with a value that would create the
- * 1,000,001st distinct entry throws BEFORE the row is buffered, the row is
+ * 2,000,001st distinct entry throws BEFORE the row is buffered, the row is
* cancellable, and the sender keeps working with already-registered values --
* the wire never carries the refused symbol.
*/
@@ -76,7 +76,7 @@ public void testSymbolPastCapThrowsAndSenderStaysUsable() throws Exception {
sender.table("t").symbol("s", "one-too-many");
Assert.fail("expected LineSenderException past the dictionary cap");
} catch (LineSenderException expected) {
- Assert.assertTrue(expected.getMessage().contains("1000000"));
+ Assert.assertTrue(expected.getMessage().contains(String.valueOf(MAX_SYMBOL_DICTIONARY_SIZE)));
}
Assert.assertEquals("the refusal must not have grown the dictionary",
MAX_SYMBOL_DICTIONARY_SIZE, dict.size());
@@ -98,6 +98,54 @@ public void testSymbolPastCapThrowsAndSenderStaysUsable() throws Exception {
});
}
+ /**
+ * A threshold configured AT the cap, with automatic reset DISABLED, must
+ * behave exactly like the undecorated cap: the refusal still fires, and
+ * its message still names the reset valve even though this particular
+ * sender has it switched off -- the valve is documented for senders that
+ * want it, not conditioned on this sender having chosen it.
+ *
+ * Out of scope here: whether {@code symbol_dict_reset=off} actually keeps
+ * {@code armIfEligible()} from arming. That only runs from the tail of a
+ * completed {@code flush()}, which this test never performs (the fill
+ * goes through the raw dictionary test accessor, and the one
+ * {@code Sender}-routed call throws inside {@code symbol()} before a row
+ * completes) -- an {@code isResetArmed()} assertion here would pass
+ * regardless of the knob, proving nothing. That arming-vs-flush property
+ * is pinned in {@code SymbolDictRecycleArmingTest.testArmsAtThreshold}.
+ */
+ @Test
+ public void testCapReachedWithResetDisabledStillThrowsAndNamesTheResetValve() throws Exception {
+ assertMemoryLeak(() -> {
+ AckAllHandler handler = new AckAllHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port
+ + ";symbol_dict_reset=off;symbol_dict_reset_threshold=" + MAX_SYMBOL_DICTIONARY_SIZE + ";")) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ GlobalSymbolDictionary dict = ws.getGlobalSymbolDictionaryForTest();
+ for (int i = 0; i < MAX_SYMBOL_DICTIONARY_SIZE; i++) {
+ dict.getOrAddSymbol("f" + i);
+ }
+
+ try {
+ sender.table("t").symbol("s", "one-too-many");
+ Assert.fail("expected LineSenderException past the dictionary cap");
+ } catch (LineSenderException expected) {
+ String message = expected.getMessage();
+ Assert.assertTrue("message names the limit: " + message,
+ message.contains(String.valueOf(MAX_SYMBOL_DICTIONARY_SIZE)));
+ Assert.assertTrue("message points at the reset valve: " + message,
+ message.contains("symbol_dict_reset") && message.contains("resetSymbolDictionary()"));
+ }
+ }
+ }
+ });
+ }
+
private static void waitFor(Condition condition, long timeoutMillis) throws Exception {
long deadline = System.currentTimeMillis() + timeoutMillis;
while (!condition.holds()) {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java
index 65d61c26..9b418a85 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java
@@ -298,14 +298,14 @@ public void testSpecialCharactersInSymbols() {
@Test
public void testGetOrAddSymbol_refusesGrowthPastProtocolCap() {
- // Pre-sized so the 1M fill does not rehash its way through the test budget.
- GlobalSymbolDictionary dict = new GlobalSymbolDictionary(1 << 21);
+ // Pre-sized so the 2M fill does not rehash its way through the test budget.
+ GlobalSymbolDictionary dict = new GlobalSymbolDictionary(1 << 22);
for (int i = 0; i < QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE; i++) {
assertEquals(i, dict.getOrAddSymbol("f" + i));
}
- // Boundary: the 1,000,000th distinct symbol (id 999_999) was ACCEPTED above --
+ // Boundary: the 2,000,000th distinct symbol (id 1_999_999) was ACCEPTED above --
// the guard must refuse growth PAST the cap, not growth TO it, because the
- // server accepts a catch-up of exactly deltaStart + deltaCount == 1_000_000.
+ // server accepts a catch-up of exactly deltaStart + deltaCount == 2_000_000.
assertEquals(QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE, dict.size());
try {
@@ -313,9 +313,12 @@ public void testGetOrAddSymbol_refusesGrowthPastProtocolCap() {
fail("expected LineSenderException past the dictionary cap");
} catch (LineSenderException expected) {
assertTrue("message names the limit: " + expected.getMessage(),
- expected.getMessage().contains("1000000"));
+ expected.getMessage().contains("2000000"));
assertTrue("message names the recovery: " + expected.getMessage(),
expected.getMessage().contains("close this sender"));
+ assertTrue("message points at the reset valve: " + expected.getMessage(),
+ expected.getMessage().contains("symbol_dict_reset")
+ && expected.getMessage().contains("resetSymbolDictionary()"));
}
// The refusal mutated nothing: size unchanged, the refused symbol absent,
@@ -333,9 +336,9 @@ public void testGetOrAddSymbol_refusesGrowthPastProtocolCap() {
@Test
public void testProtocolCapConstantPinnedToServerValue() {
// The server-side QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE (questdb OSS) is
- // 1_000_000 and the ingress decoder rejects any delta or catch-up whose
+ // 2_000_000 and the ingress decoder rejects any delta or catch-up whose
// deltaStartId + deltaCount exceeds it. If this pin fails, the server
// constant moved and both sides must move together.
- assertEquals(1_000_000, QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE);
+ assertEquals(2_000_000, QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE);
}
}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
index 5d2b9976..75540eef 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
@@ -27,12 +27,16 @@
import io.questdb.client.Sender;
import io.questdb.client.cutlass.line.LineSenderException;
import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
import io.questdb.client.test.AbstractTest;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
import io.questdb.client.test.tools.TestUtils;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
+import java.util.concurrent.TimeUnit;
+
import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
/**
@@ -269,6 +273,125 @@ public void testCatchUpCapGapMinEscalationWindowUnsetInSnapshot() {
.get("catch_up_cap_gap_min_escalation_window_millis"));
}
+ @Test
+ public void testSymbolDictResetDefaults() throws Exception {
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = new TestWebSocketServer(new TestWebSocketServer.WebSocketServerHandler() {
+ })) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ try (Sender sender = Sender.fromConfig("ws::addr=" + LOCALHOST + ":" + port + ";")) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ Assert.assertTrue(ws.isSymbolDictResetEnabled());
+ Assert.assertEquals(100_000, ws.getSymbolDictResetThreshold());
+ Assert.assertEquals(30_000L, ws.getSymbolDictResetMaxWaitMillis());
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testSymbolDictResetConfigStringRoundTrip() throws Exception {
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = new TestWebSocketServer(new TestWebSocketServer.WebSocketServerHandler() {
+ })) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ try (Sender sender = Sender.fromConfig("ws::addr=" + LOCALHOST + ":" + port
+ + ";symbol_dict_reset=off;symbol_dict_reset_threshold=500;"
+ + "symbol_dict_reset_max_wait_millis=0;")) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ Assert.assertFalse(ws.isSymbolDictResetEnabled());
+ Assert.assertEquals(500, ws.getSymbolDictResetThreshold());
+ Assert.assertEquals(0L, ws.getSymbolDictResetMaxWaitMillis());
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testSymbolDictResetThresholdRejectsBadValues() {
+ assertThrows("symbol_dict_reset_threshold must be > 0",
+ () -> Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset_threshold=0;"));
+ assertThrows("symbol_dict_reset_threshold must be > 0",
+ () -> Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset_threshold=-5;"));
+ assertThrows("symbol_dict_reset_threshold must be > 0 and <= " + QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE,
+ () -> Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset_threshold="
+ + (QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE + 1) + ";"));
+ assertThrows("symbol_dict_reset_max_wait_millis must be >= 0: -1",
+ () -> Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset_max_wait_millis=-1;"));
+ }
+
+ @Test
+ public void testSymbolDictResetRejectedForNonWebSocketTransport() {
+ assertThrows("symbol_dict_reset is only supported for WebSocket transport",
+ () -> Sender.builder("http::addr=" + LOCALHOST + ":9000;symbol_dict_reset=on;"));
+ assertThrows("symbol_dict_reset_threshold is only supported for WebSocket transport",
+ () -> Sender.builder("http::addr=" + LOCALHOST + ":9000;symbol_dict_reset_threshold=500;"));
+ assertThrows("symbol_dict_reset_max_wait_millis is only supported for WebSocket transport",
+ () -> Sender.builder("http::addr=" + LOCALHOST + ":9000;symbol_dict_reset_max_wait_millis=0;"));
+ }
+
+ /**
+ * The three fluent setters carry the same transport guard as the
+ * connect-string keys, but sit on a separate code path -- pin them
+ * directly so a guard dropped from the setters alone cannot ship green.
+ */
+ @Test
+ public void testSymbolDictResetFluentSettersRejectNonWebSocketTransport() {
+ assertThrows("symbol_dict_reset is only supported for WebSocket transport",
+ () -> Sender.builder(Sender.Transport.HTTP).symbolDictReset(true));
+ assertThrows("symbol_dict_reset_threshold is only supported for WebSocket transport",
+ () -> Sender.builder(Sender.Transport.HTTP).symbolDictResetThreshold(500));
+ assertThrows("symbol_dict_reset_max_wait_millis is only supported for WebSocket transport",
+ () -> Sender.builder(Sender.Transport.HTTP).symbolDictResetMaxWaitMillis(0));
+ }
+
+ /**
+ * {@code symbol_dict_reset=on} must survive the parse as {@code true} --
+ * distinct from the default-true path, which passes with the parse branch
+ * deleted. Contrast against {@code off} on an otherwise identical builder.
+ */
+ @Test
+ public void testSymbolDictResetOnParsesTrue() {
+ Assert.assertEquals(Boolean.TRUE,
+ Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset=on;")
+ .wsConfigSnapshotForTest()
+ .get("symbol_dict_reset"));
+ Assert.assertEquals(Boolean.FALSE,
+ Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset=off;")
+ .wsConfigSnapshotForTest()
+ .get("symbol_dict_reset"));
+ }
+
+ @Test
+ public void testSymbolDictResetRejectsInvalidValue() {
+ assertThrows("invalid symbol_dict_reset [value=banana, allowed-values=[on, off]]",
+ () -> Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset=banana;"));
+ }
+
+ /**
+ * The accepted upper edge: exactly {@code MAX_SYMBOL_DICTIONARY_SIZE}
+ * (2M) must pass validation -- a {@code >} -> {@code >=} regression at
+ * the bound would reject it. Both the connect-string and the fluent
+ * setter paths.
+ */
+ @Test
+ public void testSymbolDictResetThresholdAcceptsHardCapBoundary() {
+ Assert.assertEquals(QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE,
+ Sender.builder("ws::addr=" + LOCALHOST + ";symbol_dict_reset_threshold="
+ + QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE + ";")
+ .wsConfigSnapshotForTest()
+ .get("symbol_dict_reset_threshold"));
+ Assert.assertEquals(QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE,
+ Sender.builder(Sender.Transport.WEBSOCKET)
+ .symbolDictResetThreshold(QwpConstants.MAX_SYMBOL_DICTIONARY_SIZE)
+ .wsConfigSnapshotForTest()
+ .get("symbol_dict_reset_threshold"));
+ }
+
@Test
public void testConnectionRefused() throws Exception {
assertMemoryLeak(() -> {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWireTestUtils.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWireTestUtils.java
index 1ed78752..8b42b337 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWireTestUtils.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWireTestUtils.java
@@ -147,7 +147,7 @@ static int readVarint(byte[] buffer, int[] position) {
throw new IllegalStateException("varint truncated");
}
- static int tableCount(byte[] frame) {
+ public static int tableCount(byte[] frame) {
return (frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8);
}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleArmingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleArmingTest.java
new file mode 100644
index 00000000..d54433a8
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleArmingTest.java
@@ -0,0 +1,366 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.DelegatingFilesFacade;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * Covers the arming half of the symbol-dictionary recycle feature:
+ * {@code QwpWebSocketSender.armIfEligible()}, called at the tail of
+ * {@code resetTableBuffersAfterFlush()}, and the manual advisory API
+ * {@link Sender#resetSymbolDictionary()}.
+ */
+public class SymbolDictRecycleArmingTest {
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ @Test
+ public void testArmsAtThreshold() throws Exception {
+ // threshold=3, send rows with symbols a,b -> flush -> not armed;
+ // add c -> flush -> armed
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = ackingServer()) {
+ try (Sender sender = Sender.fromConfig(cfg(server) + "symbol_dict_reset_threshold=3;")) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.table("t").symbol("s", "a").longColumn("v", 1).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1).atNow();
+ sender.flush();
+ Assert.assertFalse(ws.isResetArmed());
+ sender.table("t").symbol("s", "c").longColumn("v", 1).atNow();
+ sender.flush();
+ Assert.assertTrue(ws.isResetArmed());
+ }
+ }
+ });
+ }
+
+ /**
+ * Decision 5: arming ignores {@code deltaDictEnabled} -- threshold-based
+ * evaluation must still run once the sender has degraded to full self-sufficient
+ * frames. Reaching a custom low {@code symbol_dict_reset_threshold} on a
+ * sender that also carries the fault-injecting {@code FilesFacade} needs
+ * {@code QwpWebSocketSender}'s widest {@code connect(List
+ * This {@code connect(...)} overload installs no {@link
+ * io.questdb.client.cutlass.qwp.client.QwpWebSocketSender.EngineRebuildFactory
+ * EngineRebuildFactory} (only {@code Sender.build()} does), so
+ * crossing the threshold must never actually arm -- {@code
+ * armIfEligible()} folds the capability check in ahead of the threshold
+ * comparison. Decision 5 is instead pinned negatively here: full-dict
+ * degradation does not change that verdict either way.
+ */
+ @Test
+ public void testDoesNotArmWithoutRebuildFactory() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("arm-full-dict-sf").toString();
+ String slot = Paths.get(sfDir, "default").toString();
+ Assert.assertEquals(0, io.questdb.client.std.Files.mkdir(sfDir,
+ io.questdb.client.std.Files.DIR_MODE_DEFAULT));
+
+ try (TestWebSocketServer server = ackingServer()) {
+ int port = server.getPort();
+
+ MmapFaultDictFacade ff = new MmapFaultDictFacade();
+ CursorSendEngine engine = new CursorSendEngine(
+ slot, 4L * 1024 * 1024, 64L * 1024 * 1024,
+ CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, ff);
+ QwpWebSocketSender sender = QwpWebSocketSender.connect(
+ Collections.singletonList(new QwpWebSocketSender.Endpoint("localhost", port)),
+ null, // tlsConfig
+ 0, 0, 0L, // autoFlushRows, autoFlushBytes, autoFlushIntervalNanos
+ null, // authorizationHeader
+ false, // requestDurableAck
+ engine,
+ 5_000L, // closeFlushTimeoutMillis
+ CursorWebSocketSendLoop.DEFAULT_RECONNECT_MAX_DURATION_MILLIS,
+ CursorWebSocketSendLoop.DEFAULT_RECONNECT_INITIAL_BACKOFF_MILLIS,
+ CursorWebSocketSendLoop.DEFAULT_RECONNECT_MAX_BACKOFF_MILLIS,
+ Sender.InitialConnectMode.OFF,
+ null, // errorHandler
+ SenderErrorDispatcher.DEFAULT_CAPACITY,
+ CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS,
+ QwpWebSocketSender.DEFAULT_AUTH_TIMEOUT_MS,
+ 0, // connectTimeoutMs
+ null, // connectionListener
+ SenderConnectionDispatcher.DEFAULT_CAPACITY,
+ CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS,
+ CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS,
+ CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS,
+ true, // symbolDictResetEnabled
+ 3, // symbolDictResetThresholdSymbols -- low, deliberately crossed below
+ QwpWebSocketSender.DEFAULT_SYMBOL_DICT_RESET_MAX_WAIT_MILLIS);
+ try {
+ ff.armed = true; // next dictionary mmap growth raises a recognised fault
+ sender.table("m").symbol("s", "a").longColumn("v", 1L).atNow();
+ try {
+ sender.flush();
+ Assert.fail("expected the injected mmap fault to fail this flush");
+ } catch (LineSenderException expected) {
+ // Same guard MmapFaultDegradesTest pins: the fault degrades the
+ // sender to full self-sufficient frames instead of propagating raw.
+ }
+ Assert.assertFalse("a recognised mmap access fault must degrade the sender "
+ + "to full-dict mode",
+ sender.isDeltaDictEnabledForTest());
+ Assert.assertFalse("dictionary has only 1 entry, below the threshold of 3",
+ sender.isResetArmed());
+
+ // The fault facade disarms itself after firing once, so this retry
+ // succeeds and clears pendingRowCount back to 0; "a" is now published.
+ sender.flush();
+ Assert.assertFalse("still degraded, dictionary still below threshold",
+ sender.isDeltaDictEnabledForTest());
+ Assert.assertFalse(sender.isResetArmed());
+
+ sender.table("m").symbol("s", "b").longColumn("v", 2L).atNow();
+ sender.flush();
+ Assert.assertFalse("dictionary has 2 entries, still below the threshold of 3",
+ sender.isResetArmed());
+
+ // No manual resetSymbolDictionary() call anywhere in this test: crossing
+ // the threshold, even while degraded, still must not arm -- this
+ // connect(...) overload installs no engineRebuildFactory,
+ // and that capability check now runs ahead of the threshold
+ // comparison in armIfEligible().
+ sender.table("m").symbol("s", "c").longColumn("v", 3L).atNow();
+ sender.flush();
+ Assert.assertFalse("a sender with no rebuild factory must never arm, even once "
+ + "the threshold is crossed in full-dict mode",
+ sender.isResetArmed());
+ } finally {
+ sender.close();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testDoesNotArmWhenDisabled() throws Exception {
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = ackingServer()) {
+ try (Sender sender = Sender.fromConfig(
+ cfg(server) + "symbol_dict_reset=off;symbol_dict_reset_threshold=2;")) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ sender.flush();
+ Assert.assertFalse("symbol_dict_reset=off must never arm", ws.isResetArmed());
+ sender.table("t").symbol("s", "c").longColumn("v", 1L).atNow();
+ sender.flush();
+ Assert.assertFalse("symbol_dict_reset=off must never arm", ws.isResetArmed());
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testManualResetRequestArms() throws Exception {
+ assertMemoryLeak(() -> {
+ // pendingRowCount == 0: resetSymbolDictionary() arms immediately.
+ try (TestWebSocketServer server = ackingServer()) {
+ try (Sender sender = Sender.fromConfig(cfg(server))) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.resetSymbolDictionary();
+ Assert.assertTrue(ws.isResetArmed());
+ }
+ }
+
+ // Mid-batch: a request while a row is buffered (pendingRowCount != 0)
+ // only arms once the next flush runs armIfEligible() at its tail.
+ try (TestWebSocketServer server = ackingServer()) {
+ try (Sender sender = Sender.fromConfig(cfg(server))) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.resetSymbolDictionary();
+ Assert.assertFalse("a mid-batch request must not arm before the next flush",
+ ws.isResetArmed());
+ sender.flush();
+ Assert.assertTrue(ws.isResetArmed());
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testReArmFloorDoublesPerSwapAndBlocksOrganicReArm() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("floor-sf").toString();
+ try (TestWebSocketServer server = ackingServer()) {
+ String config = cfg(server) + "sf_dir=" + sfDir + ";symbol_dict_reset_threshold=2;";
+ try (Sender sender = Sender.fromConfig(config)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ Assert.assertEquals("no swap yet: floor is 0", 0, ws.getResetFloorSymbolsForTesting());
+
+ // epoch 0: two symbols == threshold -> arms
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000));
+ Assert.assertTrue(ws.isResetArmed());
+
+ // swap #1 runs inside this table() with dictSizeAtSwap == 2
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+ Assert.assertEquals("floor = 2 x size-at-swap", 4, ws.getResetFloorSymbolsForTesting());
+
+ // epoch 1: c,d,e,f == floor -> arms again
+ sender.table("t").symbol("s", "d").longColumn("v", 2L).atNow();
+ sender.table("t").symbol("s", "e").longColumn("v", 2L).atNow();
+ sender.table("t").symbol("s", "f").longColumn("v", 2L).atNow();
+ Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000));
+ Assert.assertTrue("size 4 >= max(threshold 2, floor 4) must arm", ws.isResetArmed());
+
+ // swap #2 with dictSizeAtSwap == 4
+ sender.table("t").symbol("s", "g").longColumn("v", 3L).atNow();
+ Assert.assertEquals(2, ws.getSymbolDictEpoch());
+ Assert.assertEquals("floor doubles again", 8, ws.getResetFloorSymbolsForTesting());
+
+ // epoch 2: four symbols is above the threshold but below the floor
+ sender.table("t").symbol("s", "h").longColumn("v", 4L).atNow();
+ sender.table("t").symbol("s", "i").longColumn("v", 4L).atNow();
+ sender.table("t").symbol("s", "j").longColumn("v", 4L).atNow();
+ Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000));
+ Assert.assertFalse("size 4 < floor 8 must not re-arm organically", ws.isResetArmed());
+
+ sender.resetSymbolDictionary();
+ Assert.assertTrue("the advisory request bypasses the floor", ws.isResetArmed());
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testResetSymbolDictionaryOnNonWsSenderIsNoOp() throws Exception {
+ assertMemoryLeak(() -> {
+ // protocolVersion(2) skips the eager server-side settings detection
+ // connect that build() otherwise performs, so no live server is needed
+ // (see LineSenderBuilderTest.testCustomPemRootsDoNotRequirePassword).
+ try (Sender sender = Sender.builder(Sender.Transport.HTTP)
+ .address("localhost")
+ .protocolVersion(2)
+ .build()) {
+ sender.resetSymbolDictionary();
+ }
+ });
+ }
+
+ @Test
+ public void testSplitFlushPathArms() throws Exception {
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = ackingServer()) {
+ server.setAdvertisedMaxBatchSize(150); // forces the two-table batch to split
+ // Padding inflates each table past half the cap, so the combined
+ // two-table message exceeds it while each single-table split frame fits.
+ String pad = TestUtils.repeat("x", 60);
+ try (Sender sender = Sender.fromConfig(cfg(server) + "symbol_dict_reset_threshold=2;")) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ sender.table("t1").symbol("s", "a").stringColumn("p", pad).longColumn("v", 1L).atNow();
+ sender.table("t2").symbol("s", "b").stringColumn("p", pad).longColumn("v", 2L).atNow();
+ sender.flush();
+ Assert.assertTrue("the split-flush path shares resetTableBuffersAfterFlush's tail",
+ ws.isResetArmed());
+ }
+ }
+ });
+ }
+
+ private static TestWebSocketServer ackingServer() throws Exception {
+ TestWebSocketServer server = new TestWebSocketServer(new AckAllHandler());
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ return server;
+ }
+
+ private static String cfg(TestWebSocketServer server) {
+ return "ws::addr=localhost:" + server.getPort() + ";";
+ }
+
+ /**
+ * ACKs every frame it receives; does not otherwise inspect the wire.
+ */
+ private static class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ try {
+ client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Raises a RECOGNISED mmap access fault out of the persisted dictionary's next
+ * mmap growth, once, when {@link #armed}. Copied from
+ * {@code MmapFaultDegradesTest.MmapFaultDictFacade}.
+ */
+ private static final class MmapFaultDictFacade extends DelegatingFilesFacade {
+ boolean armed;
+
+ @Override
+ public boolean isMmapAllowed() {
+ return true;
+ }
+
+ @Override
+ public long mmap(int fd, long len, long offset, int flags, int memoryTag) {
+ if (armed) {
+ armed = false;
+ throw new InternalError(
+ "a fault occurred in a recent unsafe memory access operation in compiled Java code");
+ }
+ return INSTANCE.mmap(fd, len, offset, flags, memoryTag);
+ }
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleCatchUpSkipTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleCatchUpSkipTest.java
new file mode 100644
index 00000000..bb15a55d
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleCatchUpSkipTest.java
@@ -0,0 +1,380 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.std.Compat;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE;
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * Pins that the symbol-dictionary recycle's fresh connection ({@code
+ * QwpWebSocketSender.recycleForDictReset()}'s step 7 reconnect) never pays
+ * for a delta-dictionary catch-up frame, and that the state-reset it relies
+ * on to get there is not accidentally general-purpose.
+ *
+ * The catch-up mechanism itself is {@link DeltaDictCatchUpTest}'s territory
+ * ({@code CursorWebSocketSendLoop.setWireBaselineWithCatchUp}'s gate:
+ * {@code client != null && sentDictCount > 0 && hasReplayDictionaryDependency}).
+ * This suite does not re-implement or re-verify that mechanism -- it only
+ * observes the ONE fact specific to the recycle: {@code sentDictCount} on the
+ * fresh loop starts at 0 because {@code recycleForDictReset()}'s step 6
+ * rebuilds the engine on a freshly-emptied slot, whose {@code
+ * PersistedSymbolDict.recoveredSize()} is 0 -- so the loop constructor's
+ * {@code pd.recoveredSize() > 0} seed never fires, and the gate stays false
+ * for the whole first post-recycle connection. A PLAIN (non-recycle)
+ * reconnect on that same connection, by contrast, reuses the SAME loop
+ * instance whose mirror has since grown from the frames it sent -- so it DOES
+ * trip the gate. Observing both back to back in one test is the only way to
+ * prove the zero count above is the recycle's fresh-mirror property and not a
+ * blind spot in how this suite's handler counts frames.
+ *
+ * No production change is expected to make these pass. A failure here means
+ * either the fresh-mirror seeding regressed (a post-recycle connection
+ * started paying for catch-up again) or the recycle's {@code
+ * sentMaxSymbolId} reset ({@code recycleForDictReset()}'s step 6) leaked
+ * onto the ordinary reconnect path, which today never touches that
+ * baseline.
+ */
+public class SymbolDictRecycleCatchUpSkipTest {
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ /**
+ * The core scenario, SF-disk mode. {@code symbol_dict_reset_threshold=3}
+ * is deliberately higher than the 2 symbols this test registers in the
+ * new epoch before forcing the unplanned drop: epoch 0 crosses the
+ * threshold on its own (a, b, x -- 3 distinct symbols), so the recycle
+ * fires exactly once, synchronously, on the "c" call. Epoch 1 then
+ * registers only c, d (2 symbols, below the threshold) before the drop,
+ * and only e (a 3rd) after it -- staying unarmed for the whole test so no
+ * SECOND recycle can sneak in and confound the "does a plain reconnect
+ * still catch up / preserve the baseline" assertions below. (A lower
+ * threshold that let epoch 1 re-arm on c, d would turn the later {@code
+ * table("e")} call into an unwanted second recycle, landing e on a 4th
+ * connection instead of a plain reconnect's 3rd -- exactly the
+ * confounder this threshold choice avoids.)
+ */
+ @Test
+ public void testRecycleSkipsCatchUpThenUnplannedReconnectBoundsCatchUpToNewEpoch() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("catchup-skip").toString();
+ SkipCatchUpHandler handler = new SkipCatchUpHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=3;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ // Epoch 0 (connection 1): 3 distinct symbols cross threshold=3 and arm.
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "x").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: the arming batch must be acked before the recycle",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("must be armed after crossing threshold=3", ws.isResetArmed());
+ Assert.assertEquals(1, handler.connectionsAccepted.get());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ // Ring drained: this table() call recycles synchronously (steps 1-6:
+ // fresh empty engine/dictionary/epoch), and "c" is then the new
+ // epoch's own first symbol. The fresh connection (2) itself is the
+ // I/O thread's job and completes asynchronously -- confirmed below,
+ // after an acked post-recycle frame proves it is up.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertFalse("recycle must disarm", ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("epoch-1 batch must be acked before the unplanned drop",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertEquals("recycle must open a fresh connection",
+ 2, server.handshakeCount());
+
+ // --- Pin 1 + 2: zero catch-up frames, dictionary tiles from 0. ---
+ // Connection 2 is the FIRST connection after the recycle: its loop's
+ // sentDictCount mirror was seeded from the fresh engine's
+ // PersistedSymbolDict.recoveredSize() == 0 (nothing survived the
+ // recycle's slot wipe), so setWireBaselineWithCatchUp's
+ // `sentDictCount > 0` gate stays false for this whole connection.
+ Assert.assertEquals("first post-recycle connection must send zero catch-up "
+ + "(zero-table) frames",
+ 0, handler.zeroTableFramesFor(2));
+ Assert.assertEquals("connection 2's dictionary must tile ids from 0 with "
+ + "exactly the new epoch's symbols, none of epoch 0's a, b, x",
+ Arrays.asList("c", "d"), handler.dictFor(2));
+
+ // --- Positive control + pin 3: an UNPLANNED reconnect (server-side
+ // drop, no recycle involved) on this SAME connection DOES produce a
+ // catch-up frame, and that catch-up is bounded to exactly what this
+ // epoch has sent so far (c, d) -- proving both that the zero count
+ // above is a real property (not a handler blind spot) and that the
+ // recycle's fresh mirror does not somehow retain epoch 0's symbols.
+ handler.dropConnection(2);
+ waitFor(() -> handler.connectionsAccepted.get() >= 3, 5_000);
+ waitFor(() -> handler.dictFor(3).size() >= 2, 5_000);
+
+ Assert.assertTrue("an unplanned reconnect mid-epoch must still produce a "
+ + "catch-up frame",
+ handler.zeroTableFramesFor(3) >= 1);
+ Assert.assertEquals("the catch-up must bound itself to exactly this epoch's "
+ + "own symbols (c, d), never replaying epoch 0's a, b, x",
+ Arrays.asList("c", "d"), handler.dictFor(3));
+
+ // --- Pin 4: the plain reconnect preserved sentMaxSymbolId. A NEW
+ // symbol registered after it must ship with a delta start ABOVE 0.
+ // Nothing on this I/O-thread reconnect path touches sentMaxSymbolId
+ // (resetSymbolDictStateForNewConnection runs only on the foreground
+ // initial-connect path, guarded by the connected flag, and never
+ // fires here), so the producer's baseline (c, d already at ids 0, 1)
+ // survives the wire boundary and e resumes at id 2. Only
+ // recycleForDictReset()'s step 6 ever zeroes that baseline; a
+ // regression that folded the reset into a path this reconnect DOES
+ // run would re-ship the whole dictionary from deltaStart 0.
+ sender.table("t").symbol("s", "e").longColumn("v", 4L).atNow();
+ long fsn3 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-reconnect row must still get acked",
+ sender.awaitAckedFsn(fsn3, 5_000));
+ Assert.assertTrue("connection 3's post-reconnect data frame carrying the new "
+ + "symbol e must ship a delta start ABOVE the surviving "
+ + "baseline (>= 1), not 0",
+ handler.sawDeltaAboveBaselineOn(3));
+ }
+
+ Assert.assertEquals("exactly 3 connections total (epoch 0, epoch 1's first "
+ + "connection, epoch 1's unplanned reconnect)",
+ 3, handler.connectionsAccepted.get());
+ }
+ });
+ }
+
+ /**
+ * Pin 5: the recycle's step 7 reconnect funnels through {@code
+ * ensureConnected()}'s {@code ASYNC} arm exactly like any other initial
+ * connect, which ends up at the same {@code swapClient} catch-up gate as
+ * the SYNC-mode scenario above. Mirrors {@code
+ * SymbolDictRecycleMemoryModeTest#testRecycleUnderAsyncInitialConnect},
+ * but in SF-disk mode (this suite's mode throughout) rather than memory
+ * mode, and asserts the zero-catch-up property instead of just the
+ * delta-start/dictionary-content pair that test already covers.
+ */
+ @Test
+ public void testRecycleUnderAsyncInitialConnectSendsZeroCatchUpFrames() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("catchup-skip-async").toString();
+ SkipCatchUpHandler handler = new SkipCatchUpHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";initial_connect_retry=async;symbol_dict_reset_threshold=2;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ // Let the I/O thread complete the deferred initial connect before
+ // driving any traffic through it (see
+ // SymbolDictRecycleMemoryModeTest.awaitWasEverConnected).
+ awaitWasEverConnected(ws);
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: batch must be acked before the recycle",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+ Assert.assertEquals(1, handler.connectionsAccepted.get());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ // Recycles synchronously on the producer thread for steps 1-6; step
+ // 7's reconnect just re-arms the ASYNC path -- the actual handshake
+ // happens on the I/O thread and must be awaited via the ack below.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertFalse("recycle must disarm immediately (producer-side state)",
+ ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-recycle batch must still get acked once the async "
+ + "I/O thread completes the fresh handshake",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertEquals(2, handler.connectionsAccepted.get());
+ }
+
+ Assert.assertEquals("exactly 2 connections total", 2, handler.connectionsAccepted.get());
+ Assert.assertEquals("the ASYNC path funnels through the same swapClient catch-up "
+ + "gate -- the first post-recycle connection must still send zero "
+ + "catch-up frames",
+ 0, handler.zeroTableFramesFor(2));
+ Assert.assertEquals("connection 2's dictionary must hold only the post-recycle "
+ + "symbols, not a, b",
+ Arrays.asList("c", "d"), handler.dictFor(2));
+ }
+ });
+ }
+
+ /**
+ * Spins until the I/O thread has completed the deferred ASYNC initial
+ * connect (mirrors {@code SymbolDictRecycleMemoryModeTest}'s helper of
+ * the same name).
+ */
+ private static void awaitWasEverConnected(QwpWebSocketSender ws) {
+ long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+ while (!ws.wasEverConnected()) {
+ if (System.nanoTime() > deadlineNanos) {
+ throw new AssertionError("I/O thread did not complete the async initial "
+ + "connect within 5s");
+ }
+ Compat.onSpinWait();
+ }
+ }
+
+ private static void waitFor(BoolCondition cond, long timeoutMillis) {
+ long deadline = System.currentTimeMillis() + timeoutMillis;
+ while (System.currentTimeMillis() < deadline) {
+ if (cond.test()) return;
+ try {
+ Thread.sleep(20);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ Assert.fail("interrupted");
+ }
+ }
+ Assert.fail("waitFor timed out");
+ }
+
+ @FunctionalInterface
+ private interface BoolCondition {
+ boolean test();
+ }
+
+ /**
+ * Reconstructs each connection's per-connection delta dictionary (mirrors
+ * {@code DeltaDictCatchUpTest.CatchUpHandler} / {@code
+ * SymbolDictRecycleTest.RecycleHandler}), counts zero-table (catch-up)
+ * frames per connection, tracks whether any data frame on a connection
+ * carried a delta start above 0, and -- unlike the sibling handlers --
+ * exposes {@link #dropConnection(int)} so the TEST THREAD can force an
+ * unplanned drop asynchronously, independent of the ack-driven close a
+ * handler normally does from inside {@code onBinaryMessage}.
+ */
+ private static class SkipCatchUpHandler implements TestWebSocketServer.WebSocketServerHandler {
+ final AtomicInteger connectionsAccepted = new AtomicInteger();
+ private final List
+ * The wedge: {@code SegmentManager}'s trim-sync hook runs unconditionally once
+ * per service pass, so parking the worker there leaves it un-joinable exactly
+ * the way a stalled disk/NFS syscall does, and a shrunken
+ * {@code workerJoinTimeoutMillis} lets the close's bounded join give up
+ * promptly (the same recipe as {@code SlotLockReleasedContractTest}).
+ */
+public class SymbolDictRecycleDeferredCloseTest {
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ /**
+ * A transient wedge: the worker un-wedges while the recycle is parked in
+ * its deferred-close await. The recycle must ride the stall out and
+ * complete -- fresh epoch, rebuilt engine, sender fully usable -- instead
+ * of latching terminal on the retained flock.
+ */
+ @Test(timeout = 60_000L)
+ public void testRecycleSurvivesDeferredEngineClose() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("recycle-deferred-survive").toString();
+ try (TestWebSocketServer server = ackingServer()) {
+ String cfg = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir + ";";
+ CountDownLatch workerBlocked = new CountDownLatch(1);
+ CountDownLatch releaseWorker = new CountDownLatch(1);
+ AtomicBoolean wedgeFired = new AtomicBoolean();
+ AtomicReference
+ * Every test that rolls the base does so on a sender BEFORE its first connect
+ * (via {@link #createRolledSender}), never on an already-connected one:
+ * {@code rollFsnEpochBase}'s precondition forbids rolling while a live
+ * {@code CursorWebSocketSendLoop} is attached (its {@code externalFsnBase} is a
+ * construction-time snapshot, never updated on a live loop -- see that method's
+ * javadoc). Tests that need a realistic pre-roll FSN to roll by first drive a
+ * SEPARATE, ordinarily-connected sender against the same server to publish and
+ * ack a batch, close it, then hand that FSN to {@code createRolledSender} for a
+ * second, fresh sender/engine -- modelling the post-recycle engine that
+ * restarts its raw FSNs at 0.
+ */
+public class SymbolDictRecycleFsnContinuityTest {
+
+ /**
+ * Rolls a FRESH sender/engine (never published-to raw watermark starts at -1), not
+ * the already-connected one that produced {@code fsn1}: {@code rollFsnEpochBase}'s
+ * precondition forbids rolling while a live loop is attached (see its javadoc), and
+ * -- independent of that -- an already-connected sender's engine keeps its raw
+ * watermark across the roll, which would make this test pass even with the
+ * translation deleted (raw {@code ackedFsn() == fsn1 >= fsn1} regardless of any
+ * epoch math). Only a genuinely fresh engine (raw {@code ackedFsn() == -1}) makes
+ * the pre-roll short-circuit the ONLY way {@code awaitAckedFsn(fsn1, 0)} can return
+ * true here.
+ */
+ @Test
+ public void testPreRollTargetAnswersTrueAfterRoll() throws Exception {
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = ackingServer()) {
+ long fsn1;
+ try (QwpWebSocketSender sender1 = (QwpWebSocketSender) Sender.fromConfig(cfg(server))) {
+ sender1.table("t").longColumn("v", 1L).atNow();
+ fsn1 = sender1.flushAndGetSequence();
+ Assert.assertTrue("setup: the batch must actually be acked before the roll",
+ sender1.drain(5_000));
+ }
+
+ QwpWebSocketSender sender2 = createRolledSender(server, fsn1);
+ try {
+ long t0 = System.nanoTime();
+ Assert.assertTrue("a target FSN from a pre-recycle epoch must be reported acked "
+ + "immediately -- it was proven acked before the swap",
+ sender2.awaitAckedFsn(fsn1, 0));
+ long elapsedMs = (System.nanoTime() - t0) / 1_000_000;
+ Assert.assertTrue("must short-circuit, not poll: took " + elapsedMs + "ms",
+ elapsedMs < 200);
+ } finally {
+ sender2.close();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testPostRollSequencesExceedAllPreRoll() throws Exception {
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = ackingServer()) {
+ long fsn1;
+ try (QwpWebSocketSender sender1 = (QwpWebSocketSender) Sender.fromConfig(cfg(server))) {
+ sender1.table("t").longColumn("v", 1L).atNow();
+ fsn1 = sender1.flushAndGetSequence();
+ Assert.assertTrue(sender1.drain(5_000));
+ }
+
+ QwpWebSocketSender sender2 = createRolledSender(server, fsn1);
+ try {
+ long newBase = sender2.getFsnEpochBaseForTest();
+ Assert.assertEquals(fsn1 + 1, newBase);
+
+ sender2.table("t").longColumn("v", 2L).atNow();
+ long fsn2 = sender2.flushAndGetSequence();
+ Assert.assertTrue(sender2.drain(5_000));
+
+ Assert.assertTrue("post-roll FSN must exceed every pre-roll FSN: fsn2=" + fsn2
+ + " fsn1=" + fsn1,
+ fsn2 > fsn1);
+ // sender2's engine is genuinely fresh (raw publishedFsn() starts at -1), so
+ // its first-ever flush publishes raw 0. The exact-equality check is strictly
+ // stronger than ">" alone: it also catches an off-by-one in the roll formula
+ // (e.g. fsnEpochBase += lastPublishedFsn instead of + 1L), which the ">"
+ // check above would not.
+ Assert.assertEquals(newBase, fsn2);
+ } finally {
+ sender2.close();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testGetAckedFsnMonotoneAcrossRoll() throws Exception {
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = ackingServer()) {
+ long w;
+ long lastPublishedFsn;
+ try (QwpWebSocketSender sender1 = (QwpWebSocketSender) Sender.fromConfig(cfg(server))) {
+ sender1.table("t").longColumn("v", 1L).atNow();
+ long fsn1 = sender1.flushAndGetSequence();
+ Assert.assertTrue(sender1.drain(5_000));
+ w = sender1.getAckedFsn();
+ lastPublishedFsn = fsn1;
+ Assert.assertEquals("sanity: single-batch acked watermark must match its own FSN",
+ fsn1, w);
+ }
+
+ // A fresh sender/engine models the post-recycle engine that restarts its
+ // internal FSNs at 0; rolling its epoch base by the outgoing epoch's last
+ // published FSN is exactly what the recycle swap does in production.
+ QwpWebSocketSender sender2 = createRolledSender(server, lastPublishedFsn);
+ try {
+ long newBase = sender2.getFsnEpochBaseForTest();
+ Assert.assertEquals(lastPublishedFsn + 1, newBase);
+
+ Assert.assertEquals("before any new ack, getAckedFsn must read the synthetic "
+ + "watermark: one past the last external FSN the outgoing epoch "
+ + "ever reported",
+ newBase - 1, sender2.getAckedFsn());
+ Assert.assertTrue(sender2.getAckedFsn() >= w);
+
+ sender2.table("t").longColumn("v", 2L).atNow();
+ sender2.flush();
+ Assert.assertTrue(sender2.drain(5_000));
+ Assert.assertTrue("a new ack must advance the watermark past the synthetic "
+ + "post-roll value",
+ sender2.getAckedFsn() > newBase - 1);
+ } finally {
+ sender2.close();
+ }
+ }
+ });
+ }
+
+ /**
+ * The raw-feed bug test: without the {@code drain()} fix, a rolled epoch base makes
+ * the raw {@code cursorEngine.publishedFsn()} target look like it belongs to a
+ * pre-recycle epoch (its raw value is smaller than the rolled base), so the fixed
+ * {@code awaitAckedFsn} would short-circuit {@code true} on an un-rebased target --
+ * even though the frame was never actually acked. Must fail (spurious true) before
+ * {@code drain()} translates its target by {@code fsnEpochBase}.
+ */
+ @Test
+ public void testDrainAfterRollWaitsForNewFrames() throws Exception {
+ assertMemoryLeak(() -> {
+ GatedAckHandler handler = new GatedAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ // Roll well past the raw FSNs this fresh engine will ever publish, so a
+ // missing translation in drain() would make its raw target look pre-roll.
+ // Must roll before the sender's first connect (see rollFsnEpochBase's
+ // precondition: cursorSendLoop must be null).
+ QwpWebSocketSender sender = createRolledSender(server, 999L);
+ try {
+ sender.table("foo").longColumn("v", 1L).atNow();
+ boolean drainedEarly = sender.drain(200);
+ Assert.assertFalse("drain() must not spuriously report the new frame acked just "
+ + "because its raw FSN is smaller than the rolled epoch base",
+ drainedEarly);
+
+ handler.releaseAcks();
+ Assert.assertTrue("drain() must return true once the real ack arrives",
+ sender.drain(5_000));
+ } finally {
+ handler.releaseAcks();
+ sender.close();
+ }
+ }
+ });
+ }
+
+ /**
+ * {@link SenderError#getFromFsn()} / {@link SenderError#getToFsn()} surface synchronously
+ * via {@link LineSenderServerException#getServerError()}, unreachable by any
+ * dispatcher-side rebase -- the loop must rebase the span itself. Rolls the epoch base
+ * BEFORE the sender's first connect (the loop's {@code externalFsnBase} is frozen at
+ * construction) so the terminal NACK's span is built under a nonzero base.
+ */
+ @Test
+ public void testSenderErrorSpansCarryExternalFsns() throws Exception {
+ assertMemoryLeak(() -> {
+ TerminalNackHandler handler = new TerminalNackHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ AtomicReference
+ * Also covers the three permanent recycle-metrics getters ({@code getSymbolDictEpoch()},
+ * {@code getSymbolDictResetsPerformed()}, {@code getSymbolDictResetStarvationTimeouts()}).
+ */
+public class SymbolDictRecycleHealingTest {
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ @Test
+ public void testMetricsAfterTwoRecycles() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("metrics-sf").toString();
+ try (TestWebSocketServer server = ackingServer()) {
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=2;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+ Assert.assertEquals(0, ws.getSymbolDictResetsPerformed());
+ Assert.assertEquals(0, ws.getSymbolDictResetStarvationTimeouts());
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("armed: 2 distinct symbols crossed threshold=2", ws.isResetArmed());
+
+ // Ring drained -> this table() call recycles synchronously: epoch 1.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+ Assert.assertEquals(1, ws.getSymbolDictResetsPerformed());
+ Assert.assertEquals("no starvation wait was deliberately triggered",
+ 0, ws.getSymbolDictResetStarvationTimeouts());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+ // The anti-thrash floor (resetFloorSymbols = 2x the first swap's
+ // dictSizeAtSwap = 4) keeps c,d (2 symbols, == threshold but < floor)
+ // from re-arming on their own; a manual request bypasses the floor by
+ // design, so drive the second recycle through resetSymbolDictionary().
+ sender.resetSymbolDictionary();
+ Assert.assertTrue("manual reset request bypasses the re-arm floor",
+ ws.isResetArmed());
+
+ // Ring drained again -> second recycle: epoch 2.
+ sender.table("t").symbol("s", "e").longColumn("v", 4L).atNow();
+ Assert.assertEquals(2, ws.getSymbolDictEpoch());
+ Assert.assertEquals(2, ws.getSymbolDictResetsPerformed());
+ Assert.assertEquals("still no starvation wait was deliberately triggered",
+ 0, ws.getSymbolDictResetStarvationTimeouts());
+
+ long fsn3 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn3, 5_000));
+ }
+ }
+ });
+ }
+
+ /**
+ * The recovery-side sibling of {@code MmapFaultDegradesTest.testMmapAccessFaultDegradesPersistInsteadOfPropagating}:
+ * once the sender has degraded to full self-sufficient frames, the underlying fault clears,
+ * and a recycle rebuilds the engine, the fresh engine must re-derive delta-dict mode from
+ * scratch rather than staying degraded forever. Wire evidence: the first post-recycle frame
+ * (a fresh, empty dictionary) starts a delta at 0; the SECOND post-recycle frame, which
+ * introduces exactly one more symbol, starts its delta where the first one left off and
+ * carries only that one new entry -- the shape only delta mode produces. In full-dict mode
+ * every frame re-ships the whole dictionary from id 0 (see
+ * {@code QwpWebSocketSender.symbolDeltaBaseline()}: confirmedMaxId is permanently -1), so
+ * this pair of frames could not look like this if healing had not taken effect.
+ */
+ @Test
+ public void testRecycleHealsFullDictDegradeBackToDeltaMode() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("heal-sf").toString();
+ String slot = Paths.get(sfDir, "default").toString();
+ Assert.assertEquals(0, io.questdb.client.std.Files.mkdir(sfDir,
+ io.questdb.client.std.Files.DIR_MODE_DEFAULT));
+
+ CapturingAckHandler handler = new CapturingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+
+ HealableMmapFaultFacade ff = new HealableMmapFaultFacade();
+ CursorSendEngine engine = new CursorSendEngine(
+ slot, 4L * 1024 * 1024, 64L * 1024 * 1024,
+ CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, ff);
+ QwpWebSocketSender sender = buildSender(port, engine, 100_000);
+ // connect() never installs an engineRebuildFactory (only Sender.build() does),
+ // so the recycle would otherwise be a no-op. Install one that rebuilds on the
+ // SAME slot with the SAME (healable) facade -- mirroring the real factory
+ // Sender.build() installs, minus the FilesFacade seam Sender.fromConfig lacks.
+ sender.setEngineRebuildFactory(() -> new CursorSendEngine(
+ slot, 4L * 1024 * 1024, 64L * 1024 * 1024,
+ CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, ff));
+ try {
+ Assert.assertTrue("must start in delta mode", sender.isDeltaDictEnabledForTest());
+
+ // Degrade mid-life: fault the dictionary's next mmap growth.
+ ff.armed = true;
+ sender.table("m").symbol("s", "a").longColumn("v", 1L).atNow();
+ try {
+ sender.flush();
+ Assert.fail("expected the injected mmap fault to fail this flush");
+ } catch (LineSenderException expected) {
+ // same guard MmapFaultDegradesTest pins
+ Assert.assertTrue("the fault must be reported as a sender error, not a "
+ + "raw InternalError: " + expected.getMessage(),
+ expected.getMessage().contains(
+ "failed to persist symbol dictionary before publish"));
+ }
+ Assert.assertFalse("a recognised mmap access fault must degrade the sender",
+ sender.isDeltaDictEnabledForTest());
+
+ // Heal the facade. The retry below does not itself touch mmap --
+ // persistNewSymbolsBeforePublish short-circuits once !deltaDictEnabled --
+ // so healing here matters only for what the fresh post-recycle engine sees.
+ ff.armed = false;
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("the degraded retry must still ingest the row",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertEquals(0, sender.getSymbolDictEpoch());
+
+ // Drained: arm and trigger the recycle.
+ sender.resetSymbolDictionary();
+ Assert.assertTrue(sender.isResetArmed());
+ sender.table("m").symbol("s", "b").longColumn("v", 2L).atNow();
+ Assert.assertFalse("recycle must disarm", sender.isResetArmed());
+ Assert.assertEquals(1, sender.getSymbolDictEpoch());
+ Assert.assertEquals(1, sender.getSymbolDictResetsPerformed());
+
+ // The rebuilt engine re-derives delta-dict mode from scratch (a fresh,
+ // empty dictionary always opens cleanly at construction -- see this
+ // test's persistent-fault sibling for why this alone does not prove the
+ // facade was healed). The discriminating check is below, after the first
+ // post-recycle append.
+ Assert.assertTrue(sender.isDeltaDictEnabledForTest());
+
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+
+ // Discriminating check: fsn2's flush was the fresh engine's first
+ // append. A still-armed facade would have degraded it there (as the
+ // persistent-fault sibling proves against the identical setup) -- staying
+ // true here is real evidence the heal took effect, not just an artifact
+ // of fresh-engine construction never touching mmap.
+ Assert.assertTrue("a healed facade must let the fresh engine's first "
+ + "post-recycle append succeed and keep delta mode enabled",
+ sender.isDeltaDictEnabledForTest());
+
+ sender.table("m").symbol("s", "c").longColumn("v", 3L).atNow();
+ long fsn3 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn3, 5_000));
+
+ List
+ * WARNING -- recycle-only handler, do not copy into a plain-reconnect test.
+ * The per-connection sequence reset below assumes every connection change
+ * is a recycle, i.e. that a fresh engine is behind the new connection and
+ * its raw FSNs really do restart at 0. On an ordinary reconnect the SAME
+ * engine survives and keeps counting, so resetting here would ack frames
+ * the sender never published and silently advance its watermark past
+ * unsent data.
+ */
+ private static class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private TestWebSocketServer.ClientHandler currentClient;
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ if (currentClient != client) {
+ // A rebuilt engine restarts its raw FSNs at 0 (externalFsnBase absorbs the
+ // offset), and the ack sequence below is applied as a raw engine FSN -- so
+ // acking a recycle's fresh connection against the outgoing connection's
+ // sequence would ack frames that were never published. Reset per connection,
+ // matching CapturingAckHandler below.
+ currentClient = client;
+ nextSeq.set(0);
+ }
+ try {
+ client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /** ACKs every frame and records the raw bytes of every data frame, grouped by connection. */
+ private static class CapturingAckHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private final List
+ * The recycle swap's eight steps ({@code QwpWebSocketSender.recycleForDictReset()})
+ * were written against the store-and-forward slot lifecycle, but the factory's
+ * {@code slotPath == null} arm, {@code CursorSendEngine}'s file-less close, and the
+ * barrier itself are all mode-agnostic by construction -- nothing in
+ * {@code maybeRecycleForDictReset()} or the swap checks whether the sender is
+ * SF-backed. This suite pins that: every scenario {@code SymbolDictRecycleTest}
+ * proves for a disk-backed sender must hold identically for a {@code Sender.fromConfig}
+ * sender built with no {@code sf_dir} at all. No production change is expected to
+ * make these pass; a failure here means Task 5's swap accidentally gated something
+ * on store-and-forward being present.
+ */
+public class SymbolDictRecycleMemoryModeTest {
+
+ @Test
+ public void testRecycleAtEmptyBacklog() throws Exception {
+ assertMemoryLeak(() -> {
+ RecycleHandler handler = new RecycleHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ // No sf_dir: memory mode. Everything else mirrors
+ // SymbolDictRecycleTest#testRecycleAtEmptyBacklog exactly.
+ String cfg = "ws::addr=localhost:" + port + ";symbol_dict_reset_threshold=2;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: batch must be acked before the recycle",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+ Assert.assertEquals(1, handler.connectionsAccepted.get());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ // Ring drained, no row in progress: this table() call must
+ // recycle synchronously, exactly as in SF mode. The fresh
+ // WebSocket handshake is the I/O thread's job and completes
+ // asynchronously -- it is asserted below, after an acked
+ // post-recycle frame proves the connection is up.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertFalse("recycle must disarm", ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-recycle batch must still get acked",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertEquals("recycle must open a fresh connection",
+ 2, server.handshakeCount());
+ Assert.assertTrue("post-recycle FSN must exceed pre-recycle FSN "
+ + "[fsn1=" + fsn1 + ", fsn2=" + fsn2 + ']',
+ fsn2 > fsn1);
+ }
+
+ Assert.assertEquals("exactly 2 connections total", 2, handler.connectionsAccepted.get());
+ Assert.assertEquals("connection 2's first data frame must carry deltaStart == 0 "
+ + "(a fresh, empty dictionary)",
+ 0, handler.conn2FirstFrameDeltaStart);
+ Assert.assertEquals("connection 2's dictionary must hold only the post-recycle "
+ + "symbols, not a, b",
+ Arrays.asList("c", "d"), handler.dictFor(2));
+ }
+ });
+ }
+
+ /**
+ * Strengthens {@link #testRecycleAtEmptyBacklog} into a content oracle: every
+ * row before and after the recycle carries a distinct symbol value, and this
+ * asserts the server observed the FULL, exact, gap-free, duplicate-free
+ * sequence across both connections -- not just a spot check of the boundary
+ * frame. Proves the epoch swap loses (and doesn't duplicate) nothing that was
+ * ever acked, in memory mode exactly as {@code testPostRecycleSlotContents}
+ * proves the persisted-dictionary shape in SF mode.
+ */
+ @Test
+ public void testRecycleLosesNothingAcked() throws Exception {
+ assertMemoryLeak(() -> {
+ RecycleHandler handler = new RecycleHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";symbol_dict_reset_threshold=2;";
+
+ List
+ * (a) proves the recycle's step 2 ({@code cursorSendLoop.close()}) correctly
+ * joins an I/O thread that is itself mid-reconnect (not idle, not yet given
+ * up), and that step 7 no longer recovers the connection on the calling
+ * thread -- it defers to the I/O loop, so the swap returns promptly and the
+ * producer never observes the outage -- exercising
+ * {@code CursorWebSocketSendLoop.close()}'s "handles both states" contract
+ * under a real outage rather than a synthetic one.
+ *
+ * (b) proves the swap only ever tears down the producer's OWN cursor
+ * engine/I/O loop: an orphan drainer's engine and loop are entirely separate
+ * objects owned by {@code BackgroundDrainerPool}, so a recycle firing while a
+ * drain is in flight must leave the drain untouched and able to complete
+ * afterward.
+ */
+public class SymbolDictRecycleOutageTest {
+
+ private static final String ORPHAN_MARKER_SYMBOL = "orphan-marker-1";
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ /**
+ * Kills the server out from under an armed, fully-drained sender, waits
+ * for the pre-recycle I/O thread to actually enter its own reconnect
+ * loop (not just assumed via a fixed sleep) -- so the recycle's step 2
+ * ({@code cursorSendLoop.close()}) provably joins a MID-reconnect
+ * thread -- then triggers the recycle inline, on the calling thread.
+ * {@code reconnect_max_duration_millis} bounds only the sender's initial
+ * connect; under the store-and-forward contract step 7 no longer
+ * re-enters {@code connectWithRetry} on the producer thread, so the
+ * triggering {@code table()} call must return well within that budget
+ * even though the endpoint is still down when it fires. The main thread
+ * revives a fresh server on the same port after asserting the bound,
+ * mirroring {@code ReconnectTest}'s down-then-up realism.
+ */
+ @Test
+ public void testSyncModeRecycleDoesNotBlockProducerDuringOutage() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("outage-recycle").toString();
+ AckAllHandler firstHandler = new AckAllHandler();
+ int port;
+ try (TestWebSocketServer server = new TestWebSocketServer(firstHandler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=2"
+ + ";reconnect_initial_backoff_millis=20"
+ + ";reconnect_max_backoff_millis=80"
+ + ";reconnect_max_duration_millis=6000;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: the arming batch must be acked before the outage",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ // Kill the connection AND the listener -- a real outage, not
+ // just a dropped socket the same server would re-accept
+ // instantly.
+ server.close();
+
+ // Confirm the pre-recycle I/O thread actually entered its
+ // own reconnect loop against the now-refused port before we
+ // trigger the recycle -- so step 2's close() below is
+ // provably joining a MID-reconnect thread, not one that
+ // simply hasn't noticed the drop yet.
+ long attemptDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+ while (ws.getTotalReconnectAttempts() == 0 && System.nanoTime() < attemptDeadline) {
+ Thread.sleep(5);
+ }
+ Assert.assertTrue("pre-recycle I/O thread must have entered reconnect before "
+ + "the triggering table() call",
+ ws.getTotalReconnectAttempts() > 0);
+
+ // The recycle must return promptly: reconnect_max_duration_millis
+ // governs only the initial connect, and step 7 defers to the
+ // I/O loop instead of re-entering connectWithRetry on the
+ // producer thread.
+ long startNanos = System.nanoTime();
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ long elapsedMillis = (System.nanoTime() - startNanos) / 1_000_000L;
+ Assert.assertFalse("recycle must disarm", ws.isResetArmed());
+ Assert.assertEquals("recycle must complete despite the outage",
+ 1, ws.getSymbolDictEpoch());
+ Assert.assertTrue("the swap must not block the producer on the reconnect "
+ + "budget [elapsedMillis=" + elapsedMillis + ']',
+ elapsedMillis < 3_000);
+
+ long fsn2 = sender.flushAndGetSequence();
+ OutageRecycleHandler revivedHandler = new OutageRecycleHandler();
+ try (TestWebSocketServer revived =
+ new TestWebSocketServer(revivedHandler, false, null, port)) {
+ revived.start();
+ Assert.assertTrue(revived.awaitStart(5, TimeUnit.SECONDS));
+ Assert.assertTrue("the outage-window row must land once reconnected",
+ sender.awaitAckedFsn(fsn2, 10_000));
+ Assert.assertTrue(fsn2 > fsn1);
+ Assert.assertEquals(0, revivedHandler.firstFrameDeltaStart);
+ Assert.assertEquals(Collections.singletonList("c"), revivedHandler.dict());
+ }
+ }
+ }
+ });
+ }
+
+ /**
+ * Default configuration: no {@code reconnect_*} knob and no
+ * {@code initial_connect_retry}, so the builder resolves
+ * {@code initialConnectMode} to OFF. Under the store-and-forward
+ * contract, step 7 no longer opens a connection on the calling thread
+ * at all -- it defers to the I/O loop, so the triggering {@code table()}
+ * call must return normally even while the endpoint refuses
+ * connections.
+ *
+ * Proves the swap commits exactly one epoch and disarms without the
+ * caller ever observing a transport failure, that the flush right after
+ * publishes into the fresh epoch's SF slot, and that once the endpoint
+ * returns on the same port the I/O loop's own reconnect replays every
+ * row sent during the outage with zero loss -- reconnecting only, never
+ * re-running a teardown step and never swapping a second time.
+ */
+ @Test
+ public void testDefaultConfigRecycleBuffersThroughOutage() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("default-config-outage").toString();
+ AckAllHandler firstHandler = new AckAllHandler();
+ int port;
+ try (TestWebSocketServer server = new TestWebSocketServer(firstHandler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";symbol_dict_reset_threshold=2;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ Assert.assertTrue("the recycle must be on under a default configuration",
+ ws.isSymbolDictResetEnabled());
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: the arming batch must be acked before the outage",
+ sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ // Kill the listener AND the live connection. The ring is
+ // drained, so the sender-level connected flag is still true
+ // and the next table() call fires the recycle into a wire
+ // that is already down.
+ server.close();
+
+ // The ring is drained, so the next table() fires the recycle
+ // into a wire that is already down. The swap must complete AND
+ // return normally -- the reconnect is the I/O loop's job, so
+ // no transport failure may reach the producer. "c" registers
+ // into the fresh dictionary after the swap's
+ // resetSymbolDictStateForNewConnection but before the wire is
+ // up, which keeps pinning the drained-guard: a deferred
+ // connect that cleared the batch watermark would ship a row
+ // pointing at an id the server never received.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ Assert.assertTrue("wasEverConnected() must stay sticky across the recycle's "
+ + "rebuilt loop while the endpoint is still down -- the fresh "
+ + "loop must not report 'never connected' just because it is a "
+ + "new loop instance",
+ ws.wasEverConnected());
+ Assert.assertEquals("the swap must commit exactly one epoch",
+ 1, ws.getSymbolDictEpoch());
+ Assert.assertEquals(1, ws.getSymbolDictResetsPerformed());
+ Assert.assertFalse("a committed swap disarms", ws.isResetArmed());
+
+ // Producer keeps working against the dead endpoint: the
+ // flush publishes into the fresh epoch's SF slot.
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-recycle FSN must exceed pre-recycle FSN",
+ fsn2 > fsn1);
+
+ // Endpoint back on the SAME port: the I/O loop's own
+ // reconnect must land the buffered rows -- zero loss.
+ OutageRecycleHandler revivedHandler = new OutageRecycleHandler();
+ try (TestWebSocketServer revived =
+ new TestWebSocketServer(revivedHandler, false, null, port)) {
+ revived.start();
+ Assert.assertTrue(revived.awaitStart(5, TimeUnit.SECONDS));
+
+ Assert.assertTrue("rows sent during the outage must replay once "
+ + "the endpoint returns",
+ sender.awaitAckedFsn(fsn2, 10_000));
+ Assert.assertEquals("the recovery reconnects only -- no second swap",
+ 1, ws.getSymbolDictEpoch());
+ Assert.assertEquals(1, ws.getSymbolDictResetsPerformed());
+ Assert.assertEquals("the fresh connection's first frame must carry a "
+ + "fresh (empty) dictionary, not a, b",
+ 0, revivedHandler.firstFrameDeltaStart);
+ Assert.assertEquals(Collections.singletonList("c"), revivedHandler.dict());
+
+ // And the epoch keeps extending normally from there.
+ sender.table("t").symbol("s", "e").longColumn("v", 4L).atNow();
+ long fsn3 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn3, 5_000));
+ Assert.assertEquals("later batches must extend the same fresh dictionary",
+ Arrays.asList("c", "e"), revivedHandler.dict());
+ }
+ }
+ }
+ });
+ }
+
+ /**
+ * An orphan drainer's engine and I/O loop are objects entirely separate
+ * from the foreground sender's own {@code cursorEngine}/{@code
+ * cursorSendLoop} -- {@code BackgroundDrainerPool} owns them. Seeds a
+ * sibling orphan slot (mirrors {@code OrphanScanIntegrationTest}'s ghost
+ * recipe), lets the drainer adopt it and get its replay frame gated on
+ * the wire, then arms and fires a recycle on the foreground stream while
+ * the drain is provably still in flight. The recycle must leave the
+ * drain untouched: releasing the gate afterward still lets it complete,
+ * and every one of the three streams (pre-recycle foreground,
+ * post-recycle foreground, drained orphan) lands with the right symbol.
+ */
+ @Test
+ public void testOrphanDrainerSurvivesRecycleMidDrain() throws Exception {
+ assertMemoryLeak(() -> {
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("outage-orphan-drain").toString();
+
+ // Phase 1: seed a sibling orphan slot. The ghost writes one row
+ // carrying a uniquely-marked symbol and dies without ever being
+ // acked -- same recipe as OrphanScanIntegrationTest.
+ SilentHandler ghostSilent = new SilentHandler();
+ try (TestWebSocketServer ghostServer = new TestWebSocketServer(ghostSilent)) {
+ ghostServer.start();
+ Assert.assertTrue(ghostServer.awaitStart(5, TimeUnit.SECONDS));
+ String ghostCfg = "ws::addr=localhost:" + ghostServer.getPort()
+ + ";sf_dir=" + sfDir + ";sender_id=ghost;close_flush_timeout_millis=0;";
+ try (Sender ghost = Sender.fromConfig(ghostCfg)) {
+ ghost.table("orphaned").symbol("s", ORPHAN_MARKER_SYMBOL).longColumn("v", 99L).atNow();
+ ghost.flush();
+ Assert.assertTrue("ghost frame must reach the wire before close",
+ ghostSilent.awaitFrame(5, TimeUnit.SECONDS));
+ }
+ }
+ Assert.assertEquals("ghost slot must be a candidate orphan",
+ 1, OrphanScanner.scan(sfDir, "primary").size());
+
+ // Phase 2: one server serves both the primary sender and the
+ // orphan drainer it spawns. Gating is CONTENT-based (whichever
+ // connection ships the ghost's marker symbol), not
+ // connection-order-based -- the drainer's connect can race the
+ // primary's own first flush, and content-based gating stays
+ // correct regardless of which one wins that race.
+ PrimaryAndOrphanHandler handler = new PrimaryAndOrphanHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String primaryCfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";sender_id=primary;drain_orphans=on;symbol_dict_reset_threshold=2;";
+
+ try (Sender sender = Sender.fromConfig(primaryCfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ // Let the drainer discover + adopt the ghost slot and get
+ // its replay frame gated on the wire before touching the
+ // foreground stream at all -- proves the two run
+ // concurrently, not sequentially.
+ Assert.assertTrue("orphan drainer must ship its replay frame",
+ handler.awaitOrphanFrame(10, TimeUnit.SECONDS));
+
+ // Arm + fire the recycle on the foreground stream. These
+ // frames carry none of the orphan marker, so they get
+ // acked immediately regardless of the drain's state.
+ sender.table("t").symbol("s", "pre-a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "pre-b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ // Recycle fires synchronously here, tearing down + rebuilding
+ // ONLY the foreground's own cursor engine/I/O loop.
+ sender.table("t").symbol("s", "post-c").longColumn("v", 2L).atNow();
+ Assert.assertFalse("recycle must disarm", ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-recycle row must land on the fresh connection",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertTrue(fsn2 > fsn1);
+
+ // The drain must still be exactly where it was -- gated,
+ // not failed, not restarted -- proving the recycle never
+ // reached into the drainer's separate stack.
+ Assert.assertFalse("the drainer's connection must not have been touched by "
+ + "the foreground's recycle", handler.orphanAcked());
+
+ // Now release the drainer's gate: a drain that survived the
+ // recycle untouched must still be able to complete.
+ handler.releaseOrphan();
+ long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+ while (OrphanScanner.scan(sfDir, "primary").size() > 0
+ && System.nanoTime() < deadlineNanos) {
+ Thread.sleep(10);
+ }
+ Assert.assertEquals("orphan drainer must complete the drain after the recycle",
+ 0, OrphanScanner.scan(sfDir, "primary").size());
+ }
+
+ // Per-row symbol correctness for all three streams.
+ Assert.assertEquals("pre-recycle foreground stream",
+ Arrays.asList("pre-a", "pre-b"), handler.dictContaining("pre-a"));
+ Assert.assertEquals("post-recycle foreground stream",
+ Collections.singletonList("post-c"), handler.dictContaining("post-c"));
+ Assert.assertEquals("drained orphan stream",
+ Collections.singletonList(ORPHAN_MARKER_SYMBOL),
+ handler.dictContaining(ORPHAN_MARKER_SYMBOL));
+ }
+ });
+ }
+
+ /** ACKs every frame it receives immediately; does not otherwise inspect the wire. */
+ private static class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ try {
+ client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Reconstructs the single connection it expects (the recycle's
+ * post-outage reconnect) and records the delta-start id of its first
+ * data frame. Tracks by connection identity like
+ * {@code SymbolDictRecycleTest.RecycleHandler} so a partially-established
+ * retry that never sends data cannot corrupt the state of the connection
+ * that actually does.
+ */
+ private static class OutageRecycleHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private final List
+ * {@link SymbolDictRecycleTest} and {@link SymbolDictRecycleMemoryModeTest}
+ * pin the swap itself; {@link SymbolDictRecycleArmingTest} pins how
+ * {@code resetArmed} flips true; {@link SymbolDictRecycleStarvationTest} pins
+ * the bounded blocking wait for an unacked backlog. This suite pins the
+ * OPPOSITE: every condition under which an armed sender must keep working
+ * normally and NOT recycle, until that condition clears -- at which point
+ * the still-armed request fires as a positive control in the same test.
+ * Every test asserts both halves: no recycle (connection count and
+ * {@code getSymbolDictEpoch()} unchanged) AND that ingestion keeps
+ * working (a row lands and gets acked) both before and after the eventual
+ * recycle.
+ */
+public class SymbolDictRecycleRefusalTest {
+
+ /**
+ * The most basic refusal: a batch that itself crossed the arming
+ * threshold is still unacked when the very next {@code table()} call
+ * checks the barrier. {@code symbol_dict_reset_max_wait_millis=0}
+ * disables the (separately-pinned, {@link SymbolDictRecycleStarvationTest})
+ * blocking wait, so every refusal here is instant and this test stays
+ * purely about the ring-drained guard. Repeated {@code table()} calls
+ * spread over a real span of wall-clock time (not one instantaneous
+ * check) prove the recycle does not fire late, either -- the whole
+ * mechanism is synchronous and producer-thread-driven, but a bounded
+ * settle window is the only way a test can actually witness that rather
+ * than assume it.
+ */
+ @Test
+ public void testUnackedBacklogRefusesUntilAcked() throws Exception {
+ assertMemoryLeak(() -> {
+ GatedAckHandler handler = new GatedAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port
+ + ";symbol_dict_reset_threshold=2"
+ + ";symbol_dict_reset_max_wait_millis=0;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence(); // ack withheld by the handler
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+ Assert.assertEquals(1, server.handshakeCount());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ for (int i = 0; i < 5; i++) {
+ sender.table("t");
+ Assert.assertTrue("recycle must not fire while the arming batch is unacked",
+ ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+ Assert.assertEquals(1, server.handshakeCount());
+ Thread.sleep(30);
+ }
+
+ // Positive control: release the acks and prove the still-armed
+ // recycle fires on the very next drained table() call.
+ handler.releaseAcks();
+ Assert.assertTrue("setup: the arming batch must get acked once released",
+ sender.awaitAckedFsn(fsn1, 5_000));
+
+ sender.table("t");
+ Assert.assertFalse("recycle must fire once the backlog drains", ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+
+ // Ingestion continues on the fresh epoch.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-recycle batch must still get acked",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertEquals("recycle must open a fresh connection",
+ 2, server.handshakeCount());
+ Assert.assertTrue(fsn2 > fsn1);
+ }
+ }
+ });
+ }
+
+ /**
+ * Isolates the {@code pendingRowCount != 0} guard from the ring-drained
+ * guard {@link #testUnackedBacklogRefusesUntilAcked} pins: the arming
+ * batch's ack is released and awaited WHILE a third row sits buffered
+ * (committed via {@code atNow()}, but never flushed -- {@code auto_flush_rows}
+ * is set well above 1 so it does not auto-flush). By the time the
+ * settle-window loop runs, the ring itself is fully drained, so any
+ * refusal it observes can only be this guard, not the earlier one.
+ */
+ @Test
+ public void testPendingRowCountRefuses() throws Exception {
+ assertMemoryLeak(() -> {
+ GatedAckHandler handler = new GatedAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port
+ + ";symbol_dict_reset_threshold=2"
+ + ";symbol_dict_reset_max_wait_millis=0"
+ + ";auto_flush_rows=10;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence(); // ack withheld by the handler
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+
+ // Ring not drained yet: refused by the OTHER guard, which just
+ // lets execution fall through so a new row can be buffered.
+ sender.table("t");
+ Assert.assertTrue(ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ // A third row, committed but never flushed: pendingRowCount=1,
+ // far under auto_flush_rows=10, so it stays buffered.
+ sender.symbol("s", "c").longColumn("v", 2L).atNow();
+
+ // Drain the arming batch -- from here on the ring itself is
+ // fully drained, isolating the pendingRowCount guard.
+ handler.releaseAcks();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+
+ for (int i = 0; i < 5; i++) {
+ sender.table("t");
+ Assert.assertTrue("recycle must not fire while a row is buffered "
+ + "unflushed, even with the ring otherwise drained",
+ ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+ Assert.assertEquals(1, server.handshakeCount());
+ Thread.sleep(30);
+ }
+
+ // Positive control: flush the buffered row, then the
+ // still-armed recycle fires on the next table() call.
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+
+ sender.table("t");
+ Assert.assertFalse("recycle must fire once the buffered batch is flushed "
+ + "and acked",
+ ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn3 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn3, 5_000));
+ Assert.assertEquals(2, server.handshakeCount());
+ Assert.assertTrue(fsn3 > fsn2);
+ }
+ }
+ });
+ }
+
+ /**
+ * A row under construction (columns set, {@code atNow()} not yet called)
+ * refuses the barrier two different ways depending on the next
+ * {@code table()} call's table name. The same-name case is the sharper
+ * proof: {@code table()}'s resetArmed check runs BEFORE the
+ * same-table-name fast path that would otherwise skip straight past
+ * everything, so this is the only way to prove the hook actually sits
+ * ahead of that shortcut. The different-name case falls through to the
+ * pre-existing "cannot switch tables while row is in progress" guard
+ * instead -- a thrown exception, not a recycle, and not a new failure
+ * mode this feature introduced.
+ */
+ @Test
+ public void testInProgressRowRefuses() throws Exception {
+ assertMemoryLeak(() -> {
+ AckAllHandler handler = new AckAllHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ // Start a row but do not commit it: symbol() registers "a"
+ // into the dictionary immediately, yet the row itself stays
+ // in progress until atNow() runs.
+ sender.table("t").symbol("s", "a");
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ // Arm WHILE the row is in progress: pendingRowCount is still
+ // 0 (an in-progress row is not counted as pending), so the
+ // manual request arms immediately even though a row is
+ // genuinely mid-flight.
+ sender.resetSymbolDictionary();
+ Assert.assertTrue(ws.isResetArmed());
+
+ for (int i = 0; i < 3; i++) {
+ sender.table("t"); // same name -- fast path would skip past everything
+ Assert.assertTrue("recycle must not fire while a row is in progress",
+ ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+ Assert.assertEquals(1, server.handshakeCount());
+ Thread.sleep(20);
+ }
+
+ LineSenderException thrown = null;
+ try {
+ sender.table("other");
+ Assert.fail("expected 'cannot switch tables' while a row is in progress");
+ } catch (LineSenderException e) {
+ thrown = e;
+ }
+ Assert.assertNotNull(thrown);
+ Assert.assertTrue("unexpected message: " + thrown.getMessage(),
+ thrown.getMessage().contains("cannot switch tables while row is in progress"));
+ Assert.assertTrue("the failed table-switch attempt must not have consumed "
+ + "the arming",
+ ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+ Assert.assertEquals(1, server.handshakeCount());
+
+ // Complete the row: ingestion still works after both refusals.
+ sender.longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("nothing yet consumed the arming", ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+
+ // Positive control: with the row complete and the batch
+ // acked, the still-armed recycle fires on the next call.
+ sender.table("t");
+ Assert.assertFalse("recycle must fire once the row completes and the ring "
+ + "drains",
+ ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+
+ sender.table("t").symbol("s", "d").longColumn("v", 2L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertEquals(2, server.handshakeCount());
+ Assert.assertTrue(fsn2 > fsn1);
+ }
+ }
+ });
+ }
+
+ /**
+ * The one data-safety-critical refusal, mirroring
+ * {@code SymbolDictRecycleStarvationTest#testDeferredCommitGroupSkipsWait}
+ * but for the barrier itself rather than the blocking-wait futility
+ * guard: the server withholds acks for {@code FLAG_DEFER_COMMIT} frames
+ * by design until the closing commit lands, so {@code isRingDrained()}
+ * stays false for as long as the group is open, however long that is.
+ * {@code symbol_dict_reset_max_wait_millis=0} keeps this test orthogonal
+ * to the (separately-pinned) starvation-wait timing.
+ */
+ @Test
+ public void testDeferredCommitGroupRefusesUntilCommitAcked() throws Exception {
+ assertMemoryLeak(() -> {
+ DeferAwareAckHandler handler = new DeferAwareAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port
+ + ";symbol_dict_reset_threshold=2"
+ + ";symbol_dict_reset_max_wait_millis=0;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+ ws.setDeferCommit(true);
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ sender.flush(); // deferred frame -- server withholds its ack by design
+ Assert.assertTrue("must be armed after crossing threshold=2", ws.isResetArmed());
+
+ for (int i = 0; i < 5; i++) {
+ sender.table("t");
+ Assert.assertTrue("an open deferred-commit group must never let the "
+ + "recycle fire -- the server withholds its ack until "
+ + "the closing commit",
+ ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+ Assert.assertEquals(1, server.handshakeCount());
+ Thread.sleep(30);
+ }
+
+ // Positive control: close the group, wait for its
+ // (retroactive) ack, and prove the still-armed recycle
+ // fires next.
+ ws.setDeferCommit(false);
+ long commitFsn = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: the commit must get acked",
+ sender.awaitAckedFsn(commitFsn, 5_000));
+
+ sender.table("t");
+ Assert.assertFalse("recycle must fire once the group is committed and acked",
+ ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+
+ sender.table("t").symbol("s", "c").longColumn("v", 2L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertEquals(2, server.handshakeCount());
+ Assert.assertTrue(fsn2 > commitFsn);
+ }
+ }
+ });
+ }
+
+ /**
+ * A manual {@code resetSymbolDictionary()} call arms {@code resetArmed}
+ * regardless of connection state ({@code armIfEligible()} touches only
+ * producer-side fields), so it can go through before the sender has ever
+ * connected -- modelled the same way
+ * {@code SymbolDictRecycleFsnContinuityTest} builds unconnected senders:
+ * {@link QwpWebSocketSender#createForTesting} plus a manually attached
+ * engine, with {@link QwpWebSocketSender#setEngineRebuildFactory} filled
+ * in (unlike {@code createForTesting}'s production counterparts, a
+ * connect()-built sender normally has none -- see
+ * {@code SymbolDictRecycleTest#testConnectBuiltSenderNeverRecyclesWithoutFactory})
+ * so the deferred request can actually execute once connected. The very
+ * next {@code table()} call -- still pre-connect -- must defer rather
+ * than NPE: {@code !connected} refuses the barrier before it ever
+ * touches the cursor engine or I/O loop.
+ */
+ @Test
+ public void testManualResetBeforeFirstConnectDeferred() throws Exception {
+ assertMemoryLeak(() -> {
+ AckAllHandler handler = new AckAllHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+
+ QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", port);
+ try {
+ CursorSendEngine engine = new CursorSendEngine(
+ null, 4L * 1024 * 1024, 128L * 1024 * 1024,
+ CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS);
+ sender.setCursorEngine(engine, true);
+ sender.setEngineRebuildFactory(() -> new CursorSendEngine(
+ null, 4L * 1024 * 1024, 128L * 1024 * 1024,
+ CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS));
+
+ // Manual request before the sender has ever connected.
+ sender.resetSymbolDictionary();
+ Assert.assertTrue("a manual request arms immediately, independent of "
+ + "connection state",
+ sender.isResetArmed());
+ Assert.assertEquals(0, sender.getSymbolDictEpoch());
+ Assert.assertEquals(0, server.handshakeCount());
+
+ // table()'s barrier check runs here while still pre-connect
+ // (ensureConnected() only runs later, inside atNow()'s
+ // sendRow()) -- must defer quietly, not NPE.
+ sender.table("t").longColumn("v", 1L).atNow();
+ Assert.assertTrue("still armed -- deferred, not consumed",
+ sender.isResetArmed());
+ Assert.assertEquals(0, sender.getSymbolDictEpoch());
+ Assert.assertEquals(1, server.handshakeCount());
+
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertTrue("flush alone does not consume the arming -- only table() "
+ + "does",
+ sender.isResetArmed());
+ Assert.assertEquals(0, sender.getSymbolDictEpoch());
+
+ // Positive control: now connected and drained, the
+ // deferred request executes on the next table() call.
+ sender.table("t");
+ Assert.assertFalse("the deferred request must execute once connected and "
+ + "drained",
+ sender.isResetArmed());
+ Assert.assertEquals(1, sender.getSymbolDictEpoch());
+
+ sender.table("t").longColumn("v", 2L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertEquals(2, server.handshakeCount());
+ Assert.assertTrue(fsn2 > fsn1);
+ } finally {
+ sender.close();
+ }
+ }
+ });
+ }
+
+ /**
+ * {@code reset()} discards a buffered-but-never-shipped row -- including
+ * reclaiming any symbol id it registered but never sent, via the same
+ * {@code truncateTo} mechanism the {@code BatchTooLargeForCapException}
+ * remedy documents. This proves that discard is compatible with an
+ * already-armed swap: after {@code reset()} clears the in-progress row
+ * that was the ONLY thing refusing the barrier, every guard is
+ * satisfied (connected, no pending row, no in-progress row, ring
+ * drained from the earlier shipped batch), so the next {@code table()}
+ * call recycles -- observed here to fire deterministically, not
+ * probabilistically, once those guards clear. It is compatible with
+ * {@code reset()}'s own reclaim: the swap replaces the whole dictionary
+ * object outright (step 6 of {@code recycleForDictReset()}), so
+ * whatever {@code truncateTo} did to the outgoing instance is moot --
+ * the swap subsumes it.
+ */
+ @Test
+ public void testResetDiscardsBufferedRowThenArmedSwapFires() throws Exception {
+ assertMemoryLeak(() -> {
+ AckAllHandler handler = new AckAllHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+ int port = server.getPort();
+ String cfg = "ws::addr=localhost:" + port + ";symbol_dict_reset_threshold=3;";
+
+ try (Sender sender = Sender.fromConfig(cfg)) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ // A real shipped batch: two distinct symbols, below the
+ // threshold of 3, so nothing arms yet.
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ sender.table("t").symbol("s", "b").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn1, 5_000));
+ Assert.assertFalse("dictionary has only 2 entries, below the threshold of 3",
+ ws.isResetArmed());
+
+ // Start (but never commit) a third row -- registers "c",
+ // crossing the threshold, but arming is only ever
+ // evaluated at a flush's tail or by resetSymbolDictionary(),
+ // neither of which has run yet.
+ sender.table("t").symbol("s", "c").longColumn("v", 2L);
+ Assert.assertFalse(ws.isResetArmed());
+
+ // Arm explicitly while the row is still in progress --
+ // the in-progress-row guard refuses table(), exactly as
+ // testInProgressRowRefuses proves.
+ sender.resetSymbolDictionary();
+ Assert.assertTrue(ws.isResetArmed());
+ sender.table("t"); // refused: row "c" is in progress
+ Assert.assertTrue(ws.isResetArmed());
+ Assert.assertEquals(0, ws.getSymbolDictEpoch());
+ Assert.assertEquals(1, server.handshakeCount());
+
+ // Discard the buffered row -- reset() drops the
+ // in-progress row AND reclaims "c"'s never-shipped id.
+ sender.reset();
+
+ // Every barrier guard is now satisfied: connected, no
+ // pending row (reset cleared it), no in-progress row
+ // (reset cleared it), ring drained (a, b were acked
+ // before any of this). The armed swap fires here.
+ sender.table("t");
+ Assert.assertFalse("the armed swap fires once reset() clears the blocking "
+ + "in-progress row",
+ ws.isResetArmed());
+ Assert.assertEquals(1, ws.getSymbolDictEpoch());
+
+ // Ingestion continues correctly post-swap: a fresh row
+ // lands and gets acked with no exception.
+ sender.table("t").symbol("s", "d").longColumn("v", 3L).atNow();
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue(sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertEquals(2, server.handshakeCount());
+ Assert.assertTrue(fsn2 > fsn1);
+ }
+ }
+ });
+ }
+
+ /**
+ * ACKs every frame it receives; does not otherwise inspect the wire.
+ * Resets its wire sequence per new connection, mirroring
+ * {@code SymbolDictRecycleTest.RecycleHandler}, so post-recycle
+ * ingestion on the fresh connection acks correctly too.
+ */
+ private static class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private TestWebSocketServer.ClientHandler currentClient;
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ if (currentClient != client) {
+ currentClient = client;
+ nextSeq.set(0);
+ }
+ try {
+ client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Acks every non-deferred frame immediately, but withholds acks for any
+ * frame carrying {@code FLAG_DEFER_COMMIT} -- the real server's ack
+ * contract for an open deferred-commit group. Mirrors
+ * {@code SymbolDictRecycleStarvationTest.DeferAwareAckHandler}, plus a
+ * per-connection wire-sequence reset so ingestion on the post-recycle
+ * connection acks correctly too.
+ */
+ private static class DeferAwareAckHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private TestWebSocketServer.ClientHandler currentClient;
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ if (currentClient != client) {
+ currentClient = client;
+ nextSeq.set(0);
+ }
+ long seq = nextSeq.getAndIncrement();
+ boolean deferred = data.length > 5 && (data[5] & FLAG_DEFER_COMMIT) != 0;
+ if (deferred) {
+ return; // withhold the ack -- the group is still open
+ }
+ try {
+ client.sendBinary(QwpWireTestUtils.buildAck(seq));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Receives frames but withholds every ack until {@link #releaseAcks()}
+ * is called, so a refusal-guard test provably has an unacknowledged
+ * target to refuse on. Mirrors
+ * {@code SymbolDictRecycleFsnContinuityTest.GatedAckHandler} /
+ * {@code SymbolDictRecycleStarvationTest.GatedAckHandler}, plus a
+ * per-connection wire-sequence reset so ingestion on the post-recycle
+ * connection acks correctly too.
+ */
+ private static class GatedAckHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private final CountDownLatch released = new CountDownLatch(1);
+ private TestWebSocketServer.ClientHandler currentClient;
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ try {
+ if (!released.await(20, TimeUnit.SECONDS)) {
+ throw new AssertionError("refusal-guard witness never released the ack gate");
+ }
+ synchronized (this) {
+ if (currentClient != client) {
+ currentClient = client;
+ nextSeq.set(0);
+ }
+ client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement()));
+ }
+ } catch (IOException | InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
+ }
+
+ void releaseAcks() {
+ released.countDown();
+ }
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleSlotHealTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleSlotHealTest.java
new file mode 100644
index 00000000..6b89e274
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SymbolDictRecycleSlotHealTest.java
@@ -0,0 +1,514 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager;
+import io.questdb.client.std.Files;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * The two verdicts {@code QwpWebSocketSender.completeRecycleRebuild} reaches
+ * when the recycle's step-4 rebuild comes back
+ * {@code wasRecoveredFromDisk()} -- i.e. when the outgoing engine's
+ * fully-drained close did NOT leave the slot empty.
+ *
+ * The acked-leftover recipe injects the unlink failure the way
+ * {@code CursorSendEngineCloseUnlinkFailureTest} does: it drops write
+ * permission on the slot directory (POSIX unlink needs a writable parent), so
+ * that test skips on Windows and wherever permissions are not enforced (root).
+ */
+public class SymbolDictRecycleSlotHealTest {
+
+ /**
+ * Big enough for the real QWP frames the sender appends to the rebuilt
+ * engine after the heal, and identical in the prep helpers so recovery
+ * reads the doctored segments back at the size they were written with.
+ */
+ private static final long SEGMENT_BYTES = 1024L * 1024L;
+ private static final int PAYLOAD_BYTES = 32;
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build();
+
+ /**
+ * A benign fully-drained close verdict (segment unlink
+ * transiently failed; watermark retained by design) must not brick the
+ * recycle. The rebuild recovers fully-acked leftovers; the sender heals by
+ * closing the recovered engine (which retries the unlink) and rebuilding
+ * once more.
+ */
+ @Test(timeout = 60_000L)
+ public void testRecoveredFullyAckedLeftoversHealAndRecycleCompletes() throws Exception {
+ assertMemoryLeak(() -> {
+ // Phase 1: doctor a slot -- fully-acked frames whose close-time
+ // unlink failed (CursorSendEngineCloseUnlinkFailureTest's recipe).
+ String doctoredSlot = temporaryFolder.getRoot().toPath()
+ .resolve("doctored-slot").toString();
+ prepareFullyAckedLeftoverSlot(doctoredSlot);
+
+ // Phase 2: a live sender whose rebuild factory lands on that slot.
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("heal-sf").toString();
+ try (TestWebSocketServer server = ackingServer()) {
+ try (Sender sender = Sender.fromConfig(config(server, sfDir))) {
+ QwpWebSocketSender ws = (QwpWebSocketSender) sender;
+
+ sender.table("t").symbol("s", "a").longColumn("v", 1L).atNow();
+ long fsn1 = sender.flushAndGetSequence();
+ Assert.assertTrue("setup: batch must be acked before the recycle",
+ sender.awaitAckedFsn(fsn1, 5_000));
+
+ AtomicInteger rebuilds = new AtomicInteger();
+ ws.setEngineRebuildFactory(() -> {
+ rebuilds.incrementAndGet();
+ return new CursorSendEngine(doctoredSlot, SEGMENT_BYTES);
+ });
+
+ sender.resetSymbolDictionary();
+ Assert.assertTrue(ws.isResetArmed());
+ // Recycle: rebuild #1 recovers the acked leftovers -> heal
+ // -> rebuild #2 stands on a genuinely empty slot.
+ sender.table("t").symbol("s", "b").longColumn("v", 2L).atNow();
+
+ Assert.assertEquals("heal must close the recovered engine and rebuild again",
+ 2, rebuilds.get());
+ Assert.assertEquals("the recycle must have committed",
+ 1, ws.getSymbolDictEpoch());
+ Assert.assertFalse("recycle must disarm", ws.isResetArmed());
+ // The heal's close retried the unlink the outgoing close
+ // could not do, so the engine the swap committed on stands
+ // on a genuinely emptied slot -- not on the leftovers.
+ Assert.assertFalse("the recycle must commit on a non-recovered engine",
+ ws.getCursorEngineForTesting().wasRecoveredFromDisk());
+
+ long fsn2 = sender.flushAndGetSequence();
+ Assert.assertTrue("post-heal batch must still get acked",
+ sender.awaitAckedFsn(fsn2, 5_000));
+ Assert.assertTrue("post-recycle FSN must exceed pre-recycle FSN "
+ + "[fsn1=" + fsn1 + ", fsn2=" + fsn2 + ']', fsn2 > fsn1);
+ }
+ }
+ });
+ }
+
+ /**
+ * The heal closes the engine that recovered the leftovers. When that engine's
+ * SF worker is wedged, its close returns with the slot flock retained, exactly
+ * like the outgoing engine's close in step 3 -- and must be awaited the same
+ * way, or rebuild #2 collides with the retained flock.
+ */
+ @Test(timeout = 60_000L)
+ public void testHealRidesOutADeferredCloseOfTheRecoveredEngine() throws Exception {
+ assertMemoryLeak(() -> {
+ String doctoredSlot = temporaryFolder.getRoot().toPath().resolve("doctored-deferred").toString();
+ prepareFullyAckedLeftoverSlot(doctoredSlot);
+ String sfDir = temporaryFolder.getRoot().toPath().resolve("heal-deferred-sf").toString();
+ CountDownLatch workerBlocked = new CountDownLatch(1);
+ CountDownLatch releaseWorker = new CountDownLatch(1);
+ AtomicReference
+ *
+ * This method runs steps 1-3 and hands steps 4-7 to
+ * {@link #completeRecycleRebuild(int, long)}. The producer-visible swap
+ * (dictionary, counters, epoch) commits only once a fresh engine stands on
+ * the emptied slot, and a throw before that point no longer kills the
+ * sender: every frame that existed before this call was already proven
+ * acked, so nothing is at risk, and the recycle simply records how far it
+ * got ({@link #recycleResume}) and resumes from the next
+ * {@link #table(CharSequence)} or {@link #ensureConnected()} -- see
+ * {@link #resumeRecycleIfPending()}. An {@link Error} (OOM/SOE/linkage)
+ * passes through untouched, neither recorded nor wrapped -- it is not a
+ * recycle verdict.
+ *
+ *
+ * Both tests doctor a slot directly at the engine level and then point a live
+ * sender's rebuild factory at it, so the verdict is driven by real on-disk
+ * state rather than by a mocked engine.
+ *
+ * 1. lastPublishedFsn = cursorEngine.publishedFsn()
+ * 2. close cursorSendLoop (I/O thread + client)
+ * 3. cursorEngine.close() -- FULLY DRAINED (the barrier only fires the swap once
+ * isRingDrained() is true), so this unlinks every *.sfa, the ack watermark,
+ * the persisted dictionary and the logical slot lock, leaving the slot empty.
+ * 4. rollFsnEpochBase(lastPublishedFsn)
+ * 5. producer state swap: fresh GlobalSymbolDictionary, sentMaxSymbolId=-1,
+ * symbolDictEpoch++, resetArmed=false
+ * 6. cursorEngine = engineRebuildFactory.rebuild() -- a brand-new CursorSendEngine
+ * on the now-empty slot (fresh .lock/.ack-watermark/.symbol-dict/segments)
+ * 7. reconnect (ensureConnected())
+ * 8. (catch) recycleFailure latch on any throw
+ *
+ * This suite pins what a restarted sender recovers if the process dies at each
+ * of four points around that sequence, per the phase-11 brief:
+ *
+ *
+ *
+ * Why these are simulated, not paused mid-sequence
+ * {@code recycleForDictReset()} runs synchronously inside one {@code table()}
+ * call with no external hook between its steps, so a test cannot literally
+ * suspend a live sender between step 3 and step 6. And unlike
+ * {@code CursorSendEngineCrashConsistencyTest}'s bare {@code CursorSendEngine} +
+ * fault-injecting {@code FilesFacade}, a {@code Sender} built through the public
+ * API (as production always does) has no seam for a custom {@code FilesFacade}
+ * -- {@code LineSenderBuilder.constructEngineOnSlot} always goes through the real
+ * filesystem. Each arm below instead constructs the exact on-disk image a crash
+ * at that point would leave, using only real production code paths plus
+ * filesystem-level fixtures already established elsewhere in this test suite
+ * ({@code DeltaDictRecoveryTest}'s {@code writeAckWatermark}, {@code
+ * RecoveryReplayTest}'s close-fast-with-a-silent-server idiom):
+ *
+ *
+ *
+ * (b) and (c) are NOT the same recoverable state
+ * Both look empty of data and both replay nothing, but they are not
+ * byte-identical on disk, and a restarted engine can tell them apart. Arm (b)'s
+ * directory holds nothing this engine ever created -- no manifest, no segment.
+ * (The crashed sender's own fully-drained close already removed {@code
+ * sf-manifest.bin} along with the last segment, so recovery finds NO {@code
+ * .sfa} files and NO manifest, and falls straight through to {@code
+ * Recovery.empty()}.) Arm (c)'s directory holds the fresh rebuild's own
+ * {@code sf-manifest.bin} (boundaries collapsed at 0) and its zero-frame
+ * {@code sf-initial.sfa} / {@code sf-...0000.sfa} pair. {@code
+ * SegmentRing.recover()}'s manifest branch (the {@code chain.size() == 0}
+ * check) accepts a manifest whose {@code headBase == activeBase} alongside a
+ * same-based, zero-frame active segment as a RECOVERED (if empty) chain -- a
+ * different branch entirely from the one arm (b) falls through to. So {@code
+ * wasRecoveredFromDisk()} comes back {@code false} for (b) and {@code true} for
+ * (c): the pinned, distinguishing observable between the two, asserted
+ * explicitly below instead of writing two assertion-for-assertion duplicate
+ * tests.
+ *
+ * Oracle
+ * Every arm asserts the same three things about the RECOVERED sender: it keeps
+ * ingesting after recovery; the symbols it and its predecessor registered are
+ * exactly and correctly reconstructable from the wire (each fresh server
+ * handler rebuilds the per-connection delta dictionary via {@link
+ * QwpWireTestUtils#accumulateDeltaDictionary}); and no data (table-carrying)
+ * frame is delivered more than the at-least-once contract allows (each handler
+ * also counts data frames, so a spurious re-send shows up as an unexpected
+ * count).
+ */
+public class SymbolDictRecycleCrashWindowsTest {
+
+ /**
+ * The exact file set a freshly-rebuilt (never-flushed) engine's own slot
+ * holds -- matches {@code SymbolDictRecycleTest#testPostRecycleSlotContents}'s
+ * {@code freshSlotFiles}. Shared by arm (c)'s pre-snapshot wait and its
+ * post-restore assertion so the two can never drift apart.
+ */
+ private static final List