From 9cf96cb202be822ec04ebccb3529d69f3d1f7e09 Mon Sep 17 00:00:00 2001 From: Day Dreamer <87640074+HugoXK@users.noreply.github.com> Date: Thu, 19 Nov 2026 08:56:42 +0800 Subject: [PATCH] chat index: a read-only window says where its index stops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A window another host keeps read-only adds nothing to the chat index, so it answers with the rows that host indexed. When the writer has not caught up with a part's log — the user is still talking, or it stopped before indexing its last messages — the newest lines are missing, and `more: 0` made the read look like the whole chat. The three reads now carry `stop: { at }` and `read_own_chat` says where the index stops, so a missing message reads as newer than the cut instead of never said (#3194). Fixes #3194 --- changelog.d/3194.fixed.md | 1 + core/src/host/chat-index.js | 35 ++++++++-- core/src/host/groups.js | 34 +++++++--- core/test/chat-index-writers.test.js | 39 ++++++++++- docs/subsystems/core-host.md | 6 ++ patches/ds-bot/0001-core-fixes.patch | 63 +++++++++++++---- patches/ds-bot/0012-room-membership.patch | 68 ++++++++++++------- .../files/test/chat-index-writers.test.js | 39 ++++++++++- 8 files changed, 225 insertions(+), 60 deletions(-) create mode 100644 changelog.d/3194.fixed.md diff --git a/changelog.d/3194.fixed.md b/changelog.d/3194.fixed.md new file mode 100644 index 000000000..59935c2d6 --- /dev/null +++ b/changelog.d/3194.fixed.md @@ -0,0 +1 @@ +A Bot's `read_own_chat` in a read-only window (the second dsh on one team folder) used to answer with the rows the other window had indexed and read as complete, so the newest messages were missing without a word; every answer now says where that index stops and that a newer message is not in it, and a caught-up window answers exactly as before ([#3194](https://github.com/ThinkFlowLab/FlyBot/issues/3194)). diff --git a/core/src/host/chat-index.js b/core/src/host/chat-index.js index d852f6c27..7107bd5a6 100644 --- a/core/src/host/chat-index.js +++ b/core/src/host/chat-index.js @@ -218,6 +218,7 @@ export function install(rt) { from: db.prepare('SELECT text, session, seq, time, who FROM lines WHERE session = ? AND seq >= ? ORDER BY seq LIMIT ?'), countAll: db.prepare('SELECT COUNT(*) AS n FROM lines WHERE session IN (SELECT value FROM json_each(?))'), countVisible: db.prepare('SELECT COUNT(*) AS n FROM lines WHERE session IN (SELECT value FROM json_each(?)) AND NOT (session = ? AND seq >= ?)'), + newestTime: db.prepare('SELECT MAX(time) AS t FROM lines WHERE session IN (SELECT value FROM json_each(?))'), } } catch (error) { db.close() @@ -230,6 +231,27 @@ export function install(rt) { indexOpening = undefined }, 'ds-bot: chat index') + // A window another host keeps read-only adds nothing to the index: it answers with the rows + // that host wrote. When that host has not caught up with a part's own log — the user is still + // talking in it, or it stopped before indexing its last messages — the newest lines are + // missing, and a capped `more: 0` read as the whole chat. A read that stops short says where + // it stops, so a message missing from the answer is visibly newer than the cut (#P-652). + async function indexStop(index, parts) { + if (index.writable) return undefined + let behind = false + for (const part of parts) { + const agent = await rt.resolveAgent(part.sessionId).catch(() => undefined) + if (agent === undefined) continue + if ((index.mark.get(part.sessionId)?.through ?? -1) < Number(agent.session.seq) - 1) { + behind = true + break + } + } + if (!behind) return undefined + const { t } = index.newestTime.get(JSON.stringify(parts.map(part => part.sessionId))) + return { at: t === null ? undefined : rt.clock(t) } + } + // Adds what a part said since its mark. Reads of one part wait for each other within this // installation; two installations over one home meet again inside the transaction below. const catchingUp = new Map() @@ -322,19 +344,19 @@ export function install(rt) { // A part that cannot be read now is searched as far as it was indexed. for (const part of parts) await catchUp(index, part).catch(error => rt.warn('ds-bot: indexing %s failed: %s', part.sessionId, error)) const turn = askingId !== undefined && askingId === parts.at(-1)?.sessionId ? index.mark.get(askingId)?.turn : undefined - return { index, parts, hidden: turn === undefined || turn < 0 ? undefined : { session: askingId, from: turn } } + return { index, parts, stop: await indexStop(index, parts), hidden: turn === undefined || turn < 0 ? undefined : { session: askingId, from: turn } } } // Messages with all the words, best first; with none of them having all, those with // the most of them. Shown oldest first. `more` is how many hits the cap left out (#1906). async function searchOwnChat(botId, query, { askingId, limit = OWN_CHAT_SEARCH } = {}) { const terms = [...new Set(query.toLowerCase().split(/\s+/).filter(Boolean))].slice(0, SEARCH_TERMS_MAX) - const { index, parts, hidden } = await indexedParts(botId, askingId) + const { index, parts, hidden, stop } = await indexedParts(botId, askingId) if (index === null) return { ...(await scanOwnChat(botId, query, limit)), every: true } const sessions = parts.map(part => part.sessionId) const byTime = (a, b) => (a.time - b.time) || (sessions.indexOf(a.session) - sessions.indexOf(b.session)) || (a.seq - b.seq) const found = matchedRows(index, sessions, terms, limit, hidden) - return { lines: found.rows.sort(byTime).map(row => rowLine(parts, row, terms)), every: found.every, more: found.more } + return { lines: found.rows.sort(byTime).map(row => rowLine(parts, row, terms)), every: found.every, more: found.more, stop } } // The rows a query matches, best first, and how many the cap left out (#1906): the core @@ -386,7 +408,7 @@ export function install(rt) { // The latest messages across all parts, oldest first; `more` is how many the cap left out (#1906). async function latestOwnChat(botId, { askingId, limit = OWN_CHAT_READ } = {}) { - const { index, parts, hidden } = await indexedParts(botId, askingId) + const { index, parts, hidden, stop } = await indexedParts(botId, askingId) if (index === null) return scanOwnChat(botId, '', limit) const rows = [] for (const part of [...parts].reverse()) { @@ -398,7 +420,7 @@ export function install(rt) { const total = hidden === undefined ? index.countAll.get(JSON.stringify(sessions)).n : index.countVisible.get(JSON.stringify(sessions), hidden.session, hidden.from).n - return { lines: rows.reverse().map(row => rowLine(parts, row)), more: Math.max(0, total - rows.length) } + return { lines: rows.reverse().map(row => rowLine(parts, row)), more: Math.max(0, total - rows.length), stop } } // The messages around an anchor, in its part. A bare `#` — an anchor printed before @@ -406,7 +428,7 @@ export function install(rt) { // but `guessedPart` says so and `exact` is then false: the seq a Bot saved named no part, // so the message it finds may be a different one (#2710). async function aroundOwnChat(botId, anchor) { - const { index, parts } = await indexedParts(botId) + const { index, parts, stop } = await indexedParts(botId) if (index === null) return { error: 'Anchors need the chat index, which is unavailable here. Search with words instead.' } const found = /(?:part\s*)?(\d+)?\s*#\s*(\d+)/i.exec(anchor) if (found === null) return { error: `"${anchor}" is not an anchor. Anchors look like "part 2 #1234", or "#1234" in the current part.` } @@ -422,6 +444,7 @@ export function install(rt) { exact: !guessedPart && rows.some(row => row.seq === seq), guessedPart, lines: rows.map(row => rowLine(parts, row, [], row.seq === seq ? ANCHOR_LINE_MAX : OWN_CHAT_LINE_MAX)), + stop, } } diff --git a/core/src/host/groups.js b/core/src/host/groups.js index 78ef75cb6..6c117e174 100644 --- a/core/src/host/groups.js +++ b/core/src/host/groups.js @@ -979,17 +979,26 @@ export function install(rt) { void rt.save().catch(() => {}) } const partsNote = parts.length > 1 ? `\n\nYour own chat has ${parts.length} parts: ${rt.partIndex(parts)}. Call read_own_chat with part N for the note that closed an earlier part, or with words to search all parts.` : '' + // A window another dsh keeps read-only answers with the rows that window indexed, so its + // newest messages can be missing. The read says where the index stops: `more: 0` alone + // read as the whole chat while the newest line sat unindexed in the log (#P-652). + const stopped = found => { + if (found?.stop === undefined) return '' + return found.stop.at === undefined + ? '\nThe chat index in this window holds nothing yet, and this window is read-only: messages the other window has not indexed are not in this answer.' + : `\nThe chat index in this window is read-only and stops at ${found.stop.at}; the other window has not indexed anything newer, so a message after that is not in this answer.` + } if (query === '' && around !== '') { const found = await rt.aroundOwnChat(botId, around) if (found.error) return `${found.error}${partsNote}` - if (found.lines.length === 0) return `Nothing was said around ${around}.${partsNote}` + if (found.lines.length === 0) return `Nothing was said around ${around}.${stopped(found)}${partsNote}` if (found.guessedPart) { // The anchor names no part, so this read can only guess: say so, and never report // the guess as an exact hit (#2710). - return `Your own chat around ${around} in the part that is current now; that anchor names no part, so the part is a guess (name one as "part N #"):\n${found.lines.join('\n')}${partsNote}` + return `Your own chat around ${around} in the part that is current now; that anchor names no part, so the part is a guess (name one as "part N #"):\n${found.lines.join('\n')}${stopped(found)}${partsNote}` } const heading = found.exact ? `around ${around}` : `with no message at ${around}; the nearest ones` - return `Your own chat ${heading} (oldest first):\n${found.lines.join('\n')}${partsNote}` + return `Your own chat ${heading} (oldest first):\n${found.lines.join('\n')}${stopped(found)}${partsNote}` } if (query === '' && number !== undefined) { const part = parts[number - 1] @@ -1009,26 +1018,29 @@ export function install(rt) { // keeps the unchanged sentence rather than failing this read. } if (checkpoints === 0) return `Part ${number} is the current one; it is in your context.${partsNote}` - const { lines, more } = await rt.latestOwnChat(botId, { askingId: id }) + const latest = await rt.latestOwnChat(botId, { askingId: id }) + const { lines, more } = latest // The cap hides the rest, so the sentence says how many of how many this read shows: // the bare sentence let 12 of 15 read as the whole of it (#2780). const newest = more > 0 ? `its newest ${lines.length} of ${lines.length + more} messages` : `its newest ${lines.length} messages` - return `Part ${number} is the current one, but its earlier messages are condensed into [Checkpoint ${checkpoints}], so only ${newest} are still in your context:\n${lines.join('\n')}\nSearch it with words to read the rest word for word.${partsNote}` + return `Part ${number} is the current one, but its earlier messages are condensed into [Checkpoint ${checkpoints}], so only ${newest} are still in your context:\n${lines.join('\n')}\nSearch it with words to read the rest word for word.${stopped(latest)}${partsNote}` } return `Part ${number} of your own chat (${rt.clock(part.startedAt)} – ${rt.clock(part.endedAt)}). The handoff note that closed it:\n${rt.noteText(part) ?? '(none: the part ended on a context overflow)'}${partsNote}` } if (query === '') { - const { lines, more } = await rt.latestOwnChat(botId, { askingId: id }) - if (lines.length === 0) return `Your own chat is empty.${partsNote}` + const latest = await rt.latestOwnChat(botId, { askingId: id }) + const { lines, more } = latest + if (lines.length === 0) return `Your own chat is empty.${stopped(latest)}${partsNote}` const heading = more > 0 ? `Your own chat (newest ${lines.length} of ${lines.length + more}, oldest first)` : 'Your own chat (oldest first)' - return `${heading}:\n${lines.join('\n')}${partsNote}` + return `${heading}:\n${lines.join('\n')}${stopped(latest)}${partsNote}` } - const { lines, every, more } = await rt.searchOwnChat(botId, query, { askingId: id }) - if (lines.length === 0) return `Nothing in your own chat mentions "${query}".${partsNote}` + const found = await rt.searchOwnChat(botId, query, { askingId: id }) + const { lines, every, more } = found + if (lines.length === 0) return `Nothing in your own chat mentions "${query}".${stopped(found)}${partsNote}` const heading = every ? `messages mentioning "${query}"` : `no message has all of "${query}"; messages with the most of these words` // A capped search says how much it left out, so a message that is missing never reads as gone (#1906). const leftOut = more > 0 ? `\n${more} more not shown; search with narrower words, or read around one of these anchors.` : '' - return `Your own chat, ${heading} (oldest first):\n${lines.join('\n')}${leftOut}${partsNote}` + return `Your own chat, ${heading} (oldest first):\n${lines.join('\n')}${leftOut}${stopped(found)}${partsNote}` }, })) diff --git a/core/test/chat-index-writers.test.js b/core/test/chat-index-writers.test.js index 00e2e5325..f61720d58 100644 --- a/core/test/chat-index-writers.test.js +++ b/core/test/chat-index-writers.test.js @@ -142,6 +142,7 @@ async function start(t, home, { heldBy, gate, saved } = {}) { const ctx = fakeContext() const warnings = [] ctx.logger.warn = (...args) => warnings.push(args.map(part => String(part)).join(' ')) + let pages = 0 let route const registered = ctx.connection.fetch.register ctx.connection.fetch.register = (options) => { route = options; return registered(options) } @@ -152,6 +153,7 @@ async function start(t, home, { heldBy, gate, saved } = {}) { resolveAgent: async sessionId => ({ agent: { session: { id: sessionId, seq: (HISTORY[sessionId]?.at(-1)?.seq ?? -1) + 1 }, followup() {} } }), selectModel: async () => {}, page: async ({ address, beforeSeq, maxMessages } = {}) => { + pages += 1 if (gate !== undefined) await gate() const all = (HISTORY[address?.sessionId] ?? []).filter(event => beforeSeq === undefined || event.seq < beforeSeq) const records = maxMessages === undefined ? all : all.slice(-maxMessages) @@ -169,7 +171,11 @@ async function start(t, home, { heldBy, gate, saved } = {}) { // parts, returns `{lines: []}` and never calls `sessionController.page()`, so the party // this test's gate waits for never arrives. Wait for the load the host relies on. await rt.load() - const host = { home, ctx, rt, route, warnings, stop: () => ctx.dispose() } + const host = { + home, ctx, rt, route, warnings, pages: () => pages, + tool: (name, args, sessionId) => ctx.registeredTools.get(name).execute(args, { agent: { id: sessionId } }), + stop: () => ctx.dispose(), + } t.after(() => host.stop()) return host } @@ -261,9 +267,15 @@ test('a read-only window reads the index and leaves it alone (#2711)', async (t) const reader = await start(t, home, { heldBy: process.ppid }) await reader.rt.load() assert.notEqual(reader.rt.readOnly(), undefined, 'this window must be read-only for the case to mean anything') + // The writer stopped before indexing the fourth line, so its index ends at seq 2. A read + // answered from it showed three lines without a word about the cut, and `more: 0` made + // that read as the whole chat while the fourth line sat in the log (#3194). const read = await reader.rt.latestOwnChat('a', {}) assert.deepEqual(anchorsOf(read), ['0', '1', '2']) assert.equal(read.more, 0) + assert.equal(typeof read.stop?.at, 'string', 'the read says where the index stops') + const out = await reader.tool('read_own_chat', {}, 'solo') + assert.match(out, /this window is read-only and stops at \S+ \S+;/, `the read says where the index stops: ${out}`) assert.deepEqual(readFileSync(indexPath), before, 'the read-only window wrote the index') assert.equal(statSync(indexPath).mtimeMs, stamp) assert.equal(rowsOf(home), 3) @@ -283,13 +295,36 @@ test('a read-only window reads the index while the writer still holds the home ( const reader = await start(t, home, { heldBy: process.ppid }) await reader.rt.load() assert.notEqual(reader.rt.readOnly(), undefined) - assert.deepEqual(anchorsOf(await reader.rt.latestOwnChat('a', {})), ['0', '1', '2']) + // The index stops at the writer's last catch-up, and the fourth line is newer (#3194): + // every read says where it stops, so a missing message is visibly newer than the cut. + const read = await reader.rt.latestOwnChat('a', {}) + assert.deepEqual(anchorsOf(read), ['0', '1', '2']) + assert.equal(typeof read.stop?.at, 'string') + const found = await reader.rt.searchOwnChat('a', 'fourth', {}) + assert.equal(typeof found.stop?.at, 'string') + const around = await reader.rt.aroundOwnChat('a', 'part 1 #3') + assert.equal(typeof around.stop?.at, 'string') assert.equal(rowsOf(home), 3, 'the reader added a message the writer had not indexed') } finally { HISTORY.solo.pop() } }) +test('a read-only window whose index is caught up answers from the index and reads no log (#3194)', async (t) => { + const home = scratch(t) + const writer = await start(t, home) + assert.deepEqual(anchorsOf(await writer.rt.latestOwnChat('a', {})), ['0', '1', '2']) + await writer.stop() + + const reader = await start(t, home, { heldBy: process.ppid }) + await reader.rt.load() + assert.notEqual(reader.rt.readOnly(), undefined) + const read = await reader.rt.latestOwnChat('a', {}) + assert.deepEqual(anchorsOf(read), ['0', '1', '2']) + assert.equal(read.stop, undefined, 'a caught-up index needs no cut sentence') + assert.equal(reader.pages(), 0, 'the read fell back to the logs although the index was caught up') +}) + test('a read-only window creates no index where there is none (#2711)', async (t) => { const home = scratch(t) const reader = await start(t, home, { heldBy: process.ppid }) diff --git a/docs/subsystems/core-host.md b/docs/subsystems/core-host.md index f4f00c9f8..98a7e5c75 100644 --- a/docs/subsystems/core-host.md +++ b/docs/subsystems/core-host.md @@ -439,6 +439,12 @@ has lines (#2790, `core/test/room-read-floor-clamp.test.js`). - `chat-index.js` returns the count next to the lines — `searchOwnChat` answers `{ lines, every, more }`, `latestOwnChat` `{ lines, more }` — instead of the lines alone. A search counts the same match in all only once its limit was filled (`termCountQuery`; the per-term fallback uses `anyTermCountQuery`, which unions one branch per term so a line matching two counts once), and a latest read counts the part's visible lines (`countAll` / `countVisible`, the latter leaving out the asking turn). Without the SQLite index the log scan counts every match while it reads, because that count is the only way to say what was left out. - Covered by `core/test/group-work.test.js`; `core/test/history.test.js` still pins `termQuery`'s SQL. +### A read-only window's index says where it stops (#3194) + +- A second dsh window on one home reads the index but never adds to it (`chat-index.js` `catchUp` returns at once when the index is not writable), so its reads answer with the rows the writer indexed. When that writer has not caught up with a part's log — a live conversation, or a writer that stopped before indexing its last messages — the newest lines are missing, and `more: 0` made the read look like the whole chat. +- `indexStop()` compares each part's `marks.through` with the log's own last seq (`agent.session.seq - 1`) when the index is not writable, and returns `{ at }` built from the newest indexed row time (`newestTime` over the read's sessions). `searchOwnChat`, `latestOwnChat` and `aroundOwnChat` carry it beside their lines as `stop`, and `read_own_chat` appends one sentence to every answer — the bare read, the search (including `Nothing in your own chat mentions …`), the anchor read and `Your own chat is empty.` — naming the time the index stops at and saying that a message after it is not in the answer. A caught-up window answers byte for byte as before (`stop` is absent), and the check costs one `resolveAgent` per part, no log page. +- Covered by `core/test/chat-index-writers.test.js`: the two read-only cases over a part the writer never finished indexing assert the sentence, and the caught-up case asserts `stop` is absent and no log page was read. + ### Group reads say what a cap left out (#2780) - `read_group_chat` shows at most `ROOM_READ_MAX` (30) of the room lines visible to the caller, the newest ones, and the cut used to be silent: `Latest in "" (oldest first):` printed as if those 30 were the whole log, so a Bot asked "didn't we decide X?" read a truncated log as the whole log — the same failure #1906 fixed for the own-chat read. The header now names the cut, `Latest in "" (newest 30 of 45, oldest first):`, and a read that fits the cap stays byte-identical; lines before the member's `joinedSeq` are not counted as left out, and a room whose log was trimmed counts the lines it dropped (the store keeps only the newest `ROOM_LOG_MAX` lines, 60 here and 200 in `room-membership.js`, while `room.seq` keeps counting every line, so 500 lines said and 60 kept read `newest 30 of 500`). A share-seq reservation for a co-seat mention writes no line, so it does not move what the read calls the total: the reservation takes its number from `room.seq` — the receiver keys its one-bubble rule on that number, so a second counter would stamp the next line with the number the mention just went out with (#2863) — and `room.reserved` counts the numbers issued that never became a line here, which the read subtracts before it names a cut. Counting those numbers as lines read as a cap that had dropped them and grew the header by one per mention (#2855). A booking also does not count for a reader whose floor sits above it: `room.reserved` is room-wide, so subtracting every booking shrank a window that starts above them — a room at 510 with ten bookings below a floor of 300 kept 60 lines and read `newest 30 of 201`, where 300..510 are 211 lines. `takeLineSeq` keeps each booking's number in `room.reservedSeqs`, and the read discounts only the bookings at or above the caller's floor; a room saved before that list existed carries only the count and keeps the room-wide discount (#3037). The same fallback covers a room whose list is shorter than its count — `reserved > booked.length` — because only bookings taken after the list existed carry a number, so the room's placements are unknown and the read keeps the room-wide discount instead of trusting the partial list (#3046). Covered by `core/test/room-read-honest-count.test.js`, `core/test/room-read-trimmed-count.test.js`, `core/test/read-group-total-below-floor.test.js` and `core/test/group-work.test.js`. diff --git a/patches/ds-bot/0001-core-fixes.patch b/patches/ds-bot/0001-core-fixes.patch index 3bd827b2d..e4085f9a4 100644 --- a/patches/ds-bot/0001-core-fixes.patch +++ b/patches/ds-bot/0001-core-fixes.patch @@ -4401,7 +4401,7 @@ diff --git a/src/host/chat-index.js b/src/host/chat-index.js } catch (error) { if (fresh) rt.warn('ds-bot: no chat index, read_own_chat scans the logs: %s', error) } -@@ -100,24 +100,66 @@ export function install(rt) { +@@ -100,24 +100,67 @@ export function install(rt) { return opening } @@ -4477,15 +4477,37 @@ diff --git a/src/host/chat-index.js b/src/host/chat-index.js from: db.prepare('SELECT text, session, seq, time, who FROM lines WHERE session = ? AND seq >= ? ORDER BY seq LIMIT ?'), + countAll: db.prepare('SELECT COUNT(*) AS n FROM lines WHERE session IN (SELECT value FROM json_each(?))'), + countVisible: db.prepare('SELECT COUNT(*) AS n FROM lines WHERE session IN (SELECT value FROM json_each(?)) AND NOT (session = ? AND seq >= ?)'), ++ newestTime: db.prepare('SELECT MAX(time) AS t FROM lines WHERE session IN (SELECT value FROM json_each(?))'), } } catch (error) { db.close() -@@ -130,16 +130,18 @@ export function install(rt) { +@@ -130,16 +130,39 @@ export function install(rt) { indexOpening = undefined }, 'ds-bot: chat index') - // Adds what a part said since its mark. Reads of one part wait for each other, so no - // line is added twice. ++ // A window another host keeps read-only adds nothing to the index: it answers with the rows ++ // that host wrote. When that host has not caught up with a part's own log — the user is still ++ // talking in it, or it stopped before indexing its last messages — the newest lines are ++ // missing, and a capped `more: 0` read as the whole chat. A read that stops short says where ++ // it stops, so a message missing from the answer is visibly newer than the cut (#P-652). ++ async function indexStop(index, parts) { ++ if (index.writable) return undefined ++ let behind = false ++ for (const part of parts) { ++ const agent = await rt.resolveAgent(part.sessionId).catch(() => undefined) ++ if (agent === undefined) continue ++ if ((index.mark.get(part.sessionId)?.through ?? -1) < Number(agent.session.seq) - 1) { ++ behind = true ++ break ++ } ++ } ++ if (!behind) return undefined ++ const { t } = index.newestTime.get(JSON.stringify(parts.map(part => part.sessionId))) ++ return { at: t === null ? undefined : rt.clock(t) } ++ } ++ + // Adds what a part said since its mark. Reads of one part wait for each other within this + // installation; two installations over one home meet again inside the transaction below. const catchingUp = new Map() @@ -4574,7 +4596,12 @@ diff --git a/src/host/chat-index.js b/src/host/chat-index.js } const rowLine = (parts, row, terms = [], max = OWN_CHAT_LINE_MAX) => { const lower = row.text.toLowerCase() -@@ -202,19 +202,29 @@ export function install(rt) { +@@ -198,23 +198,33 @@ export function install(rt) { + // A part that cannot be read now is searched as far as it was indexed. + for (const part of parts) await catchUp(index, part).catch(error => rt.warn('ds-bot: indexing %s failed: %s', part.sessionId, error)) + const turn = askingId !== undefined && askingId === parts.at(-1)?.sessionId ? index.mark.get(askingId)?.turn : undefined +- return { index, parts, hidden: turn === undefined || turn < 0 ? undefined : { session: askingId, from: turn } } ++ return { index, parts, stop: await indexStop(index, parts), hidden: turn === undefined || turn < 0 ? undefined : { session: askingId, from: turn } } } // Messages with all the words, best first; with none of them having all, those with @@ -4582,13 +4609,14 @@ diff --git a/src/host/chat-index.js b/src/host/chat-index.js + // the most of them. Shown oldest first. `more` is how many hits the cap left out (#1906). async function searchOwnChat(botId, query, { askingId, limit = OWN_CHAT_SEARCH } = {}) { const terms = [...new Set(query.toLowerCase().split(/\s+/).filter(Boolean))].slice(0, SEARCH_TERMS_MAX) - const { index, parts, hidden } = await indexedParts(botId, askingId) +- const { index, parts, hidden } = await indexedParts(botId, askingId) - if (index === null) return { lines: await scanOwnChat(botId, query, limit), every: true } ++ const { index, parts, hidden, stop } = await indexedParts(botId, askingId) + if (index === null) return { ...(await scanOwnChat(botId, query, limit)), every: true } const sessions = parts.map(part => part.sessionId) + const byTime = (a, b) => (a.time - b.time) || (sessions.indexOf(a.session) - sessions.indexOf(b.session)) || (a.seq - b.seq) + const found = matchedRows(index, sessions, terms, limit, hidden) -+ return { lines: found.rows.sort(byTime).map(row => rowLine(parts, row, terms)), every: found.every, more: found.more } ++ return { lines: found.rows.sort(byTime).map(row => rowLine(parts, row, terms)), every: found.every, more: found.more, stop } + } + + // The rows a query matches, best first, and how many the cap left out (#1906): the core @@ -4607,7 +4635,7 @@ diff --git a/src/host/chat-index.js b/src/host/chat-index.js if (rows.length === 0 && terms.length > 1) { every = false const found = new Map() -@@ -227,11 +227,30 @@ export function install(rt) { +@@ -227,13 +227,32 @@ export function install(rt) { } } rows = [...found.values()].sort((a, b) => (b.count - a.count) || (b.row.time - a.row.time)).slice(0, limit).map(hit => hit.row) @@ -4617,8 +4645,9 @@ diff --git a/src/host/chat-index.js b/src/host/chat-index.js } - return { lines: rows.sort(byTime).map(row => rowLine(parts, row, terms)), every } + return { rows, every, more: total === undefined ? 0 : Math.max(0, total - rows.length) } -+ } -+ + } + +- // The latest messages across all parts, oldest first. + // The whole-text search behind the palette's "Messages (full text)" group (P-820): the + // outline previews stop at 50/120 characters, so a hit past that budget is invisible to + // the outline read. The sessions are the caller's to name — the channel lists the caller's @@ -4633,14 +4662,16 @@ diff --git a/src/host/chat-index.js b/src/host/chat-index.js + const sessions = [...new Set(sessionIds)] + for (const sessionId of sessions) await catchUp(index, { sessionId }).catch(error => rt.warn('ds-bot: indexing %s failed: %s', sessionId, error)) + return matchedRows(index, sessions, terms, limit, undefined) - } - -- // The latest messages across all parts, oldest first. ++ } ++ + // The latest messages across all parts, oldest first; `more` is how many the cap left out (#1906). async function latestOwnChat(botId, { askingId, limit = OWN_CHAT_READ } = {}) { - const { index, parts, hidden } = await indexedParts(botId, askingId) +- const { index, parts, hidden } = await indexedParts(botId, askingId) ++ const { index, parts, hidden, stop } = await indexedParts(botId, askingId) if (index === null) return scanOwnChat(botId, '', limit) -@@ -241,21 +241,35 @@ export function install(rt) { + const rows = [] + for (const part of [...parts].reverse()) { +@@ -241,21 +241,36 @@ export function install(rt) { rows.push(...index.before.all(part.sessionId, below, limit - rows.length)) if (rows.length >= limit) break } @@ -4649,7 +4680,7 @@ diff --git a/src/host/chat-index.js b/src/host/chat-index.js + const total = hidden === undefined + ? index.countAll.get(JSON.stringify(sessions)).n + : index.countVisible.get(JSON.stringify(sessions), hidden.session, hidden.from).n -+ return { lines: rows.reverse().map(row => rowLine(parts, row)), more: Math.max(0, total - rows.length) } ++ return { lines: rows.reverse().map(row => rowLine(parts, row)), more: Math.max(0, total - rows.length), stop } } - // The messages around an anchor, in its part. @@ -4658,7 +4689,8 @@ diff --git a/src/host/chat-index.js b/src/host/chat-index.js + // but `guessedPart` says so and `exact` is then false: the seq a Bot saved named no part, + // so the message it finds may be a different one (#2710). async function aroundOwnChat(botId, anchor) { - const { index, parts } = await indexedParts(botId) +- const { index, parts } = await indexedParts(botId) ++ const { index, parts, stop } = await indexedParts(botId) if (index === null) return { error: 'Anchors need the chat index, which is unavailable here. Search with words instead.' } const found = /(?:part\s*)?(\d+)?\s*#\s*(\d+)/i.exec(anchor) if (found === null) return { error: `"${anchor}" is not an anchor. Anchors look like "part 2 #1234", or "#1234" in the current part.` } @@ -4675,6 +4707,7 @@ diff --git a/src/host/chat-index.js b/src/host/chat-index.js + exact: !guessedPart && rows.some(row => row.seq === seq), + guessedPart, + lines: rows.map(row => rowLine(parts, row, [], row.seq === seq ? ANCHOR_LINE_MAX : OWN_CHAT_LINE_MAX)), ++ stop, + } } diff --git a/patches/ds-bot/0012-room-membership.patch b/patches/ds-bot/0012-room-membership.patch index 3302de4ea..138f92994 100644 --- a/patches/ds-bot/0012-room-membership.patch +++ b/patches/ds-bot/0012-room-membership.patch @@ -1756,7 +1756,7 @@ diff --git a/src/host/groups.js b/src/host/groups.js if (by === undefined) delete called.by else called.by = by round.slots.splice(next, 0, called) -@@ -285,23 +285,185 @@ export function install(rt) { +@@ -285,34 +285,204 @@ export function install(rt) { // Session's inbox, so it does not open another round when this one ends. Returns // the number of turns added, which the round's limit allows for. function drainRoomInbox(room, round, turns, index, agent) { @@ -1809,11 +1809,11 @@ diff --git a/src/host/groups.js b/src/host/groups.js - // The user spoke in this room, so Bots may start their own rounds again. - room.botRounds = 0 + // The user spoke in this room, so late replies may be answered again. -+ room.lateRounds = 0 -+ void rt.save().catch(() => {}) -+ return round.slots.length - before -+ } -+ + room.lateRounds = 0 + void rt.save().catch(() => {}) + return round.slots.length - before + } + + // One line that reaches the round while a member speaks (#1267). It is logged at once — the room + // shows it when it is sent, not at the next slot — and `notePostedLine` keeps the next relay from + // logging it a second time. Its pending copy leaves the relay's queue, because this round answers @@ -1826,8 +1826,8 @@ diff --git a/src/host/groups.js b/src/host/groups.js + if (at !== -1) rt.roomInbox.set(room.id, queued.toSpliced(at, 1)) + round.replies.push({ kind: 'user', text: line.text, time: Date.now(), authorId: line.authorId, authorKind: line.authorKind, name: line.authorName }) + // The user spoke in this room, so late replies may be answered again. - room.lateRounds = 0 - void rt.save().catch(() => {}) ++ room.lateRounds = 0 ++ void rt.save().catch(() => {}) + // The one line that ends work in flight (#1267 B): the slot is superseded, its turn is cancelled, + // and nothing it says when it stops is posted. The line stays in the Session's queue, so the head + // drain opens it as a fresh line (admin first) for the room to answer. @@ -1947,10 +1947,11 @@ diff --git a/src/host/groups.js b/src/host/groups.js + const uncapped = new Set(callees.filter(bot => rt.isAdminOf(room, bot.id)).map(bot => bot.id)) + callMembers(round, turns, index - 1, callees, `${call.name} mentioned you`, leads ? ROOM_ADMIN_TURNS_EACH : ROOM_TURNS_EACH, call.by, uncapped, call.text) + } - return round.slots.length - before - } - -@@ -310,9 +310,17 @@ export function install(rt) { ++ return round.slots.length - before ++ } ++ + async function runRound(room, round, signal, agent) { + let limit = round.slots.length + ROOM_EXTRA_TURNS + (room.admin ? ROOM_ADMIN_EXTRA_TURNS : 0) const turns = new Map() // The inbox check keeps a message sent during the last speaker's turn inside // this round: it drains at the loop's head even once the slots ran out. @@ -2227,17 +2228,32 @@ diff --git a/src/host/groups.js b/src/host/groups.js const parts = rt.partsOf(botId) const number = Number.isInteger(args.part) ? args.part : undefined if (room !== undefined && member !== undefined) { -@@ -507,23 +507,52 @@ export function install(rt) { +@@ -503,27 +503,68 @@ export function install(rt) { + void rt.save().catch(() => {}) + } + const partsNote = parts.length > 1 ? `\n\nYour own chat has ${parts.length} parts: ${rt.partIndex(parts)}. Call read_own_chat with part N for the note that closed an earlier part, or with words to search all parts.` : '' ++ // A window another dsh keeps read-only answers with the rows that window indexed, so its ++ // newest messages can be missing. The read says where the index stops: `more: 0` alone ++ // read as the whole chat while the newest line sat unindexed in the log (#P-652). ++ const stopped = found => { ++ if (found?.stop === undefined) return '' ++ return found.stop.at === undefined ++ ? '\nThe chat index in this window holds nothing yet, and this window is read-only: messages the other window has not indexed are not in this answer.' ++ : `\nThe chat index in this window is read-only and stops at ${found.stop.at}; the other window has not indexed anything newer, so a message after that is not in this answer.` ++ } + if (query === '' && around !== '') { const found = await rt.aroundOwnChat(botId, around) if (found.error) return `${found.error}${partsNote}` - if (found.lines.length === 0) return `Nothing was said around ${around}.${partsNote}` +- if (found.lines.length === 0) return `Nothing was said around ${around}.${partsNote}` ++ if (found.lines.length === 0) return `Nothing was said around ${around}.${stopped(found)}${partsNote}` + if (found.guessedPart) { + // The anchor names no part, so this read can only guess: say so, and never report + // the guess as an exact hit (#2710). -+ return `Your own chat around ${around} in the part that is current now; that anchor names no part, so the part is a guess (name one as "part N #"):\n${found.lines.join('\n')}${partsNote}` ++ return `Your own chat around ${around} in the part that is current now; that anchor names no part, so the part is a guess (name one as "part N #"):\n${found.lines.join('\n')}${stopped(found)}${partsNote}` + } const heading = found.exact ? `around ${around}` : `with no message at ${around}; the nearest ones` - return `Your own chat ${heading} (oldest first):\n${found.lines.join('\n')}${partsNote}` +- return `Your own chat ${heading} (oldest first):\n${found.lines.join('\n')}${partsNote}` ++ return `Your own chat ${heading} (oldest first):\n${found.lines.join('\n')}${stopped(found)}${partsNote}` } if (query === '' && number !== undefined) { const part = parts[number - 1] @@ -2258,30 +2274,34 @@ diff --git a/src/host/groups.js b/src/host/groups.js + // keeps the unchanged sentence rather than failing this read. + } + if (checkpoints === 0) return `Part ${number} is the current one; it is in your context.${partsNote}` -+ const { lines, more } = await rt.latestOwnChat(botId, { askingId: id }) ++ const latest = await rt.latestOwnChat(botId, { askingId: id }) ++ const { lines, more } = latest + // The cap hides the rest, so the sentence says how many of how many this read shows: + // the bare sentence let 12 of 15 read as the whole of it (#2780). + const newest = more > 0 ? `its newest ${lines.length} of ${lines.length + more} messages` : `its newest ${lines.length} messages` -+ return `Part ${number} is the current one, but its earlier messages are condensed into [Checkpoint ${checkpoints}], so only ${newest} are still in your context:\n${lines.join('\n')}\nSearch it with words to read the rest word for word.${partsNote}` ++ return `Part ${number} is the current one, but its earlier messages are condensed into [Checkpoint ${checkpoints}], so only ${newest} are still in your context:\n${lines.join('\n')}\nSearch it with words to read the rest word for word.${stopped(latest)}${partsNote}` + } return `Part ${number} of your own chat (${rt.clock(part.startedAt)} – ${rt.clock(part.endedAt)}). The handoff note that closed it:\n${rt.noteText(part) ?? '(none: the part ended on a context overflow)'}${partsNote}` } if (query === '') { - const lines = await rt.latestOwnChat(botId, { askingId: id }) - return lines.length === 0 ? `Your own chat is empty.${partsNote}` : `Your own chat (oldest first):\n${lines.join('\n')}${partsNote}` -+ const { lines, more } = await rt.latestOwnChat(botId, { askingId: id }) -+ if (lines.length === 0) return `Your own chat is empty.${partsNote}` ++ const latest = await rt.latestOwnChat(botId, { askingId: id }) ++ const { lines, more } = latest ++ if (lines.length === 0) return `Your own chat is empty.${stopped(latest)}${partsNote}` + const heading = more > 0 ? `Your own chat (newest ${lines.length} of ${lines.length + more}, oldest first)` : 'Your own chat (oldest first)' -+ return `${heading}:\n${lines.join('\n')}${partsNote}` ++ return `${heading}:\n${lines.join('\n')}${stopped(latest)}${partsNote}` } - const { lines, every } = await rt.searchOwnChat(botId, query, { askingId: id }) -+ const { lines, every, more } = await rt.searchOwnChat(botId, query, { askingId: id }) - if (lines.length === 0) return `Nothing in your own chat mentions "${query}".${partsNote}` +- if (lines.length === 0) return `Nothing in your own chat mentions "${query}".${partsNote}` ++ const found = await rt.searchOwnChat(botId, query, { askingId: id }) ++ const { lines, every, more } = found ++ if (lines.length === 0) return `Nothing in your own chat mentions "${query}".${stopped(found)}${partsNote}` const heading = every ? `messages mentioning "${query}"` : `no message has all of "${query}"; messages with the most of these words` - return `Your own chat, ${heading} (oldest first):\n${lines.join('\n')}${partsNote}` + // A capped search says how much it left out, so a message that is missing never reads as gone (#1906). + const leftOut = more > 0 ? `\n${more} more not shown; search with narrower words, or read around one of these anchors.` : '' -+ return `Your own chat, ${heading} (oldest first):\n${lines.join('\n')}${leftOut}${partsNote}` ++ return `Your own chat, ${heading} (oldest first):\n${lines.join('\n')}${leftOut}${stopped(found)}${partsNote}` }, })) diff --git a/patches/ds-bot/files/test/chat-index-writers.test.js b/patches/ds-bot/files/test/chat-index-writers.test.js index 00e2e5325..f61720d58 100644 --- a/patches/ds-bot/files/test/chat-index-writers.test.js +++ b/patches/ds-bot/files/test/chat-index-writers.test.js @@ -142,6 +142,7 @@ async function start(t, home, { heldBy, gate, saved } = {}) { const ctx = fakeContext() const warnings = [] ctx.logger.warn = (...args) => warnings.push(args.map(part => String(part)).join(' ')) + let pages = 0 let route const registered = ctx.connection.fetch.register ctx.connection.fetch.register = (options) => { route = options; return registered(options) } @@ -152,6 +153,7 @@ async function start(t, home, { heldBy, gate, saved } = {}) { resolveAgent: async sessionId => ({ agent: { session: { id: sessionId, seq: (HISTORY[sessionId]?.at(-1)?.seq ?? -1) + 1 }, followup() {} } }), selectModel: async () => {}, page: async ({ address, beforeSeq, maxMessages } = {}) => { + pages += 1 if (gate !== undefined) await gate() const all = (HISTORY[address?.sessionId] ?? []).filter(event => beforeSeq === undefined || event.seq < beforeSeq) const records = maxMessages === undefined ? all : all.slice(-maxMessages) @@ -169,7 +171,11 @@ async function start(t, home, { heldBy, gate, saved } = {}) { // parts, returns `{lines: []}` and never calls `sessionController.page()`, so the party // this test's gate waits for never arrives. Wait for the load the host relies on. await rt.load() - const host = { home, ctx, rt, route, warnings, stop: () => ctx.dispose() } + const host = { + home, ctx, rt, route, warnings, pages: () => pages, + tool: (name, args, sessionId) => ctx.registeredTools.get(name).execute(args, { agent: { id: sessionId } }), + stop: () => ctx.dispose(), + } t.after(() => host.stop()) return host } @@ -261,9 +267,15 @@ test('a read-only window reads the index and leaves it alone (#2711)', async (t) const reader = await start(t, home, { heldBy: process.ppid }) await reader.rt.load() assert.notEqual(reader.rt.readOnly(), undefined, 'this window must be read-only for the case to mean anything') + // The writer stopped before indexing the fourth line, so its index ends at seq 2. A read + // answered from it showed three lines without a word about the cut, and `more: 0` made + // that read as the whole chat while the fourth line sat in the log (#3194). const read = await reader.rt.latestOwnChat('a', {}) assert.deepEqual(anchorsOf(read), ['0', '1', '2']) assert.equal(read.more, 0) + assert.equal(typeof read.stop?.at, 'string', 'the read says where the index stops') + const out = await reader.tool('read_own_chat', {}, 'solo') + assert.match(out, /this window is read-only and stops at \S+ \S+;/, `the read says where the index stops: ${out}`) assert.deepEqual(readFileSync(indexPath), before, 'the read-only window wrote the index') assert.equal(statSync(indexPath).mtimeMs, stamp) assert.equal(rowsOf(home), 3) @@ -283,13 +295,36 @@ test('a read-only window reads the index while the writer still holds the home ( const reader = await start(t, home, { heldBy: process.ppid }) await reader.rt.load() assert.notEqual(reader.rt.readOnly(), undefined) - assert.deepEqual(anchorsOf(await reader.rt.latestOwnChat('a', {})), ['0', '1', '2']) + // The index stops at the writer's last catch-up, and the fourth line is newer (#3194): + // every read says where it stops, so a missing message is visibly newer than the cut. + const read = await reader.rt.latestOwnChat('a', {}) + assert.deepEqual(anchorsOf(read), ['0', '1', '2']) + assert.equal(typeof read.stop?.at, 'string') + const found = await reader.rt.searchOwnChat('a', 'fourth', {}) + assert.equal(typeof found.stop?.at, 'string') + const around = await reader.rt.aroundOwnChat('a', 'part 1 #3') + assert.equal(typeof around.stop?.at, 'string') assert.equal(rowsOf(home), 3, 'the reader added a message the writer had not indexed') } finally { HISTORY.solo.pop() } }) +test('a read-only window whose index is caught up answers from the index and reads no log (#3194)', async (t) => { + const home = scratch(t) + const writer = await start(t, home) + assert.deepEqual(anchorsOf(await writer.rt.latestOwnChat('a', {})), ['0', '1', '2']) + await writer.stop() + + const reader = await start(t, home, { heldBy: process.ppid }) + await reader.rt.load() + assert.notEqual(reader.rt.readOnly(), undefined) + const read = await reader.rt.latestOwnChat('a', {}) + assert.deepEqual(anchorsOf(read), ['0', '1', '2']) + assert.equal(read.stop, undefined, 'a caught-up index needs no cut sentence') + assert.equal(reader.pages(), 0, 'the read fell back to the logs although the index was caught up') +}) + test('a read-only window creates no index where there is none (#2711)', async (t) => { const home = scratch(t) const reader = await start(t, home, { heldBy: process.ppid })