Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/3194.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
A Bot's `read_own_chat` in a read-only window (the second dsh on one team folder) used to answer with the rows the other window had indexed and read as complete, so the newest messages were missing without a word; every answer now says where that index stops and that a newer message is not in it, and a caught-up window answers exactly as before ([#3194](https://github.com/ThinkFlowLab/FlyBot/issues/3194)).
35 changes: 29 additions & 6 deletions core/src/host/chat-index.js
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,7 @@ export function install(rt) {
from: db.prepare('SELECT text, session, seq, time, who FROM lines WHERE session = ? AND seq >= ? ORDER BY seq LIMIT ?'),
countAll: db.prepare('SELECT COUNT(*) AS n FROM lines WHERE session IN (SELECT value FROM json_each(?))'),
countVisible: db.prepare('SELECT COUNT(*) AS n FROM lines WHERE session IN (SELECT value FROM json_each(?)) AND NOT (session = ? AND seq >= ?)'),
newestTime: db.prepare('SELECT MAX(time) AS t FROM lines WHERE session IN (SELECT value FROM json_each(?))'),
}
} catch (error) {
db.close()
Expand All @@ -230,6 +231,27 @@ export function install(rt) {
indexOpening = undefined
}, 'ds-bot: chat index')

// A window another host keeps read-only adds nothing to the index: it answers with the rows
// that host wrote. When that host has not caught up with a part's own log — the user is still
// talking in it, or it stopped before indexing its last messages — the newest lines are
// missing, and a capped `more: 0` read as the whole chat. A read that stops short says where
// it stops, so a message missing from the answer is visibly newer than the cut (#P-652).
async function indexStop(index, parts) {
if (index.writable) return undefined
let behind = false
for (const part of parts) {
const agent = await rt.resolveAgent(part.sessionId).catch(() => undefined)
if (agent === undefined) continue
if ((index.mark.get(part.sessionId)?.through ?? -1) < Number(agent.session.seq) - 1) {
behind = true
break
}
}
if (!behind) return undefined
const { t } = index.newestTime.get(JSON.stringify(parts.map(part => part.sessionId)))
return { at: t === null ? undefined : rt.clock(t) }
}

// Adds what a part said since its mark. Reads of one part wait for each other within this
// installation; two installations over one home meet again inside the transaction below.
const catchingUp = new Map()
Expand Down Expand Up @@ -322,19 +344,19 @@ export function install(rt) {
// A part that cannot be read now is searched as far as it was indexed.
for (const part of parts) await catchUp(index, part).catch(error => rt.warn('ds-bot: indexing %s failed: %s', part.sessionId, error))
const turn = askingId !== undefined && askingId === parts.at(-1)?.sessionId ? index.mark.get(askingId)?.turn : undefined
return { index, parts, hidden: turn === undefined || turn < 0 ? undefined : { session: askingId, from: turn } }
return { index, parts, stop: await indexStop(index, parts), hidden: turn === undefined || turn < 0 ? undefined : { session: askingId, from: turn } }
}

// Messages with all the words, best first; with none of them having all, those with
// the most of them. Shown oldest first. `more` is how many hits the cap left out (#1906).
async function searchOwnChat(botId, query, { askingId, limit = OWN_CHAT_SEARCH } = {}) {
const terms = [...new Set(query.toLowerCase().split(/\s+/).filter(Boolean))].slice(0, SEARCH_TERMS_MAX)
const { index, parts, hidden } = await indexedParts(botId, askingId)
const { index, parts, hidden, stop } = await indexedParts(botId, askingId)
if (index === null) return { ...(await scanOwnChat(botId, query, limit)), every: true }
const sessions = parts.map(part => part.sessionId)
const byTime = (a, b) => (a.time - b.time) || (sessions.indexOf(a.session) - sessions.indexOf(b.session)) || (a.seq - b.seq)
const found = matchedRows(index, sessions, terms, limit, hidden)
return { lines: found.rows.sort(byTime).map(row => rowLine(parts, row, terms)), every: found.every, more: found.more }
return { lines: found.rows.sort(byTime).map(row => rowLine(parts, row, terms)), every: found.every, more: found.more, stop }
}

// The rows a query matches, best first, and how many the cap left out (#1906): the core
Expand Down Expand Up @@ -386,7 +408,7 @@ export function install(rt) {

// The latest messages across all parts, oldest first; `more` is how many the cap left out (#1906).
async function latestOwnChat(botId, { askingId, limit = OWN_CHAT_READ } = {}) {
const { index, parts, hidden } = await indexedParts(botId, askingId)
const { index, parts, hidden, stop } = await indexedParts(botId, askingId)
if (index === null) return scanOwnChat(botId, '', limit)
const rows = []
for (const part of [...parts].reverse()) {
Expand All @@ -398,15 +420,15 @@ export function install(rt) {
const total = hidden === undefined
? index.countAll.get(JSON.stringify(sessions)).n
: index.countVisible.get(JSON.stringify(sessions), hidden.session, hidden.from).n
return { lines: rows.reverse().map(row => rowLine(parts, row)), more: Math.max(0, total - rows.length) }
return { lines: rows.reverse().map(row => rowLine(parts, row)), more: Math.max(0, total - rows.length), stop }
}

// The messages around an anchor, in its part. A bare `#<seq>` — an anchor printed before
// the chat had more than one part — still resolves in the current part, as it always did,
// but `guessedPart` says so and `exact` is then false: the seq a Bot saved named no part,
// so the message it finds may be a different one (#2710).
async function aroundOwnChat(botId, anchor) {
const { index, parts } = await indexedParts(botId)
const { index, parts, stop } = await indexedParts(botId)
if (index === null) return { error: 'Anchors need the chat index, which is unavailable here. Search with words instead.' }
const found = /(?:part\s*)?(\d+)?\s*#\s*(\d+)/i.exec(anchor)
if (found === null) return { error: `"${anchor}" is not an anchor. Anchors look like "part 2 #1234", or "#1234" in the current part.` }
Expand All @@ -422,6 +444,7 @@ export function install(rt) {
exact: !guessedPart && rows.some(row => row.seq === seq),
guessedPart,
lines: rows.map(row => rowLine(parts, row, [], row.seq === seq ? ANCHOR_LINE_MAX : OWN_CHAT_LINE_MAX)),
stop,
}
}

Expand Down
34 changes: 23 additions & 11 deletions core/src/host/groups.js
Original file line number Diff line number Diff line change
Expand Up @@ -979,17 +979,26 @@ export function install(rt) {
void rt.save().catch(() => {})
}
const partsNote = parts.length > 1 ? `\n\nYour own chat has ${parts.length} parts: ${rt.partIndex(parts)}. Call read_own_chat with part N for the note that closed an earlier part, or with words to search all parts.` : ''
// A window another dsh keeps read-only answers with the rows that window indexed, so its
// newest messages can be missing. The read says where the index stops: `more: 0` alone
// read as the whole chat while the newest line sat unindexed in the log (#P-652).
const stopped = found => {
if (found?.stop === undefined) return ''
return found.stop.at === undefined
? '\nThe chat index in this window holds nothing yet, and this window is read-only: messages the other window has not indexed are not in this answer.'
: `\nThe chat index in this window is read-only and stops at ${found.stop.at}; the other window has not indexed anything newer, so a message after that is not in this answer.`
}
if (query === '' && around !== '') {
const found = await rt.aroundOwnChat(botId, around)
if (found.error) return `${found.error}${partsNote}`
if (found.lines.length === 0) return `Nothing was said around ${around}.${partsNote}`
if (found.lines.length === 0) return `Nothing was said around ${around}.${stopped(found)}${partsNote}`
if (found.guessedPart) {
// The anchor names no part, so this read can only guess: say so, and never report
// the guess as an exact hit (#2710).
return `Your own chat around ${around} in the part that is current now; that anchor names no part, so the part is a guess (name one as "part N #<seq>"):\n${found.lines.join('\n')}${partsNote}`
return `Your own chat around ${around} in the part that is current now; that anchor names no part, so the part is a guess (name one as "part N #<seq>"):\n${found.lines.join('\n')}${stopped(found)}${partsNote}`
}
const heading = found.exact ? `around ${around}` : `with no message at ${around}; the nearest ones`
return `Your own chat ${heading} (oldest first):\n${found.lines.join('\n')}${partsNote}`
return `Your own chat ${heading} (oldest first):\n${found.lines.join('\n')}${stopped(found)}${partsNote}`
}
if (query === '' && number !== undefined) {
const part = parts[number - 1]
Expand All @@ -1009,26 +1018,29 @@ export function install(rt) {
// keeps the unchanged sentence rather than failing this read.
}
if (checkpoints === 0) return `Part ${number} is the current one; it is in your context.${partsNote}`
const { lines, more } = await rt.latestOwnChat(botId, { askingId: id })
const latest = await rt.latestOwnChat(botId, { askingId: id })
const { lines, more } = latest
// The cap hides the rest, so the sentence says how many of how many this read shows:
// the bare sentence let 12 of 15 read as the whole of it (#2780).
const newest = more > 0 ? `its newest ${lines.length} of ${lines.length + more} messages` : `its newest ${lines.length} messages`
return `Part ${number} is the current one, but its earlier messages are condensed into [Checkpoint ${checkpoints}], so only ${newest} are still in your context:\n${lines.join('\n')}\nSearch it with words to read the rest word for word.${partsNote}`
return `Part ${number} is the current one, but its earlier messages are condensed into [Checkpoint ${checkpoints}], so only ${newest} are still in your context:\n${lines.join('\n')}\nSearch it with words to read the rest word for word.${stopped(latest)}${partsNote}`
}
return `Part ${number} of your own chat (${rt.clock(part.startedAt)} – ${rt.clock(part.endedAt)}). The handoff note that closed it:\n${rt.noteText(part) ?? '(none: the part ended on a context overflow)'}${partsNote}`
}
if (query === '') {
const { lines, more } = await rt.latestOwnChat(botId, { askingId: id })
if (lines.length === 0) return `Your own chat is empty.${partsNote}`
const latest = await rt.latestOwnChat(botId, { askingId: id })
const { lines, more } = latest
if (lines.length === 0) return `Your own chat is empty.${stopped(latest)}${partsNote}`
const heading = more > 0 ? `Your own chat (newest ${lines.length} of ${lines.length + more}, oldest first)` : 'Your own chat (oldest first)'
return `${heading}:\n${lines.join('\n')}${partsNote}`
return `${heading}:\n${lines.join('\n')}${stopped(latest)}${partsNote}`
}
const { lines, every, more } = await rt.searchOwnChat(botId, query, { askingId: id })
if (lines.length === 0) return `Nothing in your own chat mentions "${query}".${partsNote}`
const found = await rt.searchOwnChat(botId, query, { askingId: id })
const { lines, every, more } = found
if (lines.length === 0) return `Nothing in your own chat mentions "${query}".${stopped(found)}${partsNote}`
const heading = every ? `messages mentioning "${query}"` : `no message has all of "${query}"; messages with the most of these words`
// A capped search says how much it left out, so a message that is missing never reads as gone (#1906).
const leftOut = more > 0 ? `\n${more} more not shown; search with narrower words, or read around one of these anchors.` : ''
return `Your own chat, ${heading} (oldest first):\n${lines.join('\n')}${leftOut}${partsNote}`
return `Your own chat, ${heading} (oldest first):\n${lines.join('\n')}${leftOut}${stopped(found)}${partsNote}`
},
}))

Expand Down
39 changes: 37 additions & 2 deletions core/test/chat-index-writers.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ async function start(t, home, { heldBy, gate, saved } = {}) {
const ctx = fakeContext()
const warnings = []
ctx.logger.warn = (...args) => warnings.push(args.map(part => String(part)).join(' '))
let pages = 0
let route
const registered = ctx.connection.fetch.register
ctx.connection.fetch.register = (options) => { route = options; return registered(options) }
Expand All @@ -152,6 +153,7 @@ async function start(t, home, { heldBy, gate, saved } = {}) {
resolveAgent: async sessionId => ({ agent: { session: { id: sessionId, seq: (HISTORY[sessionId]?.at(-1)?.seq ?? -1) + 1 }, followup() {} } }),
selectModel: async () => {},
page: async ({ address, beforeSeq, maxMessages } = {}) => {
pages += 1
if (gate !== undefined) await gate()
const all = (HISTORY[address?.sessionId] ?? []).filter(event => beforeSeq === undefined || event.seq < beforeSeq)
const records = maxMessages === undefined ? all : all.slice(-maxMessages)
Expand All @@ -169,7 +171,11 @@ async function start(t, home, { heldBy, gate, saved } = {}) {
// parts, returns `{lines: []}` and never calls `sessionController.page()`, so the party
// this test's gate waits for never arrives. Wait for the load the host relies on.
await rt.load()
const host = { home, ctx, rt, route, warnings, stop: () => ctx.dispose() }
const host = {
home, ctx, rt, route, warnings, pages: () => pages,
tool: (name, args, sessionId) => ctx.registeredTools.get(name).execute(args, { agent: { id: sessionId } }),
stop: () => ctx.dispose(),
}
t.after(() => host.stop())
return host
}
Expand Down Expand Up @@ -261,9 +267,15 @@ test('a read-only window reads the index and leaves it alone (#2711)', async (t)
const reader = await start(t, home, { heldBy: process.ppid })
await reader.rt.load()
assert.notEqual(reader.rt.readOnly(), undefined, 'this window must be read-only for the case to mean anything')
// The writer stopped before indexing the fourth line, so its index ends at seq 2. A read
// answered from it showed three lines without a word about the cut, and `more: 0` made
// that read as the whole chat while the fourth line sat in the log (#3194).
const read = await reader.rt.latestOwnChat('a', {})
assert.deepEqual(anchorsOf(read), ['0', '1', '2'])
assert.equal(read.more, 0)
assert.equal(typeof read.stop?.at, 'string', 'the read says where the index stops')
const out = await reader.tool('read_own_chat', {}, 'solo')
assert.match(out, /this window is read-only and stops at \S+ \S+;/, `the read says where the index stops: ${out}`)
assert.deepEqual(readFileSync(indexPath), before, 'the read-only window wrote the index')
assert.equal(statSync(indexPath).mtimeMs, stamp)
assert.equal(rowsOf(home), 3)
Expand All @@ -283,13 +295,36 @@ test('a read-only window reads the index while the writer still holds the home (
const reader = await start(t, home, { heldBy: process.ppid })
await reader.rt.load()
assert.notEqual(reader.rt.readOnly(), undefined)
assert.deepEqual(anchorsOf(await reader.rt.latestOwnChat('a', {})), ['0', '1', '2'])
// The index stops at the writer's last catch-up, and the fourth line is newer (#3194):
// every read says where it stops, so a missing message is visibly newer than the cut.
const read = await reader.rt.latestOwnChat('a', {})
assert.deepEqual(anchorsOf(read), ['0', '1', '2'])
assert.equal(typeof read.stop?.at, 'string')
const found = await reader.rt.searchOwnChat('a', 'fourth', {})
assert.equal(typeof found.stop?.at, 'string')
const around = await reader.rt.aroundOwnChat('a', 'part 1 #3')
assert.equal(typeof around.stop?.at, 'string')
assert.equal(rowsOf(home), 3, 'the reader added a message the writer had not indexed')
} finally {
HISTORY.solo.pop()
}
})

test('a read-only window whose index is caught up answers from the index and reads no log (#3194)', async (t) => {
const home = scratch(t)
const writer = await start(t, home)
assert.deepEqual(anchorsOf(await writer.rt.latestOwnChat('a', {})), ['0', '1', '2'])
await writer.stop()

const reader = await start(t, home, { heldBy: process.ppid })
await reader.rt.load()
assert.notEqual(reader.rt.readOnly(), undefined)
const read = await reader.rt.latestOwnChat('a', {})
assert.deepEqual(anchorsOf(read), ['0', '1', '2'])
assert.equal(read.stop, undefined, 'a caught-up index needs no cut sentence')
assert.equal(reader.pages(), 0, 'the read fell back to the logs although the index was caught up')
})

test('a read-only window creates no index where there is none (#2711)', async (t) => {
const home = scratch(t)
const reader = await start(t, home, { heldBy: process.ppid })
Expand Down
Loading
Loading