Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/3198.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
The 12288-token output cap now applies only to the DashScope route whose 400 it encodes, so a call on any other provider keeps the output budget it asked for — or the one its model row declares — instead of being silently cut to 12288 ([#3198](https://github.com/ThinkFlowLab/FlyBot/issues/3198)).
27 changes: 22 additions & 5 deletions core/src/host/models.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,21 @@ const TEST_PROMPT = 'Reply with the single word OK.'
// answer is lost even though the rest of the call was fine (#3132). A model row can
// declare more than the endpoint accepts, so the cap is clamped on the way out.
const OUTBOUND_MAX_TOKENS = 12_288
// That ceiling is one endpoint's, not every provider's (#3198). #3132 recorded it on the
// `opencsitool` route, whose proxy answered `litellm.BadRequestError: DashscopeException …
// expected a value <= 12288` for `opencsitool/DeepSeek-V4.1-Flash-1017`, so the clamp tests
// that DSH route key. Another route carries its own endpoint's limit and DSH sends what the
// model row declares; writing 12288 there would cut an answer the provider would have given.
const DASHSCOPE_ROUTE = 'opencsitool'

// A route with no provider key cannot be placed against that evidence. A cap that is too
// low only shortens an answer, while a 400 loses it whole, so an unplaceable route keeps
// the clamp rather than risk the failure #3132 recorded.
function takesDashScopeCap(options) {
const provider = options?.provider
if (typeof provider !== 'string' || provider === '') return true
return provider === DASHSCOPE_ROUTE
}

// The outcome of a connection test as a code only (#409). A provider's own text can
// echo part of a key, so none of it leaves the host: the client words each code.
Expand Down Expand Up @@ -44,12 +59,14 @@ export function install(rt) {
ctx.on?.('llm/adapters-updated', () => { catalogCache = { at: 0, value: [] }; reasoningCache.clear() })

// The clamp sits on `llm/stream`, the seam every model call crosses — a Bot's turn and
// its checkpoint, a workflow step, the host's own request. `next` cannot take new
// options (the constraint secrets.js and images.js document), so a call that needs the
// cap set starts over with a copy and passes this listener the second time, carrying
// 12288. The caller's object is never written to: DSH compares a prepared call's config
// and throws INVALID_PREPARED_CALL when one changes underneath it.
// its checkpoint, a workflow step, the host's own request. It is the DashScope route's
// ceiling, so a call on any other route is passed straight on (see DASHSCOPE_ROUTE).
// `next` cannot take new options (the constraint secrets.js and images.js document), so
// a call that needs the cap set starts over with a copy and passes this listener the
// second time, carrying 12288. The caller's object is never written to: DSH compares a
// prepared call's config and throws INVALID_PREPARED_CALL when one changes underneath it.
ctx.on?.('llm/stream', (options, next) => {
if (!takesDashScopeCap(options)) return next()
const asked = options?.maxTokens
if (Number.isFinite(asked)) return asked > OUTBOUND_MAX_TOKENS ? ctx.llm.stream({ ...options, maxTokens: OUTBOUND_MAX_TOKENS }) : next()
return (async function* clampDeclaredCap() {
Expand Down
26 changes: 26 additions & 0 deletions core/test/llm-output-cap.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ import { fakeContext } from './fake-ctx.js'

const CEILING = 12_288
const MODEL = 'opencsitool/DeepSeek-V4.1-Flash-1017'
// #3198: the ceiling is DashScope's, reached through the `opencsitool` route that #3132
// recorded; another provider's endpoint accepts more, so the cap is not its business.
const OTHER_PROVIDER = 'deepseek-official'
const OTHER_MODEL = `${OTHER_PROVIDER}/DeepSeek-V4.1-Flash-1017`

const user = text => ({ role: 'user', source: { kind: 'user' }, content: [{ type: 'text', text }] })
const request = (overrides = {}) => ({
Expand Down Expand Up @@ -57,6 +61,13 @@ test('a model row declaring more than the endpoint takes is asked for 12288, and
assert.equal(asked.maxTokens, undefined, 'the caller\'s object is untouched')
})

test('a model row declaring more than the ceiling is left alone on a route whose endpoint accepts it', async () => {
const { ask } = bench({ info: { [OTHER_MODEL]: { defaultMaxTokens: 16_384 } } })
const { sent, count } = await ask(request({ provider: OTHER_PROVIDER }))
assert.equal('maxTokens' in sent, false, 'the declared default is filled by DSH, not written here')
assert.equal(count, 1, 'one provider request, not two')
})

test('a model row under the ceiling keeps what it declares, and nothing is invented', async () => {
const { ask } = bench({ info: { [MODEL]: { defaultMaxTokens: 4_096 } } })
const { sent } = await ask(request())
Expand All @@ -72,6 +83,21 @@ test('a caller that asks for more than the ceiling is clamped to it, and its own
assert.equal(asked.maxTokens, 20_000)
})

test('a caller asking for more than the ceiling keeps its value on a route whose endpoint accepts it', async () => {
const { ask } = bench({ info: { [OTHER_MODEL]: { defaultMaxTokens: 16_384 } } })
const asked = request({ provider: OTHER_PROVIDER, maxTokens: 20_000 })
const { sent, count } = await ask(asked)
assert.equal(sent.maxTokens, 20_000, 'the caller\'s request reaches the provider unchanged')
assert.equal(count, 1, 'one provider request, not two')
assert.equal(asked.maxTokens, 20_000)
})

test('a route that cannot be named keeps the clamp', async () => {
const { ask } = bench({ info: { [MODEL]: { defaultMaxTokens: 16_384 } } })
const { sent } = await ask(request({ provider: undefined, maxTokens: 20_000 }))
assert.equal(sent.maxTokens, CEILING, 'an unplaceable route keeps the proven-safe cap')
})

test('a caller at or under the ceiling is passed through unchanged', async () => {
const { ask } = bench({ info: { [MODEL]: { defaultMaxTokens: 16_384 } } })
for (const maxTokens of [CEILING, 4_096, 1]) {
Expand Down
Loading
Loading