diff --git a/packages/coding-agent/docs/providers.md b/packages/coding-agent/docs/providers.md index 6e7808d07..b56bcc7af 100644 --- a/packages/coding-agent/docs/providers.md +++ b/packages/coding-agent/docs/providers.md @@ -43,6 +43,12 @@ Availability depends on the selected account, workspace, and granted permissions Token usage is reported without applying API-key token prices to subscription usage. Review app limits and credits in [ChatGPT Settings - Usage](https://chatgpt.com/settings/usage). +When choosing a default, Diffex prefers `gpt-5.6-terra` if it is in the available ChatGPT catalog; otherwise it uses the first model in the account's catalog order. Available saved defaults and session models are preserved, not automatically upgraded. + +At startup or session restoration, if a saved ChatGPT model is no longer available, Diffex reports the replacement and stays with ChatGPT while that account has available models. It falls back to another configured provider only when ChatGPT has none. These fallbacks do not rewrite your saved default; use `/model` to save a new choice. + +Login selects and saves an available model when there is no current model or switching ChatGPT accounts makes the current ChatGPT model unavailable. If the newly authenticated provider has no models, login reports that instead of selecting a different provider. Logging in does not replace an available current choice or an unrelated custom model. + `/logout` stops the account's active requests, attempts remote session revocation, and clears the active tokens while retaining the host and registration for later sign-in. If remote revocation cannot be confirmed, Diffex reports that local sign-out completed and directs you to disconnect the app in ChatGPT Settings. An unusable refresh token clears the local token set and requires a new sign-in with the saved registration. diff --git a/packages/coding-agent/src/core/model-resolver.ts b/packages/coding-agent/src/core/model-resolver.ts index 7b5994164..44ee996d7 100644 --- a/packages/coding-agent/src/core/model-resolver.ts +++ b/packages/coding-agent/src/core/model-resolver.ts @@ -25,7 +25,7 @@ export const defaultModelPerProvider: Record = { anthropic: "claude-opus-4-8", openai: "gpt-5.5", "azure-openai-responses": "gpt-5.4", - "openai-codex": "gpt-5.5", + "openai-codex": "gpt-5.6-terra", nvidia: "nvidia/nemotron-3-super-120b-a12b", deepseek: "deepseek-v4-pro", google: "gemini-3.1-pro-preview", @@ -61,6 +61,26 @@ export const defaultModelPerProvider: Record = { "xiaomi-token-plan-sgp": "mimo-v2.5-pro", }; +/** + * Select a preferred model only from the supplied available catalog. + * Provider-scoped fallback preserves catalog order, including ChatGPT's account-specific order. + */ +export function findDefaultModel(availableModels: readonly Model[], provider?: string): Model | undefined { + if (provider !== undefined) { + const providerModels = availableModels.filter((model) => model.provider === provider); + return ( + providerModels.find((model) => model.id === defaultModelPerProvider[provider as KnownProvider]) ?? + providerModels[0] + ); + } + + for (const [providerId, defaultId] of Object.entries(defaultModelPerProvider)) { + const match = availableModels.find((model) => model.provider === providerId && model.id === defaultId); + if (match) return match; + } + return availableModels[0]; +} + export interface ScopedModel { model: Model; /** Thinking level if explicitly specified in pattern (e.g., "model:high"), undefined otherwise */ @@ -173,13 +193,8 @@ export interface ParsedModelResult { } function buildFallbackModel(provider: string, modelId: string, availableModels: Model[]): Model | undefined { - const providerModels = availableModels.filter((m) => m.provider === provider); - if (providerModels.length === 0) return undefined; - - const defaultId = defaultModelPerProvider[provider as KnownProvider]; - const baseModel = defaultId - ? (providerModels.find((m) => m.id === defaultId) ?? providerModels[0]) - : providerModels[0]; + const baseModel = findDefaultModel(availableModels, provider); + if (!baseModel) return undefined; return { ...baseModel, @@ -645,7 +660,7 @@ export async function findInitialModel(options: { } = options; let model: Model | undefined; - let thinkingLevel: ThinkingLevel = DEFAULT_THINKING_LEVEL; + const thinkingLevel: ThinkingLevel = DEFAULT_THINKING_LEVEL; // 1. CLI args take priority if (cliProvider && cliModel) { @@ -676,36 +691,32 @@ export async function findInitialModel(options: { const removedDefault = defaultProvider ? getRemovedProviderMessage(defaultProvider) : undefined; if (removedDefault) throw new ModelsError("provider", `${removedDefault} Update defaultProvider and defaultModel in settings.json.`); - if (defaultProvider && defaultModelId) { - const found = modelRuntime.getModel(defaultProvider, defaultModelId); - if (found && modelRuntime.hasConfiguredAuth(found.provider)) { - model = found; - if (defaultThinkingLevel) { - thinkingLevel = defaultThinkingLevel; - } - return { model, thinkingLevel, fallbackMessage: undefined }; - } - } - - // 4. Try first available model with valid API key - const availableModels = [...modelRuntime.getAvailableSnapshot()]; - - if (availableModels.length > 0) { - // Try to find a default model from known providers - for (const provider of Object.keys(defaultModelPerProvider) as KnownProvider[]) { - const defaultId = defaultModelPerProvider[provider]; - const match = availableModels.find((m) => m.provider === provider && m.id === defaultId); - if (match) { - return { model: match, thinkingLevel: DEFAULT_THINKING_LEVEL, fallbackMessage: undefined }; - } + const availableModels = modelRuntime.getAvailableSnapshot(); + if (defaultProvider) { + const found = availableModels.find((entry) => entry.provider === defaultProvider && entry.id === defaultModelId); + if (found) { + return { + model: found, + thinkingLevel: defaultThinkingLevel ?? DEFAULT_THINKING_LEVEL, + fallbackMessage: undefined, + }; } - // If no default found, use first available - return { model: availableModels[0], thinkingLevel: DEFAULT_THINKING_LEVEL, fallbackMessage: undefined }; + // A retired or account-restricted saved model should not switch providers when + // the selected provider still has usable models. Never synthesize a model ID here. + model = findDefaultModel(availableModels, defaultProvider); } - // 5. No model found - return { model: undefined, thinkingLevel: DEFAULT_THINKING_LEVEL, fallbackMessage: undefined }; + // 4. Fall back across providers only when the saved provider has no available models. + model ??= findDefaultModel(availableModels); + if (model && defaultProvider && defaultModelId) { + return { + model, + thinkingLevel: defaultThinkingLevel ?? DEFAULT_THINKING_LEVEL, + fallbackMessage: `Saved default model ${defaultProvider}/${defaultModelId} is unavailable. Using ${model.provider}/${model.id}.`, + }; + } + return { model, thinkingLevel, fallbackMessage: undefined }; } /** @@ -720,12 +731,10 @@ export async function restoreModelFromSession( ): Promise<{ model: Model | undefined; fallbackMessage: string | undefined }> { const removed = getRemovedProviderMessage(savedProvider); if (removed) throw new ModelsError("provider", `${removed} Resume with an explicit --model provider/model.`); - const restoredModel = modelRuntime.getModel(savedProvider, savedModelId); - - // Check if restored model exists and still has auth configured - const hasConfiguredAuth = restoredModel ? modelRuntime.hasConfiguredAuth(restoredModel.provider) : false; + const availableModels = modelRuntime.getAvailableSnapshot(); + const restoredModel = availableModels.find((model) => model.provider === savedProvider && model.id === savedModelId); - if (restoredModel && hasConfiguredAuth) { + if (restoredModel) { if (shouldPrintMessages) { console.log(chalk.dim(`Restored model: ${savedProvider}/${savedModelId}`)); } @@ -733,7 +742,7 @@ export async function restoreModelFromSession( } // Model not found or no API key - fall back - const reason = !restoredModel ? "model no longer exists" : "no auth configured"; + const reason = !modelRuntime.hasConfiguredAuth(savedProvider) ? "no auth configured" : "model is unavailable"; if (shouldPrintMessages) { console.error(chalk.yellow(`Warning: Could not restore model ${savedProvider}/${savedModelId} (${reason}).`)); @@ -751,25 +760,9 @@ export async function restoreModelFromSession( } // Try to find any available model - const availableModels = [...modelRuntime.getAvailableSnapshot()]; - - if (availableModels.length > 0) { - // Try to find a default model from known providers - let fallbackModel: Model | undefined; - for (const provider of Object.keys(defaultModelPerProvider) as KnownProvider[]) { - const defaultId = defaultModelPerProvider[provider]; - const match = availableModels.find((m) => m.provider === provider && m.id === defaultId); - if (match) { - fallbackModel = match; - break; - } - } - - // If no default found, use first available - if (!fallbackModel) { - fallbackModel = availableModels[0]; - } + const fallbackModel = findDefaultModel(availableModels, savedProvider) ?? findDefaultModel(availableModels); + if (fallbackModel) { if (shouldPrintMessages) { console.log(chalk.dim(`Falling back to: ${fallbackModel.provider}/${fallbackModel.id}`)); } diff --git a/packages/coding-agent/src/core/sdk.ts b/packages/coding-agent/src/core/sdk.ts index 372808908..c9ddd53bb 100644 --- a/packages/coding-agent/src/core/sdk.ts +++ b/packages/coding-agent/src/core/sdk.ts @@ -29,7 +29,7 @@ import { } from "./harness-state-runtime.ts"; import { createLessonReportContinuation } from "./lesson-reports.ts"; import { convertToLlm } from "./messages.ts"; -import { findInitialModel } from "./model-resolver.ts"; +import { findDefaultModel, findInitialModel } from "./model-resolver.ts"; import { ModelRuntime } from "./model-runtime.ts"; import { mergeProviderAttributionHeaders } from "./provider-attribution.ts"; import type { ResourceLoader } from "./resource-loader.ts"; @@ -154,7 +154,7 @@ export interface CreateAgentSessionResult { session: AgentSession; /** Extensions result (for UI context setup in interactive mode) */ extensionsResult: LoadExtensionsResult; - /** Warning if session was restored with a different model than saved */ + /** Warning if a saved session model or default could not be selected. */ modelFallbackMessage?: string; /** Non-fatal warning when optional harness state could not be activated. */ harnessStateWarning?: string; @@ -484,12 +484,17 @@ export async function createInternalAgentSession( if (!model && hasExistingSession && existingSession.model) { const removed = getRemovedProviderMessage(existingSession.model.provider); if (removed) throw new ModelsError("provider", `${removed} Resume with an explicit --model provider/model.`); - const restoredModel = modelRuntime.getModel(existingSession.model.provider, existingSession.model.modelId); - if (restoredModel && modelRuntime.hasConfiguredAuth(restoredModel.provider)) { - model = restoredModel; - } + const savedModel = existingSession.model; + const availableModels = modelRuntime.getAvailableSnapshot(); + model = availableModels.find( + (candidate) => candidate.provider === savedModel.provider && candidate.id === savedModel.modelId, + ); if (!model) { + model = findDefaultModel(availableModels, existingSession.model.provider); modelFallbackMessage = `Could not restore model ${existingSession.model.provider}/${existingSession.model.modelId}`; + if (model) { + modelFallbackMessage += `. Using ${model.provider}/${model.id}`; + } } } @@ -508,6 +513,8 @@ export async function createInternalAgentSession( modelFallbackMessage = formatNoModelsAvailableMessage(); } else if (modelFallbackMessage) { modelFallbackMessage += `. Using ${model.provider}/${model.id}`; + } else { + modelFallbackMessage = result.fallbackMessage; } } diff --git a/packages/coding-agent/src/modes/interactive/interactive-mode.ts b/packages/coding-agent/src/modes/interactive/interactive-mode.ts index cf635bda4..a5c879885 100644 --- a/packages/coding-agent/src/modes/interactive/interactive-mode.ts +++ b/packages/coding-agent/src/modes/interactive/interactive-mode.ts @@ -95,7 +95,7 @@ import { configureHttpDispatcher, formatHttpIdleTimeoutMs } from "../../core/htt import { type AppKeybinding, KeybindingsManager } from "../../core/keybindings.ts"; import { createCompactionSummaryMessage } from "../../core/messages.ts"; import { - defaultModelPerProvider, + findDefaultModel, findExactModelReferenceMatch, resolveModelScopeFromModels, } from "../../core/model-resolver.ts"; @@ -302,10 +302,6 @@ export function formatResumeCommand(sessionManager: SessionManager): string | un return args.join(" "); } -function hasDefaultModelProvider(providerId: string): providerId is keyof typeof defaultModelPerProvider { - return providerId in defaultModelPerProvider; -} - type LoginProviderCompletionOption = { id: string; name: string; @@ -6454,26 +6450,26 @@ export class InteractiveMode { let selectedModel: Model | undefined; let selectionError: string | undefined; - if (isUnknownModel(previousModel)) { - const availableModels = this.session.modelRuntime.getAvailableSnapshot(); - const providerModels = availableModels.filter((model) => model.provider === providerId); - if (!hasDefaultModelProvider(providerId)) { - selectionError = `${actionLabel}, but no default model is configured for provider "${providerId}". Use /model to select a model.`; - } else if (providerModels.length === 0) { + const availableModels = this.session.modelRuntime.getAvailableSnapshot(); + if ( + !previousModel || + isUnknownModel(previousModel) || + (previousModel.provider === providerId && + providerId === "openai-codex" && + !availableModels.some( + (model) => model.provider === previousModel.provider && model.id === previousModel.id, + )) + ) { + selectedModel = findDefaultModel(availableModels, providerId); + if (!selectedModel) { selectionError = `${actionLabel}, but no models are available for that provider. Use /model to select a model.`; } else { - const defaultModelId = defaultModelPerProvider[providerId]; - selectedModel = providerModels.find((model) => model.id === defaultModelId); - if (!selectedModel) { - selectionError = `${actionLabel}, but its default model "${defaultModelId}" is not available. Use /model to select a model.`; - } else { - try { - await this.session.setModel(selectedModel); - } catch (error: unknown) { - selectedModel = undefined; - const errorMessage = error instanceof Error ? error.message : String(error); - selectionError = `${actionLabel}, but selecting its default model failed: ${errorMessage}. Use /model to select a model.`; - } + try { + await this.session.setModel(selectedModel); + } catch (error: unknown) { + selectedModel = undefined; + const errorMessage = error instanceof Error ? error.message : String(error); + selectionError = `${actionLabel}, but selecting its default model failed: ${errorMessage}. Use /model to select a model.`; } } } diff --git a/packages/coding-agent/test/model-resolver.test.ts b/packages/coding-agent/test/model-resolver.test.ts index 391c6a613..9e94e7417 100644 --- a/packages/coding-agent/test/model-resolver.test.ts +++ b/packages/coding-agent/test/model-resolver.test.ts @@ -2,11 +2,13 @@ import type { Model } from "@diffexai/diffex-ai"; import { describe, expect, test, vi } from "vitest"; import { defaultModelPerProvider, + findDefaultModel, findInitialModel, parseModelPattern, resolveCliModel, resolveModelScope, resolveModelScopeWithDiagnostics, + restoreModelFromSession, } from "../src/core/model-resolver.ts"; // Mock models for testing @@ -693,10 +695,161 @@ describe("resolveCliModel", () => { }); }); +describe("account-specific default model selection", () => { + const codexModel = (id: string): Model<"openai-responses"> => ({ + ...mockModels[1], + id, + name: id, + api: "openai-responses", + provider: "openai-codex", + }); + const oldModel = codexModel("gpt-5.5"); + const preferredModel = codexModel("gpt-5.6-terra"); + const futureModel = codexModel("future-chatgpt-model"); + const otherProviderModel = { ...mockModels[0], id: defaultModelPerProvider.anthropic }; + const models = [otherProviderModel, oldModel, preferredModel, futureModel]; + const runtime = (available = models) => + ({ + getModels: () => models, + getModel: (provider: string, id: string) => + models.find((model) => model.provider === provider && model.id === id), + getAvailableSnapshot: () => available, + hasConfiguredAuth: (provider: string) => available.some((model) => model.provider === provider), + }) as unknown as Parameters[0]["modelRuntime"]; + + test("prefers Terra only when listed for the selected provider", () => { + expect(findDefaultModel(models, "openai-codex")).toBe(preferredModel); + expect(findDefaultModel([otherProviderModel, futureModel], "openai-codex")).toBe(futureModel); + expect(findDefaultModel([otherProviderModel], "openai-codex")).toBeUndefined(); + }); + + test("preserves provider catalog order when the preferred model is absent", () => { + expect(findDefaultModel([futureModel, oldModel], "openai-codex")).toBe(futureModel); + expect(findDefaultModel([oldModel, futureModel], "openai-codex")).toBe(oldModel); + }); + + test("preserves global provider priority and handles empty catalogs", () => { + expect(findDefaultModel([preferredModel, otherProviderModel])).toBe(otherProviderModel); + expect(findDefaultModel([futureModel, oldModel])).toBe(futureModel); + expect(findDefaultModel([])).toBeUndefined(); + expect(findDefaultModel([], "openai-codex")).toBeUndefined(); + }); + + test("new ChatGPT-only installations prefer Terra over a listed older model", async () => { + const result = await findInitialModel({ + scopedModels: [], + isContinuing: false, + modelRuntime: runtime([oldModel, preferredModel]), + }); + expect(result.model).toBe(preferredModel); + expect(result.fallbackMessage).toBeUndefined(); + }); + + test.each([oldModel, futureModel])("preserves an available saved choice: $id", async (saved) => { + const result = await findInitialModel({ + defaultProvider: "openai-codex", + defaultModelId: saved.id, + defaultThinkingLevel: "high", + scopedModels: [], + isContinuing: false, + modelRuntime: runtime(), + }); + expect(result.model).toBe(saved); + expect(result.thinkingLevel).toBe("high"); + expect(result.fallbackMessage).toBeUndefined(); + }); + + test.each([ + { available: [otherProviderModel, preferredModel, futureModel], expected: preferredModel }, + { available: [otherProviderModel, futureModel], expected: futureModel }, + { available: [otherProviderModel], expected: otherProviderModel }, + ])("falls back from a retired saved model to $expected.id", async ({ available, expected }) => { + const result = await findInitialModel({ + defaultProvider: "openai-codex", + defaultModelId: oldModel.id, + defaultThinkingLevel: "high", + scopedModels: [], + isContinuing: false, + modelRuntime: runtime(available), + }); + expect(result.model).toBe(expected); + expect(result.thinkingLevel).toBe("high"); + expect(result.fallbackMessage).toBe( + `Saved default model openai-codex/gpt-5.5 is unavailable. Using ${expected.provider}/${expected.id}.`, + ); + }); + + test("honors a saved provider even without a saved model", async () => { + const result = await findInitialModel({ + defaultProvider: "openai-codex", + scopedModels: [], + isContinuing: false, + modelRuntime: runtime(), + }); + expect(result.model).toBe(preferredModel); + expect(result.fallbackMessage).toBeUndefined(); + }); + + test("an explicit model scope still takes precedence over saved defaults", async () => { + const result = await findInitialModel({ + defaultProvider: "openai-codex", + defaultModelId: oldModel.id, + scopedModels: [{ model: futureModel, thinkingLevel: "low" }], + isContinuing: false, + modelRuntime: runtime(), + }); + expect(result.model).toBe(futureModel); + expect(result.thinkingLevel).toBe("low"); + }); + + test("does not invent a model when the account has an empty catalog", async () => { + const result = await findInitialModel({ + defaultProvider: "openai-codex", + defaultModelId: oldModel.id, + scopedModels: [], + isContinuing: false, + modelRuntime: runtime([]), + }); + expect(result.model).toBeUndefined(); + }); + + test.each([oldModel, futureModel])("restores an available session choice: $id", async (saved) => { + const result = await restoreModelFromSession("openai-codex", saved.id, undefined, false, runtime()); + expect(result.model).toBe(saved); + expect(result.fallbackMessage).toBeUndefined(); + }); + + test.each([ + { available: [otherProviderModel, preferredModel], expected: preferredModel }, + { available: [otherProviderModel, futureModel], expected: futureModel }, + { available: [otherProviderModel], expected: otherProviderModel }, + ])("restores an unavailable session model with $expected.id", async ({ available, expected }) => { + const result = await restoreModelFromSession("openai-codex", oldModel.id, undefined, false, runtime(available)); + expect(result.model).toBe(expected); + expect(result.fallbackMessage).toContain(`Using ${expected.provider}/${expected.id}.`); + }); + + test("preserves an explicit current model as the session fallback", async () => { + const result = await restoreModelFromSession( + "openai-codex", + "retired-model", + otherProviderModel, + false, + runtime(), + ); + expect(result.model).toBe(otherProviderModel); + }); + + test("session fallback returns no model for an empty catalog", async () => { + const result = await restoreModelFromSession("openai-codex", oldModel.id, undefined, false, runtime([])); + expect(result.model).toBeUndefined(); + }); +}); + describe("default model selection", () => { test("openai defaults track current models", () => { expect(defaultModelPerProvider.openai).toBe("gpt-5.5"); - expect(defaultModelPerProvider["openai-codex"]).toBe("gpt-5.5"); + expect(defaultModelPerProvider["openai-codex"]).toBe("gpt-5.6-terra"); }); test("zai, minimax, cerebras, and ant-ling defaults track current models", () => { diff --git a/packages/coding-agent/test/suite/regressions/296-chatgpt-login-defaults.test.ts b/packages/coding-agent/test/suite/regressions/296-chatgpt-login-defaults.test.ts new file mode 100644 index 000000000..52b6cf37f --- /dev/null +++ b/packages/coding-agent/test/suite/regressions/296-chatgpt-login-defaults.test.ts @@ -0,0 +1,187 @@ +import type { Api, Model } from "@diffexai/diffex-ai"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { InteractiveMode } from "../../../src/modes/interactive/interactive-mode.ts"; +import { createHarness, type Harness } from "../harness.ts"; + +const CHATGPT = "openai-codex"; +const OLD_DEFAULT = "gpt-5.5"; +const NEW_DEFAULT = "gpt-5.6-terra"; + +// Exercise login completion with a synthetic account catalog and the faux provider only. +describe("regression #296: ChatGPT login defaults", () => { + let harness: Harness; + let catalog: Model[]; + let unavailable: Set; + + beforeEach(async () => { + vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("Unexpected network request")); + harness = await createHarness(); + catalog = []; + unavailable = new Set(); + const runtime = harness.session.modelRuntime; + const fauxProvider = runtime.getProvider(harness.getModel().provider); + if (!fauxProvider) throw new Error("Faux provider is unavailable"); + runtime.registerNativeProvider({ + ...fauxProvider, + id: CHATGPT, + name: "ChatGPT fixture", + getModels: () => catalog, + filterModels: (models) => models.filter((model) => !unavailable.has(model.id)), + }); + }); + + afterEach(async () => { + try { + await harness?.cleanup(); + expect(globalThis.fetch).not.toHaveBeenCalled(); + } finally { + vi.restoreAllMocks(); + } + }); + + async function setCatalog(ids: string[], unavailableIds: string[] = []): Promise { + catalog = ids.map((id) => ({ ...harness.getModel(), provider: CHATGPT, id, name: id })); + unavailable = new Set(unavailableIds); + await harness.session.modelRuntime.refresh({ allowNetwork: false }); + } + + async function completeLogin( + previousModel: Model | undefined, + authType: "oauth" | "api_key" = "oauth", + providerId = CHATGPT, + ) { + const runtime = harness.session.modelRuntime; + const refresh = vi.spyOn(runtime, "refresh").mockResolvedValue({ aborted: false, errors: new Map() }); + const setModel = vi.spyOn(harness.session, "setModel"); + const context = { + session: harness.session, + updateAvailableProviderCount: vi.fn(), + footer: { invalidate: vi.fn() }, + updateEditorBorderColor: vi.fn(), + showStatus: vi.fn(), + showError: vi.fn(), + showWarning: vi.fn(), + maybeWarnAboutAnthropicSubscriptionAuth: vi.fn(), + checkDaxnutsEasterEgg: vi.fn(), + ui: { requestRender: vi.fn() }, + }; + const complete = Reflect.get(InteractiveMode.prototype, "completeProviderAuthentication") as ( + this: object, + providerId: string, + providerName: string, + authType: "oauth" | "api_key", + previousModel: Model | undefined, + ) => Promise; + + await complete.call(context, providerId, "ChatGPT fixture", authType, previousModel); + expect(refresh).toHaveBeenCalledExactlyOnceWith({ providers: [providerId], signal: expect.any(AbortSignal) }); + return { ...context, setModel }; + } + + it.each(["oauth", "api_key"] as const)( + "selects the current default on first %s login without a model", + async (authType) => { + await setCatalog([OLD_DEFAULT, NEW_DEFAULT]); + + const context = await completeLogin(undefined, authType); + + expect(harness.session.model).toMatchObject({ provider: CHATGPT, id: NEW_DEFAULT }); + expect(context.setModel).toHaveBeenCalledExactlyOnceWith(catalog[1]); + expect(context.showError).not.toHaveBeenCalled(); + }, + ); + + it("replaces the unknown placeholder on login", async () => { + await setCatalog([OLD_DEFAULT, NEW_DEFAULT]); + const unknownModel = { ...harness.getModel(), provider: "unknown", id: "unknown", api: "unknown" }; + harness.session.agent.state.model = unknownModel; + + const context = await completeLogin(unknownModel); + + expect(harness.session.model).toMatchObject({ provider: CHATGPT, id: NEW_DEFAULT }); + expect(context.showError).not.toHaveBeenCalled(); + }); + + it("preserves an available current and saved model instead of upgrading it", async () => { + await setCatalog([OLD_DEFAULT, NEW_DEFAULT]); + const previousModel = catalog[0]!; + await harness.session.setModel(previousModel); + + const context = await completeLogin(previousModel); + + expect(context.setModel).not.toHaveBeenCalled(); + expect(harness.session.model).toBe(previousModel); + expect(harness.settingsManager.getDefaultModel()).toBe(OLD_DEFAULT); + expect(context.showError).not.toHaveBeenCalled(); + }); + + it("preserves an available model from another provider", async () => { + await setCatalog([NEW_DEFAULT]); + const previousModel = harness.getModel(); + await harness.session.setModel(previousModel); + + const context = await completeLogin(previousModel); + + expect(context.setModel).not.toHaveBeenCalled(); + expect(harness.session.model).toBe(previousModel); + expect(harness.settingsManager.getDefaultProvider()).toBe(previousModel.provider); + expect(harness.settingsManager.getDefaultModel()).toBe(previousModel.id); + expect(context.showError).not.toHaveBeenCalled(); + }); + + it.each(["ChatGPT login", "custom provider reauthentication"])( + "preserves an uncatalogued custom model during %s", + async (login) => { + await setCatalog([NEW_DEFAULT]); + const previousModel = { ...harness.getModel(), id: "uncatalogued-custom-model" }; + await harness.session.setModel(previousModel); + expect(harness.session.modelRuntime.getModel(previousModel.provider, previousModel.id)).toBeUndefined(); + expect(harness.session.modelRuntime.getAvailableSnapshot()).not.toContainEqual(previousModel); + const providerId = login === "ChatGPT login" ? CHATGPT : previousModel.provider; + + const context = await completeLogin(previousModel, "api_key", providerId); + + expect(context.setModel).not.toHaveBeenCalled(); + expect(harness.session.model).toBe(previousModel); + expect(harness.settingsManager.getDefaultProvider()).toBe(previousModel.provider); + expect(harness.settingsManager.getDefaultModel()).toBe(previousModel.id); + expect(context.showError).not.toHaveBeenCalled(); + }, + ); + + it("replaces a known current model excluded by the newly active account", async () => { + await setCatalog([OLD_DEFAULT, NEW_DEFAULT]); + const previousModel = catalog[0]!; + await harness.session.setModel(previousModel); + await setCatalog([OLD_DEFAULT, NEW_DEFAULT], [OLD_DEFAULT]); + expect(harness.session.modelRuntime.getModel(CHATGPT, OLD_DEFAULT)).toBeDefined(); + expect(harness.session.modelRuntime.getAvailableSnapshot()).not.toContainEqual(previousModel); + + const context = await completeLogin(previousModel); + + expect(harness.session.model).toMatchObject({ provider: CHATGPT, id: NEW_DEFAULT }); + expect(context.setModel).toHaveBeenCalledExactlyOnceWith(catalog[1]); + expect(context.showError).not.toHaveBeenCalled(); + }); + + it("uses account catalog order when the preferred default is unavailable", async () => { + await setCatalog([NEW_DEFAULT, "account-first", "account-second"], [NEW_DEFAULT]); + + const context = await completeLogin(undefined); + + expect(harness.session.model).toMatchObject({ provider: CHATGPT, id: "account-first" }); + expect(context.setModel).toHaveBeenCalledExactlyOnceWith(catalog[1]); + expect(context.showError).not.toHaveBeenCalled(); + }); + + it("reports an empty account catalog without selecting another provider's model", async () => { + await setCatalog([NEW_DEFAULT], [NEW_DEFAULT]); + + const context = await completeLogin(undefined); + + expect(context.setModel).not.toHaveBeenCalled(); + expect(context.showError).toHaveBeenCalledWith( + "Logged in to ChatGPT fixture, but no models are available for that provider. Use /model to select a model.", + ); + }); +}); diff --git a/packages/coding-agent/test/suite/regressions/296-chatgpt-model-defaults.test.ts b/packages/coding-agent/test/suite/regressions/296-chatgpt-model-defaults.test.ts new file mode 100644 index 000000000..acca8f079 --- /dev/null +++ b/packages/coding-agent/test/suite/regressions/296-chatgpt-model-defaults.test.ts @@ -0,0 +1,206 @@ +import { join } from "node:path"; +import type { Api, Model } from "@diffexai/diffex-ai"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { AgentSession } from "../../../src/core/agent-session.ts"; +import { createAgentSession } from "../../../src/core/sdk.ts"; +import { SessionManager } from "../../../src/core/session-manager.ts"; +import { createHarness, type Harness } from "../harness.ts"; + +const CHATGPT = "openai-codex"; +const OLD_DEFAULT = "gpt-5.5"; +const NEW_DEFAULT = "gpt-5.6-terra"; + +// The catalog and availability policy are synthetic; all provider behavior comes from the faux harness. +describe("regression #296: ChatGPT model defaults", () => { + let harness: Harness; + let catalog: Model[]; + let unavailable: Set; + const sessions: AgentSession[] = []; + + beforeEach(async () => { + vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("Unexpected network request")); + harness = await createHarness({ models: [{ id: "fallback-model", name: "Fallback model" }] }); + catalog = []; + unavailable = new Set(); + const runtime = harness.session.modelRuntime; + const fauxModel = harness.getModel(); + const fauxProvider = runtime.getProvider(fauxModel.provider); + if (!fauxProvider) throw new Error("Faux provider is unavailable"); + runtime.registerNativeProvider({ + ...fauxProvider, + id: CHATGPT, + name: "ChatGPT fixture", + getModels: () => catalog, + filterModels: (models) => models.filter((model) => !unavailable.has(model.id)), + }); + await runtime.refresh({ allowNetwork: false }); + const getAvailableSnapshot = runtime.getAvailableSnapshot.bind(runtime); + vi.spyOn(runtime, "getAvailableSnapshot").mockImplementation(() => + getAvailableSnapshot().filter((model) => model.provider === CHATGPT || model.provider === fauxModel.provider), + ); + }); + + afterEach(async () => { + try { + for (const session of sessions.splice(0)) await session.dispose(); + await harness?.cleanup(); + expect(globalThis.fetch).not.toHaveBeenCalled(); + } finally { + vi.restoreAllMocks(); + } + }); + + async function setCatalog(ids: string[], unavailableIds: string[] = []): Promise { + catalog = ids.map((id) => ({ ...harness.getModel(), id, name: id, provider: CHATGPT })); + unavailable = new Set(unavailableIds); + await harness.session.modelRuntime.refresh({ allowNetwork: false }); + } + + async function openSession(options: { savedModelId?: string; model?: Model } = {}) { + const sessionManager = SessionManager.inMemory(harness.tempDir); + if (options.savedModelId) { + sessionManager.appendModelChange(CHATGPT, options.savedModelId); + sessionManager.appendMessage({ role: "user", content: "Previous conversation", timestamp: 1 }); + } + const result = await createAgentSession({ + cwd: harness.tempDir, + agentDir: join(harness.tempDir, "agent"), + modelRuntime: harness.session.modelRuntime, + settingsManager: harness.settingsManager, + resourceLoader: harness.session.resourceLoader, + sessionManager, + model: options.model, + subagents: false, + noTools: "all", + }); + sessions.push(result.session); + return result; + } + + it("uses the current default for startup when both old and new models are available", async () => { + await setCatalog([OLD_DEFAULT, NEW_DEFAULT]); + + const result = await openSession(); + + expect(result.session.model).toMatchObject({ provider: CHATGPT, id: NEW_DEFAULT }); + expect(result.modelFallbackMessage).toBeUndefined(); + }); + + it("keeps an available saved default instead of upgrading it", async () => { + await setCatalog([OLD_DEFAULT, NEW_DEFAULT]); + harness.settingsManager.setDefaultModelAndProvider(CHATGPT, OLD_DEFAULT); + + const result = await openSession(); + + expect(result.session.model).toMatchObject({ provider: CHATGPT, id: OLD_DEFAULT }); + expect(result.modelFallbackMessage).toBeUndefined(); + expect(harness.settingsManager.getDefaultModel()).toBe(OLD_DEFAULT); + }); + + it("replaces an unavailable saved default within ChatGPT and exposes the startup warning", async () => { + await setCatalog(["account-first", NEW_DEFAULT]); + harness.settingsManager.setDefaultModelAndProvider(CHATGPT, OLD_DEFAULT); + + const result = await openSession(); + + expect(result.session.model).toMatchObject({ provider: CHATGPT, id: NEW_DEFAULT }); + expect(result.modelFallbackMessage).toContain(`Saved default model ${CHATGPT}/${OLD_DEFAULT}`); + expect(result.modelFallbackMessage).toContain(`Using ${CHATGPT}/${NEW_DEFAULT}`); + expect(harness.settingsManager.getDefaultProvider()).toBe(CHATGPT); + expect(harness.settingsManager.getDefaultModel()).toBe(OLD_DEFAULT); + }); + + it.each([ + { ids: [OLD_DEFAULT], selected: OLD_DEFAULT }, + { ids: ["account-first", "account-second"], selected: "account-first" }, + ])("uses account catalog order when the hardcoded default is missing: $ids", async ({ ids, selected }) => { + await setCatalog(ids); + harness.settingsManager.setDefaultModelAndProvider(CHATGPT, "removed-model"); + + const result = await openSession(); + + expect(result.session.model).toMatchObject({ provider: CHATGPT, id: selected }); + expect(result.modelFallbackMessage).toContain(`Using ${CHATGPT}/${selected}`); + expect(harness.settingsManager.getDefaultModel()).toBe("removed-model"); + }); + + it("restores an available session model instead of upgrading it or taking the saved default", async () => { + await setCatalog([OLD_DEFAULT, NEW_DEFAULT]); + harness.settingsManager.setDefaultModelAndProvider(CHATGPT, NEW_DEFAULT); + + const result = await openSession({ savedModelId: OLD_DEFAULT }); + + expect(result.session.model).toMatchObject({ provider: CHATGPT, id: OLD_DEFAULT }); + expect(result.modelFallbackMessage).toBeUndefined(); + expect(harness.settingsManager.getDefaultModel()).toBe(NEW_DEFAULT); + }); + + it("keeps a missing session model on ChatGPT before considering another saved provider", async () => { + await setCatalog(["account-first", NEW_DEFAULT]); + const fallback = harness.getModel(); + harness.settingsManager.setDefaultModelAndProvider(fallback.provider, fallback.id); + + const result = await openSession({ savedModelId: OLD_DEFAULT }); + + expect(result.session.model).toMatchObject({ provider: CHATGPT, id: NEW_DEFAULT }); + expect(result.modelFallbackMessage).toContain(`Could not restore model ${CHATGPT}/${OLD_DEFAULT}`); + expect(result.modelFallbackMessage).toContain(`Using ${CHATGPT}/${NEW_DEFAULT}`); + expect(harness.settingsManager.getDefaultProvider()).toBe(fallback.provider); + expect(harness.settingsManager.getDefaultModel()).toBe(fallback.id); + }); + + it("does not restore a known model excluded by the active account's availability policy", async () => { + await setCatalog([OLD_DEFAULT, NEW_DEFAULT], [OLD_DEFAULT]); + expect(harness.session.modelRuntime.getModel(CHATGPT, OLD_DEFAULT)).toBeDefined(); + expect(harness.session.modelRuntime.hasConfiguredAuth(CHATGPT)).toBe(true); + + const result = await openSession({ savedModelId: OLD_DEFAULT }); + + expect(result.session.model).toMatchObject({ provider: CHATGPT, id: NEW_DEFAULT }); + expect(result.modelFallbackMessage).toContain(`Using ${CHATGPT}/${NEW_DEFAULT}`); + }); + + it("restores using account catalog order when neither old nor current defaults are available", async () => { + await setCatalog(["account-first", "account-second"]); + + const result = await openSession({ savedModelId: OLD_DEFAULT }); + + expect(result.session.model).toMatchObject({ provider: CHATGPT, id: "account-first" }); + expect(result.modelFallbackMessage).toContain(`Using ${CHATGPT}/account-first`); + }); + + it("retains settings fallback and the restore warning when ChatGPT has no available models", async () => { + await setCatalog([OLD_DEFAULT], [OLD_DEFAULT]); + const fallback = harness.getModel(); + harness.settingsManager.setDefaultModelAndProvider(fallback.provider, fallback.id); + + const result = await openSession({ savedModelId: OLD_DEFAULT }); + + expect(result.session.model).toMatchObject({ provider: fallback.provider, id: fallback.id }); + expect(result.modelFallbackMessage).toContain(`Could not restore model ${CHATGPT}/${OLD_DEFAULT}`); + expect(result.modelFallbackMessage).toContain(`Using ${fallback.provider}/${fallback.id}`); + }); + + it("retains the unknown placeholder and actionable guidance when no provider has available models", async () => { + await setCatalog([]); + vi.mocked(harness.session.modelRuntime.getAvailableSnapshot).mockReturnValue([]); + harness.settingsManager.setDefaultModelAndProvider(CHATGPT, OLD_DEFAULT); + + const result = await openSession({ savedModelId: OLD_DEFAULT }); + + expect(result.session.model).toMatchObject({ provider: "unknown", id: "unknown", api: "unknown" }); + expect(result.modelFallbackMessage).toContain("No models available"); + expect(result.modelFallbackMessage).toContain("/login"); + expect(harness.settingsManager.getDefaultModel()).toBe(OLD_DEFAULT); + }); + + it("preserves an explicitly supplied model over session restoration", async () => { + await setCatalog([OLD_DEFAULT, NEW_DEFAULT]); + const explicitModel = harness.getModel(); + + const result = await openSession({ savedModelId: OLD_DEFAULT, model: explicitModel }); + + expect(result.session.model).toMatchObject({ provider: explicitModel.provider, id: explicitModel.id }); + expect(result.modelFallbackMessage).toBeUndefined(); + }); +});