From bb6cf783b158a7a1db96f581619826d5a36d14e8 Mon Sep 17 00:00:00 2001 From: Oliver Bartsch Date: Tue, 27 Oct 2026 17:26:50 +0100 Subject: [PATCH] [BUGFIX] Key routing history by the configured model Providers report a dated snapshot (gpt-4o-mini-2024-07-18) or the target of an alias (claude-sonnet-4-20250514 for claude-sonnet-4-0), and the request log stored only that name, while smart routing and the auto model switch compare the history with the model a configuration names. Smart routing therefore never found enough history to downgrade a request to OpenAI or Anthropic. The request log now records the model of the configuration a request ran on (model_configured), and the performance profile is keyed by it. A rerouted request keeps the model it was routed away from in model_requested, so model_configured is the only place its target is known. Entries logged before the column existed keep counting under the reported name. Smart routing also tries the next cheaper model when no configuration takes the cheapest one in the history, instead of giving up. The profile query sums in SQL instead of multiplying averages back. Signed-off-by: Oliver Bartsch --- CHANGELOG.md | 14 +++ Classes/Controller/RequestLogController.php | 23 +++- .../Repository/RequestLogRepository.php | 49 +++++--- .../Middleware/RequestLoggingMiddleware.php | 3 + Classes/Middleware/SmartRoutingMiddleware.php | 51 ++++---- Documentation/Pipeline.md | 6 + Documentation/Usage.md | 2 +- .../Private/Partials/RequestLog/Row.html | 2 +- .../Private/Partials/RequestLogDetail.html | 2 +- .../Public/JavaScript/request-log-poll.js | 2 +- .../RequestLogControllerPollTest.php | 25 ++++ .../Repository/RequestLogRepositoryTest.php | 45 +++++++ .../Middleware/SmartRoutingDispatchTest.php | 86 ++++++++++++- .../Provider/AutoModelSwitchHistoryTest.php | 114 ++++++++++++++++++ ext_tables.sql | 1 + 15 files changed, 377 insertions(+), 48 deletions(-) create mode 100644 Tests/Functional/Provider/AutoModelSwitchHistoryTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index f5304b7..adb209e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ ## Unreleased +- **Bugfix "Smart routing with dated model names"**: Smart routing never downgraded a request to OpenAI +or Anthropic. The request log stores the model name a provider reports, a dated snapshot +(`gpt-4o-mini-2024-07-18`) or the target of an alias (`claude-sonnet-4-20250514` for `claude-sonnet-4-0`), +while the routing compared it with the name a configuration gives. The log now also records the model of +the configuration a request ran on (`model_configured`), and the routing history, the cost ranking of +the automatic model switch and the *Model Usage* dashboard widget count every request for that model, +rerouted ones included. Entries logged before the update keep counting under the reported name, so smart +routing starts downgrading once requests logged after the update reach its minimum history. It also moves +on to the next cheaper model when no configuration takes the cheapest one, instead of not downgrading at +all. The request log marks a request as "requested: …" only when it was routed to another configuration, +not when the provider reported the dated name or the alias target of the model requested. Run "Analyze +Database Structure" after updating: until the column exists, requests are answered but neither logged nor +graded, smart routing and the automatic model switch find no history, and the *Model Usage* widget fails. + - **Bugfix "Max tokens of 0"**: A request with a max tokens of 0, which a provider configuration's *Max Tokens* field passes on when it is set to 0, sent that 0 to the provider, and OpenAI rejected the request ("Invalid 'max_output_tokens' … got 0"). 0 now means no limit: the option is left out and the diff --git a/Classes/Controller/RequestLogController.php b/Classes/Controller/RequestLogController.php index 83c6bae..324eef2 100644 --- a/Classes/Controller/RequestLogController.php +++ b/Classes/Controller/RequestLogController.php @@ -79,7 +79,7 @@ public function logAction(ServerRequestInterface $request): ResponseInterface $buttonBar->addButton($shortcutButton, ButtonBar::BUTTON_POSITION_RIGHT, 3); $demand = RequestLogDemand::fromRequest($request); - $logEntries = $this->logRepository->findByDemand($demand); + $logEntries = array_map($this->withModelRequestedDiffers(...), $this->logRepository->findByDemand($demand)); $totalCount = $this->logRepository->countByDemand($demand); $paginator = new DemandedArrayPaginator( @@ -177,6 +177,7 @@ private function resolveShowViewData(int $uid): array if ($entry === null) { return ['entry' => null]; } + $entry = $this->withModelRequestedDiffers($entry); $configurationUid = (int)($entry['configuration_uid'] ?? 0); $entry['configuration_edit_url'] = $configurationUid > 0 @@ -203,7 +204,7 @@ public function pollAction(ServerRequestInterface $request): ResponseInterface } $demand = RequestLogDemand::fromRequest($request); - $logEntries = $this->logRepository->findByDemand($demand); + $logEntries = array_map($this->withModelRequestedDiffers(...), $this->logRepository->findByDemand($demand)); $totalCount = $this->logRepository->countByDemand($demand); $statistics = $this->logRepository->getStatistics(); @@ -249,6 +250,7 @@ public function pollAction(ServerRequestInterface $request): ResponseInterface : '', 'model_used' => $entry['model_used'] ?: ($entry['model_requested'] ?? ''), 'model_requested' => $entry['model_requested'] ?? '', + 'model_requested_differs' => $entry['model_requested_differs'], 'total_tokens' => (int)($entry['total_tokens'] ?? 0), 'prompt_tokens' => (int)($entry['prompt_tokens'] ?? 0), 'completion_tokens' => (int)($entry['completion_tokens'] ?? 0), @@ -282,6 +284,23 @@ public function pollAction(ServerRequestInterface $request): ResponseInterface ]); } + /** + * Whether the entry ran on another model than the one requested. Compared + * with the configured model, since the name a provider reports for it is + * often a dated snapshot or the target of an alias. + * + * @param array $entry + * @return array + */ + private function withModelRequestedDiffers(array $entry): array + { + // Entries logged before model_configured existed only have the name the provider reported + $ran = (string)($entry['model_configured'] ?? '') ?: (string)($entry['model_used'] ?? ''); + $requested = (string)($entry['model_requested'] ?? ''); + $entry['model_requested_differs'] = $ran !== '' && $requested !== '' && $requested !== $ran; + return $entry; + } + /** * Maps a demand's filters/sorting to route parameters, so a rebuilt * "aim_request_log" URL reflects the same filter/sort state (deliberately diff --git a/Classes/Domain/Repository/RequestLogRepository.php b/Classes/Domain/Repository/RequestLogRepository.php index cef0a46..023bf35 100644 --- a/Classes/Domain/Repository/RequestLogRepository.php +++ b/Classes/Domain/Repository/RequestLogRepository.php @@ -233,6 +233,8 @@ public function getStatisticsByExtension(): array * Performance profile per model for a given request type. * Used by smart routing middleware. * + * "model_used" is the model as the configuration names it (model_configured). + * * @return list */ public function getModelPerformanceProfile(string $requestType = ''): array @@ -241,21 +243,39 @@ public function getModelPerformanceProfile(string $requestType = ''): array ->executeQuery() ->fetchAllAssociative(); + $merged = []; + foreach ($rows as $row) { + // model_used is what the provider reported, often a dated snapshot + // (gpt-4o-mini-2024-07-18) or an alias target no configuration names. + // Entries logged before model_configured existed keep that name. + $model = (string)$row['model_configured'] ?: (string)$row['model_used']; + if (!isset($merged[$model])) { + $merged[$model] = ['model_used' => $model, 'request_count' => 0, 'cost_sum' => 0.0, 'duration_sum' => 0, 'tokens_sum' => 0, 'successful_requests' => 0, 'grade_score_sum' => 0.0, 'graded_count' => 0]; + } + $merged[$model]['request_count'] += (int)$row['request_count']; + $merged[$model]['cost_sum'] += (float)$row['cost_sum']; + $merged[$model]['duration_sum'] += (int)$row['duration_sum']; + $merged[$model]['tokens_sum'] += (int)$row['tokens_sum']; + $merged[$model]['successful_requests'] += (int)$row['successful_requests']; + $merged[$model]['grade_score_sum'] += (float)$row['grade_score_sum']; + $merged[$model]['graded_count'] += (int)$row['graded_count']; + } + usort($merged, static fn(array $a, array $b): int => $b['request_count'] <=> $a['request_count']); + return array_map(static function (array $row): array { - $count = (int)$row['request_count']; - $successful = (int)$row['successful_requests']; - $gradedCount = (int)$row['graded_count']; + $count = $row['request_count']; + $gradedCount = $row['graded_count']; return [ 'model_used' => $row['model_used'], 'request_count' => $count, - 'avg_cost' => round((float)$row['avg_cost'], 6), - 'avg_duration_ms' => (int)$row['avg_duration_ms'], - 'success_rate' => $count > 0 ? round($successful / $count * 100, 1) : 0, - 'avg_tokens' => (int)$row['avg_tokens'], + 'avg_cost' => round($row['cost_sum'] / $count, 6), + 'avg_duration_ms' => intdiv($row['duration_sum'], $count), + 'success_rate' => round($row['successful_requests'] / $count * 100, 1), + 'avg_tokens' => intdiv($row['tokens_sum'], $count), 'graded_count' => $gradedCount, - 'avg_grade_score' => $gradedCount > 0 ? round((float)$row['grade_score_sum'] / $gradedCount, 4) : 0.0, + 'avg_grade_score' => $gradedCount > 0 ? round($row['grade_score_sum'] / $gradedCount, 4) : 0.0, ]; - }, $rows); + }, $merged); } private function buildModelPerformanceQueryBuilder(string $requestType): QueryBuilder @@ -264,11 +284,12 @@ private function buildModelPerformanceQueryBuilder(string $requestType): QueryBu $qb = $this->getQueryBuilder(); $qb->selectLiteral( 'model_used', + 'model_configured', $qb->expr()->count('*', 'request_count'), - 'AVG(cost) AS avg_cost', - 'AVG(duration_ms) AS avg_duration_ms', + 'SUM(cost) AS cost_sum', + 'SUM(duration_ms) AS duration_sum', 'SUM(success) AS successful_requests', - 'AVG(total_tokens) AS avg_tokens', + 'SUM(total_tokens) AS tokens_sum', sprintf("SUM(CASE WHEN grade_status = '%s' THEN grade_score ELSE 0 END) AS grade_score_sum", $done), sprintf("SUM(CASE WHEN grade_status = '%s' THEN 1 ELSE 0 END) AS graded_count", $done), ); @@ -278,9 +299,7 @@ private function buildModelPerformanceQueryBuilder(string $requestType): QueryBu } else { $qb->where($qb->expr()->neq('model_used', $qb->createNamedParameter(''))); } - return $qb - ->groupBy('model_used') - ->orderBy('request_count', 'DESC'); + return $qb->groupBy('model_used', 'model_configured'); } public function getDistinctProviders(): array diff --git a/Classes/Middleware/RequestLoggingMiddleware.php b/Classes/Middleware/RequestLoggingMiddleware.php index 3f5975d..c1e0a36 100644 --- a/Classes/Middleware/RequestLoggingMiddleware.php +++ b/Classes/Middleware/RequestLoggingMiddleware.php @@ -196,6 +196,9 @@ private function logRequest( 'request_prompt' => $prompt, 'request_system_prompt' => $systemPrompt, ]; + // Routing history counts the request for this model, not for the one + // requested (rerouted away from) or the name the provider reports + $data['model_configured'] = $configuration->model; if ($response !== null) { $usage = $response->usage; diff --git a/Classes/Middleware/SmartRoutingMiddleware.php b/Classes/Middleware/SmartRoutingMiddleware.php index ba0d368..53ae679 100644 --- a/Classes/Middleware/SmartRoutingMiddleware.php +++ b/Classes/Middleware/SmartRoutingMiddleware.php @@ -347,8 +347,8 @@ private function findCheaperModel(AiRequestInterface $request, ProviderConfigura return null; } - // Find a cheaper model with good success rate - $bestCandidate = null; + // Find cheaper models with good success rate + $candidates = []; foreach ($profiles as $profile) { if ($profile['model_used'] === $currentConfig->model) { continue; @@ -377,40 +377,41 @@ private function findCheaperModel(AiRequestInterface $request, ProviderConfigura continue; } - // Cheaper + reliable — is there a config for this model? - if ($bestCandidate === null || $profile['avg_cost'] < $bestCandidate['avg_cost']) { - $bestCandidate = $profile; - } + $candidates[] = $profile; } - if ($bestCandidate === null) { + if ($candidates === []) { return null; } + usort($candidates, static fn(array $a, array $b): int => $a['avg_cost'] <=> $b['avg_cost']); - // Resolve the cheaper model's provider + config + // The cheapest candidate a configuration can take; history can name a + // model no configuration (any longer) does try { $capabilityFqcn = $this->resolveCapabilityForRequest($request); if ($capabilityFqcn === null) { return null; } $allProviders = $this->providerResolver->resolveAllForCapability($capabilityFqcn); - foreach ($allProviders as $resolved) { - if ($resolved->configuration->model === $bestCandidate['model_used'] - && !$resolved->configuration->disabled - && $resolved->configuration->acceptsReroutedRequests - && $this->configurationAccess->isAccessibleByCurrentUser($resolved->configuration) - ) { - return [ - 'provider' => $resolved->manifest->getInstance(), - 'configuration' => $resolved->configuration, - 'avg_grade_score' => $bestCandidate['avg_grade_score'], - 'graded_count' => $bestCandidate['graded_count'], - 'avg_cost' => $bestCandidate['avg_cost'], - 'avg_duration_ms' => $bestCandidate['avg_duration_ms'], - 'avg_tokens' => $bestCandidate['avg_tokens'], - 'from_cost' => $currentCost, - 'from_duration_ms' => $currentDuration ?? 0, - ]; + foreach ($candidates as $candidate) { + foreach ($allProviders as $resolved) { + if ($resolved->configuration->model === $candidate['model_used'] + && !$resolved->configuration->disabled + && $resolved->configuration->acceptsReroutedRequests + && $this->configurationAccess->isAccessibleByCurrentUser($resolved->configuration) + ) { + return [ + 'provider' => $resolved->manifest->getInstance(), + 'configuration' => $resolved->configuration, + 'avg_grade_score' => $candidate['avg_grade_score'], + 'graded_count' => $candidate['graded_count'], + 'avg_cost' => $candidate['avg_cost'], + 'avg_duration_ms' => $candidate['avg_duration_ms'], + 'avg_tokens' => $candidate['avg_tokens'], + 'from_cost' => $currentCost, + 'from_duration_ms' => $currentDuration ?? 0, + ]; + } } } } catch (\Throwable) { diff --git a/Documentation/Pipeline.md b/Documentation/Pipeline.md index dab6f61..f3756ec 100644 --- a/Documentation/Pipeline.md +++ b/Documentation/Pipeline.md @@ -15,6 +15,12 @@ The `SmartRoutingMiddleware` classifies prompt complexity using language-agnosti Classification is logged per request (`complexity_score`, `complexity_label`, `complexity_reason`). When a cheaper model has proven reliable for simple prompts (based on historical request log data with minimum 10 requests and 90%+ success rate), the middleware automatically downgrades. +The history counts a request for the model of the configuration it ran on (`model_configured` in the request +log), not for the name the provider reported, which is often a dated snapshot (`gpt-4o-mini-2024-07-18`) or +the target of an alias (`claude-sonnet-4-20250514` for `claude-sonnet-4-0`). A downgraded request counts for +the cheaper configuration it was routed to. Entries logged before `model_configured` existed count under the +reported name. When no configuration takes the cheapest model in the history, the next cheaper one is tried. + ### Quality gate "Reliable" on its own only means *the API call didn't error*. A cheap model can succeed every time while producing weak answers. When [LLM grading](#llm-grading) is enabled, smart routing also consults the recorded `grade_score`: a cheaper model is only chosen if its graded responses for that request type average at least **0.65** (the "good" boundary) across at least **10 graded requests**. diff --git a/Documentation/Usage.md b/Documentation/Usage.md index 7f91cb7..804c2d1 100644 --- a/Documentation/Usage.md +++ b/Documentation/Usage.md @@ -376,7 +376,7 @@ Providers can declare per-model capabilities via `modelCapabilities`. Models lis When a provider config has `gpt-4o` but an embedding request comes in, AiM automatically switches to the cheapest capable model (e.g. `text-embedding-3-small`) using the same API key. The selection is data-driven: if historical cost data exists in the request log, AiM picks the cheapest model with a good success rate. Otherwise it falls back to the most specialized model. The switch is: -- **Logged** with `model_requested`, `model_used`, and reroute reason +- **Logged** with `model_requested` (the model switched away from), `model_configured` (the model switched to), `model_used` (the name the provider reported), and reroute reason - **Controllable** at three levels: | Level | Setting | Default | diff --git a/Resources/Private/Partials/RequestLog/Row.html b/Resources/Private/Partials/RequestLog/Row.html index 2710e39..51b48be 100644 --- a/Resources/Private/Partials/RequestLog/Row.html +++ b/Resources/Private/Partials/RequestLog/Row.html @@ -46,7 +46,7 @@ {entry.model_used -> f:or(alternative: entry.model_requested)} - +
requested: {entry.model_requested} diff --git a/Resources/Private/Partials/RequestLogDetail.html b/Resources/Private/Partials/RequestLogDetail.html index fa178d2..213fba9 100644 --- a/Resources/Private/Partials/RequestLogDetail.html +++ b/Resources/Private/Partials/RequestLogDetail.html @@ -59,7 +59,7 @@ {entry.model_used -> f:or(alternative: entry.model_requested)} - + requested: {entry.model_requested} diff --git a/Resources/Public/JavaScript/request-log-poll.js b/Resources/Public/JavaScript/request-log-poll.js index ccf9e9a..d8e4df0 100644 --- a/Resources/Public/JavaScript/request-log-poll.js +++ b/Resources/Public/JavaScript/request-log-poll.js @@ -255,7 +255,7 @@ class RequestLogPoll { #renderModel(e) { let html = `${this.#esc(e.model_used)}`; - if (e.model_used && e.model_requested && e.model_used !== e.model_requested) { + if (e.model_requested_differs) { html += `
requested: ${this.#esc(e.model_requested)}`; } return html; diff --git a/Tests/Functional/Controller/RequestLogControllerPollTest.php b/Tests/Functional/Controller/RequestLogControllerPollTest.php index a150e60..c1737b2 100644 --- a/Tests/Functional/Controller/RequestLogControllerPollTest.php +++ b/Tests/Functional/Controller/RequestLogControllerPollTest.php @@ -180,6 +180,31 @@ public function pollActionHonoursTheFilterSortingAndPageOfTheCurrentView(): void self::assertCount(0, $this->rowsOf($this->poll(['page' => '2'])), 'The page was ignored, so a poll drops the user back to page 1.'); } + /** + * An alias is reported under a name that shares nothing with it but the + * family, so only the configured model tells a reroute apart. + */ + #[Test] + public function pollActionFlagsAnAliasedModelOnlyWhenTheRequestWasRerouted(): void + { + $this->getConnectionPool()->getConnectionForTable('be_users')->insert('be_users', [ + 'uid' => 1, + 'pid' => 0, + 'username' => 'admin', + 'admin' => 1, + ]); + $this->setUpBackendUser(1); + $GLOBALS['LANG'] = $this->get(LanguageServiceFactory::class)->createFromUserPreferences($GLOBALS['BE_USER']); + + $logRepo = $this->get(RequestLogRepository::class); + $logRepo->log(['request_type' => 'TextGenerationRequest', 'provider_identifier' => 'test', 'model_requested' => 'claude-sonnet-4-0', 'model_configured' => 'claude-sonnet-4-0', 'model_used' => 'claude-sonnet-4-20250514', 'cost' => 0.1]); + $logRepo->log(['request_type' => 'TextGenerationRequest', 'provider_identifier' => 'test', 'model_requested' => 'claude-sonnet-4-0', 'model_configured' => 'claude-3-5-haiku-latest', 'model_used' => 'claude-3-5-haiku-20241022', 'reroute_type' => 'model_switch', 'cost' => 0.2]); + + $rows = $this->rowsOf($this->poll(['orderField' => 'cost', 'orderDirection' => 'asc'])); + + self::assertSame([false, true], array_column($rows, 'model_requested_differs')); + } + /** * @return array> */ diff --git a/Tests/Functional/Domain/Repository/RequestLogRepositoryTest.php b/Tests/Functional/Domain/Repository/RequestLogRepositoryTest.php index b69639e..d0aa089 100644 --- a/Tests/Functional/Domain/Repository/RequestLogRepositoryTest.php +++ b/Tests/Functional/Domain/Repository/RequestLogRepositoryTest.php @@ -185,6 +185,51 @@ public function modelPerformanceProfileReportsZeroGradesForUngradedModel(): void self::assertSame(0.0, $model['avg_grade_score']); } + #[Test] + public function modelPerformanceProfileMergesDatedSnapshotsWeightedByRequestCount(): void + { + $logRepo = $this->get(RequestLogRepository::class); + foreach ([1, 2] as $_) { + $logRepo->log(array_replace($this->row('gpt-4o-mini', 0.1, GradeStatus::Done, 1.0), ['model_configured' => 'gpt-4o-mini', 'duration_ms' => 1000, 'total_tokens' => 100])); + } + $logRepo->log(array_replace($this->row('gpt-4o-mini-2024-07-18', 0.4, GradeStatus::Done, 0.4), ['model_configured' => 'gpt-4o-mini', 'duration_ms' => 4000, 'total_tokens' => 400, 'success' => 0])); + + $profiles = $logRepo->getModelPerformanceProfile('TextGenerationRequest'); + + self::assertSame(['gpt-4o-mini'], array_column($profiles, 'model_used')); + $model = $profiles[0]; + self::assertSame(3, $model['request_count']); + // A mean of the two per-name averages would give 0.25 and 2500 + self::assertEqualsWithDelta(0.2, $model['avg_cost'], 0.000001); + self::assertSame(2000, $model['avg_duration_ms']); + self::assertSame(200, $model['avg_tokens']); + self::assertSame(66.7, $model['success_rate']); + self::assertSame(3, $model['graded_count']); + self::assertEqualsWithDelta(0.8, $model['avg_grade_score'], 0.0001); + } + + /** + * The configured model names the request exactly, whatever the provider + * reported and wherever the request was routed from. + */ + #[Test] + public function modelPerformanceProfileIsKeyedByTheConfiguredModel(): void + { + $logRepo = $this->get(RequestLogRepository::class); + $logRepo->log(array_replace($this->row('claude-3-5-haiku-20241022', 0.0001, GradeStatus::Pending, 0.0), ['model_requested' => 'claude-sonnet-4-0', 'model_configured' => 'claude-3-5-haiku-latest', 'reroute_type' => 'model_switch'])); + $logRepo->log(array_replace($this->row('claude-3-5-haiku-20241022', 0.0003, GradeStatus::Pending, 0.0), ['model_requested' => 'claude-3-5-haiku-latest', 'model_configured' => 'claude-3-5-haiku-latest'])); + $logRepo->log(array_replace($this->row('gpt-4o-2024-08-06', 0.01, GradeStatus::Pending, 0.0), ['model_requested' => 'gpt-4o', 'model_configured' => 'gpt-4o-2024-08-06', 'reroute_type' => 'model_switch'])); + $logRepo->log(array_replace($this->row('gpt-4o-2024-11-20', 0.02, GradeStatus::Pending, 0.0), ['model_requested' => 'gpt-4o', 'model_configured' => 'gpt-4o'])); + // Logged before model_configured existed + $logRepo->log(array_replace($this->row('gpt-4o-2024-11-20', 0.03, GradeStatus::Pending, 0.0), ['model_requested' => 'gpt-4o'])); + + $profiles = $logRepo->getModelPerformanceProfile('TextGenerationRequest'); + + self::assertEqualsCanonicalizing(['claude-3-5-haiku-latest', 'gpt-4o-2024-08-06', 'gpt-4o', 'gpt-4o-2024-11-20'], array_column($profiles, 'model_used')); + self::assertSame(2, $this->profileFor($profiles, 'claude-3-5-haiku-latest')['request_count']); + self::assertEqualsWithDelta(0.0002, $this->profileFor($profiles, 'claude-3-5-haiku-latest')['avg_cost'], 0.000001); + } + private function row(string $model, float $cost, GradeStatus $status, float $gradeScore): array { return [ diff --git a/Tests/Functional/Middleware/SmartRoutingDispatchTest.php b/Tests/Functional/Middleware/SmartRoutingDispatchTest.php index 51d5a1f..e7df36e 100644 --- a/Tests/Functional/Middleware/SmartRoutingDispatchTest.php +++ b/Tests/Functional/Middleware/SmartRoutingDispatchTest.php @@ -104,6 +104,76 @@ public function aCheaperAndOnlySlightlySlowerModelIsStillRoutedTo(): void self::assertSame([$cheap], $this->provider->seen, 'The latency gate is too strict.'); } + /** + * Providers report dated model names (gpt-4o-mini-2024-07-18) where the + * configuration says gpt-4o-mini. + */ + #[Test] + public function aSimplePromptIsDowngradedWhenTheHistoryHasDatedModelNames(): void + { + $this->createConfiguration('gpt-4o', ['default' => 1]); + $cheap = $this->createConfiguration('gpt-4o-mini'); + $this->seedHistory('gpt-4o-2024-08-06', 0.01, configured: 'gpt-4o'); + $this->seedHistory('gpt-4o-mini-2024-07-18', 0.0001, configured: 'gpt-4o-mini'); + + $this->dispatch('Hi'); + + self::assertSame([$cheap], $this->provider->seen, 'Dated model names in the history kept smart routing from downgrading.'); + } + + #[Test] + public function historyUnderDatedAndUndatedNamesCountsForTheSameModel(): void + { + $this->createConfiguration('gpt-4o', ['default' => 1]); + $cheap = $this->createConfiguration('gpt-4o-mini'); + $this->seedHistory('gpt-4o', 0.01, configured: 'gpt-4o'); + // 6 + 6: neither name alone reaches the 10 requests routing needs + $this->seedHistory('gpt-4o-mini', 0.0001, 1000, 6, configured: 'gpt-4o-mini'); + $this->seedHistory('gpt-4o-mini-2024-07-18', 0.0001, 1000, 6, configured: 'gpt-4o-mini'); + + $this->dispatch('Hi'); + + self::assertSame([$cheap], $this->provider->seen); + } + + /** + * A downgrade is logged against the model it was routed away from, and + * the cheap model reports a name no configuration contains. Counted under + * that name, the downgrades formed a profile cheaper than the model's own, + * which no configuration could take, and routing stopped downgrading. + */ + #[Test] + public function downgradesKeepCountingForTheConfigurationTheyRanOn(): void + { + $this->createConfiguration('claude-sonnet-4-0', ['default' => 1]); + $cheap = $this->createConfiguration('claude-3-5-haiku-latest'); + $this->seedHistory('claude-sonnet-4-20250514', 0.01, configured: 'claude-sonnet-4-0'); + $this->seedHistory('claude-3-5-haiku-20241022', 0.0002, configured: 'claude-3-5-haiku-latest'); + $this->seedHistory('claude-3-5-haiku-20241022', 0.0001, configured: 'claude-3-5-haiku-latest', requested: 'claude-sonnet-4-0', rerouteType: 'model_switch'); + + $this->dispatch('Hi'); + + self::assertSame([$cheap], $this->provider->seen); + } + + /** + * Older log entries carry no configured model, so their history can still + * end up under a name no configuration contains. + */ + #[Test] + public function aCheaperModelWithoutAConfigurationDoesNotStopTheDowngrade(): void + { + $this->createConfiguration(self::EXPENSIVE_MODEL, ['default' => 1]); + $cheap = $this->createConfiguration(self::CHEAP_MODEL); + $this->seedHistory(self::EXPENSIVE_MODEL, 0.01); + $this->seedHistory(self::CHEAP_MODEL, 0.001); + $this->seedHistory('unconfigured-model', 0.0001); + + $this->dispatch('Hi'); + + self::assertSame([$cheap], $this->provider->seen); + } + /** * The downgrade used to be visible only in the system log: the request log * recorded a plain request against the cheaper model. @@ -122,6 +192,7 @@ public function theDowngradeIsRecordedAsARerouteWithItsNumbers(): void self::assertSame(1, (int)$row['rerouted'], 'The downgrade was not recorded as a reroute.'); self::assertSame('model_switch', $row['reroute_type']); self::assertSame(self::EXPENSIVE_MODEL, $row['model_requested'], 'The original model was not kept.'); + self::assertSame(self::CHEAP_MODEL, $row['model_configured']); self::assertStringContainsString('smart routing', $row['reroute_reason']); self::assertStringContainsString('cost 0.010000 -> 0.000100', $row['reroute_reason']); self::assertStringContainsString('duration 1000 -> 900 ms', $row['reroute_reason']); @@ -214,15 +285,26 @@ private function latestLogRow(): array return $rows[0]; } - private function seedHistory(string $model, float $cost, int $durationMs = 1000): void + private function seedHistory( + string $model, + float $cost, + int $durationMs = 1000, + int $count = 12, + string $requested = '', + string $configured = '', + string $rerouteType = '', + ): void { $logRepository = $this->get(RequestLogRepository::class); - for ($i = 0; $i < 12; $i++) { + for ($i = 0; $i < $count; $i++) { $logRepository->log([ 'crdate' => time(), 'request_type' => 'TextGenerationRequest', 'provider_identifier' => 'testprovider', + 'model_requested' => $requested, 'model_used' => $model, + 'model_configured' => $configured, + 'reroute_type' => $rerouteType, 'success' => 1, 'cost' => $cost, 'duration_ms' => $durationMs, diff --git a/Tests/Functional/Provider/AutoModelSwitchHistoryTest.php b/Tests/Functional/Provider/AutoModelSwitchHistoryTest.php new file mode 100644 index 0000000..65333c2 --- /dev/null +++ b/Tests/Functional/Provider/AutoModelSwitchHistoryTest.php @@ -0,0 +1,114 @@ +addProviderWithEmbeddingModels(['embed-small-2024-01-01', 'embed-small-2024-07-18']); + $this->seedHistory('embed-small-2024-07-18', 0.0001, 'chat-model', 'model_switch', 'embed-small-2024-07-18'); + $this->seedHistory('embed-small-2024-01-01', 0.01, 'chat-model', 'model_switch', 'embed-small-2024-01-01'); + + $resolved = $this->get(ProviderResolver::class)->resolveForCapability(EmbeddingCapableInterface::class); + + self::assertSame('embed-small-2024-07-18', $resolved->configuration->model); + } + + #[Test] + public function aDatedModelThatFailedAfterASwitchIsSkipped(): void + { + $this->addProviderWithEmbeddingModels(['embed-small-2024-01-01', 'embed-small-2024-07-18']); + $this->seedHistory('embed-small-2024-01-01', 0.0, 'chat-model', 'model_switch', 'embed-small-2024-01-01', success: 0); + + $resolved = $this->get(ProviderResolver::class)->resolveForCapability(EmbeddingCapableInterface::class); + + self::assertSame('embed-small-2024-07-18', $resolved->configuration->model); + } + + /** + * @param list $embeddingModels + */ + private function addProviderWithEmbeddingModels(array $embeddingModels): void + { + $this->get(AiProviderRegistry::class)->addProvider(new AiProviderManifest( + identifier: 'testprovider', + name: 'Test Provider', + description: '', + iconIdentifier: '', + supportedModels: [], + capabilities: [TextGenerationCapableInterface::class, EmbeddingCapableInterface::class], + serviceName: 'test.provider', + container: $this->createMock(ContainerInterface::class), + modelCapabilities: ['chat-model' => [TextGenerationCapableInterface::class]] + + array_fill_keys($embeddingModels, [EmbeddingCapableInterface::class]), + )); + $this->getConnectionPool()->getConnectionForTable('tx_aim_configuration')->insert('tx_aim_configuration', [ + 'pid' => 0, + 'ai_provider' => 'testprovider', + 'title' => 'chat', + 'model' => 'chat-model', + 'default' => 1, + 'auto_model_switch' => 1, + 'be_groups' => '', + ]); + } + + private function seedHistory( + string $modelUsed, + float $cost, + string $modelRequested, + string $rerouteType = '', + string $modelConfigured = '', + int $success = 1, + ): void + { + $logRepository = $this->get(RequestLogRepository::class); + for ($i = 0; $i < 3; $i++) { + $logRepository->log([ + 'crdate' => time(), + 'request_type' => 'EmbeddingRequest', + 'provider_identifier' => 'testprovider', + 'model_requested' => $modelRequested, + 'model_used' => $modelUsed, + 'model_configured' => $modelConfigured, + 'reroute_type' => $rerouteType, + 'success' => $success, + 'cost' => $cost, + ]); + } + } +} diff --git a/ext_tables.sql b/ext_tables.sql index 5deca95..3b7691a 100644 --- a/ext_tables.sql +++ b/ext_tables.sql @@ -68,6 +68,7 @@ CREATE TABLE tx_aim_request_log ( configuration_uid int(11) unsigned DEFAULT '0' NOT NULL, model_requested varchar(255) DEFAULT '' NOT NULL, model_used varchar(255) DEFAULT '' NOT NULL, + model_configured varchar(255) DEFAULT '' NOT NULL, extension_key varchar(255) DEFAULT '' NOT NULL, success tinyint(1) unsigned DEFAULT '0' NOT NULL, prompt_tokens int(11) unsigned DEFAULT '0' NOT NULL,