diff --git a/CHANGELOG.md b/CHANGELOG.md index f5304b7..adb209e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ ## Unreleased +- **Bugfix "Smart routing with dated model names"**: Smart routing never downgraded a request to OpenAI +or Anthropic. The request log stores the model name a provider reports, a dated snapshot +(`gpt-4o-mini-2024-07-18`) or the target of an alias (`claude-sonnet-4-20250514` for `claude-sonnet-4-0`), +while the routing compared it with the name a configuration gives. The log now also records the model of +the configuration a request ran on (`model_configured`), and the routing history, the cost ranking of +the automatic model switch and the *Model Usage* dashboard widget count every request for that model, +rerouted ones included. Entries logged before the update keep counting under the reported name, so smart +routing starts downgrading once requests logged after the update reach its minimum history. It also moves +on to the next cheaper model when no configuration takes the cheapest one, instead of not downgrading at +all. The request log marks a request as "requested: …" only when it was routed to another configuration, +not when the provider reported the dated name or the alias target of the model requested. Run "Analyze +Database Structure" after updating: until the column exists, requests are answered but neither logged nor +graded, smart routing and the automatic model switch find no history, and the *Model Usage* widget fails. + - **Bugfix "Max tokens of 0"**: A request with a max tokens of 0, which a provider configuration's *Max Tokens* field passes on when it is set to 0, sent that 0 to the provider, and OpenAI rejected the request ("Invalid 'max_output_tokens' … got 0"). 0 now means no limit: the option is left out and the diff --git a/Classes/Controller/RequestLogController.php b/Classes/Controller/RequestLogController.php index 83c6bae..324eef2 100644 --- a/Classes/Controller/RequestLogController.php +++ b/Classes/Controller/RequestLogController.php @@ -79,7 +79,7 @@ public function logAction(ServerRequestInterface $request): ResponseInterface $buttonBar->addButton($shortcutButton, ButtonBar::BUTTON_POSITION_RIGHT, 3); $demand = RequestLogDemand::fromRequest($request); - $logEntries = $this->logRepository->findByDemand($demand); + $logEntries = array_map($this->withModelRequestedDiffers(...), $this->logRepository->findByDemand($demand)); $totalCount = $this->logRepository->countByDemand($demand); $paginator = new DemandedArrayPaginator( @@ -177,6 +177,7 @@ private function resolveShowViewData(int $uid): array if ($entry === null) { return ['entry' => null]; } + $entry = $this->withModelRequestedDiffers($entry); $configurationUid = (int)($entry['configuration_uid'] ?? 0); $entry['configuration_edit_url'] = $configurationUid > 0 @@ -203,7 +204,7 @@ public function pollAction(ServerRequestInterface $request): ResponseInterface } $demand = RequestLogDemand::fromRequest($request); - $logEntries = $this->logRepository->findByDemand($demand); + $logEntries = array_map($this->withModelRequestedDiffers(...), $this->logRepository->findByDemand($demand)); $totalCount = $this->logRepository->countByDemand($demand); $statistics = $this->logRepository->getStatistics(); @@ -249,6 +250,7 @@ public function pollAction(ServerRequestInterface $request): ResponseInterface : '', 'model_used' => $entry['model_used'] ?: ($entry['model_requested'] ?? ''), 'model_requested' => $entry['model_requested'] ?? '', + 'model_requested_differs' => $entry['model_requested_differs'], 'total_tokens' => (int)($entry['total_tokens'] ?? 0), 'prompt_tokens' => (int)($entry['prompt_tokens'] ?? 0), 'completion_tokens' => (int)($entry['completion_tokens'] ?? 0), @@ -282,6 +284,23 @@ public function pollAction(ServerRequestInterface $request): ResponseInterface ]); } + /** + * Whether the entry ran on another model than the one requested. Compared + * with the configured model, since the name a provider reports for it is + * often a dated snapshot or the target of an alias. + * + * @param array $entry + * @return array + */ + private function withModelRequestedDiffers(array $entry): array + { + // Entries logged before model_configured existed only have the name the provider reported + $ran = (string)($entry['model_configured'] ?? '') ?: (string)($entry['model_used'] ?? ''); + $requested = (string)($entry['model_requested'] ?? ''); + $entry['model_requested_differs'] = $ran !== '' && $requested !== '' && $requested !== $ran; + return $entry; + } + /** * Maps a demand's filters/sorting to route parameters, so a rebuilt * "aim_request_log" URL reflects the same filter/sort state (deliberately diff --git a/Classes/Domain/Repository/RequestLogRepository.php b/Classes/Domain/Repository/RequestLogRepository.php index cef0a46..023bf35 100644 --- a/Classes/Domain/Repository/RequestLogRepository.php +++ b/Classes/Domain/Repository/RequestLogRepository.php @@ -233,6 +233,8 @@ public function getStatisticsByExtension(): array * Performance profile per model for a given request type. * Used by smart routing middleware. * + * "model_used" is the model as the configuration names it (model_configured). + * * @return list */ public function getModelPerformanceProfile(string $requestType = ''): array @@ -241,21 +243,39 @@ public function getModelPerformanceProfile(string $requestType = ''): array ->executeQuery() ->fetchAllAssociative(); + $merged = []; + foreach ($rows as $row) { + // model_used is what the provider reported, often a dated snapshot + // (gpt-4o-mini-2024-07-18) or an alias target no configuration names. + // Entries logged before model_configured existed keep that name. + $model = (string)$row['model_configured'] ?: (string)$row['model_used']; + if (!isset($merged[$model])) { + $merged[$model] = ['model_used' => $model, 'request_count' => 0, 'cost_sum' => 0.0, 'duration_sum' => 0, 'tokens_sum' => 0, 'successful_requests' => 0, 'grade_score_sum' => 0.0, 'graded_count' => 0]; + } + $merged[$model]['request_count'] += (int)$row['request_count']; + $merged[$model]['cost_sum'] += (float)$row['cost_sum']; + $merged[$model]['duration_sum'] += (int)$row['duration_sum']; + $merged[$model]['tokens_sum'] += (int)$row['tokens_sum']; + $merged[$model]['successful_requests'] += (int)$row['successful_requests']; + $merged[$model]['grade_score_sum'] += (float)$row['grade_score_sum']; + $merged[$model]['graded_count'] += (int)$row['graded_count']; + } + usort($merged, static fn(array $a, array $b): int => $b['request_count'] <=> $a['request_count']); + return array_map(static function (array $row): array { - $count = (int)$row['request_count']; - $successful = (int)$row['successful_requests']; - $gradedCount = (int)$row['graded_count']; + $count = $row['request_count']; + $gradedCount = $row['graded_count']; return [ 'model_used' => $row['model_used'], 'request_count' => $count, - 'avg_cost' => round((float)$row['avg_cost'], 6), - 'avg_duration_ms' => (int)$row['avg_duration_ms'], - 'success_rate' => $count > 0 ? round($successful / $count * 100, 1) : 0, - 'avg_tokens' => (int)$row['avg_tokens'], + 'avg_cost' => round($row['cost_sum'] / $count, 6), + 'avg_duration_ms' => intdiv($row['duration_sum'], $count), + 'success_rate' => round($row['successful_requests'] / $count * 100, 1), + 'avg_tokens' => intdiv($row['tokens_sum'], $count), 'graded_count' => $gradedCount, - 'avg_grade_score' => $gradedCount > 0 ? round((float)$row['grade_score_sum'] / $gradedCount, 4) : 0.0, + 'avg_grade_score' => $gradedCount > 0 ? round($row['grade_score_sum'] / $gradedCount, 4) : 0.0, ]; - }, $rows); + }, $merged); } private function buildModelPerformanceQueryBuilder(string $requestType): QueryBuilder @@ -264,11 +284,12 @@ private function buildModelPerformanceQueryBuilder(string $requestType): QueryBu $qb = $this->getQueryBuilder(); $qb->selectLiteral( 'model_used', + 'model_configured', $qb->expr()->count('*', 'request_count'), - 'AVG(cost) AS avg_cost', - 'AVG(duration_ms) AS avg_duration_ms', + 'SUM(cost) AS cost_sum', + 'SUM(duration_ms) AS duration_sum', 'SUM(success) AS successful_requests', - 'AVG(total_tokens) AS avg_tokens', + 'SUM(total_tokens) AS tokens_sum', sprintf("SUM(CASE WHEN grade_status = '%s' THEN grade_score ELSE 0 END) AS grade_score_sum", $done), sprintf("SUM(CASE WHEN grade_status = '%s' THEN 1 ELSE 0 END) AS graded_count", $done), ); @@ -278,9 +299,7 @@ private function buildModelPerformanceQueryBuilder(string $requestType): QueryBu } else { $qb->where($qb->expr()->neq('model_used', $qb->createNamedParameter(''))); } - return $qb - ->groupBy('model_used') - ->orderBy('request_count', 'DESC'); + return $qb->groupBy('model_used', 'model_configured'); } public function getDistinctProviders(): array diff --git a/Classes/Middleware/RequestLoggingMiddleware.php b/Classes/Middleware/RequestLoggingMiddleware.php index 3f5975d..c1e0a36 100644 --- a/Classes/Middleware/RequestLoggingMiddleware.php +++ b/Classes/Middleware/RequestLoggingMiddleware.php @@ -196,6 +196,9 @@ private function logRequest( 'request_prompt' => $prompt, 'request_system_prompt' => $systemPrompt, ]; + // Routing history counts the request for this model, not for the one + // requested (rerouted away from) or the name the provider reports + $data['model_configured'] = $configuration->model; if ($response !== null) { $usage = $response->usage; diff --git a/Classes/Middleware/SmartRoutingMiddleware.php b/Classes/Middleware/SmartRoutingMiddleware.php index ba0d368..53ae679 100644 --- a/Classes/Middleware/SmartRoutingMiddleware.php +++ b/Classes/Middleware/SmartRoutingMiddleware.php @@ -347,8 +347,8 @@ private function findCheaperModel(AiRequestInterface $request, ProviderConfigura return null; } - // Find a cheaper model with good success rate - $bestCandidate = null; + // Find cheaper models with good success rate + $candidates = []; foreach ($profiles as $profile) { if ($profile['model_used'] === $currentConfig->model) { continue; @@ -377,40 +377,41 @@ private function findCheaperModel(AiRequestInterface $request, ProviderConfigura continue; } - // Cheaper + reliable — is there a config for this model? - if ($bestCandidate === null || $profile['avg_cost'] < $bestCandidate['avg_cost']) { - $bestCandidate = $profile; - } + $candidates[] = $profile; } - if ($bestCandidate === null) { + if ($candidates === []) { return null; } + usort($candidates, static fn(array $a, array $b): int => $a['avg_cost'] <=> $b['avg_cost']); - // Resolve the cheaper model's provider + config + // The cheapest candidate a configuration can take; history can name a + // model no configuration (any longer) does try { $capabilityFqcn = $this->resolveCapabilityForRequest($request); if ($capabilityFqcn === null) { return null; } $allProviders = $this->providerResolver->resolveAllForCapability($capabilityFqcn); - foreach ($allProviders as $resolved) { - if ($resolved->configuration->model === $bestCandidate['model_used'] - && !$resolved->configuration->disabled - && $resolved->configuration->acceptsReroutedRequests - && $this->configurationAccess->isAccessibleByCurrentUser($resolved->configuration) - ) { - return [ - 'provider' => $resolved->manifest->getInstance(), - 'configuration' => $resolved->configuration, - 'avg_grade_score' => $bestCandidate['avg_grade_score'], - 'graded_count' => $bestCandidate['graded_count'], - 'avg_cost' => $bestCandidate['avg_cost'], - 'avg_duration_ms' => $bestCandidate['avg_duration_ms'], - 'avg_tokens' => $bestCandidate['avg_tokens'], - 'from_cost' => $currentCost, - 'from_duration_ms' => $currentDuration ?? 0, - ]; + foreach ($candidates as $candidate) { + foreach ($allProviders as $resolved) { + if ($resolved->configuration->model === $candidate['model_used'] + && !$resolved->configuration->disabled + && $resolved->configuration->acceptsReroutedRequests + && $this->configurationAccess->isAccessibleByCurrentUser($resolved->configuration) + ) { + return [ + 'provider' => $resolved->manifest->getInstance(), + 'configuration' => $resolved->configuration, + 'avg_grade_score' => $candidate['avg_grade_score'], + 'graded_count' => $candidate['graded_count'], + 'avg_cost' => $candidate['avg_cost'], + 'avg_duration_ms' => $candidate['avg_duration_ms'], + 'avg_tokens' => $candidate['avg_tokens'], + 'from_cost' => $currentCost, + 'from_duration_ms' => $currentDuration ?? 0, + ]; + } } } } catch (\Throwable) { diff --git a/Documentation/Pipeline.md b/Documentation/Pipeline.md index dab6f61..f3756ec 100644 --- a/Documentation/Pipeline.md +++ b/Documentation/Pipeline.md @@ -15,6 +15,12 @@ The `SmartRoutingMiddleware` classifies prompt complexity using language-agnosti Classification is logged per request (`complexity_score`, `complexity_label`, `complexity_reason`). When a cheaper model has proven reliable for simple prompts (based on historical request log data with minimum 10 requests and 90%+ success rate), the middleware automatically downgrades. +The history counts a request for the model of the configuration it ran on (`model_configured` in the request +log), not for the name the provider reported, which is often a dated snapshot (`gpt-4o-mini-2024-07-18`) or +the target of an alias (`claude-sonnet-4-20250514` for `claude-sonnet-4-0`). A downgraded request counts for +the cheaper configuration it was routed to. Entries logged before `model_configured` existed count under the +reported name. When no configuration takes the cheapest model in the history, the next cheaper one is tried. + ### Quality gate "Reliable" on its own only means *the API call didn't error*. A cheap model can succeed every time while producing weak answers. When [LLM grading](#llm-grading) is enabled, smart routing also consults the recorded `grade_score`: a cheaper model is only chosen if its graded responses for that request type average at least **0.65** (the "good" boundary) across at least **10 graded requests**. diff --git a/Documentation/Usage.md b/Documentation/Usage.md index 7f91cb7..804c2d1 100644 --- a/Documentation/Usage.md +++ b/Documentation/Usage.md @@ -376,7 +376,7 @@ Providers can declare per-model capabilities via `modelCapabilities`. Models lis When a provider config has `gpt-4o` but an embedding request comes in, AiM automatically switches to the cheapest capable model (e.g. `text-embedding-3-small`) using the same API key. The selection is data-driven: if historical cost data exists in the request log, AiM picks the cheapest model with a good success rate. Otherwise it falls back to the most specialized model. The switch is: -- **Logged** with `model_requested`, `model_used`, and reroute reason +- **Logged** with `model_requested` (the model switched away from), `model_configured` (the model switched to), `model_used` (the name the provider reported), and reroute reason - **Controllable** at three levels: | Level | Setting | Default | diff --git a/Resources/Private/Partials/RequestLog/Row.html b/Resources/Private/Partials/RequestLog/Row.html index 2710e39..51b48be 100644 --- a/Resources/Private/Partials/RequestLog/Row.html +++ b/Resources/Private/Partials/RequestLog/Row.html @@ -46,7 +46,7 @@ {entry.model_used -> f:or(alternative: entry.model_requested)} - +
requested: {entry.model_requested} diff --git a/Resources/Private/Partials/RequestLogDetail.html b/Resources/Private/Partials/RequestLogDetail.html index fa178d2..213fba9 100644 --- a/Resources/Private/Partials/RequestLogDetail.html +++ b/Resources/Private/Partials/RequestLogDetail.html @@ -59,7 +59,7 @@ {entry.model_used -> f:or(alternative: entry.model_requested)} - + requested: {entry.model_requested} diff --git a/Resources/Public/JavaScript/request-log-poll.js b/Resources/Public/JavaScript/request-log-poll.js index ccf9e9a..d8e4df0 100644 --- a/Resources/Public/JavaScript/request-log-poll.js +++ b/Resources/Public/JavaScript/request-log-poll.js @@ -255,7 +255,7 @@ class RequestLogPoll { #renderModel(e) { let html = `${this.#esc(e.model_used)}`; - if (e.model_used && e.model_requested && e.model_used !== e.model_requested) { + if (e.model_requested_differs) { html += `
requested: ${this.#esc(e.model_requested)}`; } return html; diff --git a/Tests/Functional/Controller/RequestLogControllerPollTest.php b/Tests/Functional/Controller/RequestLogControllerPollTest.php index a150e60..c1737b2 100644 --- a/Tests/Functional/Controller/RequestLogControllerPollTest.php +++ b/Tests/Functional/Controller/RequestLogControllerPollTest.php @@ -180,6 +180,31 @@ public function pollActionHonoursTheFilterSortingAndPageOfTheCurrentView(): void self::assertCount(0, $this->rowsOf($this->poll(['page' => '2'])), 'The page was ignored, so a poll drops the user back to page 1.'); } + /** + * An alias is reported under a name that shares nothing with it but the + * family, so only the configured model tells a reroute apart. + */ + #[Test] + public function pollActionFlagsAnAliasedModelOnlyWhenTheRequestWasRerouted(): void + { + $this->getConnectionPool()->getConnectionForTable('be_users')->insert('be_users', [ + 'uid' => 1, + 'pid' => 0, + 'username' => 'admin', + 'admin' => 1, + ]); + $this->setUpBackendUser(1); + $GLOBALS['LANG'] = $this->get(LanguageServiceFactory::class)->createFromUserPreferences($GLOBALS['BE_USER']); + + $logRepo = $this->get(RequestLogRepository::class); + $logRepo->log(['request_type' => 'TextGenerationRequest', 'provider_identifier' => 'test', 'model_requested' => 'claude-sonnet-4-0', 'model_configured' => 'claude-sonnet-4-0', 'model_used' => 'claude-sonnet-4-20250514', 'cost' => 0.1]); + $logRepo->log(['request_type' => 'TextGenerationRequest', 'provider_identifier' => 'test', 'model_requested' => 'claude-sonnet-4-0', 'model_configured' => 'claude-3-5-haiku-latest', 'model_used' => 'claude-3-5-haiku-20241022', 'reroute_type' => 'model_switch', 'cost' => 0.2]); + + $rows = $this->rowsOf($this->poll(['orderField' => 'cost', 'orderDirection' => 'asc'])); + + self::assertSame([false, true], array_column($rows, 'model_requested_differs')); + } + /** * @return array> */ diff --git a/Tests/Functional/Domain/Repository/RequestLogRepositoryTest.php b/Tests/Functional/Domain/Repository/RequestLogRepositoryTest.php index b69639e..d0aa089 100644 --- a/Tests/Functional/Domain/Repository/RequestLogRepositoryTest.php +++ b/Tests/Functional/Domain/Repository/RequestLogRepositoryTest.php @@ -185,6 +185,51 @@ public function modelPerformanceProfileReportsZeroGradesForUngradedModel(): void self::assertSame(0.0, $model['avg_grade_score']); } + #[Test] + public function modelPerformanceProfileMergesDatedSnapshotsWeightedByRequestCount(): void + { + $logRepo = $this->get(RequestLogRepository::class); + foreach ([1, 2] as $_) { + $logRepo->log(array_replace($this->row('gpt-4o-mini', 0.1, GradeStatus::Done, 1.0), ['model_configured' => 'gpt-4o-mini', 'duration_ms' => 1000, 'total_tokens' => 100])); + } + $logRepo->log(array_replace($this->row('gpt-4o-mini-2024-07-18', 0.4, GradeStatus::Done, 0.4), ['model_configured' => 'gpt-4o-mini', 'duration_ms' => 4000, 'total_tokens' => 400, 'success' => 0])); + + $profiles = $logRepo->getModelPerformanceProfile('TextGenerationRequest'); + + self::assertSame(['gpt-4o-mini'], array_column($profiles, 'model_used')); + $model = $profiles[0]; + self::assertSame(3, $model['request_count']); + // A mean of the two per-name averages would give 0.25 and 2500 + self::assertEqualsWithDelta(0.2, $model['avg_cost'], 0.000001); + self::assertSame(2000, $model['avg_duration_ms']); + self::assertSame(200, $model['avg_tokens']); + self::assertSame(66.7, $model['success_rate']); + self::assertSame(3, $model['graded_count']); + self::assertEqualsWithDelta(0.8, $model['avg_grade_score'], 0.0001); + } + + /** + * The configured model names the request exactly, whatever the provider + * reported and wherever the request was routed from. + */ + #[Test] + public function modelPerformanceProfileIsKeyedByTheConfiguredModel(): void + { + $logRepo = $this->get(RequestLogRepository::class); + $logRepo->log(array_replace($this->row('claude-3-5-haiku-20241022', 0.0001, GradeStatus::Pending, 0.0), ['model_requested' => 'claude-sonnet-4-0', 'model_configured' => 'claude-3-5-haiku-latest', 'reroute_type' => 'model_switch'])); + $logRepo->log(array_replace($this->row('claude-3-5-haiku-20241022', 0.0003, GradeStatus::Pending, 0.0), ['model_requested' => 'claude-3-5-haiku-latest', 'model_configured' => 'claude-3-5-haiku-latest'])); + $logRepo->log(array_replace($this->row('gpt-4o-2024-08-06', 0.01, GradeStatus::Pending, 0.0), ['model_requested' => 'gpt-4o', 'model_configured' => 'gpt-4o-2024-08-06', 'reroute_type' => 'model_switch'])); + $logRepo->log(array_replace($this->row('gpt-4o-2024-11-20', 0.02, GradeStatus::Pending, 0.0), ['model_requested' => 'gpt-4o', 'model_configured' => 'gpt-4o'])); + // Logged before model_configured existed + $logRepo->log(array_replace($this->row('gpt-4o-2024-11-20', 0.03, GradeStatus::Pending, 0.0), ['model_requested' => 'gpt-4o'])); + + $profiles = $logRepo->getModelPerformanceProfile('TextGenerationRequest'); + + self::assertEqualsCanonicalizing(['claude-3-5-haiku-latest', 'gpt-4o-2024-08-06', 'gpt-4o', 'gpt-4o-2024-11-20'], array_column($profiles, 'model_used')); + self::assertSame(2, $this->profileFor($profiles, 'claude-3-5-haiku-latest')['request_count']); + self::assertEqualsWithDelta(0.0002, $this->profileFor($profiles, 'claude-3-5-haiku-latest')['avg_cost'], 0.000001); + } + private function row(string $model, float $cost, GradeStatus $status, float $gradeScore): array { return [ diff --git a/Tests/Functional/Middleware/SmartRoutingDispatchTest.php b/Tests/Functional/Middleware/SmartRoutingDispatchTest.php index 51d5a1f..e7df36e 100644 --- a/Tests/Functional/Middleware/SmartRoutingDispatchTest.php +++ b/Tests/Functional/Middleware/SmartRoutingDispatchTest.php @@ -104,6 +104,76 @@ public function aCheaperAndOnlySlightlySlowerModelIsStillRoutedTo(): void self::assertSame([$cheap], $this->provider->seen, 'The latency gate is too strict.'); } + /** + * Providers report dated model names (gpt-4o-mini-2024-07-18) where the + * configuration says gpt-4o-mini. + */ + #[Test] + public function aSimplePromptIsDowngradedWhenTheHistoryHasDatedModelNames(): void + { + $this->createConfiguration('gpt-4o', ['default' => 1]); + $cheap = $this->createConfiguration('gpt-4o-mini'); + $this->seedHistory('gpt-4o-2024-08-06', 0.01, configured: 'gpt-4o'); + $this->seedHistory('gpt-4o-mini-2024-07-18', 0.0001, configured: 'gpt-4o-mini'); + + $this->dispatch('Hi'); + + self::assertSame([$cheap], $this->provider->seen, 'Dated model names in the history kept smart routing from downgrading.'); + } + + #[Test] + public function historyUnderDatedAndUndatedNamesCountsForTheSameModel(): void + { + $this->createConfiguration('gpt-4o', ['default' => 1]); + $cheap = $this->createConfiguration('gpt-4o-mini'); + $this->seedHistory('gpt-4o', 0.01, configured: 'gpt-4o'); + // 6 + 6: neither name alone reaches the 10 requests routing needs + $this->seedHistory('gpt-4o-mini', 0.0001, 1000, 6, configured: 'gpt-4o-mini'); + $this->seedHistory('gpt-4o-mini-2024-07-18', 0.0001, 1000, 6, configured: 'gpt-4o-mini'); + + $this->dispatch('Hi'); + + self::assertSame([$cheap], $this->provider->seen); + } + + /** + * A downgrade is logged against the model it was routed away from, and + * the cheap model reports a name no configuration contains. Counted under + * that name, the downgrades formed a profile cheaper than the model's own, + * which no configuration could take, and routing stopped downgrading. + */ + #[Test] + public function downgradesKeepCountingForTheConfigurationTheyRanOn(): void + { + $this->createConfiguration('claude-sonnet-4-0', ['default' => 1]); + $cheap = $this->createConfiguration('claude-3-5-haiku-latest'); + $this->seedHistory('claude-sonnet-4-20250514', 0.01, configured: 'claude-sonnet-4-0'); + $this->seedHistory('claude-3-5-haiku-20241022', 0.0002, configured: 'claude-3-5-haiku-latest'); + $this->seedHistory('claude-3-5-haiku-20241022', 0.0001, configured: 'claude-3-5-haiku-latest', requested: 'claude-sonnet-4-0', rerouteType: 'model_switch'); + + $this->dispatch('Hi'); + + self::assertSame([$cheap], $this->provider->seen); + } + + /** + * Older log entries carry no configured model, so their history can still + * end up under a name no configuration contains. + */ + #[Test] + public function aCheaperModelWithoutAConfigurationDoesNotStopTheDowngrade(): void + { + $this->createConfiguration(self::EXPENSIVE_MODEL, ['default' => 1]); + $cheap = $this->createConfiguration(self::CHEAP_MODEL); + $this->seedHistory(self::EXPENSIVE_MODEL, 0.01); + $this->seedHistory(self::CHEAP_MODEL, 0.001); + $this->seedHistory('unconfigured-model', 0.0001); + + $this->dispatch('Hi'); + + self::assertSame([$cheap], $this->provider->seen); + } + /** * The downgrade used to be visible only in the system log: the request log * recorded a plain request against the cheaper model. @@ -122,6 +192,7 @@ public function theDowngradeIsRecordedAsARerouteWithItsNumbers(): void self::assertSame(1, (int)$row['rerouted'], 'The downgrade was not recorded as a reroute.'); self::assertSame('model_switch', $row['reroute_type']); self::assertSame(self::EXPENSIVE_MODEL, $row['model_requested'], 'The original model was not kept.'); + self::assertSame(self::CHEAP_MODEL, $row['model_configured']); self::assertStringContainsString('smart routing', $row['reroute_reason']); self::assertStringContainsString('cost 0.010000 -> 0.000100', $row['reroute_reason']); self::assertStringContainsString('duration 1000 -> 900 ms', $row['reroute_reason']); @@ -214,15 +285,26 @@ private function latestLogRow(): array return $rows[0]; } - private function seedHistory(string $model, float $cost, int $durationMs = 1000): void + private function seedHistory( + string $model, + float $cost, + int $durationMs = 1000, + int $count = 12, + string $requested = '', + string $configured = '', + string $rerouteType = '', + ): void { $logRepository = $this->get(RequestLogRepository::class); - for ($i = 0; $i < 12; $i++) { + for ($i = 0; $i < $count; $i++) { $logRepository->log([ 'crdate' => time(), 'request_type' => 'TextGenerationRequest', 'provider_identifier' => 'testprovider', + 'model_requested' => $requested, 'model_used' => $model, + 'model_configured' => $configured, + 'reroute_type' => $rerouteType, 'success' => 1, 'cost' => $cost, 'duration_ms' => $durationMs, diff --git a/Tests/Functional/Provider/AutoModelSwitchHistoryTest.php b/Tests/Functional/Provider/AutoModelSwitchHistoryTest.php new file mode 100644 index 0000000..65333c2 --- /dev/null +++ b/Tests/Functional/Provider/AutoModelSwitchHistoryTest.php @@ -0,0 +1,114 @@ +addProviderWithEmbeddingModels(['embed-small-2024-01-01', 'embed-small-2024-07-18']); + $this->seedHistory('embed-small-2024-07-18', 0.0001, 'chat-model', 'model_switch', 'embed-small-2024-07-18'); + $this->seedHistory('embed-small-2024-01-01', 0.01, 'chat-model', 'model_switch', 'embed-small-2024-01-01'); + + $resolved = $this->get(ProviderResolver::class)->resolveForCapability(EmbeddingCapableInterface::class); + + self::assertSame('embed-small-2024-07-18', $resolved->configuration->model); + } + + #[Test] + public function aDatedModelThatFailedAfterASwitchIsSkipped(): void + { + $this->addProviderWithEmbeddingModels(['embed-small-2024-01-01', 'embed-small-2024-07-18']); + $this->seedHistory('embed-small-2024-01-01', 0.0, 'chat-model', 'model_switch', 'embed-small-2024-01-01', success: 0); + + $resolved = $this->get(ProviderResolver::class)->resolveForCapability(EmbeddingCapableInterface::class); + + self::assertSame('embed-small-2024-07-18', $resolved->configuration->model); + } + + /** + * @param list $embeddingModels + */ + private function addProviderWithEmbeddingModels(array $embeddingModels): void + { + $this->get(AiProviderRegistry::class)->addProvider(new AiProviderManifest( + identifier: 'testprovider', + name: 'Test Provider', + description: '', + iconIdentifier: '', + supportedModels: [], + capabilities: [TextGenerationCapableInterface::class, EmbeddingCapableInterface::class], + serviceName: 'test.provider', + container: $this->createMock(ContainerInterface::class), + modelCapabilities: ['chat-model' => [TextGenerationCapableInterface::class]] + + array_fill_keys($embeddingModels, [EmbeddingCapableInterface::class]), + )); + $this->getConnectionPool()->getConnectionForTable('tx_aim_configuration')->insert('tx_aim_configuration', [ + 'pid' => 0, + 'ai_provider' => 'testprovider', + 'title' => 'chat', + 'model' => 'chat-model', + 'default' => 1, + 'auto_model_switch' => 1, + 'be_groups' => '', + ]); + } + + private function seedHistory( + string $modelUsed, + float $cost, + string $modelRequested, + string $rerouteType = '', + string $modelConfigured = '', + int $success = 1, + ): void + { + $logRepository = $this->get(RequestLogRepository::class); + for ($i = 0; $i < 3; $i++) { + $logRepository->log([ + 'crdate' => time(), + 'request_type' => 'EmbeddingRequest', + 'provider_identifier' => 'testprovider', + 'model_requested' => $modelRequested, + 'model_used' => $modelUsed, + 'model_configured' => $modelConfigured, + 'reroute_type' => $rerouteType, + 'success' => $success, + 'cost' => $cost, + ]); + } + } +} diff --git a/ext_tables.sql b/ext_tables.sql index 5deca95..3b7691a 100644 --- a/ext_tables.sql +++ b/ext_tables.sql @@ -68,6 +68,7 @@ CREATE TABLE tx_aim_request_log ( configuration_uid int(11) unsigned DEFAULT '0' NOT NULL, model_requested varchar(255) DEFAULT '' NOT NULL, model_used varchar(255) DEFAULT '' NOT NULL, + model_configured varchar(255) DEFAULT '' NOT NULL, extension_key varchar(255) DEFAULT '' NOT NULL, success tinyint(1) unsigned DEFAULT '0' NOT NULL, prompt_tokens int(11) unsigned DEFAULT '0' NOT NULL,