Smart routing with dated model names

CHANGELOG.md

@@ -2,6 +2,20 @@
 
 ## Unreleased
 
+- **Bugfix "Smart routing with dated model names"**: Smart routing never downgraded a request to OpenAI
+or Anthropic. The request log stores the model name a provider reports, a dated snapshot
+(`gpt-4o-mini-2024-07-18`) or the target of an alias (`claude-sonnet-4-20250514` for `claude-sonnet-4-0`),
+while the routing compared it with the name a configuration gives. The log now also records the model of
+the configuration a request ran on (`model_configured`), and the routing history, the cost ranking of
+the automatic model switch and the *Model Usage* dashboard widget count every request for that model,
+rerouted ones included. Entries logged before the update keep counting under the reported name, so smart
+routing starts downgrading once requests logged after the update reach its minimum history. It also moves
+on to the next cheaper model when no configuration takes the cheapest one, instead of not downgrading at
+all. The request log marks a request as "requested: …" only when it was routed to another configuration,
+not when the provider reported the dated name or the alias target of the model requested. Run "Analyze
+Database Structure" after updating: until the column exists, requests are answered but neither logged nor
+graded, smart routing and the automatic model switch find no history, and the *Model Usage* widget fails.
+
 - **Bugfix "Max tokens of 0"**: A request with a max tokens of 0, which a provider configuration's
 *Max Tokens* field passes on when it is set to 0, sent that 0 to the provider, and OpenAI rejected the
 request ("Invalid 'max_output_tokens' … got 0"). 0 now means no limit: the option is left out and the

Classes/Controller/RequestLogController.php

@@ -79,7 +79,7 @@ public function logAction(ServerRequestInterface $request): ResponseInterface
         $buttonBar->addButton($shortcutButton, ButtonBar::BUTTON_POSITION_RIGHT, 3);
 
         $demand = RequestLogDemand::fromRequest($request);
-        $logEntries = $this->logRepository->findByDemand($demand);
+        $logEntries = array_map($this->withModelRequestedDiffers(...), $this->logRepository->findByDemand($demand));
         $totalCount = $this->logRepository->countByDemand($demand);
 
         $paginator = new DemandedArrayPaginator(
@@ -177,6 +177,7 @@ private function resolveShowViewData(int $uid): array
         if ($entry === null) {
             return ['entry' => null];
         }
+        $entry = $this->withModelRequestedDiffers($entry);
 
         $configurationUid = (int)($entry['configuration_uid'] ?? 0);
         $entry['configuration_edit_url'] = $configurationUid > 0
@@ -203,7 +204,7 @@ public function pollAction(ServerRequestInterface $request): ResponseInterface
         }
 
         $demand = RequestLogDemand::fromRequest($request);
-        $logEntries = $this->logRepository->findByDemand($demand);
+        $logEntries = array_map($this->withModelRequestedDiffers(...), $this->logRepository->findByDemand($demand));
         $totalCount = $this->logRepository->countByDemand($demand);
         $statistics = $this->logRepository->getStatistics();
 
@@ -249,6 +250,7 @@ public function pollAction(ServerRequestInterface $request): ResponseInterface
                     : '',
                 'model_used' => $entry['model_used'] ?: ($entry['model_requested'] ?? ''),
                 'model_requested' => $entry['model_requested'] ?? '',
+                'model_requested_differs' => $entry['model_requested_differs'],
                 'total_tokens' => (int)($entry['total_tokens'] ?? 0),
                 'prompt_tokens' => (int)($entry['prompt_tokens'] ?? 0),
                 'completion_tokens' => (int)($entry['completion_tokens'] ?? 0),
@@ -282,6 +284,23 @@ public function pollAction(ServerRequestInterface $request): ResponseInterface
         ]);
     }
 
+    /**
+     * Whether the entry ran on another model than the one requested. Compared
+     * with the configured model, since the name a provider reports for it is
+     * often a dated snapshot or the target of an alias.
+     *
+     * @param array<string, mixed> $entry
+     * @return array<string, mixed>
+     */
+    private function withModelRequestedDiffers(array $entry): array
+    {
+        // Entries logged before model_configured existed only have the name the provider reported
+        $ran = (string)($entry['model_configured'] ?? '') ?: (string)($entry['model_used'] ?? '');
+        $requested = (string)($entry['model_requested'] ?? '');
+        $entry['model_requested_differs'] = $ran !== '' && $requested !== '' && $requested !== $ran;
+        return $entry;
+    }
+
     /**
      * Maps a demand's filters/sorting to route parameters, so a rebuilt
      * "aim_request_log" URL reflects the same filter/sort state (deliberately

Classes/Domain/Repository/RequestLogRepository.php

@@ -233,6 +233,8 @@ public function getStatisticsByExtension(): array
      * Performance profile per model for a given request type.
      * Used by smart routing middleware.
      *
+     * "model_used" is the model as the configuration names it (model_configured).
+     *
      * @return list<array{model_used: string, request_count: int, avg_cost: float, avg_duration_ms: int, success_rate: float, avg_tokens: int, graded_count: int, avg_grade_score: float}>
      */
     public function getModelPerformanceProfile(string $requestType = ''): array
@@ -241,21 +243,39 @@ public function getModelPerformanceProfile(string $requestType = ''): array
             ->executeQuery()
             ->fetchAllAssociative();
 
+        $merged = [];
+        foreach ($rows as $row) {
+            // model_used is what the provider reported, often a dated snapshot
+            // (gpt-4o-mini-2024-07-18) or an alias target no configuration names.
+            // Entries logged before model_configured existed keep that name.
+            $model = (string)$row['model_configured'] ?: (string)$row['model_used'];
+            if (!isset($merged[$model])) {
+                $merged[$model] = ['model_used' => $model, 'request_count' => 0, 'cost_sum' => 0.0, 'duration_sum' => 0, 'tokens_sum' => 0, 'successful_requests' => 0, 'grade_score_sum' => 0.0, 'graded_count' => 0];
+            }
+            $merged[$model]['request_count'] += (int)$row['request_count'];
+            $merged[$model]['cost_sum'] += (float)$row['cost_sum'];
+            $merged[$model]['duration_sum'] += (int)$row['duration_sum'];
+            $merged[$model]['tokens_sum'] += (int)$row['tokens_sum'];
+            $merged[$model]['successful_requests'] += (int)$row['successful_requests'];
+            $merged[$model]['grade_score_sum'] += (float)$row['grade_score_sum'];
+            $merged[$model]['graded_count'] += (int)$row['graded_count'];
+        }
+        usort($merged, static fn(array $a, array $b): int => $b['request_count'] <=> $a['request_count']);
+
         return array_map(static function (array $row): array {
-            $count = (int)$row['request_count'];
-            $successful = (int)$row['successful_requests'];
-            $gradedCount = (int)$row['graded_count'];
+            $count = $row['request_count'];
+            $gradedCount = $row['graded_count'];
             return [
                 'model_used' => $row['model_used'],
                 'request_count' => $count,
-                'avg_cost' => round((float)$row['avg_cost'], 6),
-                'avg_duration_ms' => (int)$row['avg_duration_ms'],
-                'success_rate' => $count > 0 ? round($successful / $count * 100, 1) : 0,
-                'avg_tokens' => (int)$row['avg_tokens'],
+                'avg_cost' => round($row['cost_sum'] / $count, 6),
+                'avg_duration_ms' => intdiv($row['duration_sum'], $count),
+                'success_rate' => round($row['successful_requests'] / $count * 100, 1),
+                'avg_tokens' => intdiv($row['tokens_sum'], $count),
                 'graded_count' => $gradedCount,
-                'avg_grade_score' => $gradedCount > 0 ? round((float)$row['grade_score_sum'] / $gradedCount, 4) : 0.0,
+                'avg_grade_score' => $gradedCount > 0 ? round($row['grade_score_sum'] / $gradedCount, 4) : 0.0,
             ];
-        }, $rows);
+        }, $merged);
     }
 
     private function buildModelPerformanceQueryBuilder(string $requestType): QueryBuilder
@@ -264,11 +284,12 @@ private function buildModelPerformanceQueryBuilder(string $requestType): QueryBu
         $qb = $this->getQueryBuilder();
         $qb->selectLiteral(
             'model_used',
+            'model_configured',
             $qb->expr()->count('*', 'request_count'),
-            'AVG(cost) AS avg_cost',
-            'AVG(duration_ms) AS avg_duration_ms',
+            'SUM(cost) AS cost_sum',
+            'SUM(duration_ms) AS duration_sum',
             'SUM(success) AS successful_requests',
-            'AVG(total_tokens) AS avg_tokens',
+            'SUM(total_tokens) AS tokens_sum',
             sprintf("SUM(CASE WHEN grade_status = '%s' THEN grade_score ELSE 0 END) AS grade_score_sum", $done),
             sprintf("SUM(CASE WHEN grade_status = '%s' THEN 1 ELSE 0 END) AS graded_count", $done),
         );
@@ -278,9 +299,7 @@ private function buildModelPerformanceQueryBuilder(string $requestType): QueryBu
         } else {
             $qb->where($qb->expr()->neq('model_used', $qb->createNamedParameter('')));
         }
-        return $qb
-            ->groupBy('model_used')
-            ->orderBy('request_count', 'DESC');
+        return $qb->groupBy('model_used', 'model_configured');
     }
 
     public function getDistinctProviders(): array

Classes/Middleware/RequestLoggingMiddleware.php

@@ -196,6 +196,9 @@ private function logRequest(
             'request_prompt' => $prompt,
             'request_system_prompt' => $systemPrompt,
         ];
+        // Routing history counts the request for this model, not for the one
+        // requested (rerouted away from) or the name the provider reports
+        $data['model_configured'] = $configuration->model;
 
         if ($response !== null) {
             $usage = $response->usage;

Classes/Middleware/SmartRoutingMiddleware.php

@@ -347,8 +347,8 @@ private function findCheaperModel(AiRequestInterface $request, ProviderConfigura
             return null;
         }
 
-        // Find a cheaper model with good success rate
-        $bestCandidate = null;
+        // Find cheaper models with good success rate
+        $candidates = [];
         foreach ($profiles as $profile) {
             if ($profile['model_used'] === $currentConfig->model) {
                 continue;
@@ -377,40 +377,41 @@ private function findCheaperModel(AiRequestInterface $request, ProviderConfigura
                 continue;
             }
 
-            // Cheaper + reliable — is there a config for this model?
-            if ($bestCandidate === null || $profile['avg_cost'] < $bestCandidate['avg_cost']) {
-                $bestCandidate = $profile;
-            }
+            $candidates[] = $profile;
         }
 
-        if ($bestCandidate === null) {
+        if ($candidates === []) {
             return null;
         }
+        usort($candidates, static fn(array $a, array $b): int => $a['avg_cost'] <=> $b['avg_cost']);
 
-        // Resolve the cheaper model's provider + config
+        // The cheapest candidate a configuration can take; history can name a
+        // model no configuration (any longer) does
         try {
             $capabilityFqcn = $this->resolveCapabilityForRequest($request);
             if ($capabilityFqcn === null) {
                 return null;
             }
             $allProviders = $this->providerResolver->resolveAllForCapability($capabilityFqcn);
-            foreach ($allProviders as $resolved) {
-                if ($resolved->configuration->model === $bestCandidate['model_used']
-                    && !$resolved->configuration->disabled
-                    && $resolved->configuration->acceptsReroutedRequests
-                    && $this->configurationAccess->isAccessibleByCurrentUser($resolved->configuration)
-                ) {
-                    return [
-                        'provider' => $resolved->manifest->getInstance(),
-                        'configuration' => $resolved->configuration,
-                        'avg_grade_score' => $bestCandidate['avg_grade_score'],
-                        'graded_count' => $bestCandidate['graded_count'],
-                        'avg_cost' => $bestCandidate['avg_cost'],
-                        'avg_duration_ms' => $bestCandidate['avg_duration_ms'],
-                        'avg_tokens' => $bestCandidate['avg_tokens'],
-                        'from_cost' => $currentCost,
-                        'from_duration_ms' => $currentDuration ?? 0,
-                    ];
+            foreach ($candidates as $candidate) {
+                foreach ($allProviders as $resolved) {
+                    if ($resolved->configuration->model === $candidate['model_used']
+                        && !$resolved->configuration->disabled
+                        && $resolved->configuration->acceptsReroutedRequests
+                        && $this->configurationAccess->isAccessibleByCurrentUser($resolved->configuration)
+                    ) {
+                        return [
+                            'provider' => $resolved->manifest->getInstance(),
+                            'configuration' => $resolved->configuration,
+                            'avg_grade_score' => $candidate['avg_grade_score'],
+                            'graded_count' => $candidate['graded_count'],
+                            'avg_cost' => $candidate['avg_cost'],
+                            'avg_duration_ms' => $candidate['avg_duration_ms'],
+                            'avg_tokens' => $candidate['avg_tokens'],
+                            'from_cost' => $currentCost,
+                            'from_duration_ms' => $currentDuration ?? 0,
+                        ];
+                    }
                 }
             }
         } catch (\Throwable) {

Documentation/Pipeline.md

@@ -15,6 +15,12 @@ The `SmartRoutingMiddleware` classifies prompt complexity using language-agnosti
 
 Classification is logged per request (`complexity_score`, `complexity_label`, `complexity_reason`). When a cheaper model has proven reliable for simple prompts (based on historical request log data with minimum 10 requests and 90%+ success rate), the middleware automatically downgrades.
 
+The history counts a request for the model of the configuration it ran on (`model_configured` in the request
+log), not for the name the provider reported, which is often a dated snapshot (`gpt-4o-mini-2024-07-18`) or
+the target of an alias (`claude-sonnet-4-20250514` for `claude-sonnet-4-0`). A downgraded request counts for
+the cheaper configuration it was routed to. Entries logged before `model_configured` existed count under the
+reported name. When no configuration takes the cheapest model in the history, the next cheaper one is tried.
+
 ### Quality gate
 
 "Reliable" on its own only means *the API call didn't error*. A cheap model can succeed every time while producing weak answers. When [LLM grading](#llm-grading) is enabled, smart routing also consults the recorded `grade_score`: a cheaper model is only chosen if its graded responses for that request type average at least **0.65** (the "good" boundary) across at least **10 graded requests**.

Documentation/Usage.md

@@ -376,7 +376,7 @@ Providers can declare per-model capabilities via `modelCapabilities`. Models lis
 When a provider config has `gpt-4o` but an embedding request comes in, AiM automatically switches to the cheapest capable model (e.g. `text-embedding-3-small`) using the same API key. The selection is data-driven: if historical cost data exists in the request log, AiM picks the cheapest model with a good success rate. Otherwise it falls back to the most specialized model.
 
 The switch is:
-- **Logged** with `model_requested`, `model_used`, and reroute reason
+- **Logged** with `model_requested` (the model switched away from), `model_configured` (the model switched to), `model_used` (the name the provider reported), and reroute reason
 - **Controllable** at three levels:
 
 | Level | Setting | Default |

Resources/Private/Partials/RequestLog/Row.html

@@ -46,7 +46,7 @@
     </td>
     <td>
         <span class="badge badge-secondary">{entry.model_used -> f:or(alternative: entry.model_requested)}</span>
-        <f:if condition="{entry.model_used} && {entry.model_requested} && {entry.model_used} != {entry.model_requested}">
+        <f:if condition="{entry.model_requested_differs}">
             <br /><small class="text-body-secondary">requested: {entry.model_requested}</small>
         </f:if>
     </td>

Resources/Private/Partials/RequestLogDetail.html

@@ -59,7 +59,7 @@
                         <th><f:translate key="LLL:EXT:aim/Resources/Private/Language/locallang_module.xlf:requestLog.column.model" /></th>
                         <td>
                             <span class="badge badge-secondary">{entry.model_used -> f:or(alternative: entry.model_requested)}</span>
-                            <f:if condition="{entry.model_used} && {entry.model_requested} && {entry.model_used} != {entry.model_requested}">
+                            <f:if condition="{entry.model_requested_differs}">
                                 <small class="text-body-secondary">requested: {entry.model_requested}</small>
                             </f:if>
                         </td>

Resources/Public/JavaScript/request-log-poll.js

@@ -255,7 +255,7 @@ class RequestLogPoll {
 
   #renderModel(e) {
     let html = `<span class="badge badge-secondary">${this.#esc(e.model_used)}</span>`;
-    if (e.model_used && e.model_requested && e.model_used !== e.model_requested) {
+    if (e.model_requested_differs) {
       html += `<br><small class="text-body-secondary">requested: ${this.#esc(e.model_requested)}</small>`;
     }
     return html;

Tests/Functional/Controller/RequestLogControllerPollTest.php

@@ -180,6 +180,31 @@ public function pollActionHonoursTheFilterSortingAndPageOfTheCurrentView(): void
         self::assertCount(0, $this->rowsOf($this->poll(['page' => '2'])), 'The page was ignored, so a poll drops the user back to page 1.');
     }
 
+    /**
+     * An alias is reported under a name that shares nothing with it but the
+     * family, so only the configured model tells a reroute apart.
+     */
+    #[Test]
+    public function pollActionFlagsAnAliasedModelOnlyWhenTheRequestWasRerouted(): void
+    {
+        $this->getConnectionPool()->getConnectionForTable('be_users')->insert('be_users', [
+            'uid' => 1,
+            'pid' => 0,
+            'username' => 'admin',
+            'admin' => 1,
+        ]);
+        $this->setUpBackendUser(1);
+        $GLOBALS['LANG'] = $this->get(LanguageServiceFactory::class)->createFromUserPreferences($GLOBALS['BE_USER']);
+
+        $logRepo = $this->get(RequestLogRepository::class);
+        $logRepo->log(['request_type' => 'TextGenerationRequest', 'provider_identifier' => 'test', 'model_requested' => 'claude-sonnet-4-0', 'model_configured' => 'claude-sonnet-4-0', 'model_used' => 'claude-sonnet-4-20250514', 'cost' => 0.1]);
+        $logRepo->log(['request_type' => 'TextGenerationRequest', 'provider_identifier' => 'test', 'model_requested' => 'claude-sonnet-4-0', 'model_configured' => 'claude-3-5-haiku-latest', 'model_used' => 'claude-3-5-haiku-20241022', 'reroute_type' => 'model_switch', 'cost' => 0.2]);
+
+        $rows = $this->rowsOf($this->poll(['orderField' => 'cost', 'orderDirection' => 'asc']));
+
+        self::assertSame([false, true], array_column($rows, 'model_requested_differs'));
+    }
+
     /**
      * @return array<int, array<string, mixed>>
      */