Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion llm_inference/model_inference.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import logging
from typing import Dict, Any
from openai import OpenAI
import re
import tiktoken

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -194,6 +195,7 @@ def _get_provider(self, model_name: str) -> str:
"qwen/qwen3-next-80b-a3b-instruct": "openrouter",
"Qwen/Qwen3-Coder-Next": "openrouter",
"deepseek/deepseek-v4-flash": "openrouter",
"deepseek/deepseek-v4.1-flash-1017-reasoning-low": "openrouter",
"x-ai/grok-4.1-fast": "openrouter",
"mistralai/devstral-2512:free": "openrouter",
"meta-llama/llama-3.3-70b-instruct": "openrouter",
Expand Down Expand Up @@ -321,8 +323,20 @@ def _call_openrouter(self, model_name: str, prompt: str) -> Dict[str, Any]:
api_key=openrouter_api_key,
)

# "<model>-reasoning-<effort>" selects the same model with an explicit
# OpenRouter reasoning effort (e.g. deepseek/deepseek-v4.1-flash-1017-reasoning-low).
api_model, extra_body = model_name, None
match = re.fullmatch(
r"(.+)-reasoning-(none|minimal|low|medium|high)", model_name
)
if match:
api_model = match.group(1)
extra_body = {"reasoning": {"effort": match.group(2)}}

response = client.chat.completions.create(
model=model_name, messages=[{"role": "user", "content": prompt}]
model=api_model,
messages=[{"role": "user", "content": prompt}],
extra_body=extra_body,
)

usage = getattr(response, "usage", None)
Expand Down
4 changes: 4 additions & 0 deletions model_cost/model_cost.json
Original file line number Diff line number Diff line change
Expand Up @@ -386,5 +386,9 @@
"google/gemma-4-31b-it": {
"input_token_price_per_million": 0.08,
"output_token_price_per_million": 0.35
},
"deepseek/deepseek-v4.1-flash-1017-reasoning-low": {
"input_token_price_per_million": 0.15,
"output_token_price_per_million": 0.6
}
}
18 changes: 18 additions & 0 deletions router_inference/config/compass-corerouter.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"pipeline_params": {
"router_name": "compass-corerouter",
"router_cls_name": "CompassCoreRouter",
"models": [
"qwen/qwen3-235b-a22b-2507",
"google/gemma-4-31b-it",
"gemini-3-flash-preview",
"deepseek/deepseek-v4-flash",
"qwen/qwen3-next-80b-a3b-instruct",
"deepseek/deepseek-v4.1-flash-1017-reasoning-low"
],
"router_version": "corerouter-v2-hybrid-agreement-critic-veto-reasoning-tier-prompt-only",
"routing_input": "prompt text only: a TF-IDF + logistic-regression task classifier trained on our own practice prompts predicts the task type",
"policy": "per-task model pick fitted on our own practice data (5-fold CV, lambda 0.02); where the pick is gemini-3-flash-preview, Qwen3-235B and Gemma-4-31B answer first; Gemini is called when their final answers disagree, or when they agree but a self-hosted answer critic (fine-tuned on our own practice data; it reads only the prompt and our models' answers) scores the shared answer below 0.5; for the MMLU-Pro task type the escalation model is DeepSeek V4.1 Flash 1017 with reasoning effort low instead of Gemini, and the AIME and MATH task types go to it directly (choices fitted by the same cross-validation on our own practice data)",
"cost_accounting": "every call is charged on the returned row: other calls' dollar cost is added as returned-model input-price tokens; see generated_result.cascade_accounting"
}
}
Loading
Loading