diff --git a/CHANGELOG.md b/CHANGELOG.md index 1861515..63c7b0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,29 @@ All notable changes to blockrun-llm will be documented in this file. ## Unreleased +### Fixed +- **`router_core` re-synced to upstream `@blockrun/router-core` `273c2af`** + (was `8773a99`), which includes the 2026-11 router audit. All 88 + cross-language snapshot decisions match the TypeScript engine. + - Chinese debug, refactor, extraction and proof prompts are no longer + classified as chat. Japanese prompts no longer get the Mandarin + extraction band. + - With tools attached, a bare language name ("java coffee prices online") + is no longer routed to the code-agent pool. + - Tool-name verbs, airline reservation ids and the multi-step signal no + longer fire on look-alike text ("add" in "address", "CANCEL", a sentence + ending in a number). Full-width `?` now counts as a question. + - An unpriced candidate no longer outranks priced ones on cost. NaN prices + and malformed `model_performance` entries no longer leak NaN into costs + or the ranking. `calculate_model_cost` reports no savings for an + unpriced model. + - A one-character or non-numeric `prompt_truncation_chars` no longer makes + the router scan the whole prompt. +- Capability snapshot synced to the 2026-11-08 catalog. `moonshot/kimi-k3`, + which is still routed and served, is retained. DeepSeek chat, reasoner and + V4.1 Flash stay text-only until an image probe confirms the catalog's new + vision tag. + ## 1.21.5 — 2026-11-07 ### Fixed diff --git a/blockrun_llm/router_core/__init__.py b/blockrun_llm/router_core/__init__.py index 4e1e20e..24e853d 100644 --- a/blockrun_llm/router_core/__init__.py +++ b/blockrun_llm/router_core/__init__.py @@ -2,7 +2,7 @@ Router Core — deterministic, constraint-first model routing. Python port of `@blockrun/router-core `_ -(upstream commit ``8773a99``), the same routing engine the TypeScript SDK and +(upstream commit ``273c2af``), the same routing engine the TypeScript SDK and the BlockRun gateway use. The package is deliberately product-neutral: task classification, hard capability filtering, portfolio scoring, ordered fallbacks, and routing configuration. It contains no wallet, gateway client, diff --git a/blockrun_llm/router_core/config.py b/blockrun_llm/router_core/config.py index 0ed0be4..2161b9e 100644 --- a/blockrun_llm/router_core/config.py +++ b/blockrun_llm/router_core/config.py @@ -2,7 +2,7 @@ Default Routing Config Python port of ``@blockrun/router-core`` ``config.ts`` (upstream commit -``8773a99``; the file is unchanged since ``e38958b``, 2026-09-29, so it is the +``273c2af``; the file is unchanged since ``e38958b``, 2026-09-29, so it is the same table the TypeScript SDK bundles). V3.5: every tier rung is a public catalog id. diff --git a/blockrun_llm/router_core/model_capabilities.py b/blockrun_llm/router_core/model_capabilities.py index aac1d6f..1322107 100644 --- a/blockrun_llm/router_core/model_capabilities.py +++ b/blockrun_llm/router_core/model_capabilities.py @@ -8,7 +8,7 @@ product catalog is temporarily unavailable, without importing product code. GENERATED upstream by ``scripts/sync-model-capabilities.mjs`` from the public -catalog (GET https://blockrun.ai/api/v1/models) on 2026-10-20; ``supports_tools`` +catalog (GET https://blockrun.ai/api/v1/models) on 2026-11-08; ``supports_tools`` comes from a live function-calling probe. Re-sync from ``model-capabilities.ts`` rather than editing by hand — a hand edit is lost on the next sync. """ @@ -35,9 +35,9 @@ "supports_vision": True, }, # override: The public catalog's `categories` omit "vision" for this Anthropic model even - # though the gateway accepts image input for it (the prior hand-maintained snapshot had - # it, and Anthropic's model card lists it). Without this the vision filter would silently - # drop it — reported against the catalog; remove once the categories carry vision. + # though the gateway accepts image input for it (the prior hand-maintained snapshot had it, + # and Anthropic's model card lists it). Without this the vision filter would silently drop it + # — reported against the catalog; remove once the categories carry vision. "anthropic/claude-haiku-4.5": { "context_window": 200_000, "max_output_tokens": 64_000, @@ -81,9 +81,9 @@ "supports_vision": True, }, # override: The public catalog's `categories` omit "vision" for this Anthropic model even - # though the gateway accepts image input for it (the prior hand-maintained snapshot had - # it, and Anthropic's model card lists it). Without this the vision filter would silently - # drop it — reported against the catalog; remove once the categories carry vision. + # though the gateway accepts image input for it (the prior hand-maintained snapshot had it, + # and Anthropic's model card lists it). Without this the vision filter would silently drop it + # — reported against the catalog; remove once the categories carry vision. "anthropic/claude-sonnet-4.6": { "context_window": 1_000_000, "max_output_tokens": 128_000, @@ -109,12 +109,24 @@ "supports_tools": False, "supports_vision": False, }, + # override: The 2026-11-08 catalog began tagging deepseek/deepseek-chat "vision" (it was text- + # only through 2026-10-20), and DeepSeek still ships image input as a separate experimental id + # (deepseek/deepseek-v4-flash-vision-exp). No image probe has confirmed the tag. Routing an + # image turn to a model that silently drops the image returns a confident 200 the caller + # cannot branch on, so this fails closed until a correctly sized image probe passes on both + # chains; then remove this entry. "deepseek/deepseek-chat": { "context_window": 1_048_576, "max_output_tokens": 65_536, "supports_tools": True, "supports_vision": False, }, + # override: The 2026-11-08 catalog began tagging deepseek/deepseek-reasoner "vision" (it was + # text-only through 2026-10-20), and DeepSeek still ships image input as a separate + # experimental id (deepseek/deepseek-v4-flash-vision-exp). No image probe has confirmed the + # tag. Routing an image turn to a model that silently drops the image returns a confident 200 + # the caller cannot branch on, so this fails closed until a correctly sized image probe passes + # on both chains; then remove this entry. "deepseek/deepseek-reasoner": { "context_window": 1_048_576, "max_output_tokens": 65_536, @@ -133,7 +145,12 @@ "supports_tools": True, "supports_vision": False, }, - # supportsTools: not probed — fails closed + # override: The 2026-11-08 catalog began tagging deepseek/deepseek-v4.1-flash "vision" (it was + # text-only through 2026-10-20), and DeepSeek still ships image input as a separate + # experimental id (deepseek/deepseek-v4-flash-vision-exp). No image probe has confirmed the + # tag. Routing an image turn to a model that silently drops the image returns a confident 200 + # the caller cannot branch on, so this fails closed until a correctly sized image probe passes + # on both chains; then remove this entry. "deepseek/deepseek-v4.1-flash": { "context_window": 1_048_576, "max_output_tokens": 65_536, @@ -200,6 +217,7 @@ "supports_tools": True, "supports_vision": True, }, + # supportsTools: not probed — fails closed "google/gemini-4-argon": { "context_window": 1_048_576, "max_output_tokens": 65_536, @@ -225,6 +243,8 @@ "supports_tools": False, "supports_vision": True, }, + # absent from the 2026-11-08 listing but still routed — previous entry retained; probe before + # removing "moonshot/kimi-k3": { "context_window": 1_048_576, "max_output_tokens": 65_536, @@ -232,26 +252,32 @@ "supports_vision": True, }, # supportsTools: not probed — fails closed - "nvidia/llama-3.2-11b-vision": { - "context_window": 128_000, - "max_output_tokens": 16_384, + "moonshot/kimi-k3.1-pro": { + "context_window": 1_048_576, + "max_output_tokens": 65_536, "supports_tools": False, "supports_vision": True, }, # supportsTools: not probed — fails closed + "nvidia/minimax-m3.1-flash": { + "context_window": 1_048_576, + "max_output_tokens": 16_384, + "supports_tools": False, + "supports_vision": False, + }, + # supportsTools: not probed — fails closed "nvidia/muse-glimmer-30b": { "context_window": 131_072, "max_output_tokens": 16_384, "supports_tools": False, "supports_vision": False, }, - # override: The catalog tags this model "vision", but a correctly sized probe does not - # hold up (ClawRouter, 2026-08-31): a 64x64 solid-red PNG was named correctly 1 of 4 - # times on Base, and on Solana the image was silently dropped and a text model answered - # "white". An HTTP 200 with a confident wrong answer gives the caller nothing to branch - # on, so image turns must not be routed here. It is ecoTiers.SIMPLE.fallback[0] since - # nemotron-3-nano-30b was delisted (2026-09-08); remove once a probe of this size comes - # back right on both chains. + # override: The catalog tags this model "vision", but a correctly sized probe does not hold up + # (ClawRouter, 2026-08-31): a 64x64 solid-red PNG was named correctly 1 of 4 times on Base, + # and on Solana the image was silently dropped and a text model answered "white". An HTTP 200 + # with a confident wrong answer gives the caller nothing to branch on, so image turns must not + # be routed here. It is ecoTiers.SIMPLE.fallback[0] since nemotron-3-nano-30b was delisted + # (2026-09-08); remove once a probe of this size comes back right on both chains. "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning": { "context_window": 256_000, "max_output_tokens": 16_384, @@ -259,6 +285,13 @@ "supports_vision": False, }, # supportsTools: not probed — fails closed + "nvidia/nemotron-3-super-120b": { + "context_window": 131_072, + "max_output_tokens": 16_384, + "supports_tools": False, + "supports_vision": True, + }, + # supportsTools: not probed — fails closed "nvidia/nemotron-3-ultra-550b": { "context_window": 1_000_000, "max_output_tokens": 16_384, @@ -266,6 +299,13 @@ "supports_vision": False, }, # supportsTools: not probed — fails closed + "nvidia/nemotron-3.5-bolt": { + "context_window": 131_072, + "max_output_tokens": 16_384, + "supports_tools": False, + "supports_vision": True, + }, + # supportsTools: not probed — fails closed "nvidia/nemotron-3.5-lightning": { "context_window": 1_000_000, "max_output_tokens": 16_384, @@ -290,12 +330,6 @@ "supports_tools": True, "supports_vision": False, }, - "openai/gpt-4.1-nano": { - "context_window": 128_000, - "max_output_tokens": 32_768, - "supports_tools": True, - "supports_vision": False, - }, "openai/gpt-4o": { "context_window": 128_000, "max_output_tokens": 16_384, @@ -333,12 +367,12 @@ "supports_tools": False, "supports_vision": True, }, - # supportsTools: gateway unavailable at probe time — fails closed; override: 2026-08-29 - # probe: every request (6 plain + 3 tool attempts) returned a gateway 500, so the probe - # measured an incident, not the model. Codex's function calling is established by the - # 2026-07 Terminal-Bench / tau2 calibration trajectories in portfolio.ts. Hosts observing - # the 500s should drop it with RouterOptions.unavailableModels rather than this snapshot - # claiming the model cannot call tools. + # supportsTools: gateway unavailable at probe time — fails closed; override: 2026-08-29 probe: + # every request (6 plain + 3 tool attempts) returned a gateway 500, so the probe measured an + # incident, not the model. Codex's function calling is established by the 2026-07 Terminal- + # Bench / tau2 calibration trajectories in portfolio.ts. Hosts observing the 500s should drop + # it with RouterOptions.unavailableModels rather than this snapshot claiming the model cannot + # call tools. "openai/gpt-5.3-codex": { "context_window": 400_000, "max_output_tokens": 128_000, @@ -437,7 +471,6 @@ "supports_tools": True, "supports_vision": True, }, - # supportsTools: not probed — fails closed "openai/gpt-6.1-sol": { "context_window": 1_050_000, "max_output_tokens": 128_000, diff --git a/blockrun_llm/router_core/portfolio.py b/blockrun_llm/router_core/portfolio.py index 6cead63..cbfa88d 100644 --- a/blockrun_llm/router_core/portfolio.py +++ b/blockrun_llm/router_core/portfolio.py @@ -18,7 +18,7 @@ from ._js import as_utc, js_bool, js_regex, parse_date from .model_capabilities import DEFAULT_MODEL_CAPABILITIES from .model_profiles import HISTORICAL_MODEL_PROFILES, LIVE_MODEL_PROFILES -from .selector import get_fallback_chain, select_model +from .selector import get_fallback_chain, select_model, usable_pricing from .strategy import RulesStrategy from .tokens import estimate_tokens, sample_for_scan, scan_limit from .tool_intent import infer_tool_requirement @@ -118,7 +118,10 @@ class TaskFeatures: r"|天气|气象|温度|降雪|新闻|报告", ignorecase=True, ) -_MULTI_LOCATION_CONNECTOR = js_regex(r"\b(?:and also|both|y|e)\b|还有|以及|和|、", ignorecase=True) +# Spanish/Italian/Portuguese "y"/"e" only between words, so "e.g." does not count. +_MULTI_LOCATION_CONNECTOR = js_regex( + r"\b(?:and also|both)\b|\s(?:y|e)\s|还有|以及|和|、", ignorecase=True +) _COMMA = js_regex(r"[,,]") _ASCII_COMMA = js_regex(r",") _DISTINCT_ORDER_PARTS = js_regex( @@ -204,7 +207,10 @@ class TaskFeatures: r"|另一(?:个)?(?:订单|预订|账户|地址)|其他(?:订单|预订|账户|地址)", ignorecase=True, ) -_RESERVATION_ID = js_regex(r"\b[A-Z0-9]{6}\b") +# Reservation locators mix letters and digits (ZFA04Y, 4WQ150). Requiring both +# keeps shouted words ("CANCEL MY FLIGHT") and plain six-digit numbers from +# reading as two reservations. +_RESERVATION_ID = js_regex(r"\b(?=[A-Z0-9]*\d)(?=[A-Z0-9]*[A-Z])[A-Z0-9]{6}\b") _CROSS_RESERVATION_BATCH = js_regex( r"\b(?:two|three|multiple|several)(?:\s+of\s+(?:my|our|the))?\s+(?:upcoming\s+)?" r"(?:reservations?|bookings?)\b" @@ -297,6 +303,16 @@ class TaskFeatures: ignorecase=True, ) _HAN = js_regex(r"[\u3400-\u9fff]") +# Han characters alone also spell Japanese; kana marks the prompt as not Mandarin. +_KANA = js_regex(r"[\u3040-\u30ff]") +# A language name alone ("java coffee prices", "python snake habitat") is not +# code work, even on a tool turn; it joins the code-agent pool only together +# with a coding action. +_CODING_ACTION = js_regex( + r"\b(?:write|implement|refactor|debug|fix|patch|compile|build|run|test|script|program|code" + r"|function|class|module|repo(?:sitory)?)\b|编写|实现|重构|调试|修复|编译|脚本|代码", + ignorecase=True, +) _MULTIPLE_CHOICE = js_regex(r"(?:^|\n)\s*[A-D][.)]\s+", ignorecase=True, multiline=True) _NUMERIC = js_regex(r"-?\d+(?:[.,]\d+)?") _MATH_MARKERS = js_regex( @@ -304,15 +320,17 @@ class TaskFeatures: ignorecase=True, ) _TRAILING_QUESTION = js_regex(r"[??]\s*\Z") +# The Chinese alternatives sit outside \b: an ASCII word boundary never fires +# between two Han characters, so inside it they could not match at all. _DEBUG_TASK = js_regex( - r"\b(?:bug|debug|error|failure|failing|regression|crash|修复|报错|错误|调试)\b", ignorecase=True + r"\b(?:bug|debug|error|failure|failing|regression|crash)\b|修复|报错|错误|调试", ignorecase=True ) _CODE_EDIT_TASK = js_regex( - r"\b(?:refactor|implement|patch|edit|rewrite|重构|实现|修改)\b", ignorecase=True + r"\b(?:refactor|implement|patch|edit|rewrite)\b|重构|实现|修改", ignorecase=True ) -_EXTRACTION_TASK = js_regex(r"\b(?:extract|json|schema|csv|字段|提取)\b", ignorecase=True) +_EXTRACTION_TASK = js_regex(r"\b(?:extract|json|schema|csv)\b|字段|提取", ignorecase=True) _REASONING_TASK = js_regex( - r"\b(?:prove|derive|theorem|formal|mathematical|reasoning|证明|推导|定理|数学)\b", + r"\b(?:prove|derive|theorem|formal|mathematical|reasoning)\b|证明|推导|定理|数学", ignorecase=True, ) @@ -350,7 +368,11 @@ def _likely_needs_parallel_tool_calls( token for name in (tool_names or []) for token in _TOOL_NAME_SPLIT.split(name.lower()) - if token in _OPERATION_TOKENS and token in lowered + # Whole words only, with an optional inflection: a substring test read + # "add" out of "address", "book" out of "facebook" and "close" out of + # "closet" (the same failure rules.py fixed for its keyword lists). + if token in _OPERATION_TOKENS + and js_regex(rf"\b{token}(?:s|es|d|ed|ing)?\b").search(lowered) } # A single workflow naturally mentions domain nouns like order/item plus one # action. Upgrade only when two different visible operation verbs are @@ -594,7 +616,7 @@ def classify_task(prompt: str, system_prompt: str | None, options: RouterOptions and len(prompt) <= 1_000 and simple_terminal_artifact ) - language = "zh" if _HAN.search(scanned_prompt) else "other" + language = "zh" if _HAN.search(scanned_prompt) and not _KANA.search(scanned_prompt) else "other" multiple_choice_signals = len(_MULTIPLE_CHOICE.findall(scanned_prompt)) numeric_signals = len(_NUMERIC.findall(scanned_prompt)) compact_math_problem = ( @@ -613,7 +635,11 @@ def classify_task(prompt: str, system_prompt: str | None, options: RouterOptions task_type = "vision" elif estimated_input_tokens > 80_000: task_type = "long_context" - elif needs_tools and (has_code or implicit_terminal_code): + elif needs_tools and ( + code_artifact + or (has_code and _CODING_ACTION.search(scanned_prompt)) + or implicit_terminal_code + ): task_type = "code_agent" elif needs_tools and likely_parallel_tool_calls and not complex_multi_tool_plan: task_type = "tool_agent_parallel" @@ -1141,7 +1167,7 @@ def is_eligible( def estimated_cost( model_id: str, options: RouterOptions, input_tokens: int, output_tokens: int ) -> float: - price = options["model_pricing"].get(model_id) + price = usable_pricing(options["model_pricing"].get(model_id)) if not price: return math.inf flat = price.get("flat_price") @@ -1161,6 +1187,17 @@ class _ProfileScore: freshness: float +def _valid_metric(value: object, *, optional: bool) -> bool: + if value is None: + return optional + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + and value >= 0 + ) + + def profile_score(model_id: str, options: RouterOptions, now: datetime) -> _ProfileScore | None: """Weak speed/reliability priors, decayed by age and sample count.""" host_performance = options.get("model_performance") or {} @@ -1174,6 +1211,18 @@ def profile_score(model_id: str, options: RouterOptions, now: datetime) -> _Prof measured_at = parse_date(profile.get("measured_at", "")) if measured_at is None: return None + # A host-injected observation with a missing or non-numeric field would turn + # the whole score into NaN, and NaN compares as "equal" in the sort, freezing + # that candidate wherever it landed. Treat it as no observation. + if not ( + _valid_metric(profile.get("latency_ms"), optional=False) + and _valid_metric(profile.get("output_tokens_per_second"), optional=False) + and _valid_metric(profile.get("p95_latency_ms"), optional=True) + and _valid_metric(profile.get("intelligence_index"), optional=True) + and _valid_metric(profile.get("error_rate"), optional=True) + and _valid_metric(profile.get("samples"), optional=True) + ): + return None age_days = max(0.0, (now - measured_at).total_seconds() / 86_400) # A 30-day half-life makes old data a tie-breaker only. Small probe runs are # also weak evidence: three quick samples should not overturn a curated tier @@ -1370,11 +1419,15 @@ def route( cost = estimated_cost( model, options, features.estimated_input_tokens, max_output_tokens ) - cost_score = ( - 1 - (cost - min_cost) / (max_cost - min_cost) - if math.isfinite(cost) and max_cost > min_cost - else 0.5 - ) + # An unpriced candidate costs an unknown amount, so it scores as the + # most expensive rather than a neutral 0.5 that let it outrank priced + # models. Pricing presence still never filters a candidate. + if not math.isfinite(cost): + cost_score = 0.0 + elif max_cost > min_cost: + cost_score = 1 - (cost - min_cost) / (max_cost - min_cost) + else: + cost_score = 0.5 capability_score = 1.0 if model in eligible_set else 0.0 profile = profile_score(model, options, now) # Fresh observations can refine affinity. Historical observations diff --git a/blockrun_llm/router_core/rules.py b/blockrun_llm/router_core/rules.py index 29774d8..4d931c0 100644 --- a/blockrun_llm/router_core/rules.py +++ b/blockrun_llm/router_core/rules.py @@ -23,9 +23,11 @@ _MULTI_STEP_PATTERNS = [ js_regex(r"first.*then", ignorecase=True), js_regex(r"step \d", ignorecase=True), - js_regex(r"\d\.\s"), + # A numbered list item must start a line: a bare ``\d\.\s`` also matched + # every sentence that ends in a number ("born in 1990. How old…"). + js_regex(r"(?:^|\n)\s*\d+[.)]\s"), ] -_QUESTION_MARK = js_regex(r"\?") +_QUESTION_MARK = js_regex(r"[??]") # ─── Keyword Matching ─── diff --git a/blockrun_llm/router_core/selector.py b/blockrun_llm/router_core/selector.py index d63364d..6ccaec1 100644 --- a/blockrun_llm/router_core/selector.py +++ b/blockrun_llm/router_core/selector.py @@ -9,6 +9,7 @@ from __future__ import annotations +import math from collections.abc import Callable, Iterable, Mapping from .types import Capacity, Method, ModelPricing, RoutingDecision, Tier, TierConfig @@ -45,13 +46,40 @@ def _flat_price(pricing: ModelPricing | None) -> float | None: return float(flat) if flat else None +def _usable_price(value: object) -> bool: + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + and value >= 0 + ) + + +def usable_pricing(pricing: ModelPricing | None) -> ModelPricing | None: + """The pricing entry, or ``None`` when a price it carries is not a finite, non-negative number. + + A hand-built price map with a NaN would otherwise put NaN into + ``cost_estimate`` and ``savings``; a malformed price is an unknown price. + """ + if not pricing: + return None + flat = pricing.get("flat_price") + if flat: + return pricing if _usable_price(flat) else None + for key in ("input_price", "output_price"): + value = pricing.get(key) + if value is not None and not _usable_price(value): + return None + return pricing + + def _baseline_cost( model_pricing: Mapping[str, ModelPricing], estimated_input_tokens: int, max_output_tokens: int, ) -> float: """What the premium reference model would cost for the same request.""" - opus_pricing = model_pricing.get(BASELINE_MODEL_ID) + opus_pricing = usable_pricing(model_pricing.get(BASELINE_MODEL_ID)) opus_input_price = (opus_pricing or {}).get("input_price", BASELINE_INPUT_PRICE) opus_output_price = (opus_pricing or {}).get("output_price", BASELINE_OUTPUT_PRICE) baseline_input = (estimated_input_tokens / 1_000_000) * opus_input_price @@ -83,7 +111,7 @@ def select_model( """Select the primary model for a tier and build the RoutingDecision.""" tier_config = tier_configs[tier] model = tier_config["primary"] - pricing = model_pricing.get(model) + pricing = usable_pricing(model_pricing.get(model)) flat = _flat_price(pricing) if flat is not None: @@ -134,7 +162,7 @@ def calculate_model_cost( Includes the server margin and the facilitator minimum so the estimate matches the actual x402 charge. """ - pricing = model_pricing.get(model) + pricing = usable_pricing(model_pricing.get(model)) flat = _flat_price(pricing) if flat is not None: @@ -154,7 +182,10 @@ def calculate_model_cost( return { "cost_estimate": cost_estimate, "baseline_cost": baseline_cost, - "savings": _savings(cost_estimate, baseline_cost, routing_profile), + # Same rule as select_model: an unpriced model has unknown, not 100%, savings. + "savings": ( + 0.0 if pricing is None else _savings(cost_estimate, baseline_cost, routing_profile) + ), } diff --git a/blockrun_llm/router_core/tokens.py b/blockrun_llm/router_core/tokens.py index e2aadc8..2ee5e1c 100644 --- a/blockrun_llm/router_core/tokens.py +++ b/blockrun_llm/router_core/tokens.py @@ -33,9 +33,13 @@ def estimate_tokens(text: str) -> int: return cjk + math.ceil((len(text) - cjk) / 4) -def scan_limit(prompt_truncation_chars: int) -> int: +def scan_limit(prompt_truncation_chars: float) -> int: """The scan window the feature regexes read, bounded so a pasted document cannot stall routing.""" - return max(1, min(8_000, prompt_truncation_chars)) + if not isinstance(prompt_truncation_chars, (int, float)) or not math.isfinite( + prompt_truncation_chars + ): + return 8_000 + return max(1, min(8_000, math.floor(prompt_truncation_chars))) def sample_for_scan(value: str, limit: int) -> str: @@ -49,6 +53,8 @@ def sample_for_scan(value: str, limit: int) -> str: return value prefix_length = math.ceil(limit / 2) suffix_length = limit - prefix_length - # JS ``slice(-0)`` is the whole string, not an empty one. - suffix = value[-suffix_length:] if suffix_length else value - return f"{value[:prefix_length]}\n{suffix}" + # A one-character limit leaves no suffix; never fall back to the whole text + # (upstream's ``slice(-0)`` used to, which defeated the scan bound). + if suffix_length <= 0: + return value[:prefix_length] + return f"{value[:prefix_length]}\n{value[-suffix_length:]}" diff --git a/tests/unit/router_core_decisions.snapshot.json b/tests/unit/router_core_decisions.snapshot.json index 8a9758e..9afeec4 100644 --- a/tests/unit/router_core_decisions.snapshot.json +++ b/tests/unit/router_core_decisions.snapshot.json @@ -499,31 +499,27 @@ { "prompt": 12, "profile": "auto", - "model": "google/gemini-3.5-flash", - "tier": "MEDIUM", - "confidence": 0.5, + "model": "google/gemini-2.5-flash", + "tier": "SIMPLE", + "confidence": 0.708477270409738, "method": "portfolio", - "reasoning": "score=-0.01 | short (32 tokens), multi-step, references (following) | ambiguous -> default: MEDIUM | v3 task=vision agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=7", - "costEstimate": 0.0018304000000000003, + "reasoning": "score=-0.07 | short (32 tokens), references (following) | v3 task=vision agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=3", + "costEstimate": 0.00019519999999999997, "baselineCost": 0.00656, - "savings": 0.7209756097560975, + "savings": 0.9702439024390245, "agenticScore": 0, "candidates": [ - "google/gemini-3.5-flash", - "google/gemini-3.6-flash", - "zai/glm-5.3-flash", - "openai/gpt-5.6-terra", - "google/gemini-3-flash-preview", "google/gemini-2.5-flash", + "google/gemini-3-flash-preview", "openai/gpt-5.6-luna" ], "candidateScores": [ { - "model": "google/gemini-3.5-flash", - "score": 0.7275477432793295, + "model": "google/gemini-2.5-flash", + "score": 0.7269260539502254, "quality": 0.68, "cost": 0.5, - "speed": 0.5421106182761342, + "speed": 0.5332293421460749, "reliability": 1 } ], @@ -535,9 +531,9 @@ "profile": "auto", "model": "xai/grok-4.5", "tier": "REASONING", - "confidence": 0.5, + "confidence": 0.7231218051243898, "method": "portfolio", - "reasoning": "score=-0.02 | short (13 tokens), multi-step | ambiguous -> default: MEDIUM | v3 task=reasoning_math agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=10", + "reasoning": "score=-0.08 | short (13 tokens) | upgraded to MEDIUM (structured output) | v3 task=reasoning_math agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=10", "costEstimate": 0.0005406000000000001, "baselineCost": 0.032065, "savings": 0.9831404958677685, @@ -1611,18 +1607,17 @@ "prompt": 12, "profile": "eco", "model": "zai/glm-5.3-flash", - "tier": "MEDIUM", - "confidence": 0.5, + "tier": "SIMPLE", + "confidence": 0.708477270409738, "method": "portfolio", - "reasoning": "score=-0.01 | short (32 tokens), multi-step, references (following) | ambiguous -> default: MEDIUM | eco | v3 task=vision agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=3", + "reasoning": "score=-0.07 | short (32 tokens), references (following) | eco | v3 task=vision agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=2", "costEstimate": 0.0022784000000000003, "baselineCost": 0.00656, "savings": 0.6526829268292683, "agenticScore": 0, "candidates": [ "zai/glm-5.3-flash", - "openai/gpt-5.6-luna", - "google/gemini-2.5-flash" + "openai/gpt-5.6-luna" ], "candidateScores": [ { @@ -1642,9 +1637,9 @@ "profile": "eco", "model": "xai/grok-4.5", "tier": "REASONING", - "confidence": 0.5, + "confidence": 0.7231218051243898, "method": "portfolio", - "reasoning": "score=-0.02 | short (13 tokens), multi-step | ambiguous -> default: MEDIUM | eco | v3 task=reasoning_math agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=9", + "reasoning": "score=-0.08 | short (13 tokens) | upgraded to MEDIUM (structured output) | eco | v3 task=reasoning_math agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=9", "costEstimate": 0.0005406000000000001, "baselineCost": 0.032065, "savings": 0.9831404958677685, @@ -2685,31 +2680,27 @@ { "prompt": 12, "profile": "auto", - "model": "google/gemini-3.5-flash", - "tier": "MEDIUM", - "confidence": 0.5, + "model": "google/gemini-2.5-flash", + "tier": "SIMPLE", + "confidence": 0.708477270409738, "method": "portfolio", - "reasoning": "score=-0.01 | short (32 tokens), multi-step, references (following) | ambiguous -> default: MEDIUM | v3 task=vision agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=7", - "costEstimate": 0.0018304000000000003, + "reasoning": "score=-0.07 | short (32 tokens), references (following) | v3 task=vision agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=3", + "costEstimate": 0.00019519999999999997, "baselineCost": 0.00656, - "savings": 0.7209756097560975, + "savings": 0.9702439024390245, "agenticScore": 0, "candidates": [ - "google/gemini-3.5-flash", - "google/gemini-3.6-flash", - "zai/glm-5.3-flash", - "openai/gpt-5.6-terra", - "google/gemini-3-flash-preview", "google/gemini-2.5-flash", + "google/gemini-3-flash-preview", "openai/gpt-5.6-luna" ], "candidateScores": [ { - "model": "google/gemini-3.5-flash", - "score": 0.7275477432793295, + "model": "google/gemini-2.5-flash", + "score": 0.7269260539502254, "quality": 0.68, "cost": 0.5, - "speed": 0.5421106182761342, + "speed": 0.5332293421460749, "reliability": 1 } ], @@ -2721,9 +2712,9 @@ "profile": "auto", "model": "xai/grok-4.5", "tier": "REASONING", - "confidence": 0.5, + "confidence": 0.7231218051243898, "method": "portfolio", - "reasoning": "score=-0.02 | short (13 tokens), multi-step | ambiguous -> default: MEDIUM | v3 task=reasoning_math agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=10", + "reasoning": "score=-0.08 | short (13 tokens) | upgraded to MEDIUM (structured output) | v3 task=reasoning_math agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=10", "costEstimate": 0.0005406000000000001, "baselineCost": 0.032065, "savings": 0.9831404958677685, @@ -3722,40 +3713,28 @@ { "prompt": 12, "profile": "premium", - "model": "anthropic/claude-sonnet-4.6", - "tier": "MEDIUM", - "confidence": 0.5, + "model": "google/gemini-3.5-flash", + "tier": "SIMPLE", + "confidence": 0.708477270409738, "method": "portfolio", - "reasoning": "score=-0.01 | short (32 tokens), multi-step, references (following) | ambiguous -> default: MEDIUM | premium | v3 task=vision agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=8", - "costEstimate": 0.0017856000000000003, + "reasoning": "score=-0.07 | short (32 tokens), references (following) | premium | v3 task=vision agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=4", + "costEstimate": 0.0018304000000000003, "baselineCost": 0.00656, "savings": 0, "agenticScore": 0, "candidates": [ - "anthropic/claude-sonnet-4.6", - "moonshot/kimi-k3", - "anthropic/claude-sonnet-5", - "google/gemini-3.6-flash", "google/gemini-3.5-flash", - "google/gemini-2.5-pro", - "xai/grok-4.5", - "openai/gpt-5.6-terra" + "google/gemini-3.6-flash", + "anthropic/claude-haiku-4.5", + "google/gemini-2.5-flash" ], "candidateScores": [ { - "model": "anthropic/claude-sonnet-4.6", - "score": 0.8885954266748617, - "quality": 0.9, - "cost": 1, - "speed": 0.4432571112476951, - "reliability": 1 - }, - { - "model": "moonshot/kimi-k3", - "score": 0.8246386868652877, - "quality": 0.9, - "cost": 0, - "speed": 0.37731144775479714, + "model": "google/gemini-3.5-flash", + "score": 0.7469266370965681, + "quality": 0.68, + "cost": 0.5, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -3767,9 +3746,9 @@ "profile": "premium", "model": "google/gemini-3.5-flash", "tier": "REASONING", - "confidence": 0.5, + "confidence": 0.7231218051243898, "method": "portfolio", - "reasoning": "score=-0.02 | short (13 tokens), multi-step | ambiguous -> default: MEDIUM | premium | v3 task=reasoning_math agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=12", + "reasoning": "score=-0.08 | short (13 tokens) | upgraded to MEDIUM (structured output) | premium | v3 task=reasoning_math agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=12", "costEstimate": 0.008622800000000002, "baselineCost": 0.032065, "savings": 0, diff --git a/tests/unit/test_router_core_audit.py b/tests/unit/test_router_core_audit.py new file mode 100644 index 0000000..e6ed4ab --- /dev/null +++ b/tests/unit/test_router_core_audit.py @@ -0,0 +1,306 @@ +""" +Regression tests for the Router Core re-sync to upstream ``273c2af``. + +1:1 ports of upstream ``hardening.test.ts`` (the 2026-11 router-core audit: +inputs that produced a wrong classification, a NaN ranking, an unbounded scan +or a fabricated figure) plus the 2026-11-08 capability snapshot checks. The +upstream cases added before ``8773a99`` live in ``test_router_core.py``; +``test_router_core_snapshot.py`` holds the full-decision parity. +""" + +from __future__ import annotations + +import math +from datetime import datetime, timezone + +import pytest + +from blockrun_llm.router_core import ( + DEFAULT_MODEL_CAPABILITIES, + DEFAULT_ROUTING_CONFIG, + RulesStrategy, + calculate_model_cost, + classify_by_rules, + route, +) +from blockrun_llm.router_core.selector import select_model +from blockrun_llm.router_core.tokens import sample_for_scan, scan_limit + +SCORING = DEFAULT_ROUTING_CONFIG["scoring"] + + +def _price(input_price: float, output_price: float) -> dict[str, float]: + return {"input_price": input_price, "output_price": output_price} + + +# ─── model-capabilities: 2026-11-08 sync ─── + + +def test_snapshot_retains_a_routed_model_absent_from_the_listing(): + # kimi-k3 left the 2026-11-08 listing but is still served and still routed. + assert DEFAULT_MODEL_CAPABILITIES["moonshot/kimi-k3"]["supports_tools"] is True + + +@pytest.mark.parametrize( + "model", + ["deepseek/deepseek-chat", "deepseek/deepseek-reasoner", "deepseek/deepseek-v4.1-flash"], +) +def test_snapshot_fails_closed_on_unprobed_vision_tags(model): + assert DEFAULT_MODEL_CAPABILITIES[model]["supports_vision"] is False + + +# ─── hardening.test.ts: 2026-11 audit regressions ─── + +_HARDENING_PRICING = { + "anthropic/claude-sonnet-5": _price(3, 15), + "anthropic/claude-opus-4.8": _price(5, 25), + "openai/gpt-5.3-codex": _price(1.75, 14), + "openai/gpt-5-mini": _price(0.25, 2), + "google/gemini-3.5-flash": _price(1.5, 9), + "google/gemini-2.5-flash": _price(0.3, 2.5), + "deepseek/deepseek-v4-pro": _price(0.435, 0.87), + "moonshot/kimi-k3": _price(3, 15), +} + + +def _hardening(**extra): + return {"config": DEFAULT_ROUTING_CONFIG, "model_pricing": _HARDENING_PRICING, **extra} + + +@pytest.mark.parametrize( + ("prompt", "task_type"), + [ + ("请修复这个报错", "debug"), + ("请重构这个模块", "code_edit"), + ("提取所有字段", "extraction"), + ("证明这个定理", "reasoning"), + ], +) +def test_chinese_task_keywords_classify(prompt, task_type): + assert route(prompt, None, 512, _hardening())["task_type"] == task_type + + +class TestCodeAgentDetection: + WEB = {"has_tools": True, "requires_tools": True, "tool_count": 1, "tool_names": ["web_search"]} + + @pytest.mark.parametrize( + "prompt", + [ + "look up current java coffee bean prices online", + "search the web for python snake habitat facts", + ], + ) + def test_bare_language_name_in_a_web_lookup_is_not_code(self, prompt): + decision = route(prompt, None, 512, _hardening(**self.WEB)) + assert decision["task_type"] == "tool_agent" + assert decision["model"] != "openai/gpt-5.3-codex" + + @pytest.mark.parametrize( + "prompt", + [ + "Fix the TypeScript payment retry bug, run tests, and update the patch.", + "Write a python script that parses the CSV and run it", + "```ts\nconst x = 1\n``` refactor this", + ], + ) + def test_real_code_work_with_tools_stays_code_agent(self, prompt): + decision = route(prompt, None, 512, _hardening(has_tools=True, requires_tools=True)) + assert decision["task_type"] == "code_agent" + + +class TestParallelToolDetection: + def test_does_not_read_add_out_of_address(self): + decision = route( + "Please update my address to 12 Main St and send me a confirmation", + None, + 512, + _hardening( + has_tools=True, + requires_tools=True, + tool_count=2, + tool_names=["add_item", "send_email"], + ), + ) + assert decision["task_type"] == "tool_agent" + + def test_still_detects_two_operation_verbs(self): + decision = route( + "Add milk to my list and delete the eggs entry", + None, + 512, + _hardening( + has_tools=True, + requires_tools=True, + tool_count=2, + tool_names=["add_task", "delete_task"], + ), + ) + assert decision["task_type"] == "tool_agent_parallel" + + +def test_shouted_words_are_not_reservation_ids(): + airline = _hardening( + has_tools=True, + requires_tools=True, + tool_count=3, + tool_names=["get_reservation_details", "cancel_reservation", "search_flights"], + ) + assert "agentRisk=standard" in route("CANCEL MY FLIGHT please", None, 512, airline)["reasoning"] + two_ids = route("Cancel reservations ZFA04Y and 4WQ150", None, 512, airline) + assert "agentRisk=high" in two_ids["reasoning"] + + +class TestRankingRobustness: + def test_unpriced_candidate_does_not_win_on_cost(self): + pricing = dict(_HARDENING_PRICING) + del pricing["deepseek/deepseek-v4-pro"] + decision = route( + "Return the pricier of my two orders and exchange the other one for a larger size", + None, + 1024, + _hardening( + model_pricing=pricing, + has_tools=True, + requires_tools=True, + tool_count=4, + tool_names=[ + "get_order_details", + "return_delivered_order_items", + "exchange_delivered_order_items", + "get_product_details", + ], + ), + ) + unpriced = next( + row + for row in decision["candidate_scores"] + if row["model"] == "deepseek/deepseek-v4-pro" + ) + assert unpriced["cost"] == 0 + + def test_ignores_a_malformed_performance_profile(self): + decision = route( + "Summarize the plot of Hamlet in two sentences.", + None, + 512, + _hardening( + model_performance={ + "google/gemini-2.5-flash": { + "measured_at": "2026-11-01T00:00:00Z", + "latency_ms": 900, + "output_tokens_per_second": math.nan, + "samples": math.nan, + } + } + ), + ) + for row in decision["candidate_scores"]: + assert math.isfinite(row["score"]) + + +class TestCostFields: + TIERS = { + tier: {"primary": "acme/nan", "fallback": []} + for tier in ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING") + } + BAD = {"acme/nan": {"input_price": math.nan, "output_price": 1}} + + def test_non_finite_price_is_unknown_not_nan(self): + decision = select_model("SIMPLE", 1, "rules", "", self.TIERS, self.BAD, 1000, 1000) + assert math.isfinite(decision["cost_estimate"]) + assert decision["savings"] == 0 + cost = calculate_model_cost("acme/nan", self.BAD, 1000, 1000) + assert math.isfinite(cost["cost_estimate"]) + assert cost["savings"] == 0 + + def test_no_savings_for_an_unpriced_model(self): + assert calculate_model_cost("acme/unknown", {}, 1000, 1000)["savings"] == 0 + + +class TestPromotions: + NOW = datetime(2026, 11, 8, tzinfo=timezone.utc) + + @staticmethod + def _config(start: str, end: str): + return { + **DEFAULT_ROUTING_CONFIG, + "strategy": "rules", + "promotions": [ + { + "name": "typo", + "start_date": start, + "end_date": end, + "tier_overrides": {"SIMPLE": {"primary": "acme/promo"}}, + } + ], + } + + @pytest.mark.parametrize( + ("start", "end"), [("2026-13-01", "2026-12-01"), ("2026-11-01", "soon")] + ) + def test_ignores_unparseable_dates(self, start, end): + decision = RulesStrategy().route( + "hi", + None, + 64, + { + "config": self._config(start, end), + "model_pricing": _HARDENING_PRICING, + "now": self.NOW, + }, + ) + assert decision["model"] != "acme/promo" + + def test_applies_a_valid_window(self): + decision = RulesStrategy().route( + "hi", + None, + 64, + { + "config": self._config("2026-11-01", "2026-11-15"), + "model_pricing": _HARDENING_PRICING, + "now": self.NOW, + }, + ) + assert decision["tier_configs"][decision["tier"]]["primary"] == "acme/promo" + + +class TestScanBounds: + def test_bounds_the_sample_at_the_smallest_limit(self): + long = "x" * 100_000 + assert len(sample_for_scan(long, 1)) <= 2 + assert len(sample_for_scan(long, scan_limit(0))) <= 2 + + def test_non_numeric_truncation_uses_the_default_bound(self): + assert scan_limit(math.nan) == 8_000 + assert len(sample_for_scan("x" * 100_000, scan_limit(math.nan))) <= 8_001 + + +class TestRulesSignals: + @staticmethod + def _joined(prompt: str) -> str: + return " | ".join(classify_by_rules(prompt, None, 20, SCORING)["signals"]) + + def test_sentence_ending_in_a_number_is_not_multi_step(self): + assert "multi-step" not in self._joined("I was born in 1990. How old am I?") + + def test_numbered_list_is_multi_step(self): + assert "multi-step" in self._joined("Do this:\n1. install it\n2. configure it") + + def test_counts_full_width_question_marks(self): + assert "4 questions" in self._joined("这是什么?为什么?怎么做?什么时候?") + + +def test_japanese_extraction_does_not_get_the_mandarin_band(): + decision = route( + "次の文章から氏名と住所を抽出して、JSONで出力してください。山田太郎、東京都。", + None, + 512, + _hardening(requires_structured_output=True), + ) + assert decision["task_type"] == "extraction" + kimi = next( + (row for row in decision["candidate_scores"] if row["model"] == "moonshot/kimi-k3"), None + ) + if kimi is not None: + assert kimi["quality"] < 1 diff --git a/tests/unit/test_router_core_snapshot.py b/tests/unit/test_router_core_snapshot.py index 214bae6..bbc56bf 100644 --- a/tests/unit/test_router_core_snapshot.py +++ b/tests/unit/test_router_core_snapshot.py @@ -2,7 +2,7 @@ Cross-language decision-snapshot parity. ``router_core_decisions.snapshot.json`` is a verbatim copy of upstream -``decisions.snapshot.json`` at commit ``8773a99`` — 88 complete decisions the +``src/decisions.snapshot.json`` at commit ``273c2af`` — 88 complete decisions the TypeScript engine produced for a frozen corpus (22 prompts x 4 profiles with rotating tool/vision/structured-output shapes, frozen pricing, frozen clock). This test recomputes every decision with the Python port and compares field