Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 71 additions & 15 deletions src/canvas/ModelServiceForm.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,11 @@ import {
} from "../backend/rules.ts";
import { ModelCapabilityCard } from "./ModelCapabilityCard";
import {
LOCAL_OMNI_PRESET_ID,
PROVIDER_PRESETS,
matchingConnections,
presetProtocol,
presetStages,
providerPresetFor,
} from "./providerPresets";
import { CanvasIcon } from "./CanvasIcon";
Expand Down Expand Up @@ -68,33 +71,40 @@ export function ModelServiceForm({
const t = useT();
const connectionFieldId = useId();
const [connection, setConnection] = useState(existing);
const [presetId, setPresetId] = useState(
existing
? providerPresetFor(existing)
: kind === "video"
? "vllm-omni-video"
: kind === "image"
? "vllm-omni-image"
: kind === "audio"
? "vllm-omni-speech"
: "custom",
const initialPresetId = existing
? providerPresetFor(existing)
: kind === "video" || kind === "image" || kind === "audio"
? LOCAL_OMNI_PRESET_ID
: "custom";
const initialPreset = PROVIDER_PRESETS.find(
(item) => item.id === initialPresetId,
);
// The custom preset shows a sample hosted endpoint; leave it empty so the
// user enters their own service. Self-hosted presets carry real defaults.
const initialDefaults =
initialPreset?.connectionType === "self_hosted" ? initialPreset : undefined;
const [presetId, setPresetId] = useState(initialPresetId);
const [modelOptions, setModelOptions] = useState<
Record<string, ModelOptions>
>({});
const [modelErrors, setModelErrors] = useState<Record<string, string>>({});
const [value, setValue] = useState<ModelConfigInput>({
kind,
url: existing?.url ?? "",
url: existing?.url ?? initialDefaults?.url ?? "",
modelId: existing?.modelId ?? "",
apiKey: "",
connectionType: existing?.connectionType ?? "self_hosted",
connectionType:
existing?.connectionType ??
initialDefaults?.connectionType ??
"self_hosted",
capabilityAutoCheck: existing?.capabilityAutoCheck,
protocol:
existing && isSegmentationModel(existing)
? "vllm_omni_segmentation"
: (existing?.protocol ?? PROTOCOLS[kind][0][0]),
authMode: existing?.authMode ?? "none",
: (existing?.protocol ??
presetProtocol(initialDefaults, kind) ??
PROTOCOLS[kind][0][0]),
authMode: existing?.authMode ?? initialDefaults?.authMode ?? "none",
proxyMode: existing ? (existing.proxyMode ?? "system") : "direct",
videoAdapter: existing?.videoAdapter ?? "auto",
inputModalities:
Expand Down Expand Up @@ -127,6 +137,17 @@ export function ModelServiceForm({
const segmentation = value.protocol === "vllm_omni_segmentation";
const depth = value.protocol === "vllm_omni_depth";
const score = value.protocol === "vllm_omni_score";
const selectedPreset = PROVIDER_PRESETS.find((item) => item.id === presetId);
const stages = presetStages(selectedPreset, kind);
const activeStage = stages.find(
(stage) => stage.protocol === value.protocol,
);
const protocolLabel = (protocol: ModelConfigInput["protocol"]) => {
const option = PROTOCOLS[kind].find(
([candidate]) => candidate === protocol,
);
return option ? t(option[1]) : protocol;
};
const hasSavedKey = Boolean(
connection &&
connection.authMode === "bearer" &&
Expand Down Expand Up @@ -269,7 +290,7 @@ export function ModelServiceForm({
apiKey: "",
videoAdapter: "auto",
url: preset?.url ?? "",
protocol: preset?.protocol ?? PROTOCOLS[kind][0][0],
protocol: presetProtocol(preset, kind) ?? PROTOCOLS[kind][0][0],
connectionType: preset?.connectionType ?? "self_hosted",
authMode: preset?.authMode ?? "none",
proxyMode: "direct",
Expand Down Expand Up @@ -358,6 +379,41 @@ export function ModelServiceForm({
disabled={locked}
onChange={chooseProvider}
/>
{stages.length > 1 && (
<fieldset
className="cw-model-connection cw-provider-stage"
disabled={locked}
aria-label={t("models.service.stage")}
>
<div className="cw-model-connection-field">
<label>
{t("models.service.stage")}
<select
value={value.protocol}
onChange={(event) =>
update({
protocol: event.target
.value as ModelConfigInput["protocol"],
})
}
>
{stages.map((stage) => (
<option key={stage.protocol} value={stage.protocol}>
{protocolLabel(stage.protocol)}
</option>
))}
</select>
</label>
{activeStage && (
<p className="cw-model-note">
{t("models.service.stageEndpoints", {
paths: activeStage.paths.join(", "),
})}
</p>
)}
</div>
</fieldset>
)}
{!connection && connections.length > 0 && (
<div className="cw-provider-reuse">
<span>{t("models.service.existingConnections")}</span>
Expand Down
1 change: 1 addition & 0 deletions src/canvas/model-settings.css
Original file line number Diff line number Diff line change
Expand Up @@ -660,6 +660,7 @@
.cw-model-picker.has-capability-cards .cw-model-options { max-height: none; }

.cw-model-editor-scroll > .cw-model-connection-details { margin: 0 0 16px; }
.cw-model-editor-scroll > .cw-provider-stage { margin: 0 0 16px; }
.cw-model-editor-scroll > .cw-provider-key { margin: 0 0 18px; }

.cw-model-availability { padding: 12px 0; font-size: var(--settings-label-size, 13px); }
Expand Down
156 changes: 112 additions & 44 deletions src/canvas/providerPresets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,23 +16,11 @@ const EN_NAMES: Record<string, string> = {
"MiniMax 官方语音": "MiniMax Official Speech",
"MOSS 开放平台": "MOSS Open Platform",
"火山方舟 / Agent Plan": "Volcano Ark / Agent Plan",
"vLLM-Omni 图片接口": "vLLM-Omni image API",
"vLLM-Omni 语音合成接口": "vLLM-Omni speech synthesis API",
"vLLM-Omni 音乐生成接口": "vLLM-Omni music generation API",
"vLLM-Omni 扒谱接口": "vLLM-Omni transcription API",
"vLLM-Omni 视频接口": "vLLM-Omni video API",
"vLLM-Omni 分割接口": "vLLM-Omni segmentation API",
"vLLM-Omni 深度接口": "vLLM-Omni depth API",
"vLLM-Omni 本地服务": "vLLM-Omni local service",
};

const EN_OPTION_LABELS: Record<string, string> = {
图片接口: "Image API",
语音合成接口: "Speech synthesis API",
音乐生成接口: "Music generation API",
扒谱接口: "Transcription API",
视频接口: "Video API",
分割接口: "Segmentation API",
深度接口: "Depth API",
本地服务: "Local service",
};

const EN_GROUPS: Record<string, string> = {
Expand All @@ -55,6 +43,13 @@ function groupLabel(value: string | undefined): string | undefined {
return getActiveLocale() === "en-US" ? (EN_GROUPS[value] ?? value) : value;
}

/** One stage of a multi-stage self-hosted service: the endpoints it serves. */
export interface ProviderStage {
/** Endpoint paths appended to the connection URL, e.g. `/images/generations`. */
paths: readonly string[];
protocol: ModelConfigInput["protocol"];
}

export interface ProviderPreset {
id: string;
name: string;
Expand All @@ -68,6 +63,13 @@ export interface ProviderPreset {
connectionType: ModelConfigInput["connectionType"];
authMode: ModelConfigInput["authMode"];
modelsByKind?: Partial<Record<ModelKind, readonly string[]>>;
/**
* Per-kind stages of a service that serves several endpoints behind one
* connection. The first stage is the kind's default. Absent for
* single-purpose presets, where `url` + `protocol` describe the whole
* connection.
*/
stages?: Partial<Record<ModelKind, readonly ProviderStage[]>>;
}

export const PROVIDER_PRESETS: ProviderPreset[] = [
Expand Down Expand Up @@ -257,40 +259,96 @@ export const PROVIDER_PRESETS: ProviderPreset[] = [
connectionType: "api",
authMode: "bearer",
},
...(
[
["vllm-omni-image", "vLLM-Omni 图片接口", "openai", ["image"]],
["vllm-omni-speech", "vLLM-Omni 语音合成接口", "openai", ["audio"]],
[
"vllm-omni-music",
"vLLM-Omni 音乐生成接口",
"vllm_omni_music",
["audio"],
{
id: "vllm-omni-local",
logoOrganization: "vllm-project",
name: "vLLM-Omni 本地服务",
optionLabel: "本地服务",
group: "vLLM-Omni",
kinds: ["agent", "image", "video", "audio", "vision"],
url: "http://127.0.0.1:8000/v1",
protocol: "vllm_omni",
connectionType: "self_hosted",
authMode: "none",
// One localhost service serves several stages; each stage keeps the
// protocol its adapter expects, so a single connection is enough. The
// first stage of each kind is the default.
//
// ASR is deliberately absent: vllm-omni does not serve
// /audio/transcriptions yet (S1 in #10007) and this application has no ASR
// adapter. SheetSage2 score shares the path with its own protocol.
stages: {
agent: [{ paths: ["/chat/completions"], protocol: "openai" }],
image: [
{ paths: ["/images/generations", "/images/edits"], protocol: "openai" },
{
paths: ["/images/segmentations", "/videos/segmentations"],
protocol: "vllm_omni_segmentation",
},
],
["vllm-omni-score", "vLLM-Omni 扒谱接口", "vllm_omni_score", ["audio"]],
["vllm-omni-video", "vLLM-Omni 视频接口", "vllm_omni", ["video"]],
[
"vllm-omni-segmentation",
"vLLM-Omni 分割接口",
"vllm_omni_segmentation",
["image", "video", "vision"],
video: [
{ paths: ["/videos"], protocol: "vllm_omni" },
{
paths: ["/images/segmentations", "/videos/segmentations"],
protocol: "vllm_omni_segmentation",
},
{ paths: ["/videos/depth"], protocol: "vllm_omni_depth" },
],
["vllm-omni-depth", "vLLM-Omni 深度接口", "vllm_omni_depth", ["video"]],
] as const
).map(([id, name, protocol, kinds]) => ({
id,
name,
group: "vLLM-Omni",
logoOrganization: "vllm-project",
optionLabel: name.replace("vLLM-Omni ", ""),
protocol,
kinds: [...kinds],
url: "",
connectionType: "self_hosted" as const,
authMode: "none" as const,
})),
audio: [
{ paths: ["/audio/speech"], protocol: "openai" },
{ paths: ["/audio/speech"], protocol: "vllm_omni_music" },
{ paths: ["/audio/transcriptions"], protocol: "vllm_omni_score" },
],
vision: [
{
paths: ["/images/segmentations", "/videos/segmentations"],
protocol: "vllm_omni_segmentation",
},
],
},
},
];

export const LOCAL_OMNI_PRESET_ID = "vllm-omni-local";

/** The stages a preset serves for one kind; empty for single-purpose presets. */
export function presetStages(
preset: ProviderPreset | undefined,
kind: ModelKind,
): readonly ProviderStage[] {
return preset?.stages?.[kind] ?? [];
}

/**
* The protocol a connection uses for one kind. A protocol that names one of
* the preset's stages is kept, so reopening a saved specialized connection
* does not silently switch it back to the kind's default stage.
*/
export function presetProtocol(
preset: ProviderPreset | undefined,
kind: ModelKind,
protocol?: ModelConfigInput["protocol"] | null,
): ModelConfigInput["protocol"] | undefined {
const stages = presetStages(preset, kind);
if (protocol && stages.some((stage) => stage.protocol === protocol))
return protocol;
return stages[0]?.protocol ?? preset?.protocol;
}

/**
* `openai` is shared by generic self-hosted services, so it identifies this
* preset only at the documented loopback default. The specialized vLLM-Omni
* protocols are unique and need no URL rule.
*/
function isLoopbackUrl(value: string): boolean {
try {
const host = new URL(value).hostname.replace(/^\[|\]$/g, "");
return host === "localhost" || host === "127.0.0.1" || host === "::1";
} catch {
return false;
}
}

export function providerPresetFor(model: ModelConfig): string {
const explicit = PROVIDER_PRESETS.find(
(preset) =>
Expand All @@ -300,6 +358,16 @@ export function providerPresetFor(model: ModelConfig): string {
preset.kinds.includes(model.kind),
);
if (explicit) return explicit.id;
const staged = PROVIDER_PRESETS.find(
(preset) =>
model.connectionType === "self_hosted" &&
preset.kinds.includes(model.kind) &&
presetStages(preset, model.kind).some(
(stage) => stage.protocol === model.protocol,
) &&
(model.protocol !== "openai" || isLoopbackUrl(model.url)),
);
if (staged) return staged.id;
const deployed = PROVIDER_PRESETS.find(
(preset) =>
preset.connectionType === "self_hosted" &&
Expand Down
2 changes: 2 additions & 0 deletions src/i18n/messages/en-US/models.ts
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,8 @@ export const models = {
"models.service.saving": "Saving…",
"models.service.cancel": "Cancel",
"models.service.advancedConnection": "Advanced connection settings",
"models.service.stage": "Stage",
"models.service.stageEndpoints": "Endpoints: {paths}",
"models.service.existingConnections":
"Connections with the same address",
"models.service.reuseConnection": "Reuse saved connection · {model}",
Expand Down
2 changes: 2 additions & 0 deletions src/i18n/messages/zh-CN/models.ts
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@ export const models = {
"models.service.saving": "保存中…",
"models.service.cancel": "取消",
"models.service.advancedConnection": "高级连接设置",
"models.service.stage": "接口阶段",
"models.service.stageEndpoints": "接口路径:{paths}",
"models.service.existingConnections": "已有相同地址的连接",
"models.service.reuseConnection": "复用已保存连接 · {model}",
"models.service.reuseHint": "不复用则保存为独立连接。",
Expand Down
6 changes: 5 additions & 1 deletion tests/browser/capability_discovery.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -778,7 +778,11 @@ try {
.getByText("Fixture catalogue unavailable", { exact: true }).waitFor();
await page.unroute("**/api/model-configs");
await page.getByRole("button", { name: "Close settings", exact: true }).click();
await page.locator(`.react-flow__node[data-id="${itemId}"]`).click();
const canvasNode = page.locator(
`.react-flow__node[data-id="${itemId}"]`,
);
await canvasNode.waitFor({ timeout: 30000 });
await canvasNode.click();
assert.equal(await page.locator(".cw-generation-composer button.cw-run-button").isEnabled(), true);
assert.match(await page.locator(".cw-generation-composer").innerText(), /Preserve this prompt/);
await page.getByRole("button", { name: "Settings", exact: true }).click();
Expand Down
4 changes: 2 additions & 2 deletions tests/browser/kling_video.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ const copy = {
video: "Video",
add: "Add service",
provider: "Provider",
defaultProvider: "vLLM-Omni video API",
defaultProvider: "vLLM-Omni local service",
kling: "Kling official video",
url: "Service URL (Base URL)",
save: "Save service",
Expand All @@ -69,7 +69,7 @@ const copy = {
video: "视频",
add: "添加服务",
provider: "提供商",
defaultProvider: "vLLM-Omni 视频接口",
defaultProvider: "vLLM-Omni 本地服务",
kling: "可灵官方视频",
url: "服务地址(Base URL)",
save: "保存服务",
Expand Down
Loading
Loading