diff --git a/.github/workflows/publish-skills.yml b/.github/workflows/publish-skills.yml new file mode 100644 index 00000000..6064c40f --- /dev/null +++ b/.github/workflows/publish-skills.yml @@ -0,0 +1,27 @@ +# Poke the FC publish-skills flow after skills/ changes land. +# The FC side reconciles this repo's skills/ directory against OSS +# (bailian-wiki/skills/) using the repo HEAD snapshot as the only +# source of truth — the request itself carries no content. Both the +# repo and branch params are validated against FC-side whitelists +# (PUBLISH_REPOS / PUBLISH_BRANCHES). +# +# feat/cli-skill-sync is temporary for end-to-end testing; remove it +# (here and from the FC PUBLISH_BRANCHES whitelist) once the sync +# link is verified on main. +name: Publish skills to OSS + +on: + push: + branches: + - main + - feat/cli-skill-sync + paths: + - "skills/**" + +jobs: + poke: + runs-on: ubuntu-latest + steps: + - name: Trigger FC publish-skills + run: | + curl -sf -X POST "${{ vars.FC_TRIGGER_URL }}/publish-skills?repo=modelstudioai/cli&branch=${{ github.ref_name }}" diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index b772ba34..57268420 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -67,6 +67,7 @@ import { finetuneTextCreate, finetuneAudioCreate, finetuneImageCreate, + finetuneVideoCreate, finetuneList, finetuneGet, finetuneCancel, @@ -76,6 +77,7 @@ import { finetuneExport, finetuneWatch, finetuneCapability, + finetunePrice, deployTextCreate, deployAudioCreate, deployImageCreate, @@ -85,6 +87,8 @@ import { deployScale, deployUpdate, deployDelete, + deployPause, + deployResume, tokenPlanListSeats, tokenPlanCreateKey, tokenPlanAssignSeats, @@ -191,6 +195,7 @@ export const commands: Record = { "finetune text create": finetuneTextCreate, "finetune audio create": finetuneAudioCreate, "finetune image create": finetuneImageCreate, + "finetune video create": finetuneVideoCreate, "finetune list": finetuneList, "finetune get": finetuneGet, "finetune cancel": finetuneCancel, @@ -200,6 +205,7 @@ export const commands: Record = { "finetune export": finetuneExport, "finetune watch": finetuneWatch, "finetune capability": finetuneCapability, + "finetune price": finetunePrice, "deploy text create": deployTextCreate, "deploy audio create": deployAudioCreate, "deploy image create": deployImageCreate, @@ -209,6 +215,8 @@ export const commands: Record = { "deploy scale": deployScale, "deploy update": deployUpdate, "deploy delete": deployDelete, + "deploy pause": deployPause, + "deploy resume": deployResume, "token-plan list-seats": tokenPlanListSeats, "token-plan create-key": tokenPlanCreateKey, "token-plan assign-seats": tokenPlanAssignSeats, diff --git a/packages/commands/src/commands/dataset/delete.ts b/packages/commands/src/commands/dataset/delete.ts index 5e18766a..4c1e888b 100644 --- a/packages/commands/src/commands/dataset/delete.ts +++ b/packages/commands/src/commands/dataset/delete.ts @@ -1,5 +1,5 @@ -import { defineCommand, detectOutputFormat, deleteDataset, type FlagsDef } from "bailian-cli-core"; -import { emitResult, emitBare, emitRequestId } from "bailian-cli-runtime"; +import { defineCommand, deleteDataset, type FlagsDef } from "bailian-cli-core"; +import { emitResult, emitBare } from "bailian-cli-runtime"; const DELETE_FLAGS = { fileId: { @@ -19,20 +19,18 @@ export default defineCommand({ async run(ctx) { const { settings, flags } = ctx; const fileId = flags.fileId; - const format = detectOutputFormat(settings.output); if (settings.dryRun) { - emitResult({ action: "dataset.delete", file_id: fileId }, format); + emitResult({ action: "dataset.delete", file_id: fileId }, "json"); return; } const response = await deleteDataset(ctx.client, fileId); - if (settings.quiet || format === "text") { - emitBare(`Deleted ${fileId}.`); - emitRequestId(response.request_id, settings.quiet); + if (settings.quiet) { + emitBare(fileId); } else { - emitResult(response, format); + emitResult(response, "json"); } }, }); diff --git a/packages/commands/src/commands/dataset/get.ts b/packages/commands/src/commands/dataset/get.ts index 2c1cf714..f9a41451 100644 --- a/packages/commands/src/commands/dataset/get.ts +++ b/packages/commands/src/commands/dataset/get.ts @@ -1,5 +1,5 @@ -import { defineCommand, detectOutputFormat, getDataset, type FlagsDef } from "bailian-cli-core"; -import { emitResult, emitBare, emitRequestId } from "bailian-cli-runtime"; +import { defineCommand, getDataset, type FlagsDef } from "bailian-cli-core"; +import { emitResult, emitBare } from "bailian-cli-runtime"; const GET_FLAGS = { fileId: { @@ -19,10 +19,9 @@ export default defineCommand({ async run(ctx) { const { settings, flags } = ctx; const fileId = flags.fileId; - const format = detectOutputFormat(settings.output); if (settings.dryRun) { - emitResult({ action: "dataset.get", file_id: fileId }, format); + emitResult({ action: "dataset.get", file_id: fileId }, "json"); return; } @@ -45,19 +44,10 @@ export default defineCommand({ description: file.description ?? "", }; - if (format === "json") { - emitResult({ ...item, request_id: response.request_id }, format); - return; + if (settings.quiet) { + emitBare(item.file_id); + } else { + emitResult({ ...item, request_id: response.request_id }, "json"); } - - // text / quiet - emitBare(`file_id: ${item.file_id}`); - emitBare(`name: ${item.name}`); - emitBare(`size: ${item.size}`); - if (item.md5) emitBare(`md5: ${item.md5}`); - if (item.purpose) emitBare(`purpose: ${item.purpose}`); - if (item.created_at) emitBare(`created_at: ${item.created_at}`); - if (item.description) emitBare(`description: ${item.description}`); - emitRequestId(response.request_id, settings.quiet); }, }); diff --git a/packages/commands/src/commands/dataset/list.ts b/packages/commands/src/commands/dataset/list.ts index 7bc858b4..63bbfeb2 100644 --- a/packages/commands/src/commands/dataset/list.ts +++ b/packages/commands/src/commands/dataset/list.ts @@ -1,5 +1,5 @@ -import { defineCommand, detectOutputFormat, listDatasets, type FlagsDef } from "bailian-cli-core"; -import { emitResult, emitBare, emitRequestId, formatTable } from "bailian-cli-runtime"; +import { defineCommand, listDatasets, type FlagsDef } from "bailian-cli-core"; +import { emitResult, emitBare } from "bailian-cli-runtime"; const LIST_FLAGS = { page: { type: "number", valueHint: "", description: "Page number (default: 1)" }, @@ -23,7 +23,6 @@ export default defineCommand({ exampleArgs: ["", "--purpose fine-tune", "--purpose evaluation --page-size 20", "--output json"], async run(ctx) { const { settings, flags } = ctx; - const format = detectOutputFormat(settings.output); if (settings.dryRun) { emitResult( @@ -33,7 +32,7 @@ export default defineCommand({ page_size: flags.pageSize, purpose: flags.purpose, }, - format, + "json", ); return; } @@ -46,7 +45,6 @@ export default defineCommand({ const files = response.data?.files ?? []; const total = response.data?.total; - // Normalize to consistent structure for both text/json output. const items = files.map((item) => ({ file_id: item.file_id ?? "", name: item.name ?? "", @@ -54,20 +52,10 @@ export default defineCommand({ purpose: item.purpose ?? "", })); - if (format === "json") { - emitResult({ items, total, request_id: response.request_id }, format); - return; - } - - // text / quiet - if (items.length === 0) { - emitBare("No dataset files found."); - return; + if (settings.quiet) { + for (const item of items) emitBare(item.file_id); + } else { + emitResult({ items, total, request_id: response.request_id }, "json"); } - const headers = ["FILE_ID", "NAME", "SIZE", "PURPOSE"]; - const rows = items.map((i) => [i.file_id, i.name, i.size, i.purpose]); - for (const line of formatTable(headers, rows)) emitBare(line); - if (total !== undefined) emitBare(`\nTotal: ${total}`); - emitRequestId(response.request_id, settings.quiet); }, }); diff --git a/packages/commands/src/commands/dataset/upload.ts b/packages/commands/src/commands/dataset/upload.ts index a3c61045..822c5a9f 100644 --- a/packages/commands/src/commands/dataset/upload.ts +++ b/packages/commands/src/commands/dataset/upload.ts @@ -1,23 +1,23 @@ import { defineCommand, - detectOutputFormat, uploadDataset, validateDataset, parseDatasetSchemaFlag, formatIssue, MAX_DATASET_BYTES, + MAX_CPT_BYTES, MAX_MEDIA_ZIP_BYTES, BailianError, ExitCode, type FlagsDef, } from "bailian-cli-core"; -import { emitResult, emitBare, emitRequestId } from "bailian-cli-runtime"; +import { emitResult, emitBare } from "bailian-cli-runtime"; const UPLOAD_FLAGS = { file: { type: "string", valueHint: "", - description: "Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image)", + description: "Local dataset file (.jsonl or .zip; ≤200MB SFT/DPO, ≤300MB CPT, ≤2GB media zip)", required: true, }, purpose: { @@ -29,7 +29,7 @@ const UPLOAD_FLAGS = { type: "string", valueHint: "", description: - 'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record.', + 'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record.', }, noValidate: { type: "switch", @@ -45,7 +45,7 @@ export default defineCommand({ description: "Upload a dataset file (.jsonl or .zip) to Bailian", auth: "apiKey", usageArgs: - "--file [--purpose ] [--schema ] [--no-validate] [--full-validate]", + "--file [--purpose ] [--schema ] [--no-validate] [--full-validate]", flags: UPLOAD_FLAGS, exampleArgs: [ "--file train.jsonl", @@ -58,13 +58,14 @@ export default defineCommand({ ], notes: [ "Supports .jsonl (text) and .zip (audio/image archives with a data.jsonl", - "manifest). Five record schemas are recognized: chatml = {messages:[...]}", + "manifest). Six record schemas are recognized: chatml = {messages:[...]}", '(SFT); dpo = {messages:[...], chosen, rejected}; cpt = {text:"..."}', '(continual pre-training, raw text); tts = {wav_fn:"train/xxx.wav",', 'text:"..."} (audio fine-tuning); image = {img_path:"..."} (image', - "generation). With no --schema, a record carrying wav_fn is validated as", - "TTS, img_path as image, chosen/rejected as DPO, text (no messages) as CPT,", - "otherwise ChatML. Upload cap: 300MB text, 1GB image. Upload uses the", + "generation); video = {first_frame_path:...} (video generation). With no", + "--schema, a record carrying wav_fn is validated as TTS, img_path as image,", + "chosen/rejected as DPO, text (no messages) as CPT, otherwise ChatML.", + "Upload cap: 200MB SFT/DPO text, 300MB CPT, 2GB media zip. Upload uses the", "OpenAI-compatible /compatible-mode/v1/files endpoint so the purpose tag is", "persisted (the DashScope-native /api/v1/files drops it).", ], @@ -73,19 +74,15 @@ export default defineCommand({ const filePath = flags.file; const purpose = flags.purpose || "fine-tune"; const schema = parseDatasetSchemaFlag(flags.schema); - if (schema === "video") { - throw new BailianError( - `--schema video is not supported.`, - ExitCode.USAGE, - `Supported schemas: chatml, dpo, cpt, tts, image.`, - ); - } - const format = detectOutputFormat(settings.output); - // Image schema allows larger ZIPs (1 GB vs 300 MB for text). - const isMediaSchema = schema === "image"; + // Size caps differ per training type: SFT/DPO 200MB, CPT 300MB, media ZIP 2GB. + const isMediaSchema = schema === "image" || schema === "video"; + const maxBytes = isMediaSchema + ? MAX_MEDIA_ZIP_BYTES + : schema === "cpt" + ? MAX_CPT_BYTES + : MAX_DATASET_BYTES; if (!flags.noValidate) { - const maxBytes = isMediaSchema ? MAX_MEDIA_ZIP_BYTES : MAX_DATASET_BYTES; const result = await validateDataset(filePath, { fullValidate: flags.fullValidate, schema, @@ -125,11 +122,11 @@ export default defineCommand({ action: "dataset.upload", file: filePath, purpose, - max_bytes: isMediaSchema ? MAX_MEDIA_ZIP_BYTES : MAX_DATASET_BYTES, + max_bytes: maxBytes, validate: !flags.noValidate, schema: schema ?? "auto", }, - format, + "json", ); return; } @@ -142,11 +139,8 @@ export default defineCommand({ if (settings.quiet) { emitBare(file.file_id); - } else if (format === "text") { - emitBare(`Uploaded ${file.name} → file_id=${file.file_id}`); - emitRequestId(request_id, settings.quiet); } else { - emitResult({ ...file, request_id }, format); + emitResult({ ...file, request_id }, "json"); } }, }); diff --git a/packages/commands/src/commands/dataset/validate.ts b/packages/commands/src/commands/dataset/validate.ts index 8bf0d292..b9e7f62b 100644 --- a/packages/commands/src/commands/dataset/validate.ts +++ b/packages/commands/src/commands/dataset/validate.ts @@ -1,26 +1,13 @@ import { defineCommand, - detectOutputFormat, validateDataset, parseDatasetSchemaFlag, - formatIssue, BailianError, ExitCode, - type ValidationResult, type FlagsDef, } from "bailian-cli-core"; import { emitResult, emitBare } from "bailian-cli-runtime"; -function formatStats(result: ValidationResult): string[] { - const out: string[] = []; - if (result.stats.totalRecords !== undefined) out.push(`records: ${result.stats.totalRecords}`); - if (result.stats.sampledRecords !== undefined) - out.push(`sampled: ${result.stats.sampledRecords}`); - if (result.stats.bytes !== undefined) out.push(`bytes: ${result.stats.bytes}`); - if (result.stats.durationMs !== undefined) out.push(`took: ${result.stats.durationMs}ms`); - return out; -} - const VALIDATE_FLAGS = { file: { type: "string", @@ -36,7 +23,7 @@ const VALIDATE_FLAGS = { type: "string", valueHint: "", description: - 'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record.', + 'Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record.', }, } satisfies FlagsDef; @@ -44,13 +31,14 @@ export default defineCommand({ description: "Locally validate a dataset file (.jsonl or .zip) without uploading", // 纯本地校验,不触网、不需 API key(与 `pipeline validate` 一致)。 auth: "none", - usageArgs: "--file [--full-validate] [--schema ]", + usageArgs: "--file [--full-validate] [--schema ]", flags: VALIDATE_FLAGS, exampleArgs: [ "--file train.jsonl", "--file dpo.jsonl --schema dpo", "--file cpt.jsonl --schema cpt", "--file audio.zip --schema tts", + "--file wan-i2v-training-dataset.zip --schema video", "--file eval.jsonl --full-validate", "--file train.jsonl --output json", ], @@ -60,27 +48,20 @@ export default defineCommand({ "Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen,", 'rejected}; cpt = {text:"..."} (continual pre-training, raw text);', 'tts = {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning);', - 'image = {img_path:"..."} (image generation). With no --schema, a record', - "carrying wav_fn is validated as TTS, img_path as image, chosen/rejected", - "as DPO, text (no messages) as CPT, otherwise ChatML. Pass --schema to", - "require a specific shape on every record. ZIP archives (.zip) are", - "validated structurally (data.jsonl present, media references resolve) in", - "addition to per-record content checks. Use --full-validate to JSON.parse", - "every line.", + 'image = {img_path:"..."} (image generation);', + 'video = {first_frame_path:"...", video_path:"..."} (video generation,', + "i2v first-frame or kf2v first+last-frame with last_frame_path). With no", + "--schema, a record carrying wav_fn is validated as TTS, img_path as image,", + "first_frame_path/video_path as video, chosen/rejected as DPO, text (no", + "messages) as CPT, otherwise ChatML. Pass --schema to require a specific", + "shape on every record. ZIP archives (.zip) are validated structurally", + "(data.jsonl present, media references resolve) in addition to per-record", + "content checks. Use --full-validate to JSON.parse every line.", ], async run(ctx) { const { settings, flags } = ctx; const filePath = flags.file; const schema = parseDatasetSchemaFlag(flags.schema); - if (schema === "video") { - throw new BailianError( - `--schema video is not supported.`, - ExitCode.USAGE, - `Supported schemas: chatml, dpo, cpt, tts, image.`, - ); - } - const format = detectOutputFormat(settings.output); - if (settings.dryRun) { emitResult( { @@ -89,38 +70,17 @@ export default defineCommand({ full: flags.fullValidate, schema: schema ?? "auto", }, - format, + "json", ); return; } const result = await validateDataset(filePath, { fullValidate: flags.fullValidate, schema }); - if (format === "json") { - // For json output we always emit the structured result, exit code conveys validity. - emitResult(result, format); - } else if (settings.quiet) { + if (settings.quiet) { emitBare(result.valid ? "ok" : "fail"); } else { - const status = result.valid ? "PASSED" : "FAILED"; - emitBare(`Dataset validation ${status} for ${result.filePath}`); - const stats = formatStats(result); - if (stats.length) emitBare(` ${stats.join(" · ")}`); - - if (result.errors.length) { - emitBare(`Errors (${result.errors.length}):`); - for (const error of result.errors.slice(0, 20)) emitBare(formatIssue(error)); - if (result.errors.length > 20) { - emitBare(` … and ${result.errors.length - 20} more.`); - } - } - if (result.warnings.length) { - emitBare(`Warnings (${result.warnings.length}):`); - for (const warning of result.warnings.slice(0, 10)) emitBare(formatIssue(warning)); - if (result.warnings.length > 10) { - emitBare(` … and ${result.warnings.length - 10} more.`); - } - } + emitResult(result, "json"); } if (!result.valid) { diff --git a/packages/commands/src/commands/deploy/create.ts b/packages/commands/src/commands/deploy/create.ts index 96c75d92..de468fcb 100644 --- a/packages/commands/src/commands/deploy/create.ts +++ b/packages/commands/src/commands/deploy/create.ts @@ -1,6 +1,5 @@ import { defineCommand, - detectOutputFormat, createDeployment, pickPlanStrategy, STRATEGIES, @@ -11,16 +10,16 @@ import { type CommandContext, type FlagsDef, } from "bailian-cli-core"; -import { emitResult, emitBare, emitRequestId } from "bailian-cli-runtime"; +import { emitResult, emitBare } from "bailian-cli-runtime"; const CREATE_FLAGS = { - model: { + modelName: { type: "string", - valueHint: "", - description: "Model name (catalog model or fine-tuned output) (required)", + valueHint: "", + description: "Model to deploy — fine-tuned output name or catalog model (required)", required: true, }, - name: { + displayName: { type: "string", valueHint: "", description: "Console display name for the deployment (required)", @@ -64,7 +63,7 @@ const CREATE_FLAGS = { } satisfies FlagsDef; const CREATE_USAGE = - "--model --name [--plan ] [--deploy-spec ] [--capacity ] [--billing-method ] [--input-tpm ] [--output-tpm ] [--thinking-output-tpm ]"; + "--model-name --display-name [--plan ] [--deploy-spec ] [--capacity ] [--billing-method ] [--input-tpm ] [--output-tpm ] [--thinking-output-tpm ]"; const CREATE_NOTES = [ "Plan defaults to `lora` (Token-billed) for text/image and `mu` (model-unit-", @@ -78,14 +77,11 @@ const CREATE_NOTES = [ "Use `bl deploy models --source base` to inspect available templates.", "After creation, status starts at PENDING and transitions to RUNNING.", "Invoke the deployed model with: bl text chat --model ", - "WARNING: --model is overloaded across commands and refers to DIFFERENT", - "values. `bl deploy create --model` takes the exported model_name", - "(e.g. `qwen3-8b-ft-...`), but the create response also returns a", - "`deployed_model` field (the deployment instance id, e.g.", - "`qwen3-8b-5ecb5f068d79`). The inference call `bl text chat --model` must use", - "the `deployed_model` from the create response — NOT the `model_name` you", - "passed to `deploy create`. Do not reuse the value across the two", - "commands.", + "NOTE: --model-name is the model being deployed (e.g. `qwen3-8b-ft-...`).", + "The create response also returns a `deployed_model` field — the deployment", + "instance id (e.g. `qwen3-8b-5ecb5f068d79`). Use that id for inference", + "(`bl text chat --model `) and lifecycle commands", + "(`deploy get/scale/pause/resume/delete --deployed-model `).", ]; /** @@ -119,10 +115,9 @@ async function runCreate( ctx: CommandContext, ): Promise { const { identity, settings, flags } = ctx; - const model = flags.model as string; - const name = flags.name as string; + const model = flags.modelName as string; + const name = flags.displayName as string; const plan = (flags.plan as string | undefined) || defaultDeployPlan(modality); - const format = detectOutputFormat(settings.output); // Plan-specific behaviour is owned by core `plans.ts`. The strategy resolves // the plan-specific body fragment (mu may auto-pick a template from the @@ -146,7 +141,7 @@ async function runCreate( }; if (settings.dryRun) { - emitResult({ action: "deploy.create", body }, format); + emitResult({ action: "deploy.create", body }, "json"); return; } @@ -155,17 +150,8 @@ async function runCreate( if (settings.quiet) { emitBare(deployment?.deployed_model ?? ""); - } else if (format === "text") { - emitBare(`Created deployment.`); - if (deployment?.deployed_model) emitBare(` deployed_model: ${deployment.deployed_model}`); - if (deployment?.status) emitBare(` status: ${deployment.status}`); - if (deployment?.plan) emitBare(` plan: ${deployment.plan}`); - emitBare( - `\nNext: track readiness with: ${identity.binName} deploy get --deployed-model ${deployment?.deployed_model ?? ""}`, - ); - emitRequestId(response.request_id, settings.quiet); } else { - emitResult(response, format); + emitResult(response, "json"); } } @@ -176,10 +162,10 @@ export const deployTextCreate = defineCommand({ usageArgs: CREATE_USAGE, flags: CREATE_FLAGS, exampleArgs: [ - "--model my-qwen-sft --name my-sft-test", - "--model qwen3.6-flash-2026-04-16 --name my-flash --plan ptu --input-tpm 10000 --output-tpm 1000", - "--model qwen3-8b --name my-qwen3-mu --plan mu", - "--model qwen3-8b --name my-qwen3 --plan mu --deploy-spec MU1 --capacity 2", + "--model-name my-qwen-sft --display-name my-sft-test", + "--model-name qwen3.6-flash-2026-04-16 --display-name my-flash --plan ptu --input-tpm 10000 --output-tpm 1000", + "--model-name qwen3-8b --display-name my-qwen3-mu --plan mu", + "--model-name qwen3-8b --display-name my-qwen3 --plan mu --deploy-spec MU1 --capacity 2", ], notes: CREATE_NOTES, validate: (flags) => validateCreate("text", flags), @@ -193,9 +179,9 @@ export const deployAudioCreate = defineCommand({ usageArgs: CREATE_USAGE, flags: CREATE_FLAGS, exampleArgs: [ - "--model my-cosyvoice-ft --name my-tts", - "--model my-cosyvoice-ft --name my-tts --deploy-spec dps-xxxx --capacity 1", - "--model my-cosyvoice-ft --name my-tts --dry-run", + "--model-name my-cosyvoice-ft --display-name my-tts", + "--model-name my-cosyvoice-ft --display-name my-tts --deploy-spec dps-xxxx --capacity 1", + "--model-name my-cosyvoice-ft --display-name my-tts --dry-run", ], notes: CREATE_NOTES, validate: (flags) => validateCreate("audio", flags), @@ -209,9 +195,9 @@ export const deployImageCreate = defineCommand({ usageArgs: CREATE_USAGE, flags: CREATE_FLAGS, exampleArgs: [ - "--model my-wan-ft --name my-wan", - "--model my-wan-ft --name my-wan-mu --plan mu", - "--model my-wan-ft --name my-wan --dry-run", + "--model-name my-wan-ft --display-name my-wan", + "--model-name my-wan-ft --display-name my-wan-mu --plan mu", + "--model-name my-wan-ft --display-name my-wan --dry-run", ], notes: CREATE_NOTES, validate: (flags) => validateCreate("image", flags), diff --git a/packages/commands/src/commands/deploy/delete.ts b/packages/commands/src/commands/deploy/delete.ts index bd6dd85e..d3f386d8 100644 --- a/packages/commands/src/commands/deploy/delete.ts +++ b/packages/commands/src/commands/deploy/delete.ts @@ -1,13 +1,12 @@ import { defineCommand, - detectOutputFormat, deleteDeployment, getDeployment, BailianError, ExitCode, type FlagsDef, } from "bailian-cli-core"; -import { emitResult, emitBare, emitRequestId } from "bailian-cli-runtime"; +import { emitResult, emitBare } from "bailian-cli-runtime"; const DELETE_FLAGS = { deployedModel: { @@ -38,10 +37,9 @@ export default defineCommand({ async run(ctx) { const { settings, flags } = ctx; const deployedModel = flags.deployedModel; - const format = detectOutputFormat(settings.output); if (settings.dryRun) { - emitResult({ action: "deploy.delete", deployed_model: deployedModel }, format); + emitResult({ action: "deploy.delete", deployed_model: deployedModel }, "json"); return; } @@ -55,7 +53,8 @@ export default defineCommand({ if (status && status !== "STOPPED" && status !== "FAILED") { throw new BailianError( `Deployment ${deployedModel} is ${status}. Only STOPPED / FAILED deployments can be deleted. ` + - `Stop it first via the platform console, or pass --skip-precheck to attempt deletion anyway.`, + `Run \`bl deploy pause --deployed-model ${deployedModel}\` to pause it first, ` + + `or pass --skip-precheck to attempt deletion anyway.`, ExitCode.USAGE, ); } @@ -69,11 +68,8 @@ export default defineCommand({ if (settings.quiet) { emitBare(deployedModel); - } else if (format === "text") { - emitBare(`Deleted ${deployedModel}.`); - emitRequestId(response.request_id, settings.quiet); } else { - emitResult(response, format); + emitResult(response, "json"); } }, }); diff --git a/packages/commands/src/commands/deploy/get.ts b/packages/commands/src/commands/deploy/get.ts index 9c53e96f..09bff852 100644 --- a/packages/commands/src/commands/deploy/get.ts +++ b/packages/commands/src/commands/deploy/get.ts @@ -1,5 +1,5 @@ -import { defineCommand, detectOutputFormat, getDeployment, type FlagsDef } from "bailian-cli-core"; -import { emitResult, emitBare, emitRequestId } from "bailian-cli-runtime"; +import { defineCommand, getDeployment, type FlagsDef } from "bailian-cli-core"; +import { emitResult } from "bailian-cli-runtime"; const GET_FLAGS = { deployedModel: { @@ -22,10 +22,9 @@ export default defineCommand({ async run(ctx) { const { settings, flags } = ctx; const deployedModel = flags.deployedModel; - const format = detectOutputFormat(settings.output); if (settings.dryRun) { - emitResult({ action: "deploy.get", deployed_model: deployedModel }, format); + emitResult({ action: "deploy.get", deployed_model: deployedModel }, "json"); return; } @@ -33,7 +32,7 @@ export default defineCommand({ const deployment = response.output ?? response.data; if (!deployment) { - emitBare(`No data returned for ${deployedModel}`); + emitResult({ deployed_model: deployedModel, request_id: response.request_id }, "json"); return; } @@ -57,18 +56,6 @@ export default defineCommand({ if (deployment.gmt_create) item.created_at = deployment.gmt_create; if (deployment.gmt_modified) item.updated_at = deployment.gmt_modified; - if (format === "json") { - emitResult({ ...item, request_id: response.request_id }, format); - return; - } - - // text / quiet — fixed-width label column for alignment - const label = (key: string) => `${key}:`.padEnd(18); - for (const [key, value] of Object.entries(item)) { - if (value === "" || value === undefined) continue; - const display = typeof value === "string" ? value : JSON.stringify(value); - emitBare(`${label(key)}${display}`); - } - emitRequestId(response.request_id, settings.quiet); + emitResult({ ...item, request_id: response.request_id }, "json"); }, }); diff --git a/packages/commands/src/commands/deploy/list.ts b/packages/commands/src/commands/deploy/list.ts index e7c29173..944038a5 100644 --- a/packages/commands/src/commands/deploy/list.ts +++ b/packages/commands/src/commands/deploy/list.ts @@ -1,10 +1,5 @@ -import { - defineCommand, - detectOutputFormat, - listDeployments, - type FlagsDef, -} from "bailian-cli-core"; -import { emitResult, emitBare, emitRequestId, formatTable } from "bailian-cli-runtime"; +import { defineCommand, listDeployments, type FlagsDef } from "bailian-cli-core"; +import { emitResult } from "bailian-cli-runtime"; const LIST_FLAGS = { page: { type: "number", valueHint: "", description: "Page number (default: 1)" }, @@ -28,13 +23,12 @@ export default defineCommand({ exampleArgs: ["", "--status RUNNING", "--page-size 20 --output json"], async run(ctx) { const { settings, flags } = ctx; - const format = detectOutputFormat(settings.output); const status = flags.status || undefined; if (settings.dryRun) { emitResult( { action: "deploy.list", page: flags.page, page_size: flags.pageSize, status }, - format, + "json", ); return; } @@ -57,27 +51,6 @@ export default defineCommand({ created_at: item.gmt_create ?? "", })); - if (format === "json") { - emitResult({ items, total, request_id: response.request_id }, format); - return; - } - - // text / quiet - if (items.length === 0) { - emitBare("No deployments found."); - return; - } - const headers = ["DEPLOYED_MODEL", "MODEL_NAME", "STATUS", "PLAN", "CAPACITY", "CREATED_AT"]; - const rows = items.map((item) => [ - item.deployed_model, - item.model_name, - item.status, - item.plan, - item.capacity, - item.created_at, - ]); - for (const line of formatTable(headers, rows)) emitBare(line); - if (total !== undefined) emitBare(`\nTotal: ${total}`); - emitRequestId(response.request_id, settings.quiet); + emitResult({ items, total, request_id: response.request_id }, "json"); }, }); diff --git a/packages/commands/src/commands/deploy/models.ts b/packages/commands/src/commands/deploy/models.ts index d1ea972e..b2c81d4c 100644 --- a/packages/commands/src/commands/deploy/models.ts +++ b/packages/commands/src/commands/deploy/models.ts @@ -1,10 +1,5 @@ -import { - defineCommand, - detectOutputFormat, - listDeployableModels, - type FlagsDef, -} from "bailian-cli-core"; -import { emitResult, emitBare, emitRequestId, formatTable } from "bailian-cli-runtime"; +import { defineCommand, listDeployableModels, type FlagsDef } from "bailian-cli-core"; +import { emitResult } from "bailian-cli-runtime"; const MODELS_FLAGS = { page: { type: "number", valueHint: "", description: "Page number (default: 1)" }, @@ -39,7 +34,6 @@ export default defineCommand({ ], async run(ctx) { const { settings, flags } = ctx; - const format = detectOutputFormat(settings.output); // Default version to v1.0 — without it, the API returns the legacy catalog // (only old fine-tune outputs). Pass --catalog-version "" to opt out. const version = flags.catalogVersion === "" ? undefined : (flags.catalogVersion ?? "v1.0"); @@ -54,7 +48,7 @@ export default defineCommand({ version, model_source: modelSource, }, - format, + "json", ); return; } @@ -72,102 +66,55 @@ export default defineCommand({ // Two response shapes: // - custom (fine-tuned): top-level supported_plans: string[] // - base (catalog): plans: [{plan, templates?, cu_specs?}] - // For json: surface the deployment-relevant fields preserved as a tree, so + // Surface the deployment-relevant fields preserved as a tree, so // downstream tooling can drive `bl deploy create --deploy-spec <…>` - // without a second round-trip. For text: keep the compact one-line summary. - if (format === "json") { - const items = models.map((model) => { - const out: Record = { - model_name: model.model_name ?? "", - }; - if (model.base_model) out.base_model = model.base_model; - if (model.model_source) out.model_source = model.model_source; - if (model.supported_plans && model.supported_plans.length > 0) { - out.supported_plans = model.supported_plans; - } - if (model.plans && model.plans.length > 0) { - out.plans = model.plans.map((plan) => { - const planEntry: Record = { plan: plan.plan ?? "" }; - if (plan.cu_specs && plan.cu_specs.length > 0) { - planEntry.cu_specs = plan.cu_specs; - } - if (plan.templates && plan.templates.length > 0) { - // Pull the top 6 fields most useful for `bl deploy create`. - // Drop noisy/redundant: template_source, template_type, - // template_version, deploy_spec (typically == template_id). - planEntry.templates = plan.templates.map((template) => { - const tpl: Record = {}; - if (template.template_id) tpl.template_id = template.template_id; - if (template.template_name) tpl.template_name = template.template_name; - if (template.charge_type) tpl.charge_type = template.charge_type; - // Flatten roles.unified for the common COUPLED case. - const unified = template.roles?.unified; - if (unified?.model_unit_spec) tpl.model_unit_spec = unified.model_unit_spec; - if (unified?.capacity_unit_per_instance !== undefined) - tpl.capacity_unit_per_instance = unified.capacity_unit_per_instance; - // Preserve split-role configs (SEPERATED) as-is so callers - // can still drive prefill/decode sizing. - if (template.roles?.prefill || template.roles?.decode) { - tpl.roles = { - prefill: template.roles?.prefill, - decode: template.roles?.decode, - }; - } - if (template.template_desc) tpl.template_desc = template.template_desc; - return tpl; - }); - } - return planEntry; - }); - } - return out; - }); - emitResult({ items, total, request_id: response.request_id }, format); - return; - } - - // text / quiet — keep the compact single-line summary table. - const textItems = models.map((model) => { - let plansSummary = ""; - if (model.supported_plans && model.supported_plans.length > 0) { - plansSummary = model.supported_plans.join(","); - } else if (model.plans && model.plans.length > 0) { - plansSummary = model.plans - .map((plan) => { - const planName = plan.plan ?? "?"; - if (plan.templates && plan.templates.length > 0) { - return `${planName}(${plan.templates.length}t)`; - } - if (plan.cu_specs && plan.cu_specs.length > 0) { - return `${planName}(${plan.cu_specs.join("/")})`; - } - return planName; - }) - .join(","); - } else { - plansSummary = "-"; - } - return { + // without a second round-trip. + const items = models.map((model) => { + const out: Record = { model_name: model.model_name ?? "", - base_model: model.base_model ?? "", - source: model.model_source ?? "", - plans: plansSummary, }; + if (model.base_model) out.base_model = model.base_model; + if (model.model_source) out.model_source = model.model_source; + if (model.supported_plans && model.supported_plans.length > 0) { + out.supported_plans = model.supported_plans; + } + if (model.plans && model.plans.length > 0) { + out.plans = model.plans.map((plan) => { + const planEntry: Record = { plan: plan.plan ?? "" }; + if (plan.cu_specs && plan.cu_specs.length > 0) { + planEntry.cu_specs = plan.cu_specs; + } + if (plan.templates && plan.templates.length > 0) { + // Pull the top 6 fields most useful for `bl deploy create`. + // Drop noisy/redundant: template_source, template_type, + // template_version, deploy_spec (typically == template_id). + planEntry.templates = plan.templates.map((template) => { + const tpl: Record = {}; + if (template.template_id) tpl.template_id = template.template_id; + if (template.template_name) tpl.template_name = template.template_name; + if (template.charge_type) tpl.charge_type = template.charge_type; + // Flatten roles.unified for the common COUPLED case. + const unified = template.roles?.unified; + if (unified?.model_unit_spec) tpl.model_unit_spec = unified.model_unit_spec; + if (unified?.capacity_unit_per_instance !== undefined) + tpl.capacity_unit_per_instance = unified.capacity_unit_per_instance; + // Preserve split-role configs (SEPERATED) as-is so callers + // can still drive prefill/decode sizing. + if (template.roles?.prefill || template.roles?.decode) { + tpl.roles = { + prefill: template.roles?.prefill, + decode: template.roles?.decode, + }; + } + if (template.template_desc) tpl.template_desc = template.template_desc; + return tpl; + }); + } + return planEntry; + }); + } + return out; }); - - if (textItems.length === 0) { - emitBare("No deployable models found."); - return; - } - const headers = ["MODEL_NAME", "BASE_MODEL", "SOURCE", "PLANS"]; - const rows = textItems.map((item) => [ - item.model_name, - item.base_model, - item.source, - item.plans, - ]); - for (const line of formatTable(headers, rows)) emitBare(line); - if (total !== undefined) emitBare(`\nTotal: ${total}`); - emitRequestId(response.request_id, settings.quiet); + emitResult({ items, total, request_id: response.request_id }, "json"); }, }); diff --git a/packages/commands/src/commands/deploy/pause.ts b/packages/commands/src/commands/deploy/pause.ts new file mode 100644 index 00000000..23d36c69 --- /dev/null +++ b/packages/commands/src/commands/deploy/pause.ts @@ -0,0 +1,85 @@ +import { + defineCommand, + stopModelService, + listIndependentDeployedModels, + findDeploymentEntry, + BailianError, + ExitCode, + type FlagsDef, +} from "bailian-cli-core"; +import { emitResult, emitBare } from "bailian-cli-runtime"; + +const PAUSE_FLAGS = { + deployedModel: { + type: "string", + valueHint: "", + description: "Deployed model identifier (required)", + required: true, + }, + skipPrecheck: { + type: "switch", + description: "Skip the local RUNNING/PENDING status precheck", + }, +} satisfies FlagsDef; + +/** + * `bl deploy pause` — pause a running deployment. + * + * Takes the model service offline so it no longer serves inference requests. + * For mu/ptu plans, billing stops while paused. + * Precheck: status must be RUNNING or PENDING. + */ +export default defineCommand({ + description: "Pause a running model deployment (stops billing for mu/ptu)", + auth: "console", + usageArgs: "--deployed-model [--skip-precheck]", + flags: PAUSE_FLAGS, + exampleArgs: [ + "--deployed-model dep-...", + "--deployed-model dep-... --skip-precheck", + "--deployed-model dep-... --dry-run", + ], + notes: [ + "While paused, billing ceases for mu/ptu plans. Use `deploy resume` to bring it back online or `deploy delete` to remove.", + "Precheck verifies status is RUNNING/PENDING before issuing the pause; pass --skip-precheck to bypass.", + ], + async run(ctx) { + const { settings, flags } = ctx; + const deployedModel = flags.deployedModel; + + if (settings.dryRun) { + emitResult({ action: "deploy.pause", deployed_model: deployedModel }, "json"); + return; + } + + // Precheck: verify the deployment is in a pausable state. + if (!flags.skipPrecheck) { + try { + const entries = await listIndependentDeployedModels(ctx.client); + const entry = findDeploymentEntry(entries, deployedModel); + if (entry) { + const status = (entry.status ?? "").toUpperCase(); + if (status && status !== "RUNNING" && status !== "PENDING") { + throw new BailianError( + `Deployment ${deployedModel} is ${status}. Only RUNNING / PENDING deployments can be paused. ` + + `Pass --skip-precheck to attempt the pause anyway.`, + ExitCode.USAGE, + ); + } + } + // If entry not found in list, proceed — the server will surface the real error. + } catch (error) { + if (error instanceof BailianError) throw error; + // If the list call itself failed, proceed and let the API call surface the error. + } + } + + const response = await stopModelService(ctx.client, deployedModel); + + if (settings.quiet) { + emitBare(deployedModel); + } else { + emitResult({ deployed_model: deployedModel, action: "pause", ...response }, "json"); + } + }, +}); diff --git a/packages/commands/src/commands/deploy/resume.ts b/packages/commands/src/commands/deploy/resume.ts new file mode 100644 index 00000000..7e0cb287 --- /dev/null +++ b/packages/commands/src/commands/deploy/resume.ts @@ -0,0 +1,84 @@ +import { + defineCommand, + startModelService, + listIndependentDeployedModels, + findDeploymentEntry, + BailianError, + ExitCode, + type FlagsDef, +} from "bailian-cli-core"; +import { emitResult, emitBare } from "bailian-cli-runtime"; + +const RESUME_FLAGS = { + deployedModel: { + type: "string", + valueHint: "", + description: "Deployed model identifier (required)", + required: true, + }, + skipPrecheck: { + type: "switch", + description: "Skip the local STOPPED status precheck", + }, +} satisfies FlagsDef; + +/** + * `bl deploy resume` — resume a paused deployment. + * + * Brings the model service back online so it can serve inference requests. + * Precheck: status must be STOPPED. + */ +export default defineCommand({ + description: "Resume a paused model deployment (brings service back online)", + auth: "console", + usageArgs: "--deployed-model [--skip-precheck]", + flags: RESUME_FLAGS, + exampleArgs: [ + "--deployed-model dep-...", + "--deployed-model dep-... --skip-precheck", + "--deployed-model dep-... --dry-run", + ], + notes: [ + "Precheck verifies status is STOPPED before issuing the resume; pass --skip-precheck to bypass.", + "For mu/ptu plans, billing resumes once the service is back online.", + ], + async run(ctx) { + const { settings, flags } = ctx; + const deployedModel = flags.deployedModel; + + if (settings.dryRun) { + emitResult({ action: "deploy.resume", deployed_model: deployedModel }, "json"); + return; + } + + // Precheck: verify the deployment is in a resumable state. + if (!flags.skipPrecheck) { + try { + const entries = await listIndependentDeployedModels(ctx.client); + const entry = findDeploymentEntry(entries, deployedModel); + if (entry) { + const status = (entry.status ?? "").toUpperCase(); + if (status && status !== "STOPPED") { + throw new BailianError( + `Deployment ${deployedModel} is ${status}. Only STOPPED deployments can be resumed. ` + + `Pass --skip-precheck to attempt the resume anyway.`, + ExitCode.USAGE, + ); + } + } + // If entry not found in list, proceed — the server will surface the real error. + } catch (error) { + if (error instanceof BailianError) throw error; + // If the list call itself failed, proceed and let the API call surface the error. + } + } + + const response = await startModelService(ctx.client, deployedModel); + + if (settings.quiet) { + emitBare(deployedModel); + } else { + emitResult({ deployed_model: deployedModel, action: "resume", ...response }, "json"); + } + }, +}); diff --git a/packages/commands/src/commands/deploy/scale.ts b/packages/commands/src/commands/deploy/scale.ts index 805f9c57..c6400a4a 100644 --- a/packages/commands/src/commands/deploy/scale.ts +++ b/packages/commands/src/commands/deploy/scale.ts @@ -1,10 +1,5 @@ -import { - defineCommand, - detectOutputFormat, - scaleDeployment, - type FlagsDef, -} from "bailian-cli-core"; -import { emitResult, emitBare, emitRequestId } from "bailian-cli-runtime"; +import { defineCommand, scaleDeployment, type FlagsDef } from "bailian-cli-core"; +import { emitResult, emitBare } from "bailian-cli-runtime"; const SCALE_FLAGS = { deployedModel: { @@ -52,7 +47,6 @@ export default defineCommand({ async run(ctx) { const { settings, flags } = ctx; const deployedModel = flags.deployedModel; - const format = detectOutputFormat(settings.output); const body: Record = {}; if (flags.capacity !== undefined) body.capacity = flags.capacity; @@ -60,21 +54,16 @@ export default defineCommand({ if (flags.outputTpm !== undefined) body.output_tpm = flags.outputTpm; if (settings.dryRun) { - emitResult({ action: "deploy.scale", deployed_model: deployedModel, body }, format); + emitResult({ action: "deploy.scale", deployed_model: deployedModel, body }, "json"); return; } const response = await scaleDeployment(ctx.client, deployedModel, body); - const deployment = response.output ?? response.data; if (settings.quiet) { emitBare(deployedModel); - } else if (format === "text") { - const cap = deployment?.capacity !== undefined ? ` (capacity=${deployment.capacity})` : ""; - emitBare(`Scaled ${deployedModel}${cap}.`); - emitRequestId(response.request_id, settings.quiet); } else { - emitResult(response, format); + emitResult(response, "json"); } }, }); diff --git a/packages/commands/src/commands/deploy/update.ts b/packages/commands/src/commands/deploy/update.ts index 0bbb29d2..6c3ab896 100644 --- a/packages/commands/src/commands/deploy/update.ts +++ b/packages/commands/src/commands/deploy/update.ts @@ -1,10 +1,5 @@ -import { - defineCommand, - detectOutputFormat, - updateDeployment, - type FlagsDef, -} from "bailian-cli-core"; -import { emitResult, emitBare, emitRequestId } from "bailian-cli-runtime"; +import { defineCommand, updateDeployment, type FlagsDef } from "bailian-cli-core"; +import { emitResult, emitBare } from "bailian-cli-runtime"; const UPDATE_FLAGS = { deployedModel: { @@ -48,31 +43,22 @@ export default defineCommand({ async run(ctx) { const { settings, flags } = ctx; const deployedModel = flags.deployedModel; - const format = detectOutputFormat(settings.output); const body: Record = {}; if (flags.rpmLimit !== undefined) body.rpm_limit = flags.rpmLimit; if (flags.tpmLimit !== undefined) body.tpm_limit = flags.tpmLimit; if (settings.dryRun) { - emitResult({ action: "deploy.update", deployed_model: deployedModel, body }, format); + emitResult({ action: "deploy.update", deployed_model: deployedModel, body }, "json"); return; } const response = await updateDeployment(ctx.client, deployedModel, body); - const deployment = response.output ?? response.data; if (settings.quiet) { emitBare(deployedModel); - } else if (format === "text") { - const parts: string[] = []; - if (deployment?.rpm_limit !== undefined) parts.push(`rpm_limit=${deployment.rpm_limit}`); - if (deployment?.tpm_limit !== undefined) parts.push(`tpm_limit=${deployment.tpm_limit}`); - const summary = parts.length ? ` (${parts.join(", ")})` : ""; - emitBare(`Updated ${deployedModel}${summary}.`); - emitRequestId(response.request_id, settings.quiet); } else { - emitResult(response, format); + emitResult(response, "json"); } }, }); diff --git a/packages/commands/src/commands/finetune/cancel.ts b/packages/commands/src/commands/finetune/cancel.ts index 789a3b03..966ce3e7 100644 --- a/packages/commands/src/commands/finetune/cancel.ts +++ b/packages/commands/src/commands/finetune/cancel.ts @@ -1,5 +1,5 @@ -import { defineCommand, detectOutputFormat, cancelFineTune, type FlagsDef } from "bailian-cli-core"; -import { emitResult, emitBare, emitRequestId } from "bailian-cli-runtime"; +import { defineCommand, cancelFineTune, type FlagsDef } from "bailian-cli-core"; +import { emitResult, emitBare } from "bailian-cli-runtime"; const CANCEL_FLAGS = { jobId: { @@ -23,24 +23,18 @@ export default defineCommand({ async run(ctx) { const { settings, flags } = ctx; const jobId = flags.jobId; - const format = detectOutputFormat(settings.output); if (settings.dryRun) { - emitResult({ action: "finetune.cancel", job_id: jobId }, format); + emitResult({ action: "finetune.cancel", job_id: jobId }, "json"); return; } const response = await cancelFineTune(ctx.client, jobId); - const job = response.output ?? response.data; if (settings.quiet) { emitBare(jobId); - } else if (format === "text") { - const status = job?.status ? ` (status=${job.status})` : ""; - emitBare(`Cancelled ${jobId}${status}.`); - emitRequestId(response.request_id, settings.quiet); } else { - emitResult(response, format); + emitResult(response, "json"); } }, }); diff --git a/packages/commands/src/commands/finetune/capability.ts b/packages/commands/src/commands/finetune/capability.ts index a92e6ee0..3ba704ae 100644 --- a/packages/commands/src/commands/finetune/capability.ts +++ b/packages/commands/src/commands/finetune/capability.ts @@ -1,6 +1,5 @@ import { defineCommand, - detectOutputFormat, fetchModelListAll, fetchModelCapability, listSupportedTrainingTypes, @@ -27,19 +26,8 @@ async function fetchAllFoundationModels(settings: Settings): Promise = { - full: "full-parameter", - lora: "LoRA", -}; - -function describeTrainingType(value: string): string { - if (!isTrainingTypeCli(value)) return value; - const { method, variant } = trainingTypeMethodVariant(value); - return `${VARIANT_LABEL[variant] ?? variant} ${method.toUpperCase()}`; -} - const CAPABILITY_FLAGS = { - model: { + baseModel: { type: "string", valueHint: "", description: "List training types supported by this base model.", @@ -55,31 +43,31 @@ export default defineCommand({ description: "Query fine-tune training capability — by model (which training types it supports) or by training type (which models support it)", auth: "none", - usageArgs: "--model | --training-type ", + usageArgs: "--base-model | --training-type ", flags: CAPABILITY_FLAGS, exampleArgs: [ - "--model qwen3-8b", + "--base-model qwen3-8b", "--training-type sft-lora", "--training-type cpt --output json", "--training-type sft --quiet", ], notes: [ - "Exactly one of --model / --training-type is required.", + "Exactly one of --base-model / --training-type is required.", "Training-type values use the `` / `-lora` convention:", "sft | sft-lora | dpo | dpo-lora | cpt. (cpt has no -lora variant server-side.)", "Queries listFoundationModels, a public API — no console login needed.", ], validate: (f) => { - if (f.model && f.trainingType) - return "--model and --training-type are mutually exclusive; pass one."; - if (!f.model && !f.trainingType) return "one of --model / --training-type is required."; + if (f.baseModel && f.trainingType) + return "--base-model and --training-type are mutually exclusive; pass one."; + if (!f.baseModel && !f.trainingType) + return "one of --base-model / --training-type is required."; return undefined; }, async run(ctx) { const { settings, flags } = ctx; - const model = flags.model || undefined; + const model = flags.baseModel || undefined; const trainingType = flags.trainingType || undefined; - const format = detectOutputFormat(settings.output); if (settings.dryRun) { emitResult( @@ -88,7 +76,7 @@ export default defineCommand({ model, training_type: trainingType, }, - format, + "json", ); return; } @@ -97,7 +85,7 @@ export default defineCommand({ if (model) { const capability = await fetchModelCapability(settings, model); if (!capability) { - emitBare(`No foundation model found matching "${model}".`); + emitResult({ model, error: `No foundation model found matching "${model}".` }, "json"); return; } const supported = listSupportedTrainingTypes(capability); @@ -105,23 +93,15 @@ export default defineCommand({ for (const value of supported) emitBare(value); return; } - if (format !== "text") { - emitResult( - { - model: capability.model ?? model, - supported, - supports: capability.supports, - trainingTypes: capability.trainingTypes, - }, - format, - ); - return; - } - emitBare(`${capability.model ?? model}`); - emitBare(supported.length ? "Supported training types:" : "No supported training types."); - for (const value of supported) { - emitBare(` ${value.padEnd(10)} ${describeTrainingType(value)}`); - } + emitResult( + { + model: capability.model ?? model, + supported, + supports: capability.supports, + trainingTypes: capability.trainingTypes, + }, + "json", + ); return; } @@ -146,20 +126,15 @@ export default defineCommand({ for (const entry of matched) emitBare(entry.model); return; } - if (format !== "text") { - emitResult( - { - training_type: trainingType, - method, - variant, - count: matched.length, - models: matched, - }, - format, - ); - return; - } - emitBare(`Models supporting ${trainingType} (${method} / ${variant}): ${matched.length}`); - for (const entry of matched) emitBare(` ${entry.model}`); + emitResult( + { + training_type: trainingType, + method, + variant, + count: matched.length, + models: matched, + }, + "json", + ); }, }); diff --git a/packages/commands/src/commands/finetune/checkpoints.ts b/packages/commands/src/commands/finetune/checkpoints.ts index 7c05a363..190110d6 100644 --- a/packages/commands/src/commands/finetune/checkpoints.ts +++ b/packages/commands/src/commands/finetune/checkpoints.ts @@ -1,10 +1,5 @@ -import { - defineCommand, - detectOutputFormat, - listCheckpoints, - type FlagsDef, -} from "bailian-cli-core"; -import { emitResult, emitBare, emitRequestId, formatTable } from "bailian-cli-runtime"; +import { defineCommand, listCheckpoints, type FlagsDef } from "bailian-cli-core"; +import { emitResult } from "bailian-cli-runtime"; const CHECKPOINTS_FLAGS = { jobId: { @@ -15,6 +10,8 @@ const CHECKPOINTS_FLAGS = { }, } satisfies FlagsDef; +const EXPIRY_WARN_THRESHOLD_MS = 72 * 60 * 60 * 1000; // 72 hours + export default defineCommand({ description: "List checkpoints produced by a fine-tune job", auth: "apiKey", @@ -22,16 +19,15 @@ export default defineCommand({ flags: CHECKPOINTS_FLAGS, exampleArgs: ["--job-id ft-xxx", "--job-id ft-xxx --output json"], notes: [ - "Use the returned `checkpoint` value with `finetune export` to publish", - "a deployable model.", + "`model_name` (shown for SUCCEEDED checkpoints) is the direct input for `deploy create --model-name`.", + "Checkpoints expire ~15 days after creation; `expire_time` shows the deadline. Export or deploy before expiry.", ], async run(ctx) { const { settings, flags } = ctx; const jobId = flags.jobId; - const format = detectOutputFormat(settings.output); if (settings.dryRun) { - emitResult({ action: "finetune.checkpoints", job_id: jobId }, format); + emitResult({ action: "finetune.checkpoints", job_id: jobId }, "json"); return; } @@ -44,22 +40,26 @@ export default defineCommand({ checkpoint: item.checkpoint ?? item.checkpoint_id ?? "", step: item.step !== undefined ? String(item.step) : "", status: item.status ?? "", + model_name: item.model_name ?? "", + expire_time: item.expire_time ?? "", })); - if (format === "json") { - emitResult({ items, total, request_id: response.request_id }, format); - return; - } + emitResult({ items, total, request_id: response.request_id }, "json"); - // text / quiet - if (items.length === 0) { - emitBare("No checkpoints found."); - return; + // Near-expiry warning: check if any non-expired checkpoint is within 72h of expiry. + const now = Date.now(); + const expiringSoon = items.filter((item) => { + if (!item.expire_time) return false; + const deadline = new Date(item.expire_time).getTime(); + if (Number.isNaN(deadline)) return false; + const remaining = deadline - now; + return remaining > 0 && remaining < EXPIRY_WARN_THRESHOLD_MS; + }); + if (expiringSoon.length > 0) { + process.stderr.write( + `\n[warning] ${expiringSoon.length} checkpoint(s) will expire within 72 hours. ` + + "Export or deploy before expiry to avoid losing the model artifact.\n", + ); } - const headers = ["CHECKPOINT", "STEP", "STATUS"]; - const rows = items.map((i) => [i.checkpoint, i.step, i.status]); - for (const line of formatTable(headers, rows)) emitBare(line); - emitBare(`\nTotal: ${total}`); - emitRequestId(response.request_id, settings.quiet); }, }); diff --git a/packages/commands/src/commands/finetune/create.ts b/packages/commands/src/commands/finetune/create.ts index 6fbb54f0..834168c0 100644 --- a/packages/commands/src/commands/finetune/create.ts +++ b/packages/commands/src/commands/finetune/create.ts @@ -1,6 +1,5 @@ import { defineCommand, - detectOutputFormat, createFineTune, getDataset, uploadDataset, @@ -27,7 +26,7 @@ import { } from "bailian-cli-core"; import { existsSync, statSync } from "fs"; import { basename } from "path"; -import { emitResult, emitBare, emitRequestId } from "bailian-cli-runtime"; +import { emitResult, emitBare } from "bailian-cli-runtime"; /** * A `--datasets` / `--validations` token is treated as a local file to upload @@ -208,7 +207,7 @@ async function uploadResolvedLocal( } /** The modality a `finetune create` subcommand is bound to. */ -type CommandModality = "text" | "audio" | "image"; +type CommandModality = "text" | "audio" | "image" | "video"; /** * Flags shared by every `finetune create` subcommand: what to train @@ -216,10 +215,10 @@ type CommandModality = "text" | "audio" | "image"; * output. Every modality's model consumes these. */ const COMMON_FLAGS = { - model: { + baseModel: { type: "string", valueHint: "", - description: "Base model to fine-tune", + description: "Base model to fine-tune (e.g. qwen3-8b; not the output model name)", required: true, }, datasets: { @@ -317,13 +316,41 @@ const IMAGE_FLAGS = { } satisfies FlagsDef; const TEXT_USAGE = - "--model --datasets [--validations ] [--model-name ] [--suffix ] [--n-epochs ] [--batch-size ] [--learning-rate ] [--max-length ] [--training-type ]"; + "--base-model --datasets [--validations ] [--model-name ] [--suffix ] [--n-epochs ] [--batch-size ] [--learning-rate ] [--max-length ] [--training-type ]"; const AUDIO_USAGE = - "--model --datasets [--validations ] [--model-name ] [--suffix ]"; + "--base-model --datasets [--validations ] [--model-name ] [--suffix ]"; const IMAGE_USAGE = - "--model --datasets [--validations ] [--model-name ] [--suffix ] [--generation-type ] [--learning-rate ]"; + "--base-model --datasets [--validations ] [--model-name ] [--suffix ] [--generation-type ] [--learning-rate ]"; + +/** + * Video (Wan i2v/kf2v) flags: exposes the three hyper-parameters that the + * video API supports and users may want to override. Defaults are model-specific + * (resolved by the sft-lora profile: wan2.7 → batch_size 1 / max_pixels 102400, + * wan2.5 → 4 / 36864, wan2.2 → 4 / 262144). + */ +const VIDEO_FLAGS = { + ...COMMON_FLAGS, + nEpochs: { + type: "number", + valueHint: "", + description: "Training epochs (default: 50)", + }, + batchSize: { + type: "number", + valueHint: "", + description: "Batch size (default: model-specific, 1 for wan2.7, 4 for wan2.5/2.2)", + }, + learningRate: { + type: "string", + valueHint: "", + description: 'Learning rate as a string to preserve precision (default: "2e-5")', + }, +} satisfies FlagsDef; + +const VIDEO_USAGE = + "--base-model --datasets [--validations ] [--model-name ] [--suffix ] [--n-epochs ] [--batch-size ] [--learning-rate ]"; const COMMON_NOTES = [ "Creating a job uploads any local datasets and consumes training quota.", @@ -383,7 +410,7 @@ async function runCreate( ): Promise { const { identity, settings } = ctx; const flags = ctx.flags as Record; - const model = flags.model as string; + const model = flags.baseModel as string; const datasetsRaw = flags.datasets as string; // CosyVoice audio fine-tuning accepts exactly one training file @@ -441,6 +468,10 @@ async function runCreate( if (detected === "image-i2i") modality = "image-i2i"; } } + if (commandModality === "video" && firstLocalPath && !settings.dryRun) { + const detected = await detectModality(firstLocalPath); + if (detected === "video-kf2v") modality = "video-kf2v"; + } const training = await analyzeDatasetTokens( settings, @@ -606,8 +637,6 @@ async function runCreate( if (modelName) body.model_name = modelName; if (suffix) body.finetuned_output_suffix = suffix; - const format = detectOutputFormat(settings.output); - if (settings.dryRun) { const pending = [ ...training.localPaths.map((path) => ({ field: "datasets", path })), @@ -617,7 +646,7 @@ async function runCreate( pending.length > 0 ? { action: "finetune.create", body, pending_uploads: pending } : { action: "finetune.create", body }, - format, + "json", ); return; } @@ -627,16 +656,8 @@ async function runCreate( if (settings.quiet) { if (job?.job_id) emitBare(job.job_id); - } else if (format === "text") { - if (job?.job_id) { - emitBare(`Created fine-tune job: ${job.job_id}`); - if (job.status) emitBare(`Status: ${job.status}`); - emitRequestId(response.request_id, settings.quiet); - } else { - emitResult(response, format); - } } else { - emitResult(response, format); + emitResult(response, "json"); } } @@ -647,14 +668,14 @@ export const finetuneTextCreate = defineCommand({ usageArgs: TEXT_USAGE, flags: TEXT_FLAGS, exampleArgs: [ - "--model qwen3-8b --datasets file-xxx", - "--model qwen3-8b --datasets ./train.jsonl", - "--model qwen3-8b --datasets ./train.jsonl --validations ./eval.jsonl", - "--model qwen3-8b --datasets file-aaa,./extra.jsonl", - "--model qwen3-8b --datasets ./train.jsonl --training-type sft", - '--model qwen3-8b --datasets file-xxx --learning-rate "1.6e-5" --n-epochs 4', - "--model qwen3-8b --datasets file-xxx --output json", - "--model qwen3-8b --datasets file-xxx --dry-run", + "--base-model qwen3-8b --datasets file-xxx", + "--base-model qwen3-8b --datasets ./train.jsonl", + "--base-model qwen3-8b --datasets ./train.jsonl --validations ./eval.jsonl", + "--base-model qwen3-8b --datasets file-aaa,./extra.jsonl", + "--base-model qwen3-8b --datasets ./train.jsonl --training-type sft", + '--base-model qwen3-8b --datasets file-xxx --learning-rate "1.6e-5" --n-epochs 4', + "--base-model qwen3-8b --datasets file-xxx --output json", + "--base-model qwen3-8b --datasets file-xxx --dry-run", ], notes: TEXT_NOTES, run: (ctx) => runCreate("text", ctx), @@ -667,11 +688,11 @@ export const finetuneAudioCreate = defineCommand({ usageArgs: AUDIO_USAGE, flags: AUDIO_FLAGS, exampleArgs: [ - "--model cosyvoice-v3-flash --datasets ./audio.zip", - "--model cosyvoice-v3-flash --datasets file-xxx", - "--model cosyvoice-v3-flash --datasets ./audio.zip --model-name my-tts", - "--model cosyvoice-v3-flash --datasets file-xxx --output json", - "--model cosyvoice-v3-flash --datasets ./audio.zip --dry-run", + "--base-model cosyvoice-v3-flash --datasets ./audio.zip", + "--base-model cosyvoice-v3-flash --datasets file-xxx", + "--base-model cosyvoice-v3-flash --datasets ./audio.zip --model-name my-tts", + "--base-model cosyvoice-v3-flash --datasets file-xxx --output json", + "--base-model cosyvoice-v3-flash --datasets ./audio.zip --dry-run", ], notes: AUDIO_NOTES, run: (ctx) => runCreate("audio", ctx), @@ -684,13 +705,38 @@ export const finetuneImageCreate = defineCommand({ usageArgs: IMAGE_USAGE, flags: IMAGE_FLAGS, exampleArgs: [ - "--model wan2.7-image-pro --datasets ./images.zip", - "--model wan2.7-image-pro --datasets file-xxx", - "--model wan2.7-image-pro --datasets file-xxx --generation-type i2i", - "--model wan2.7-image-pro --datasets ./images.zip --model-name my-wan", - "--model wan2.7-image-pro --datasets file-xxx --output json", - "--model wan2.7-image-pro --datasets ./images.zip --dry-run", + "--base-model wan2.7-image-pro --datasets ./images.zip", + "--base-model wan2.7-image-pro --datasets file-xxx", + "--base-model wan2.7-image-pro --datasets file-xxx --generation-type i2i", + "--base-model wan2.7-image-pro --datasets ./images.zip --model-name my-wan", + "--base-model wan2.7-image-pro --datasets file-xxx --output json", + "--base-model wan2.7-image-pro --datasets ./images.zip --dry-run", ], notes: IMAGE_NOTES, run: (ctx) => runCreate("image", ctx), }); + +const VIDEO_NOTES = [ + ...COMMON_NOTES, + "Video generation training (Wan i2v/kf2v) runs efficient_sft with model-", + "specific defaults: wan2.7 (batch_size=1, max_pixels=102400), wan2.5/2.2", + "(batch_size=4, max_pixels per model). Override with --batch-size/--n-epochs.", + "Datasets are .zip archives with data.jsonl + frame images + videos.", + "Recommended: ≥10 training samples, 20-100 for stable results.", +]; + +/** `bl finetune video create` — fine-tune a video generation model. Datasets are `.zip`. */ +export const finetuneVideoCreate = defineCommand({ + description: "Create a video generation model fine-tune job (Wan i2v/kf2v, efficient_sft)", + auth: "apiKey", + usageArgs: VIDEO_USAGE, + flags: VIDEO_FLAGS, + exampleArgs: [ + "--base-model wan2.7-i2v --datasets file-xxx", + "--base-model wan2.7-i2v --datasets ./i2v-data.zip", + "--base-model wan2.2-kf2v-flash --datasets file-xxx --n-epochs 100", + "--base-model wan2.7-i2v --datasets file-xxx --dry-run", + ], + notes: VIDEO_NOTES, + run: (ctx) => runCreate("video", ctx), +}); diff --git a/packages/commands/src/commands/finetune/delete.ts b/packages/commands/src/commands/finetune/delete.ts index 102fda61..f18a13f7 100644 --- a/packages/commands/src/commands/finetune/delete.ts +++ b/packages/commands/src/commands/finetune/delete.ts @@ -1,5 +1,5 @@ -import { defineCommand, detectOutputFormat, deleteFineTune, type FlagsDef } from "bailian-cli-core"; -import { emitResult, emitBare, emitRequestId } from "bailian-cli-runtime"; +import { defineCommand, deleteFineTune, type FlagsDef } from "bailian-cli-core"; +import { emitResult, emitBare } from "bailian-cli-runtime"; const DELETE_FLAGS = { jobId: { @@ -23,10 +23,9 @@ export default defineCommand({ async run(ctx) { const { settings, flags } = ctx; const jobId = flags.jobId; - const format = detectOutputFormat(settings.output); if (settings.dryRun) { - emitResult({ action: "finetune.delete", job_id: jobId }, format); + emitResult({ action: "finetune.delete", job_id: jobId }, "json"); return; } @@ -34,11 +33,8 @@ export default defineCommand({ if (settings.quiet) { emitBare(jobId); - } else if (format === "text") { - emitBare(`Deleted ${jobId}.`); - emitRequestId(response.request_id, settings.quiet); } else { - emitResult(response, format); + emitResult(response, "json"); } }, }); diff --git a/packages/commands/src/commands/finetune/export.ts b/packages/commands/src/commands/finetune/export.ts index 7e52de0b..97bb0aa7 100644 --- a/packages/commands/src/commands/finetune/export.ts +++ b/packages/commands/src/commands/finetune/export.ts @@ -1,10 +1,5 @@ -import { - defineCommand, - detectOutputFormat, - exportCheckpoint, - type FlagsDef, -} from "bailian-cli-core"; -import { emitResult, emitBare, emitRequestId } from "bailian-cli-runtime"; +import { defineCommand, exportCheckpoint, type FlagsDef } from "bailian-cli-core"; +import { emitResult, emitBare } from "bailian-cli-runtime"; const EXPORT_FLAGS = { jobId: { @@ -39,11 +34,10 @@ export default defineCommand({ "explicit export is the canonical path for non-best checkpoints.", ], async run(ctx) { - const { identity, settings, flags } = ctx; + const { settings, flags } = ctx; const jobId = flags.jobId; const checkpoint = flags.checkpoint; const modelName = flags.modelName; - const format = detectOutputFormat(settings.output); if (settings.dryRun) { emitResult( @@ -53,7 +47,7 @@ export default defineCommand({ checkpoint, model_name: modelName, }, - format, + "json", ); return; } @@ -64,14 +58,8 @@ export default defineCommand({ if (settings.quiet) { emitBare(exported); - } else if (format === "text") { - emitBare(`Exported ${jobId} / ${checkpoint} → model_name=${exported}`); - emitBare( - `Next: ${identity.binName} deploy text create --model ${exported} --name `, - ); - emitRequestId(response.request_id, settings.quiet); } else { - emitResult(response, format); + emitResult(response, "json"); } }, }); diff --git a/packages/commands/src/commands/finetune/fee.ts b/packages/commands/src/commands/finetune/fee.ts new file mode 100644 index 00000000..f1325686 --- /dev/null +++ b/packages/commands/src/commands/finetune/fee.ts @@ -0,0 +1,105 @@ +/** + * Best-effort actual training fee calculation using the model catalog's + * "ft" (fine-tune) price entry. Pure API-key domain — no console auth needed. + * + * The model catalog (`listFoundationModels` via public gateway) returns a + * `prices[]` array **only when `queryPrice: true` is passed** (the same flag + * `fetchModelDetail` uses). Combined with the job's `output.usage` (actual + * consumed tokens, present on SUCCEEDED / CANCELED), this gives the exact + * training cost without any console-domain login. + */ +import { + callConsoleGateway, + effectiveConsoleGatewayConfig, + unwrapResponse, + MODEL_LIST_API, + type Settings, + type ModelPriceInfo, +} from "bailian-cli-core"; + +export interface ActualFee { + cost: number; + unitPrice: number; + priceUnit: string; +} + +/** + * Fetch the model's training price from the public catalog gateway. + * Uses the same anonymous gateway path as `fetchModelCapability` (no console + * token required), but adds `queryPrice: true` to include the prices array. + */ +async function fetchTrainingPrice( + settings: Settings, + model: string, +): Promise { + const eff = effectiveConsoleGatewayConfig(settings); + const result = await callConsoleGateway( + { region: eff.consoleRegion, site: eff.consoleSite, switchAgent: eff.consoleSwitchAgent }, + settings.timeout, + { + api: MODEL_LIST_API, + data: { + input: { + pageNo: 1, + pageSize: 10, + group: true, + model, + queryPrice: true, + querySampleCode: false, + queryGroupByModel: true, + queryQuota: false, + queryQpmInfo: false, + queryApplyStatus: false, + queryPermissions: false, + queryActivationStatus: false, + }, + }, + }, + ); + const responseData = unwrapResponse(result as Record); + const list = (responseData.list as Record[]) ?? []; + // The response is grouped; find the exact model in items. + for (const group of list) { + const items = (group.items as Record[]) ?? []; + for (const item of items) { + if (item.model === model) { + const prices = (item.prices as ModelPriceInfo[]) ?? []; + return prices.find((entry) => entry.type === "ft") ?? null; + } + } + // Flat response fallback (no items nesting). + if (group.model === model) { + const prices = (group.prices as ModelPriceInfo[]) ?? []; + return prices.find((entry) => entry.type === "ft") ?? null; + } + } + return null; +} + +/** + * Compute the actual training fee from the model catalog's "ft" price entry. + * Returns null when the price is unavailable (network error, model not in + * catalog, or no "ft" entry). Never throws. + * + * Only uses the public model catalog (model metadata) — does NOT call + * console-domain pricing APIs (modelCenter.getModelPrice). Models whose + * catalog entry lacks a "ft" price (e.g. CosyVoice) will simply omit the + * training_cost field until the platform adds it to the catalog. + */ +export async function computeActualFee( + settings: Settings, + model: string, + usageTokens: number, +): Promise { + try { + const ftEntry = await fetchTrainingPrice(settings, model); + const unitPrice = Number(ftEntry?.price); + if (!Number.isFinite(unitPrice) || unitPrice <= 0) return null; + const priceUnit = ftEntry?.priceUnit ?? "每百万tokens"; + // Catalog price is yuan per million tokens. + const cost = (usageTokens / 1_000_000) * unitPrice; + return { cost: Number(cost.toFixed(4)), unitPrice, priceUnit }; + } catch { + return null; + } +} diff --git a/packages/commands/src/commands/finetune/get.ts b/packages/commands/src/commands/finetune/get.ts index 0ba01147..0ec4d5a9 100644 --- a/packages/commands/src/commands/finetune/get.ts +++ b/packages/commands/src/commands/finetune/get.ts @@ -1,5 +1,6 @@ -import { defineCommand, detectOutputFormat, getFineTune, type FlagsDef } from "bailian-cli-core"; -import { emitResult, emitBare, emitRequestId } from "bailian-cli-runtime"; +import { defineCommand, getFineTune, type FlagsDef } from "bailian-cli-core"; +import { emitResult } from "bailian-cli-runtime"; +import { computeActualFee } from "./fee.ts"; const GET_FLAGS = { jobId: { @@ -17,12 +18,11 @@ export default defineCommand({ flags: GET_FLAGS, exampleArgs: ["--job-id ft-xxx", "--job-id ft-xxx --output json"], async run(ctx) { - const { identity, settings, flags } = ctx; + const { settings, flags } = ctx; const jobId = flags.jobId; - const format = detectOutputFormat(settings.output); if (settings.dryRun) { - emitResult({ action: "finetune.get", job_id: jobId }, format); + emitResult({ action: "finetune.get", job_id: jobId }, "json"); return; } @@ -30,18 +30,24 @@ export default defineCommand({ const job = response.output ?? response.data; if (!job) { - emitBare(`No data returned for ${jobId}`); + emitResult({ job_id: jobId, error: "No data returned" }, "json"); return; } - const hp = job.hyper_parameters; + const hyperParameters = job.hyper_parameters; const hyperParts: string[] = []; - if (hp?.n_epochs !== undefined) hyperParts.push(`n_epochs=${hp.n_epochs}`); - if (hp?.batch_size !== undefined) hyperParts.push(`batch_size=${hp.batch_size}`); - if (hp?.learning_rate !== undefined) hyperParts.push(`learning_rate=${hp.learning_rate}`); - if (hp?.max_length !== undefined) hyperParts.push(`max_length=${hp.max_length}`); + if (hyperParameters?.n_epochs !== undefined) + hyperParts.push(`n_epochs=${hyperParameters.n_epochs}`); + if (hyperParameters?.batch_size !== undefined) + hyperParts.push(`batch_size=${hyperParameters.batch_size}`); + if (hyperParameters?.learning_rate !== undefined) + hyperParts.push(`learning_rate=${hyperParameters.learning_rate}`); + if (hyperParameters?.max_length !== undefined) + hyperParts.push(`max_length=${hyperParameters.max_length}`); - const item = { + const usageTokens = typeof job.usage === "number" ? job.usage : undefined; + + const item: Record = { job_id: job.job_id ?? jobId, base_model: job.model ?? "", status: job.status ?? "", @@ -53,29 +59,20 @@ export default defineCommand({ model_name: job.model_name ?? "", created_at: job.create_time ?? job.gmt_create ?? "", updated_at: job.end_time ?? job.gmt_modified ?? "", + usage_tokens: usageTokens ?? "", + charge_type: typeof job.charge_type === "string" ? job.charge_type : "", }; - if (format === "json") { - emitResult({ ...item, request_id: response.request_id }, format); - return; + // Actual fee: only when the platform reports a concrete token count + // (SUCCEEDED / CANCELED). Best-effort — silently omitted on lookup failure. + if (usageTokens !== undefined && usageTokens > 0 && job.model) { + const fee = await computeActualFee(settings, job.model, usageTokens); + if (fee) { + item.training_cost = fee.cost; + item.cost_basis = `${fee.unitPrice} 元/${fee.priceUnit}`; + } } - // text / quiet - emitBare(`job_id: ${item.job_id}`); - if (item.base_model) emitBare(`base_model: ${item.base_model}`); - if (item.status) emitBare(`status: ${item.status}`); - if (item.training_type) emitBare(`training_type: ${item.training_type}`); - if (item.training_files.length) emitBare(`training_files: ${item.training_files.join(", ")}`); - if (item.validation_files.length) - emitBare(`validation_files: ${item.validation_files.join(", ")}`); - if (item.hyper_params) emitBare(`hyper_params: ${item.hyper_params}`); - if (item.output_model) - emitBare( - `output_model: ${item.output_model} (→ ${identity.binName} deploy text create --model)`, - ); - if (item.model_name) emitBare(`model_name: ${item.model_name}`); - if (item.created_at) emitBare(`created_at: ${item.created_at}`); - if (item.updated_at) emitBare(`updated_at: ${item.updated_at}`); - emitRequestId(response.request_id, settings.quiet); + emitResult({ ...item, request_id: response.request_id }, "json"); }, }); diff --git a/packages/commands/src/commands/finetune/list.ts b/packages/commands/src/commands/finetune/list.ts index ca4edddf..d37fc5be 100644 --- a/packages/commands/src/commands/finetune/list.ts +++ b/packages/commands/src/commands/finetune/list.ts @@ -1,5 +1,5 @@ -import { defineCommand, detectOutputFormat, listFineTunes, type FlagsDef } from "bailian-cli-core"; -import { emitResult, emitBare, emitRequestId, formatTable } from "bailian-cli-runtime"; +import { defineCommand, listFineTunes, type FlagsDef } from "bailian-cli-core"; +import { emitResult } from "bailian-cli-runtime"; const LIST_FLAGS = { page: { type: "number", valueHint: "", description: "Page number (default: 1)" }, @@ -13,71 +13,48 @@ const LIST_FLAGS = { valueHint: "", description: "Filter by status (PENDING / RUNNING / SUCCEEDED / FAILED / CANCELED)", }, + baseModel: { + type: "string", + valueHint: "", + description: "Filter by base model ID (server-side)", + }, } satisfies FlagsDef; export default defineCommand({ description: "List fine-tune jobs", auth: "apiKey", - usageArgs: "[--page ] [--page-size ] [--status ]", + usageArgs: "[--page ] [--page-size ] [--status ] [--base-model ]", flags: LIST_FLAGS, - exampleArgs: ["", "--status RUNNING", "--page-size 20 --output json"], + exampleArgs: ["", "--status RUNNING", "--base-model qwen3-8b", "--page-size 20"], async run(ctx) { - const { identity, settings, flags } = ctx; - const format = detectOutputFormat(settings.output); + const { settings, flags } = ctx; const pageNo = flags.page; const pageSize = flags.pageSize; const status = flags.status || undefined; + const model = flags.baseModel || undefined; if (settings.dryRun) { - emitResult({ action: "finetune.list", page: pageNo, page_size: pageSize, status }, format); + emitResult( + { action: "finetune.list", page: pageNo, page_size: pageSize, status, model }, + "json", + ); return; } - const response = await listFineTunes(ctx.client, { pageNo, pageSize, status }); + const response = await listFineTunes(ctx.client, { pageNo, pageSize, status, model }); const payload = response.output ?? response.data; const jobs = payload?.jobs ?? []; const total = payload?.total; - const items = jobs.map((item) => ({ - job_id: item.job_id ?? "", - base_model: item.model ?? "", - status: item.status ?? "", - training_type: item.training_type ?? "", - output_model: item.finetuned_output ?? "", - created_at: item.create_time ?? item.gmt_create ?? "", + const items = jobs.map((job) => ({ + job_id: job.job_id ?? "", + base_model: job.model ?? "", + status: job.status ?? "", + training_type: job.training_type ?? "", + output_model: job.finetuned_output ?? "", + created_at: job.create_time ?? job.gmt_create ?? "", })); - if (format === "json") { - emitResult({ items, total, request_id: response.request_id }, format); - return; - } - - // text / quiet - if (items.length === 0) { - emitBare("No fine-tune jobs found."); - return; - } - const headers = [ - "JOB_ID", - "BASE_MODEL", - "STATUS", - "TRAINING_TYPE", - "OUTPUT_MODEL", - "CREATED_AT", - ]; - const rows = items.map((i) => [ - i.job_id, - i.base_model, - i.status, - i.training_type, - i.output_model, - i.created_at, - ]); - for (const line of formatTable(headers, rows)) emitBare(line); - if (total !== undefined) emitBare(`\nTotal: ${total}`); - emitBare( - `Tip: OUTPUT_MODEL is the input for \`${identity.binName} deploy text create --model\``, - ); - emitRequestId(response.request_id, settings.quiet); + emitResult({ items, total, request_id: response.request_id }, "json"); }, }); diff --git a/packages/commands/src/commands/finetune/logs.ts b/packages/commands/src/commands/finetune/logs.ts index bd0802ed..320c07cb 100644 --- a/packages/commands/src/commands/finetune/logs.ts +++ b/packages/commands/src/commands/finetune/logs.ts @@ -1,25 +1,24 @@ import { defineCommand, - detectOutputFormat, getFineTuneLogs, type Client, type FineTuneLogEntry, type FlagsDef, } from "bailian-cli-core"; -import { emitResult, emitBare, emitRequestId } from "bailian-cli-runtime"; +import { emitResult } from "bailian-cli-runtime"; /** - * Render a single log entry as a single line (mirrors the flatten logic used - * for non-search text output: prefer common fields, fall back to JSON). + * Render a single log entry as a single line (used for search matching: + * prefer common fields, fall back to JSON). */ function renderEntry(entry: FineTuneLogEntry | string): string { if (typeof entry === "string") return entry; const record = entry as Record; - const ts = (record.timestamp ?? record.time ?? record.create_time ?? "") as string; + const timestamp = (record.timestamp ?? record.time ?? record.create_time ?? "") as string; const level = (record.level ?? "") as string; - const msg = (record.message ?? record.msg ?? record.log ?? "") as string; - if (msg || ts || level) { - return [ts, level, msg].filter(Boolean).join("\t"); + const message = (record.message ?? record.msg ?? record.log ?? "") as string; + if (message || timestamp || level) { + return [timestamp, level, message].filter(Boolean).join("\t"); } return JSON.stringify(entry); } @@ -48,16 +47,16 @@ async function fetchAllLogs( let total = 0; // Hard cap to avoid an unbounded loop if the server misreports `total`. const maxPages = 200; - for (let i = 0; i < maxPages; i++) { + for (let page = 0; page < maxPages; page++) { const response = await getFineTuneLogs(client, jobId, { pageNo, pageSize }); const payload = response.output ?? response.data; - const page = payload?.logs ?? []; + const logs = payload?.logs ?? []; total = payload?.total ?? total; - if (page.length === 0) break; - entries.push(...page); + if (logs.length === 0) break; + entries.push(...logs); // Stop once we've collected everything the server claims exists. if (total && entries.length >= total) break; - if (page.length < pageSize) break; + if (logs.length < pageSize) break; pageNo++; } return { entries, total }; @@ -110,7 +109,6 @@ export default defineCommand({ const pageSize = flags.pageSize; const search = flags.search || undefined; const tail = flags.tail; - const format = detectOutputFormat(settings.output); if (settings.dryRun) { emitResult( @@ -122,7 +120,7 @@ export default defineCommand({ search, tail, }, - format, + "json", ); return; } @@ -147,18 +145,6 @@ export default defineCommand({ const result = tailApplied !== undefined ? scanned.slice(scanned.length - tailApplied) : scanned; - if (settings.quiet || format === "text") { - if (result.length === 0) { - emitBare(search ? `No logs matched "${search}".` : "No logs returned."); - return; - } - for (const entry of result) emitBare(renderEntry(entry)); - const parts: string[] = [`${result.length} shown`]; - if (matched !== undefined) parts.push(`matched ${matched}`); - parts.push(`of ${entries.length}` + (total ? ` (total ${total})` : "")); - emitBare(`\n${parts.join(", ")}`); - return; - } emitResult( { ...(matched !== undefined ? { matched } : {}), @@ -168,28 +154,13 @@ export default defineCommand({ ...(tailApplied !== undefined ? { tail: tailApplied } : {}), logs: result, }, - format, + "json", ); return; } // Default: single page, verbatim response. const response = await getFineTuneLogs(ctx.client, jobId, { pageNo, pageSize }); - const payload = response.output ?? response.data; - const logs = payload?.logs ?? []; - - if (settings.quiet || format === "text") { - if (logs.length === 0) { - emitBare("No logs returned."); - return; - } - for (const entry of logs) { - emitBare(renderEntry(entry)); - } - if (payload?.total !== undefined) emitBare(`\nTotal: ${payload.total}`); - emitRequestId(response.request_id, settings.quiet); - } else { - emitResult(response, format); - } + emitResult(response, "json"); }, }); diff --git a/packages/commands/src/commands/finetune/price.ts b/packages/commands/src/commands/finetune/price.ts new file mode 100644 index 00000000..32dd7562 --- /dev/null +++ b/packages/commands/src/commands/finetune/price.ts @@ -0,0 +1,139 @@ +import { + defineCommand, + fetchTrainingModelPrice, + estimateSftDpoTokens, + estimateCptTokens, + BailianError, + ExitCode, + type FlagsDef, +} from "bailian-cli-core"; +import { emitResult } from "bailian-cli-runtime"; + +const PRICE_FLAGS = { + baseModel: { + type: "string", + valueHint: "", + description: "Base model to fine-tune (e.g. qwen3-8b; not the output model name)", + required: true, + }, + datasets: { + type: "string", + valueHint: "", + description: "Training dataset file IDs, comma-separated (required)", + required: true, + }, + trainingType: { + type: "string", + valueHint: "", + description: "Training type: sft | dpo | cpt (default: sft)", + }, + nEpochs: { + type: "number", + valueHint: "", + description: "Number of training epochs (default: 3)", + }, +} satisfies FlagsDef; + +const SUPPORTED_TRAINING_TYPES = ["sft", "dpo", "cpt"]; + +// Fixed hyper-parameters used for estimation. Only n_epochs materially affects +// the estimate; the rest are held at representative defaults (not exposed as +// flags to keep the command surface minimal). +const ESTIMATE_BATCH_SIZE = 16; +const ESTIMATE_MAX_LENGTH = 8192; +const DEFAULT_N_EPOCHS = 3; + +export default defineCommand({ + description: "Estimate the training cost for a fine-tune job (token billing)", + auth: "console", + usageArgs: "--base-model --datasets [--training-type ] [--n-epochs ]", + flags: PRICE_FLAGS, + exampleArgs: [ + "--base-model qwen3-8b --datasets file-ft-xxx", + "--base-model qwen3-8b --datasets file-ft-xxx,file-ft-yyy --n-epochs 2", + "--base-model qwen3-8b --datasets file-ft-xxx --training-type cpt", + ], + notes: [ + "Estimate only — the server computes token usage from the datasets; final cost is subject to the bill.", + "Covers token billing for sft / dpo / cpt. Training-unit (MTU) billing is not supported by this command.", + "Hyper-parameters other than --n-epochs are fixed at representative defaults for estimation.", + ], + async run(ctx) { + const { settings, flags } = ctx; + const model = flags.baseModel; + const datasetIds = flags.datasets + .split(",") + .map((datasetId) => datasetId.trim()) + .filter(Boolean); + const trainingType = (flags.trainingType ?? "sft").toLowerCase(); + const nEpochs = flags.nEpochs ?? DEFAULT_N_EPOCHS; + + if (!SUPPORTED_TRAINING_TYPES.includes(trainingType)) { + throw new BailianError( + `Unsupported training type "${trainingType}". Supported: ${SUPPORTED_TRAINING_TYPES.join(", ")}.`, + ExitCode.USAGE, + ); + } + if (datasetIds.length === 0) { + throw new BailianError("--datasets must contain at least one file ID.", ExitCode.USAGE); + } + + if (settings.dryRun) { + emitResult( + { action: "finetune.price", model, datasets: datasetIds, trainingType, nEpochs }, + "json", + ); + return; + } + + // Unit price (yuan per 千Token). + const priceInfo = await fetchTrainingModelPrice(ctx.client, model); + const unitPrice = Number(priceInfo.price); + if (!Number.isFinite(unitPrice)) { + throw new BailianError( + `No training price found for model "${model}".`, + ExitCode.GENERAL, + undefined, + { rawResponse: JSON.stringify(priceInfo) }, + ); + } + + // Per-epoch token estimate (min/max range). + const estimate = + trainingType === "cpt" + ? await estimateCptTokens(ctx.client, model, datasetIds.join(","), nEpochs) + : await estimateSftDpoTokens(ctx.client, datasetIds, { + nEpochs, + batchSize: ESTIMATE_BATCH_SIZE, + maxLength: ESTIMATE_MAX_LENGTH, + }); + + const minPerEpoch = estimate.estimatedDatasetConsumedTokensMinPerEpoch ?? 0; + const maxPerEpoch = estimate.estimatedDatasetConsumedTokensMaxPerEpoch ?? 0; + const mixedMinPerEpoch = estimate.estimatedMixedConsumedTokensMinPerEpoch ?? 0; + const mixedMaxPerEpoch = estimate.estimatedMixedConsumedTokensMaxPerEpoch ?? 0; + + const minTokens = (minPerEpoch + mixedMinPerEpoch) * nEpochs; + const maxTokens = (maxPerEpoch + mixedMaxPerEpoch) * nEpochs; + // price is yuan per 1000 tokens. + const minFee = (minTokens / 1000) * unitPrice; + const maxFee = (maxTokens / 1000) * unitPrice; + + emitResult( + { + model, + training_type: trainingType, + n_epochs: nEpochs, + unit_price: unitPrice, + price_unit: priceInfo.priceUnit ?? "千Token", + estimated_tokens: { min: minTokens, max: maxTokens }, + estimated_fee_yuan: { + min: Number(minFee.toFixed(4)), + max: Number(maxFee.toFixed(4)), + }, + disclaimer: "Server-side estimate; final cost is subject to the bill.", + }, + "json", + ); + }, +}); diff --git a/packages/commands/src/commands/finetune/watch.ts b/packages/commands/src/commands/finetune/watch.ts index c02b2079..eb802a5d 100644 --- a/packages/commands/src/commands/finetune/watch.ts +++ b/packages/commands/src/commands/finetune/watch.ts @@ -1,12 +1,12 @@ import { defineCommand, - detectOutputFormat, getFineTune, BailianError, ExitCode, type FlagsDef, } from "bailian-cli-core"; -import { emitResult, emitBare, emitRequestId } from "bailian-cli-runtime"; +import { emitResult, emitBare } from "bailian-cli-runtime"; +import { computeActualFee } from "./fee.ts"; const DEFAULT_INTERVAL_SEC = 10; const MIN_INTERVAL_SEC = 1; @@ -103,7 +103,6 @@ export default defineCommand({ const follow = flags.follow; const intervalSec = Math.max(MIN_INTERVAL_SEC, flags.interval ?? DEFAULT_INTERVAL_SEC); const pollTimeoutSec = flags.pollTimeout; - const format = detectOutputFormat(settings.output); if (settings.dryRun) { emitResult( @@ -114,7 +113,7 @@ export default defineCommand({ interval: intervalSec, timeout: pollTimeoutSec, }, - format, + "json", ); return; } @@ -132,16 +131,24 @@ export default defineCommand({ if (settings.quiet) { // Just the status word — ideal for `status=$(... finetune watch ... --quiet)`. emitBare(status || "UNKNOWN"); - } else if (format === "text") { - emitBare(`${nowStamp()} ${jobId} ${status || "UNKNOWN"}`); - if (status === "SUCCEEDED") emitBare(`✓ ${jobId} ${status}`); - emitRequestId(response.request_id, settings.quiet); } else { - // json: a compact, purpose-built status probe. - emitResult( - { job_id: jobId, status: status || "UNKNOWN", terminal, request_id: response.request_id }, - format, - ); + const output: Record = { + job_id: jobId, + status: status || "UNKNOWN", + terminal, + request_id: response.request_id, + }; + // Enrich terminal output with actual fee when usage is reported. + const usageTokens = typeof job?.usage === "number" ? job.usage : undefined; + if (terminal && usageTokens && usageTokens > 0 && job?.model) { + output.usage_tokens = usageTokens; + const fee = await computeActualFee(settings, job.model as string, usageTokens); + if (fee) { + output.training_cost = fee.cost; + output.cost_basis = `${fee.unitPrice} 元/${fee.priceUnit}`; + } + } + emitResult(output, "json"); } if (terminal && status !== "SUCCEEDED") { @@ -168,18 +175,28 @@ export default defineCommand({ const job = response.output ?? response.data; const status = String(job?.status ?? "").toUpperCase(); - if (format === "text" && !settings.quiet && status !== lastStatus) { - emitBare(`${nowStamp()} ${jobId} ${status || "UNKNOWN"}`); + if (!settings.quiet && status !== lastStatus) { + process.stderr.write(`${nowStamp()} ${jobId} ${status || "UNKNOWN"}\n`); lastStatus = status; } if (TERMINAL_STATUSES.has(status)) { const elapsed = Date.now() - startedAt; - if (format !== "text" || settings.quiet) { - emitResult(response, format); - } else if (status === "SUCCEEDED") { - emitBare(`\n✓ ${jobId} ${status} (elapsed ${formatElapsed(elapsed)})`); - emitRequestId(response.request_id, settings.quiet); + if (settings.quiet) { + emitBare(status || "UNKNOWN"); + } else { + // Enrich the raw response with actual fee when usage is available. + const usageTokens = typeof job?.usage === "number" ? job.usage : undefined; + const enriched: Record = { ...response }; + if (usageTokens && usageTokens > 0 && job?.model) { + const fee = await computeActualFee(settings, job.model as string, usageTokens); + if (fee) { + enriched.training_cost = fee.cost; + enriched.usage_tokens = usageTokens; + enriched.cost_basis = `${fee.unitPrice} 元/${fee.priceUnit}`; + } + } + emitResult(enriched, "json"); } if (status !== "SUCCEEDED") { throw new BailianError( @@ -205,7 +222,7 @@ export default defineCommand({ // Any other error (including the BailianError thrown above) propagates to // the central handler. if (controller.signal.aborted) { - emitBare("\nInterrupted."); + process.stderr.write("\nInterrupted.\n"); return; } throw error; diff --git a/packages/commands/src/commands/video/generate.ts b/packages/commands/src/commands/video/generate.ts index be3f89af..c58cc4e9 100644 --- a/packages/commands/src/commands/video/generate.ts +++ b/packages/commands/src/commands/video/generate.ts @@ -1,6 +1,7 @@ import { defineCommand, videoGeneratePath, + image2videoPath, taskPath, detectOutputFormat, type DashScopeVideoRequest, @@ -43,6 +44,11 @@ export default defineCommand({ valueHint: "", description: "Input image URL for image-to-video generation", }, + lastFrame: { + type: "string", + valueHint: "", + description: "Last frame image URL (with --image, enables kf2v first+last frame mode)", + }, negativePrompt: { type: "string", valueHint: "", @@ -110,12 +116,20 @@ export default defineCommand({ const format = detectOutputFormat(settings.output); const imageUrl = flags.image; + const lastFrameUrl = flags.lastFrame as string | undefined; // Auto-upload local image file for i2v let resolvedImageUrl: string | undefined; if (imageUrl) { resolvedImageUrl = await ctx.client.resolveImageInput(imageUrl, model); } + let resolvedLastFrameUrl: string | undefined; + if (lastFrameUrl) { + resolvedLastFrameUrl = await ctx.client.resolveImageInput(lastFrameUrl, model); + } + + // kf2v mode: both --image and --last-frame provided. + const isKf2v = Boolean(resolvedImageUrl && resolvedLastFrameUrl); const watermark = resolveWatermark(flags.watermark); const promptExtend = resolveBooleanFlag(flags.promptExtend, undefined, "prompt-extend"); @@ -125,10 +139,16 @@ export default defineCommand({ input: { prompt: prompt, negative_prompt: flags.negativePrompt || undefined, - // i2v models (happyhorse-1.1-i2v) require input.media with type 'first_frame' - ...(resolvedImageUrl - ? { media: [{ type: "first_frame" as const, url: resolvedImageUrl }] } - : {}), + // kf2v: first+last frame flat fields via image2video endpoint. + // wan2.1~2.6 i2v: flat img_url via video-generation endpoint. + // wan2.7+ / happyhorse i2v: media[] via video-generation endpoint. + ...(isKf2v + ? { first_frame_url: resolvedImageUrl, last_frame_url: resolvedLastFrameUrl } + : resolvedImageUrl + ? /wan[x]?2\.[1-6]/i.test(model) + ? { img_url: resolvedImageUrl } + : { media: [{ type: "first_frame" as const, url: resolvedImageUrl }] } + : {}), }, parameters: { resolution: flags.resolution || undefined, @@ -141,15 +161,28 @@ export default defineCommand({ }; if (settings.dryRun) { - const previewBody = resolvedImageUrl - ? { - ...body, - input: { - ...body.input, - media: [{ type: "first_frame" as const, url: redactDataUri(resolvedImageUrl) }], - }, - } - : body; + let previewBody = body; + if (isKf2v) { + previewBody = { + ...body, + input: { + ...body.input, + first_frame_url: redactDataUri(resolvedImageUrl ?? ""), + last_frame_url: redactDataUri(resolvedLastFrameUrl ?? ""), + }, + }; + } else if (resolvedImageUrl) { + const redactedUrl = redactDataUri(resolvedImageUrl); + previewBody = { + ...body, + input: { + ...body.input, + ...(/wan[x]?2\.[1-6]/i.test(model) + ? { img_url: redactedUrl } + : { media: [{ type: "first_frame" as const, url: redactedUrl }] }), + }, + }; + } emitResult({ request: previewBody }, format); return; } @@ -162,7 +195,7 @@ export default defineCommand({ settings, () => ctx.client.requestJson({ - path: videoGeneratePath(), + path: isKf2v ? image2videoPath() : videoGeneratePath(), method: "POST", body, async: true, diff --git a/packages/commands/src/index.ts b/packages/commands/src/index.ts index 19aae774..d0740d90 100644 --- a/packages/commands/src/index.ts +++ b/packages/commands/src/index.ts @@ -71,6 +71,7 @@ export { finetuneTextCreate, finetuneAudioCreate, finetuneImageCreate, + finetuneVideoCreate, } from "./commands/finetune/create.ts"; export { default as finetuneList } from "./commands/finetune/list.ts"; export { default as finetuneGet } from "./commands/finetune/get.ts"; @@ -81,6 +82,7 @@ export { default as finetuneCheckpoints } from "./commands/finetune/checkpoints. export { default as finetuneExport } from "./commands/finetune/export.ts"; export { default as finetuneWatch } from "./commands/finetune/watch.ts"; export { default as finetuneCapability } from "./commands/finetune/capability.ts"; +export { default as finetunePrice } from "./commands/finetune/price.ts"; export { deployTextCreate, deployAudioCreate, @@ -92,6 +94,8 @@ export { default as deployModels } from "./commands/deploy/models.ts"; export { default as deployScale } from "./commands/deploy/scale.ts"; export { default as deployUpdate } from "./commands/deploy/update.ts"; export { default as deployDelete } from "./commands/deploy/delete.ts"; +export { default as deployPause } from "./commands/deploy/pause.ts"; +export { default as deployResume } from "./commands/deploy/resume.ts"; export { default as tokenPlanListSeats } from "./commands/token-plan/list-seats.ts"; export { default as tokenPlanCreateKey } from "./commands/token-plan/create-key.ts"; export { default as tokenPlanAssignSeats } from "./commands/token-plan/assign-seats.ts"; diff --git a/packages/commands/tests/e2e/deploy.e2e.test.ts b/packages/commands/tests/e2e/deploy.e2e.test.ts index 6fb5deda..47d5b814 100644 --- a/packages/commands/tests/e2e/deploy.e2e.test.ts +++ b/packages/commands/tests/e2e/deploy.e2e.test.ts @@ -30,7 +30,7 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: deploy (offline)", () => { "--help", ]); expect(exitCode, stderr).toBe(0); - expect(stderr).toMatch(/--model|--name/i); + expect(stderr).toMatch(/--model-name|--display-name/i); }); test("deploy create --dry-run 构造 lora 部署请求体", async () => { @@ -38,9 +38,9 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: deploy (offline)", () => { "deploy", "text", "create", - "--model", + "--model-name", "qwen-plus-2025-12-01", - "--name", + "--display-name", "my-qwen-plus", "--dry-run", "--output", @@ -68,9 +68,9 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: deploy (offline)", () => { "deploy", "text", "create", - "--model", + "--model-name", "qwen3-8b", - "--name", + "--display-name", "my-qwen3-mu", "--plan", "mu", @@ -102,9 +102,9 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: deploy (offline)", () => { "deploy", "audio", "create", - "--model", + "--model-name", "my-cosyvoice-ft", - "--name", + "--display-name", "my-tts", "--dry-run", "--output", diff --git a/packages/commands/tests/e2e/finetune.e2e.test.ts b/packages/commands/tests/e2e/finetune.e2e.test.ts index 59e9374d..53ee29a6 100644 --- a/packages/commands/tests/e2e/finetune.e2e.test.ts +++ b/packages/commands/tests/e2e/finetune.e2e.test.ts @@ -31,7 +31,7 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: finetune (offline)", () => { "--help", ]); expect(exitCode, stderr).toBe(0); - expect(stderr).toMatch(/--model|--datasets/i); + expect(stderr).toMatch(/--base-model|--datasets/i); }); test("finetune create --dry-run 构造 SFT 默认请求体", async () => { @@ -39,7 +39,7 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: finetune (offline)", () => { "finetune", "text", "create", - "--model", + "--base-model", "qwen3-8b", "--datasets", "file-aaa,file-bbb", @@ -73,7 +73,7 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: finetune (offline)", () => { "finetune", "text", "create", - "--model", + "--base-model", "qwen3-8b", "--datasets", "file-aaa", @@ -135,7 +135,7 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: finetune (offline)", () => { "finetune", "text", "create", - "--model", + "--base-model", "qwen3-8b", "--datasets", "file-aaa", @@ -157,7 +157,7 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: finetune (offline)", () => { "finetune", "text", "create", - "--model", + "--base-model", "qwen3-8b", "--datasets", "file-aaa", @@ -176,7 +176,7 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: finetune (offline)", () => { "finetune", "text", "create", - "--model", + "--base-model", "qwen3-8b", "--datasets", `${localPath},file-bbb`, @@ -207,7 +207,7 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: finetune (offline)", () => { "finetune", "text", "create", - "--model", + "--base-model", "qwen3-8b", "--datasets", " , ", @@ -228,7 +228,7 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: finetune (offline)", () => { "finetune", "text", "create", - "--model", + "--base-model", "qwen3-8b", "--datasets", localPath, @@ -250,7 +250,7 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: finetune (offline)", () => { "finetune", "text", "create", - "--model", + "--base-model", "qwen3-8b", "--datasets", localPath, @@ -272,7 +272,7 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: finetune (offline)", () => { ["cancel", ["--job-id", "ft-xxx"]], ["delete", ["--job-id", "ft-xxx"]], ["watch", ["--job-id", "ft-xxx"]], - ["capability", ["--model", "qwen3-8b"]], + ["capability", ["--base-model", "qwen3-8b"]], ])("finetune %s --dry-run 发出结构化动作", async (sub, extra) => { const { stdout, stderr, exitCode } = await runCommandE2e(FINETUNE_ROUTES, [ "finetune", @@ -292,7 +292,7 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: finetune (offline)", () => { "finetune", "text", "create", - "--model", + "--base-model", "qwen3-8b", "--datasets", " file-a , ,file-b ", @@ -314,7 +314,7 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: finetune (offline)", () => { "finetune", "audio", "create", - "--model", + "--base-model", "cosyvoice-v3-flash", "--datasets", "file-audio", @@ -343,7 +343,7 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: finetune (offline)", () => { "--help", ]); expect(exitCode, stderr).toBe(0); - expect(stderr).toMatch(/--model|--datasets/i); + expect(stderr).toMatch(/--base-model|--datasets/i); expect(stderr).not.toMatch(/--training-type|--n-epochs|--batch-size|--max-length/); }); @@ -352,7 +352,7 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: finetune (offline)", () => { "finetune", "image", "create", - "--model", + "--base-model", "wan2.7-image-pro", "--datasets", "file-image", @@ -365,6 +365,104 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: finetune (offline)", () => { expect(data.action).toBe("finetune.create"); expect(data.body.training_type).toBe("efficient_sft"); }); + + test("finetune video create --help 暴露视频超参且不含文本超参", async () => { + // Video exposes --n-epochs / --batch-size / --learning-rate; the text-only + // --training-type / --max-length surface is not offered. + const { stderr, exitCode } = await runCommandE2e(FINETUNE_ROUTES, [ + "finetune", + "video", + "create", + "--help", + ]); + expect(exitCode, stderr).toBe(0); + expect(stderr).toMatch(/--base-model/); + expect(stderr).toMatch(/--n-epochs/); + expect(stderr).toMatch(/--batch-size/); + expect(stderr).toMatch(/--learning-rate/); + expect(stderr).not.toMatch(/--training-type|--max-length/); + }); + + test("finetune video create --datasets 缺失时退出为用法错误 (2)", async () => { + const { stderr, exitCode } = await runCommandE2e(FINETUNE_ROUTES, [ + "finetune", + "video", + "create", + "--base-model", + "wan2.7-i2v", + "--quiet", + ]); + expect(exitCode).toBe(2); + expect(stderr).toMatch(/--datasets|Missing required/i); + }); + + test.each([ + // Model-family-specific defaults resolved by the sft-lora video profile. + ["wan2.7-i2v", 1, 102400], + ["wan2.5-i2v-preview", 4, 36864], + ["wan2.2-kf2v-flash", 4, 262144], + ])( + "finetune video create --dry-run %s 解析 batch_size=%i / max_pixels=%i", + async (baseModel, batchSize, maxPixels) => { + const { stdout, stderr, exitCode } = await runCommandE2e(FINETUNE_ROUTES, [ + "finetune", + "video", + "create", + "--base-model", + baseModel, + "--datasets", + "file-video", + "--dry-run", + "--output", + "json", + ]); + expect(exitCode, stderr).toBe(0); + const data = parseStdoutJson<{ + action: string; + body: { + model: string; + training_type: string; + hyper_parameters: Record; + }; + }>(stdout); + expect(data.action).toBe("finetune.create"); + expect(data.body.model).toBe(baseModel); + expect(data.body.training_type).toBe("efficient_sft"); + expect(data.body.hyper_parameters.batch_size).toBe(batchSize); + expect(data.body.hyper_parameters.max_pixels).toBe(maxPixels); + expect(data.body.hyper_parameters.learning_rate).toBe("2e-5"); + expect(data.body.hyper_parameters.lora_rank).toBe(32); + }, + ); + + test("finetune video create --dry-run 转发超参覆盖且不做 clamp", async () => { + const { stdout, stderr, exitCode } = await runCommandE2e(FINETUNE_ROUTES, [ + "finetune", + "video", + "create", + "--base-model", + "wan2.7-i2v", + "--datasets", + "file-video", + "--n-epochs", + "100", + "--batch-size", + "2", + "--learning-rate", + "1e-5", + "--dry-run", + "--output", + "json", + ]); + expect(exitCode, stderr).toBe(0); + const data = parseStdoutJson<{ + body: { hyper_parameters: Record }; + }>(stdout); + // Video overrides are forwarded verbatim (no [8, 1024] text clamp). + expect(data.body.hyper_parameters.n_epochs).toBe(100); + expect(data.body.hyper_parameters.batch_size).toBe(2); + expect(data.body.hyper_parameters.learning_rate).toBe("1e-5"); + }); }); describe.skipIf(!isDashScopeE2EReady())("e2e: finetune (DashScope)", () => { diff --git a/packages/commands/tests/e2e/topic-routes.ts b/packages/commands/tests/e2e/topic-routes.ts index 6df26efa..945048dd 100644 --- a/packages/commands/tests/e2e/topic-routes.ts +++ b/packages/commands/tests/e2e/topic-routes.ts @@ -145,6 +145,7 @@ export const FINETUNE_ROUTES: E2eRouteExports = { "finetune text create": "finetuneTextCreate", "finetune audio create": "finetuneAudioCreate", "finetune image create": "finetuneImageCreate", + "finetune video create": "finetuneVideoCreate", "finetune list": "finetuneList", "finetune get": "finetuneGet", "finetune cancel": "finetuneCancel", diff --git a/packages/commands/tests/e2e/video-generate-i2v.e2e.test.ts b/packages/commands/tests/e2e/video-generate-i2v.e2e.test.ts index 5c194865..3b686571 100644 --- a/packages/commands/tests/e2e/video-generate-i2v.e2e.test.ts +++ b/packages/commands/tests/e2e/video-generate-i2v.e2e.test.ts @@ -112,6 +112,62 @@ describe("e2e: video generate (i2v)", () => { }>(stdout); expect(data.request?.input?.media?.[0]?.url).toBe("data:image/png;base64,"); }); + + test.each([ + // wan2.1~2.6 (legacy) use flat img_url; wan2.7+ and happyhorse use media[]. + ["wan2.5-i2v-preview", "img_url"], + ["wan2.6-i2v", "img_url"], + ["wan2.7-i2v", "media"], + ["happyhorse-1.1-i2v", "media"], + ])("video generate --dry-run %s 首帧走 %s 字段", async (model, field) => { + const configDir = makeE2eOutputDir(`video-i2v-input-shape-${model}`); + writeFileSync( + join(configDir, "config.json"), + JSON.stringify({ + "token-plan": { + api_key: "sk-sp-e2e-placeholder", + base_url: "https://token-plan.cn-beijing.maas.aliyuncs.com", + }, + }), + ); + + const { stdout, stderr, exitCode } = await runCommandE2e( + VIDEO_ROUTES, + [ + "video", + "generate", + "--config", + "token-plan", + "--dry-run", + "--model", + model, + "--image", + "https://example.com/placeholder.png", + "--prompt", + "干跑校验", + "--output", + "json", + ], + { + BAILIAN_CONFIG_DIR: configDir, + DASHSCOPE_API_KEY: "", + DASHSCOPE_BASE_URL: "", + }, + ); + expect(exitCode, stderr).toBe(0); + const data = parseStdoutJson<{ + request?: { + input?: { img_url?: string; media?: Array<{ type?: string; url?: string }> }; + }; + }>(stdout); + if (field === "img_url") { + expect(data.request?.input?.img_url).toBe("https://example.com/placeholder.png"); + expect(data.request?.input?.media).toBeUndefined(); + } else { + expect(data.request?.input?.media?.[0]?.type).toBe("first_frame"); + expect(data.request?.input?.img_url).toBeUndefined(); + } + }); }); describe.skipIf(!isBailianE2EVideoEnabled() || !isDashScopeE2EReady())( diff --git a/packages/core/src/client/endpoints.ts b/packages/core/src/client/endpoints.ts index 8ec1d307..faf0a067 100644 --- a/packages/core/src/client/endpoints.ts +++ b/packages/core/src/client/endpoints.ts @@ -37,6 +37,11 @@ export function videoGeneratePath(): string { return "/api/v1/services/aigc/video-generation/video-synthesis"; } +/** POST /api/v1/services/aigc/image2video/video-synthesis — kf2v (first+last frame). */ +export function image2videoPath(): string { + return "/api/v1/services/aigc/image2video/video-synthesis"; +} + // ---- Async Task Query ---- export function taskPath(taskId: string): string { return `/api/v1/tasks/${encodeURIComponent(taskId)}`; diff --git a/packages/core/src/client/index.ts b/packages/core/src/client/index.ts index 5164e477..523394cc 100644 --- a/packages/core/src/client/index.ts +++ b/packages/core/src/client/index.ts @@ -22,6 +22,7 @@ export { taskPath, userProfilePath, videoGeneratePath, + image2videoPath, } from "./endpoints.ts"; export { isLegacyImage2ImageModel, diff --git a/packages/core/src/dataset/index.ts b/packages/core/src/dataset/index.ts index cc605c44..b6e581c2 100644 --- a/packages/core/src/dataset/index.ts +++ b/packages/core/src/dataset/index.ts @@ -7,6 +7,7 @@ export { registerValidator, listSupportedFormats, MAX_DATASET_BYTES, + MAX_CPT_BYTES, MAX_MEDIA_ZIP_BYTES, parseDatasetSchemaFlag, formatIssue, diff --git a/packages/core/src/dataset/validate/common.ts b/packages/core/src/dataset/validate/common.ts index 17bdbad5..212fe7ba 100644 --- a/packages/core/src/dataset/validate/common.ts +++ b/packages/core/src/dataset/validate/common.ts @@ -12,18 +12,24 @@ import { ExitCode } from "../../errors/codes.ts"; import type { DatasetSchema, ValidationIssue, ValidationStats } from "./types.ts"; /** - * The platform caps dataset uploads at 300MB per file. `bl dataset upload` - * enforces this client-side so users learn early. Update if the platform - * raises the cap or differentiates per-purpose limits. + * The platform caps SFT/DPO text dataset uploads at 200MB per file. + * `bl dataset upload` enforces this client-side so users learn early. + * CPT uses 300MB (see MAX_CPT_BYTES); API general upload is also 300MB. */ -export const MAX_DATASET_BYTES = 300 * 1024 * 1024; +export const MAX_DATASET_BYTES = 200 * 1024 * 1024; /** - * Image / video ZIP size cap — 1 GB per the platform docs (vs 300 MB for - * text / audio). Used by `bl dataset upload` for media schemas and by the - * `sft-lora` training profile for image / video validation. + * CPT text dataset size cap — 300 MB per the platform docs. + * CPT requires at least 50M tokens; larger files are expected. */ -export const MAX_MEDIA_ZIP_BYTES = 1024 * 1024 * 1024; +export const MAX_CPT_BYTES = 300 * 1024 * 1024; + +/** + * Image / video ZIP size cap — 2 GB per the platform docs. Used by + * `bl dataset upload` for media schemas and by the `sft-lora` training + * profile for image / video validation. + */ +export const MAX_MEDIA_ZIP_BYTES = 2 * 1024 * 1024 * 1024; export interface PreflightResult { bytes: number; diff --git a/packages/core/src/dataset/validate/index.ts b/packages/core/src/dataset/validate/index.ts index 42df4421..169a32c7 100644 --- a/packages/core/src/dataset/validate/index.ts +++ b/packages/core/src/dataset/validate/index.ts @@ -4,7 +4,12 @@ export { registerValidator, listSupportedFormats, } from "./registry.ts"; -export { MAX_DATASET_BYTES, MAX_MEDIA_ZIP_BYTES, parseDatasetSchemaFlag } from "./common.ts"; +export { + MAX_DATASET_BYTES, + MAX_CPT_BYTES, + MAX_MEDIA_ZIP_BYTES, + parseDatasetSchemaFlag, +} from "./common.ts"; export { formatIssue } from "./format.ts"; export type { ValidatorSpec, diff --git a/packages/core/src/dataset/validate/schemas/chatml.ts b/packages/core/src/dataset/validate/schemas/chatml.ts index d4c032e6..00b4ca20 100644 --- a/packages/core/src/dataset/validate/schemas/chatml.ts +++ b/packages/core/src/dataset/validate/schemas/chatml.ts @@ -5,12 +5,281 @@ * and no more specific schema matches, ChatML is selected. `inspectMessageObject` * lives here because it is the canonical per-message check; the DPO schema * imports it to validate `chosen` / `rejected` preference messages. + * + * Content format: supports both legacy plain-string content (`"content": "…"`) + * and the current platform array format (`"content": [{"text": "…"}, …]`). + * The array format may also carry `image` / `video` items for VL multimodal + * understanding data. + * + * Tool calling: supports `role: "tool"` messages with `tool_call_id`, and + * `assistant.tool_calls` arrays. Validates id correspondence. */ import { makeIssue } from "../common.ts"; import type { ValidationIssue } from "../types.ts"; import type { RecordSchemaSpec } from "./types.ts"; -const VALID_ROLES = new Set(["system", "user", "assistant"]); +const VALID_ROLES = new Set(["system", "user", "assistant", "tool"]); + +/** Platform bounds for video sampling rate params (`fps` / `sample_fps`). */ +const VIDEO_FPS_MIN = 0.1; +const VIDEO_FPS_MAX = 10; + +/** + * Validate the sampling/clipping params carried by a video content item. + * Mode rules (platform spec): + * - path mode (video: string): `fps`, `video_start`, `video_end` allowed; `sample_fps` is not + * - frame-list mode (video: string[]): `sample_fps` allowed; `fps` / `video_start` / `video_end` are not + * `fps` / `sample_fps` must be numbers within [0.1, 10] when present. + */ +function inspectVideoParams( + item: Record, + isFrameList: boolean, + lineNo: number, + itemPath: string, +): ValidationIssue[] { + const out: ValidationIssue[] = []; + const checkFpsRange = (field: "fps" | "sample_fps"): void => { + if (!(field in item)) return; + const value = item[field]; + if (typeof value !== "number" || value < VIDEO_FPS_MIN || value > VIDEO_FPS_MAX) { + out.push( + makeIssue( + "error", + "INVALID_VIDEO_FPS", + `"${field}" must be a number between ${VIDEO_FPS_MIN} and ${VIDEO_FPS_MAX} (got ${JSON.stringify(value)}).`, + { line: lineNo, path: `${itemPath}.${field}` }, + ), + ); + } + }; + checkFpsRange("fps"); + checkFpsRange("sample_fps"); + + const wrongModeFields = isFrameList ? ["fps", "video_start", "video_end"] : ["sample_fps"]; + const modeName = isFrameList ? "frame-list" : "file-path"; + for (const field of wrongModeFields) { + if (field in item) { + out.push( + makeIssue( + "warning", + "VIDEO_PARAM_MODE_MISMATCH", + `"${field}" does not apply to ${modeName} video mode and will be ignored by the platform.`, + { line: lineNo, path: `${itemPath}.${field}` }, + ), + ); + } + } + + for (const field of ["video_start", "video_end"] as const) { + if (field in item && !isFrameList && typeof item[field] !== "number") { + out.push( + makeIssue("error", "INVALID_VIDEO_CLIP_TIME", `"${field}" must be a number (seconds).`, { + line: lineNo, + path: `${itemPath}.${field}`, + }), + ); + } + } + return out; +} + +/** + * Validate a content field that may be: + * - A plain string (legacy format) + * - An array of content items: `[{text: "…"}, {image: "…"}, {video: "…"|"…"}, …]` + * + * Returns issues found. `path` scopes the location for error reporting. + */ +export function inspectContentField( + content: unknown, + lineNo: number, + path: string, +): ValidationIssue[] { + const out: ValidationIssue[] = []; + if (typeof content === "string") { + // Legacy string format — always valid. + return out; + } + if (!Array.isArray(content)) { + out.push( + makeIssue( + "error", + "INVALID_CONTENT", + `"content" must be a string or an array of content items (got ${typeof content}).`, + { line: lineNo, path }, + ), + ); + return out; + } + if (content.length === 0) { + out.push( + makeIssue("error", "EMPTY_CONTENT_ARRAY", `"content" array must not be empty.`, { + line: lineNo, + path, + }), + ); + return out; + } + for (let idx = 0; idx < content.length; idx++) { + const item = content[idx]; + const itemPath = `${path}[${idx}]`; + if (item === null || typeof item !== "object" || Array.isArray(item)) { + out.push( + makeIssue("error", "INVALID_CONTENT_ITEM", `Content item must be an object.`, { + line: lineNo, + path: itemPath, + }), + ); + continue; + } + const obj = item as Record; + const hasText = "text" in obj; + const hasImage = "image" in obj; + const hasVideo = "video" in obj; + if (!hasText && !hasImage && !hasVideo) { + out.push( + makeIssue( + "error", + "CONTENT_ITEM_NO_KNOWN_FIELD", + `Content item must contain at least one of: "text", "image", "video".`, + { line: lineNo, path: itemPath }, + ), + ); + continue; + } + if (hasText && typeof obj.text !== "string") { + out.push( + makeIssue("error", "INVALID_CONTENT_TEXT", `"text" in content item must be a string.`, { + line: lineNo, + path: `${itemPath}.text`, + }), + ); + } + if (hasImage && typeof obj.image !== "string") { + out.push( + makeIssue("error", "INVALID_CONTENT_IMAGE", `"image" in content item must be a string.`, { + line: lineNo, + path: `${itemPath}.image`, + }), + ); + } + if (hasVideo) { + // video can be a string (file path) or an array of strings (frame list) + const video = obj.video; + if (typeof video !== "string" && !Array.isArray(video)) { + out.push( + makeIssue( + "error", + "INVALID_CONTENT_VIDEO", + `"video" in content item must be a string (file path) or an array of strings (frame list).`, + { line: lineNo, path: `${itemPath}.video` }, + ), + ); + } else { + if (Array.isArray(video)) { + for (let frameIdx = 0; frameIdx < video.length; frameIdx++) { + if (typeof video[frameIdx] !== "string") { + out.push( + makeIssue( + "error", + "INVALID_VIDEO_FRAME", + `Video frame list item at index ${frameIdx} must be a string.`, + { line: lineNo, path: `${itemPath}.video[${frameIdx}]` }, + ), + ); + } + } + } + out.push(...inspectVideoParams(obj, Array.isArray(video), lineNo, itemPath)); + } + } + } + return out; +} + +/** + * Validate `tool_calls` array on an assistant message. + * Each entry: `{id: string, type: "function", function: {name: string, arguments: string}}`. + */ +function inspectToolCalls(toolCalls: unknown, lineNo: number, path: string): ValidationIssue[] { + const out: ValidationIssue[] = []; + if (!Array.isArray(toolCalls)) { + out.push( + makeIssue("error", "INVALID_TOOL_CALLS", `"tool_calls" must be an array.`, { + line: lineNo, + path, + }), + ); + return out; + } + for (let idx = 0; idx < toolCalls.length; idx++) { + const call = toolCalls[idx]; + const callPath = `${path}[${idx}]`; + if (call === null || typeof call !== "object" || Array.isArray(call)) { + out.push( + makeIssue("error", "INVALID_TOOL_CALL", `tool_calls item must be an object.`, { + line: lineNo, + path: callPath, + }), + ); + continue; + } + const obj = call as Record; + if (typeof obj.id !== "string" || obj.id.length === 0) { + out.push( + makeIssue("error", "TOOL_CALL_MISSING_ID", `tool_calls item must have a non-empty "id".`, { + line: lineNo, + path: `${callPath}.id`, + }), + ); + } + if (obj.type !== "function") { + out.push( + makeIssue( + "warning", + "TOOL_CALL_TYPE_NOT_FUNCTION", + `tool_calls item "type" should be "function" (got "${String(obj.type)}").`, + { line: lineNo, path: `${callPath}.type` }, + ), + ); + } + const fn = obj.function; + if (fn === null || typeof fn !== "object" || Array.isArray(fn)) { + out.push( + makeIssue( + "error", + "TOOL_CALL_MISSING_FUNCTION", + `tool_calls item must have a "function" object.`, + { + line: lineNo, + path: `${callPath}.function`, + }, + ), + ); + } else { + const fnObj = fn as Record; + if (typeof fnObj.name !== "string" || fnObj.name.length === 0) { + out.push( + makeIssue("error", "TOOL_CALL_FN_NO_NAME", `tool_calls function must have a "name".`, { + line: lineNo, + path: `${callPath}.function.name`, + }), + ); + } + if (typeof fnObj.arguments !== "string") { + out.push( + makeIssue( + "error", + "TOOL_CALL_FN_ARGS_NOT_STRING", + `tool_calls function "arguments" must be a JSON string.`, + { line: lineNo, path: `${callPath}.function.arguments` }, + ), + ); + } + } + } + return out; +} /** * Structural checks for a single message object `{role, content}`. Shared by @@ -35,25 +304,74 @@ export function inspectMessageObject( } const record = msg as Record; const role = record.role; - const content = record.content; if (typeof role !== "string" || !VALID_ROLES.has(role)) { out.push( makeIssue( "error", "INVALID_ROLE", - `Invalid role "${String(role)}". Expected one of: system, user, assistant.`, + `Invalid role "${String(role)}". Expected one of: system, user, assistant, tool.`, { line: lineNo, path: `${path}.role` }, ), ); } - if (typeof content !== "string") { + + // tool role: must have tool_call_id + if (role === "tool") { + if (typeof record.tool_call_id !== "string" || record.tool_call_id.length === 0) { + out.push( + makeIssue( + "error", + "TOOL_MISSING_CALL_ID", + `A "tool" role message must have a non-empty "tool_call_id".`, + { line: lineNo, path: `${path}.tool_call_id` }, + ), + ); + } + } + + // content validation: string or array format + if (!("content" in record)) { + // assistant messages with tool_calls may omit content + if (role !== "assistant" || !("tool_calls" in record)) { + out.push( + makeIssue("error", "MISSING_CONTENT", `"content" field is missing.`, { + line: lineNo, + path: `${path}.content`, + }), + ); + } + } else { + out.push(...inspectContentField(record.content, lineNo, `${path}.content`)); + } + + // tool_calls on assistant + if ("tool_calls" in record) { + out.push(...inspectToolCalls(record.tool_calls, lineNo, `${path}.tool_calls`)); + } + + // OpenAI migration guard: the platform rejects data carrying name / weight + if ("name" in record) { + out.push( + makeIssue( + "error", + "UNSUPPORTED_FIELD_NAME", + `Field "name" is not supported by Bailian and must be removed when migrating from OpenAI/Azure.`, + { line: lineNo, path: `${path}.name` }, + ), + ); + } + if ("weight" in record) { out.push( - makeIssue("error", "INVALID_CONTENT", `"content" must be a string (got ${typeof content}).`, { - line: lineNo, - path: `${path}.content`, - }), + makeIssue( + "error", + "UNSUPPORTED_FIELD_WEIGHT", + `Field "weight" is not supported by Bailian and must be removed when migrating from OpenAI/Azure. ` + + `All assistant outputs are trained; per-line importance uses "loss_weight" (invite-only).`, + { line: lineNo, path: `${path}.weight` }, + ), ); } + return out; } @@ -91,19 +409,24 @@ export function inspectChatMLRecord( let sawSystem = false; let lastRole: string | undefined; - for (let i = 0; i < messages.length; i++) { - const msg = messages[i]; - const path = `messages[${i}]`; + let lastAssistantIdx = -1; + const toolCallIds = new Set(); + const toolResponseIds = new Set(); + + for (let idx = 0; idx < messages.length; idx++) { + const msg = messages[idx]; + const path = `messages[${idx}]`; out.push(...inspectMessageObject(msg, lineNo, path)); - const role = (msg as Record | null)?.role; + const msgObj = msg as Record | null; + const role = msgObj?.role; if (role === "system") { - if (i !== 0) { + if (idx !== 0) { out.push( makeIssue( "warning", "SYSTEM_NOT_FIRST", - `"system" message should appear at index 0; found at index ${i}.`, + `"system" message should appear at index 0; found at index ${idx}.`, { line: lineNo, path: `${path}.role` }, ), ); @@ -111,6 +434,27 @@ export function inspectChatMLRecord( sawSystem = true; } + if (role === "assistant") { + lastAssistantIdx = idx; + // Collect tool_calls ids + if (msgObj && Array.isArray(msgObj.tool_calls)) { + for (const call of msgObj.tool_calls) { + const callObj = call as Record | null; + if (callObj && typeof callObj.id === "string") { + toolCallIds.add(callObj.id); + } + } + } + } + + if (role === "tool") { + const callId = msgObj?.tool_call_id; + if (typeof callId === "string" && callId.length > 0) { + toolResponseIds.add(callId); + } + } + + // Consecutive same-role warning (skip tool — multiple tool responses are normal) if (lastRole === role && (role === "user" || role === "assistant")) { out.push( makeIssue( @@ -123,6 +467,7 @@ export function inspectChatMLRecord( } if (typeof role === "string") lastRole = role; } + // Soft check: messages without any user role almost certainly indicate a bug. if (!messages.some((m) => (m as Record).role === "user")) { out.push( @@ -140,9 +485,132 @@ export function inspectChatMLRecord( }), ); } + + // tool_call_id correspondence must be one-to-one (platform spec): + // every tool response must reference a known call id (hard error), and every + // tool_call should receive a response (advisory — trailing calls are dubious + // in training data but we cannot rule out platform-side tolerance). + for (const responseId of toolResponseIds) { + if (!toolCallIds.has(responseId)) { + out.push( + makeIssue( + "error", + "TOOL_CALL_ID_UNMATCHED", + `tool message references tool_call_id "${responseId}" which does not match any assistant tool_calls[].id.`, + { line: lineNo, path: "messages" }, + ), + ); + } + } + for (const callId of toolCallIds) { + if (!toolResponseIds.has(callId)) { + out.push( + makeIssue( + "warning", + "TOOL_CALL_NO_RESPONSE", + `assistant tool_calls[].id "${callId}" has no matching tool response message.`, + { line: lineNo, path: "messages" }, + ), + ); + } + } + + // thinking tag check: should only appear in the last + // assistant message. Exemption (platform spec, tool+thinking combo): an + // assistant that carries tool_calls may legitimately hold a block + // even when it is not the last assistant message. + if (lastAssistantIdx >= 0) { + for (let idx = 0; idx < messages.length; idx++) { + if (idx === lastAssistantIdx) continue; + const msg = messages[idx] as Record | null; + if (msg?.role !== "assistant") continue; + if (msg && Array.isArray(msg.tool_calls)) continue; + const content = msg?.content; + if (contentHasThinkTag(content)) { + out.push( + makeIssue( + "warning", + "THINK_TAG_NOT_LAST", + `Thinking tags () should only appear in the last assistant message ` + + `(or an assistant message carrying tool_calls), found at messages[${idx}].`, + { line: lineNo, path: `messages[${idx}].content` }, + ), + ); + } + } + } + + // loss_weight validation (invite-only parameter). + // Range is enforced wherever the field appears (record level and message + // level); placement follows the spec: only the LAST assistant message line + // supports loss_weight — misplaced occurrences are advisory (invite-only + // semantics are account-specific, so we do not hard-fail). + const checkLossWeightRange = (value: unknown, path: string): void => { + if (typeof value !== "number" || value < 0 || value > 1) { + out.push( + makeIssue( + "error", + "INVALID_LOSS_WEIGHT", + `"loss_weight" must be a number between 0.0 and 1.0 (got ${JSON.stringify(value)}).`, + { line: lineNo, path }, + ), + ); + } + }; + if ("loss_weight" in record) { + checkLossWeightRange(record.loss_weight, "loss_weight"); + } + for (let idx = 0; idx < messages.length; idx++) { + const msg = messages[idx] as Record | null; + if (!msg || !("loss_weight" in msg)) continue; + checkLossWeightRange(msg.loss_weight, `messages[${idx}].loss_weight`); + if (!(msg.role === "assistant" && idx === lastAssistantIdx)) { + out.push( + makeIssue( + "warning", + "LOSS_WEIGHT_PLACEMENT", + `"loss_weight" is only supported on the last assistant message; found at messages[${idx}] (role "${String(msg.role)}").`, + { line: lineNo, path: `messages[${idx}].loss_weight` }, + ), + ); + } + } + + // OpenAI migration guard at record level (message-level occurrences are + // handled by inspectMessageObject above) + if ("weight" in record) { + out.push( + makeIssue( + "error", + "UNSUPPORTED_FIELD_WEIGHT", + `Record-level field "weight" is not supported by Bailian and must be removed when migrating from OpenAI/Azure.`, + { line: lineNo, path: "weight" }, + ), + ); + } + return out; } +/** Check whether content (string or array) contains a tag. */ +function contentHasThinkTag(content: unknown): boolean { + if (typeof content === "string") { + return content.includes(""); + } + if (Array.isArray(content)) { + return content.some((item) => { + if (item && typeof item === "object" && "text" in item) { + return ( + typeof (item as Record).text === "string" && + ((item as Record).text as string).includes("") + ); + } + return false; + }); + } + return false; +} + /** * ChatML / SFT schema. The auto-detect predicate is `true` so it acts as the * registry fallback — any record that isn't picked up by a more specific diff --git a/packages/core/src/dataset/validate/schemas/dpo.ts b/packages/core/src/dataset/validate/schemas/dpo.ts index 5dc69370..86de3aa9 100644 --- a/packages/core/src/dataset/validate/schemas/dpo.ts +++ b/packages/core/src/dataset/validate/schemas/dpo.ts @@ -18,6 +18,76 @@ function inspectDPORecord(record: Record, lineNo: number): Vali const messages = record.messages; if (!Array.isArray(messages) || messages.length === 0) return out; + /** image / video content items are outside the DPO support matrix */ + const mediaIssues = (content: unknown, basePath: string): ValidationIssue[] => { + if (!Array.isArray(content)) return []; + const found: ValidationIssue[] = []; + for (let itemIdx = 0; itemIdx < content.length; itemIdx++) { + const item = content[itemIdx] as Record | null; + if (!item || typeof item !== "object") continue; + for (const mediaField of ["image", "video"] as const) { + if (mediaField in item) { + found.push( + makeIssue( + "error", + "DPO_UNSUPPORTED_ELEMENT", + `DPO training data does not support ${mediaField} inputs; found at ${basePath}.content[${itemIdx}].`, + { line: lineNo, path: `${basePath}.content[${itemIdx}].${mediaField}` }, + ), + ); + } + } + } + return found; + }; + + // Support matrix (platform spec): DPO is text + thinking ONLY — no image / + // video inputs and no tool calling. Reject multimodal items and tool fields + // that the SFT-oriented ChatML inspector would otherwise accept. + if ("tools" in record) { + out.push( + makeIssue( + "error", + "DPO_UNSUPPORTED_ELEMENT", + `DPO training data does not support tool calling; remove the "tools" definition.`, + { line: lineNo, path: "tools" }, + ), + ); + } + for (let idx = 0; idx < messages.length; idx++) { + const msg = messages[idx] as Record | null; + if (!msg) continue; + const msgPath = `messages[${idx}]`; + if (msg.role === "tool" || "tool_calls" in msg) { + out.push( + makeIssue( + "error", + "DPO_UNSUPPORTED_ELEMENT", + `DPO training data does not support tool calling; found ${ + msg.role === "tool" ? `role "tool"` : `"tool_calls"` + } at ${msgPath}.`, + { line: lineNo, path: msgPath }, + ), + ); + } + out.push(...mediaIssues(msg.content, msgPath)); + } + + // DPO trains the preference for the LAST user input — messages ending with + // any other role make the chosen/rejected pair semantically meaningless. + const lastMsg = messages[messages.length - 1] as Record | null; + if (lastMsg && lastMsg.role !== "user") { + out.push( + makeIssue( + "error", + "DPO_LAST_MSG_NOT_USER", + `DPO "messages" must end with a "user" message (the prompt for chosen/rejected). ` + + `Got "${String(lastMsg.role)}" as the last message.`, + { line: lineNo, path: `messages[${messages.length - 1}].role` }, + ), + ); + } + const hasChosen = "chosen" in record; const hasRejected = "rejected" in record; @@ -39,6 +109,7 @@ function inspectDPORecord(record: Record, lineNo: number): Vali } if (hasChosen) { out.push(...inspectMessageObject(record.chosen, lineNo, "chosen")); + out.push(...mediaIssues((record.chosen as Record | null)?.content, "chosen")); const role = (record.chosen as Record | null)?.role; if (typeof role === "string" && role !== "assistant") { out.push( @@ -53,6 +124,9 @@ function inspectDPORecord(record: Record, lineNo: number): Vali } if (hasRejected) { out.push(...inspectMessageObject(record.rejected, lineNo, "rejected")); + out.push( + ...mediaIssues((record.rejected as Record | null)?.content, "rejected"), + ); const role = (record.rejected as Record | null)?.role; if (typeof role === "string" && role !== "assistant") { out.push( diff --git a/packages/core/src/dataset/validate/schemas/image.ts b/packages/core/src/dataset/validate/schemas/image.ts index daef3804..248619c7 100644 --- a/packages/core/src/dataset/validate/schemas/image.ts +++ b/packages/core/src/dataset/validate/schemas/image.ts @@ -16,7 +16,15 @@ import type { ValidationIssue } from "../types.ts"; import type { RecordSchemaSpec } from "./types.ts"; /** Accepted image file extensions (lower-case, with dot). */ -export const IMAGE_EXTENSIONS = new Set([".png", ".jpg", ".jpeg", ".bmp", ".webp", ".tiff"]); +export const IMAGE_EXTENSIONS = new Set([ + ".png", + ".jpg", + ".jpeg", + ".bmp", + ".tif", + ".tiff", + ".webp", +]); /** * Check that a path string ends with an accepted image extension. diff --git a/packages/core/src/dataset/validate/zip.ts b/packages/core/src/dataset/validate/zip.ts index 43091337..232a5f95 100644 --- a/packages/core/src/dataset/validate/zip.ts +++ b/packages/core/src/dataset/validate/zip.ts @@ -3,14 +3,15 @@ * * A training data ZIP must have: * - `data.jsonl` at the root — the manifest mapping media files to labels. - * - A `train/` subfolder (or media files at the root) referenced by the - * manifest entries. + * The platform requires data.jsonl to be directly visible when opening the + * ZIP (no wrapping folder). + * - Media files referenced by the manifest entries. * * This validator owns the **ZIP-level structural checks** (entries present, - * references resolve). The **per-record JSONL content validation** is delegated - * to the existing `jsonlValidator` — we extract `data.jsonl` to a temp file, - * run the full pipeline (quickScan + deepCheck + schema dispatch), and stitch - * the results together. + * references resolve, filename constraints). The **per-record JSONL content + * validation** is delegated to the existing `jsonlValidator` — we extract + * `data.jsonl` to a temp file, run the full pipeline (quickScan + deepCheck + + * schema dispatch), and stitch the results together. * * The schema for `data.jsonl` records is passed via `opts.schema` (typically * `"tts"` for audio). The profile layer decides which schema to use based on @@ -92,6 +93,139 @@ function collectZipEntries(zipPath: string): Promise { }); } +/** + * Platform filename constraints: + * - Allowed charset: ASCII letters (a-z, A-Z), digits (0-9), underscore (_), hyphen (-) + * - Filename (without extension) ≤ 120 characters + * - Filenames must be globally unique (ignoring extension) + */ +const FILENAME_CHARSET_RE = /^[a-zA-Z0-9_-]+$/; +const MAX_FILENAME_BASE_LENGTH = 120; + +/** Extract the base name (no extension) from a path segment. */ +function basenameNoExt(segment: string): string { + const dot = segment.lastIndexOf("."); + return dot > 0 ? segment.slice(0, dot) : segment; +} + +/** + * macOS Finder/zip metadata entries (`__MACOSX/` resource forks, `.DS_Store`, + * AppleDouble `._*` files). They are packaging noise, not training data: + * exclude them from filename constraints and media counting so Mac-created + * archives don't fail on artifacts the user never sees. + */ +function isZipMetadataEntry(entry: string): boolean { + if (entry === "__MACOSX" || entry.startsWith("__MACOSX/")) return true; + const lastSegment = + entry + .split("/") + .filter((segment) => segment.length > 0) + .pop() ?? ""; + return lastSegment === ".DS_Store" || lastSegment.startsWith("._"); +} + +/** + * Validate ZIP entry filenames against platform constraints. + * Returns issues for charset violations, over-length names, and duplicates. + * Exported for direct unit testing (not re-exported by the barrel). + */ +export function validateZipFilenames(entries: string[]): ValidationIssue[] { + const out: ValidationIssue[] = []; + const seenBasenames = new Map(); // basename (no ext) → first full path + const MAX_REPORTED = 10; + + for (const entry of entries) { + // Skip directory entries and macOS packaging metadata + if (entry.endsWith("/")) continue; + if (isZipMetadataEntry(entry)) continue; + + // Check each path segment (folder names + file name) + const segments = entry.split("/").filter((s) => s.length > 0); + for (const segment of segments) { + // Strip extension for the charset check on the base part + const base = basenameNoExt(segment); + const ext = segment.slice(base.length); // includes dot, e.g. ".jpg" + + // Charset check on base name (extension checked separately) + if (base.length > 0 && !FILENAME_CHARSET_RE.test(base)) { + if (out.length < MAX_REPORTED) { + out.push( + makeIssue( + "error", + "INVALID_FILENAME_CHARSET", + `File/folder name "${segment}" contains invalid characters. ` + + `Only a-z, A-Z, 0-9, underscore (_), and hyphen (-) are allowed.`, + { path: entry }, + ), + ); + } + } + + // Extension charset (allow dot + alphanumeric) + if (ext.length > 0 && !/^\.[a-zA-Z0-9]+$/.test(ext)) { + if (out.length < MAX_REPORTED) { + out.push( + makeIssue( + "error", + "INVALID_FILENAME_CHARSET", + `File extension "${ext}" in "${segment}" contains invalid characters.`, + { path: entry }, + ), + ); + } + } + } + + // Filename length check (base name without extension) + const fileName = segments[segments.length - 1] ?? ""; + const baseName = basenameNoExt(fileName); + if (baseName.length > MAX_FILENAME_BASE_LENGTH) { + if (out.length < MAX_REPORTED) { + out.push( + makeIssue( + "error", + "FILENAME_TOO_LONG", + `Filename "${fileName}" (without extension) exceeds ${MAX_FILENAME_BASE_LENGTH} characters ` + + `(got ${baseName.length}). Shorten the name and re-upload.`, + { path: entry }, + ), + ); + } + } + + // Global uniqueness check (ignoring extension, case-sensitive) + if (baseName.length > 0) { + const existing = seenBasenames.get(baseName); + if (existing !== undefined) { + if (out.length < MAX_REPORTED) { + out.push( + makeIssue( + "error", + "DUPLICATE_FILENAME", + `Filename "${fileName}" conflicts with "${existing}" — names must be globally unique ` + + `(ignoring extension) even across different folders.`, + { path: entry }, + ), + ); + } + } else { + seenBasenames.set(baseName, entry); + } + } + } + + if (out.length >= MAX_REPORTED) { + out.push( + makeIssue( + "warning", + "FILENAME_ISSUES_TRUNCATED", + `More filename issues exist but reporting is capped at ${MAX_REPORTED}.`, + ), + ); + } + return out; +} + /** * Extract a single entry from a ZIP archive to a destination path. */ @@ -201,19 +335,37 @@ export const zipValidator: ValidatorSpec = { }; } - // --- 2. Check for data.jsonl --- - const hasDataJsonl = entries.some( - (entry) => entry === "data.jsonl" || entry.endsWith("/data.jsonl"), - ); - if (!hasDataJsonl) { - errors.push( - makeIssue( - "error", - "MISSING_DATA_JSONL", - `ZIP archive must contain "data.jsonl" at the root. ` + - `This file maps media files (e.g. .wav) to their labels.`, - ), - ); + // --- 2. Check for data.jsonl (must be at ZIP root) --- + const hasRootDataJsonl = entries.some((entry) => entry === "data.jsonl"); + const nestedDataJsonl = + !hasRootDataJsonl && entries.find((entry) => entry.endsWith("/data.jsonl")); + if (!hasRootDataJsonl) { + if (nestedDataJsonl) { + errors.push( + makeIssue( + "error", + "DATA_JSONL_NOT_AT_ROOT", + `"data.jsonl" must be at the ZIP root (found "${nestedDataJsonl}"). ` + + `Re-package so that opening the ZIP shows data.jsonl directly, without a wrapping folder.`, + ), + ); + } else { + errors.push( + makeIssue( + "error", + "MISSING_DATA_JSONL", + `ZIP archive must contain "data.jsonl" at the root. ` + + `This file maps media files (e.g. .wav, .jpg) to their labels.`, + ), + ); + } + } + + // --- 2b. Filename constraints (charset, length, uniqueness) --- + const filenameIssues = validateZipFilenames(entries); + for (const issue of filenameIssues) { + if (issue.severity === "error") errors.push(issue); + else warnings.push(issue); } // --- 3. Check for train/ directory (modality-aware) --- @@ -249,6 +401,7 @@ export const zipValidator: ValidatorSpec = { const imageFiles = entries.filter((entry) => { if (entry === "data.jsonl" || entry.endsWith("/data.jsonl")) return false; if (entry.endsWith("/")) return false; // directory entries + if (isZipMetadataEntry(entry)) return false; // __MACOSX/._x.jpg is not an image const dot = entry.lastIndexOf("."); const ext = dot >= 0 ? entry.slice(dot).toLowerCase() : ""; return IMAGE_EXTENSIONS.has(ext); @@ -265,8 +418,8 @@ export const zipValidator: ValidatorSpec = { } } - // If data.jsonl is missing, we can't do JSONL content validation. - if (!hasDataJsonl) { + // If data.jsonl is missing entirely, we can't do JSONL content validation. + if (!hasRootDataJsonl && !nestedDataJsonl) { return { valid: false, format: "zip", @@ -278,9 +431,11 @@ export const zipValidator: ValidatorSpec = { } // --- 4. Extract data.jsonl to a temp file and run jsonlValidator --- - const dataJsonlEntry = entries.find( - (entry) => entry === "data.jsonl" || entry.endsWith("/data.jsonl"), - )!; + // Prefer root data.jsonl; fall back to nested for content validation even + // though we already reported the root-placement error above. + const dataJsonlEntry = hasRootDataJsonl + ? "data.jsonl" + : entries.find((entry) => entry.endsWith("/data.jsonl"))!; const tmpDir = join(tmpdir(), `bl-zip-${randomBytes(6).toString("hex")}`); mkdirSync(tmpDir, { recursive: true }); const tmpJsonl = join(tmpDir, "data.jsonl"); diff --git a/packages/core/src/deploy/index.ts b/packages/core/src/deploy/index.ts index e0515c13..ce3909ac 100644 --- a/packages/core/src/deploy/index.ts +++ b/packages/core/src/deploy/index.ts @@ -2,3 +2,4 @@ export * from "./api.ts"; export * from "./types.ts"; export * from "./constants.ts"; export * from "./plans.ts"; +export * from "./lifecycle.ts"; diff --git a/packages/core/src/deploy/lifecycle.ts b/packages/core/src/deploy/lifecycle.ts new file mode 100644 index 00000000..9f647d53 --- /dev/null +++ b/packages/core/src/deploy/lifecycle.ts @@ -0,0 +1,99 @@ +/** + * Deployment lifecycle operations via the **console gateway**. + * + * Unlike the DashScope REST endpoints in `api.ts`, start/stop/list-independent + * are console-domain APIs (`zeldaEasy.broadscope-platform.modelInstance.*`). + * Commands using these must declare `auth: "console"`. + */ +import type { Client } from "../client/client.ts"; +import { unwrapResponse } from "../console/models.ts"; + +// --------------------------------------------------------------------------- +// API names +// --------------------------------------------------------------------------- + +export const DEPLOY_START_API = "zeldaEasy.broadscope-platform.modelInstance.startModelService"; +export const DEPLOY_STOP_API = "zeldaEasy.broadscope-platform.modelInstance.stopModelService"; +export const DEPLOY_LIST_INDEPENDENT_API = + "zeldaEasy.broadscope-platform.modelInstance.listIndependentDeployedModel"; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface ModelServiceEntry { + modelServiceId?: string; + deployedModel?: string; + deployed_model?: string; + status?: string; + modelName?: string; + model_name?: string; + plan?: string; + [key: string]: unknown; +} + +// --------------------------------------------------------------------------- +// API wrappers +// --------------------------------------------------------------------------- + +/** Start (bring online) a stopped deployment. */ +export async function startModelService( + client: Client, + modelServiceId: string, +): Promise> { + const raw = await client.console>(DEPLOY_START_API, { + input: { modelServiceId }, + }); + return unwrapResponse(raw); +} + +/** Stop (take offline) a running deployment. Stops billing for mu/ptu plans. */ +export async function stopModelService( + client: Client, + modelServiceId: string, +): Promise> { + const raw = await client.console>(DEPLOY_STOP_API, { + input: { modelServiceId }, + }); + return unwrapResponse(raw); +} + +/** + * List independently deployed models (console domain). + * Used for precheck status verification and ID mapping. + * Paginates internally to return all entries. + */ +export async function listIndependentDeployedModels(client: Client): Promise { + const allEntries: ModelServiceEntry[] = []; + let page = 1; + + while (true) { + const raw = await client.console>(DEPLOY_LIST_INDEPENDENT_API, { + input: { pageNo: page, pageSize: 50 }, + }); + const resp = unwrapResponse(raw); + const records = (resp.records ?? []) as ModelServiceEntry[]; + allEntries.push(...records); + const pageCount = (resp.pageCount as number) ?? 1; + if (page >= pageCount || records.length === 0) break; + page++; + } + + return allEntries; +} + +/** + * Find a deployment entry by its identifier in the console-domain list. + * Matches against `modelServiceId`, `deployedModel`, or `deployed_model`. + */ +export function findDeploymentEntry( + entries: ModelServiceEntry[], + deployedModel: string, +): ModelServiceEntry | undefined { + return entries.find( + (entry) => + entry.modelServiceId === deployedModel || + entry.deployedModel === deployedModel || + entry.deployed_model === deployedModel, + ); +} diff --git a/packages/core/src/finetune/api.ts b/packages/core/src/finetune/api.ts index 1126c9dd..757c0440 100644 --- a/packages/core/src/finetune/api.ts +++ b/packages/core/src/finetune/api.ts @@ -43,6 +43,8 @@ export interface ListFineTunesParams { pageNo?: number; pageSize?: number; status?: string; + /** Filter by base model ID (server-side). */ + model?: string; signal?: AbortSignal; } @@ -55,6 +57,7 @@ export async function listFineTunes( if (params.pageNo !== undefined) qs.set("page_no", String(params.pageNo)); if (params.pageSize !== undefined) qs.set("page_size", String(params.pageSize)); if (params.status) qs.set("status", params.status); + if (params.model) qs.set("model", params.model); const base = finetuneJobsPath(); const path = qs.toString() ? `${base}?${qs.toString()}` : base; return client.requestJson({ diff --git a/packages/core/src/finetune/index.ts b/packages/core/src/finetune/index.ts index e0b064b6..3b82e583 100644 --- a/packages/core/src/finetune/index.ts +++ b/packages/core/src/finetune/index.ts @@ -3,3 +3,4 @@ export * from "./api.ts"; export * from "./capability.ts"; export * from "./preflight.ts"; export * from "./profiles/index.ts"; +export * from "./price.ts"; diff --git a/packages/core/src/finetune/price.ts b/packages/core/src/finetune/price.ts new file mode 100644 index 00000000..64651ae5 --- /dev/null +++ b/packages/core/src/finetune/price.ts @@ -0,0 +1,121 @@ +/** + * Training price estimation via the **console gateway**. + * + * These are console-domain APIs (`zeldaEasy.broadscope-platform.*`); commands + * using them must declare `auth: "console"`. Two different argument wrappers + * exist: `getModelPrice` takes a top-level `query`, while the token-estimation + * APIs take a top-level `input`. + */ +import type { Client } from "../client/client.ts"; +import { unwrapResponse } from "../console/models.ts"; + +// --------------------------------------------------------------------------- +// API names +// --------------------------------------------------------------------------- + +export const TRAINING_MODEL_PRICE_API = "zeldaEasy.broadscope-platform.modelCenter.getModelPrice"; +export const CALC_DATASETS_TOKENS_API = + "zeldaEasy.broadscope-platform.modelInstance.calculateDatasetsTotalTokens"; +export const ESTIMATE_FINETUNE_TOKENS_API = + "zeldaEasy.broadscope-platform.modelInstance.estimateFinetuneTokens"; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface TrainingModelPrice { + price?: string; + priceUnit?: string; + modelId?: string; + [key: string]: unknown; +} + +export interface TokenEstimate { + estimatedDatasetConsumedTokensMinPerEpoch?: number; + estimatedDatasetConsumedTokensMaxPerEpoch?: number; + estimatedMixedConsumedTokensMinPerEpoch?: number; + estimatedMixedConsumedTokensMaxPerEpoch?: number; + [key: string]: unknown; +} + +// --------------------------------------------------------------------------- +// API wrappers +// --------------------------------------------------------------------------- + +/** + * Training unit price for a model. `price` is denominated in `priceUnit` + * (typically "千Token" — yuan per 1000 tokens). + */ +export async function fetchTrainingModelPrice( + client: Client, + modelId: string, +): Promise { + const raw = await client.console>(TRAINING_MODEL_PRICE_API, { + query: { type: 0, modelId }, + }); + return unwrapResponse(raw) as TrainingModelPrice; +} + +/** + * Estimate training tokens for SFT / DPO jobs. + * Returns a per-epoch min/max range; multiply by `n_epochs` for the total. + */ +export async function estimateSftDpoTokens( + client: Client, + datasetIds: string[], + hyperParams: { nEpochs: number; batchSize: number; maxLength: number }, +): Promise { + const raw = await client.console>(CALC_DATASETS_TOKENS_API, { + input: { trainDatasetIds: datasetIds, hyperParams }, + }); + return unwrapResponse(raw) as TokenEstimate; +} + +/** + * Estimate training tokens for CPT jobs. + * + * The console API requires `hyperParams` as a **JSON string** with a full + * `userDefinedObj` payload (captured from the console frontend), plus several + * top-level fields (`algorithmType`, `bizType`, `priority`, …). Only + * `n_epochs` / `max_length` materially affect the estimate; the remaining + * hyper-parameters are fixed defaults. + */ +export async function estimateCptTokens( + client: Client, + model: string, + datasetIdsCsv: string, + nEpochs: number, +): Promise { + const userDefinedObj = { + batch_size: 16, + eval_steps: 50, + learning_rate: "7e-6", + lr_scheduler_type: "linear", + max_length: 8192, + n_epochs: nEpochs, + split: 0.9, + save_total_limit: "3", + resume_from_checkpoint: false, + save_strategy: "epoch", + }; + const hyperParams = JSON.stringify({ + useDefault: false, + userDefinedObj, + useQwenMixedStrategy: false, + }); + const raw = await client.console>(ESTIMATE_FINETUNE_TOKENS_API, { + input: { + trainingType: "cpt", + instanceName: `${model}_cli_estimate`, + algorithmType: 100, + bizType: 100, + trainDatasetIds: datasetIdsCsv, + hyperParams, + bailianTrainModel: model, + validationDatasetIds: "", + jobName: `${model}_cli_estimate`, + priority: "L0", + }, + }); + return unwrapResponse(raw) as TokenEstimate; +} diff --git a/packages/core/src/finetune/profiles/cpt.ts b/packages/core/src/finetune/profiles/cpt.ts index ab5aba7f..f9629445 100644 --- a/packages/core/src/finetune/profiles/cpt.ts +++ b/packages/core/src/finetune/profiles/cpt.ts @@ -1,7 +1,39 @@ /** * `cpt` profile — Continual Pre-Training (full-parameter). * Maps to the server's `cpt` training type. CPT record schema. + * CPT allows larger files (300 MB) compared to SFT/DPO (200 MB). */ -import { textProfile } from "./common.ts"; +import type { TrainingProfile, DataModality } from "./types.ts"; +import type { ValidateOpts, ValidationResult } from "../../dataset/validate/types.ts"; +import { validateDataset } from "../../dataset/validate/registry.ts"; +import { resolveTextHyperParameters } from "./common.ts"; +import { MAX_CPT_BYTES } from "../../dataset/validate/common.ts"; -export const cptProfile = textProfile("cpt", "cpt", "cpt"); +export const cptProfile: TrainingProfile = { + clientTrainingType: "cpt", + serverTrainingType: "cpt", + acceptedExtensions: [".jsonl"], + + async validate( + filePath: string, + _modality: DataModality, + opts: ValidateOpts, + ): Promise { + return validateDataset(filePath, { ...opts, schema: "cpt", maxBytes: MAX_CPT_BYTES }); + }, + + resolveHyperParameters( + _modality: DataModality, + flags: Record, + ): Record { + return resolveTextHyperParameters(flags); + }, + + shouldSkipGate(_gate: string, _modality: DataModality): boolean { + return false; + }, + + shouldSkipCapabilityCheck(_modality: DataModality): boolean { + return false; + }, +}; diff --git a/packages/core/src/finetune/profiles/sft-lora.ts b/packages/core/src/finetune/profiles/sft-lora.ts index d442dc5e..538543bf 100644 --- a/packages/core/src/finetune/profiles/sft-lora.ts +++ b/packages/core/src/finetune/profiles/sft-lora.ts @@ -75,20 +75,24 @@ const IMAGE_HYPER_PARAMS_I2I: Record = { * * Shared across all video models; `batch_size` and `max_pixels` differ by model * family (resolved per model in `resolveHyperParameters`): - * - wan2.5 (e.g. wan2.5-i2v-preview): batch_size 2, max_pixels 36864 - * - wan2.2 (i2v-flash / kf2v-flash): batch_size 4, max_pixels 262144 + * - wan2.7 (e.g. wan2.7-i2v): batch_size 1, max_pixels 102400 + * - wan2.5 (e.g. wan2.5-i2v-preview): batch_size 4, max_pixels 36864 + * - wan2.2 (i2v-flash / kf2v-flash): batch_size 4, max_pixels 262144 * * `learning_rate` is a string to avoid JSON-number precision loss (consistent * with the image defaults). `split` (0.9) + `max_split_val_dataset_sample` * (5) drive the automatic train/validation split when no explicit * validation_file_ids are provided. + * + * Values aligned with the official user guide (2026-07): + * n_epochs 50, eval_epochs 20 (≥ n_epochs/10). */ const VIDEO_HYPER_PARAMS_BASE: Record = { - n_epochs: 400, + n_epochs: 50, learning_rate: "2e-5", - split: 0.9, + split: 0.5, max_split_val_dataset_sample: 5, - eval_epochs: 50, + eval_epochs: 20, save_total_limit: 10, lora_rank: 32, lora_alpha: 32, @@ -109,6 +113,11 @@ function isWan25(model: string | undefined): boolean { return typeof model === "string" && /wan2\.5/i.test(model); } +/** wan2.7 family uses batch_size 1 and max_pixels 102400. */ +function isWan27(model: string | undefined): boolean { + return typeof model === "string" && /wan2\.7/i.test(model); +} + export const sftLoraProfile: TrainingProfile = { clientTrainingType: "sft-lora", serverTrainingType: "efficient_sft", @@ -195,15 +204,18 @@ export const sftLoraProfile: TrainingProfile = { } if (isVideo(modality)) { // Video: shared defaults + model-family-specific batch_size / max_pixels. - // wan2.5 uses batch_size 2 / max_pixels 36864; wan2.2 uses 4 / 262144. - const wan25 = isWan25(flags.model as string | undefined); + // wan2.7: batch_size 1, max_pixels 102400 + // wan2.5: batch_size 4, max_pixels 36864 + // wan2.2: batch_size 4, max_pixels 262144 + const model = (flags.model ?? flags.baseModel) as string | undefined; const hp: Record = { ...VIDEO_HYPER_PARAMS_BASE, - batch_size: 4, - max_pixels: wan25 ? 36864 : 262144, + batch_size: isWan27(model) ? 1 : 4, + max_pixels: isWan27(model) ? 102400 : isWan25(model) ? 36864 : 262144, }; // Optional overrides (no clamping — video batch_size is intentionally small). if (flags.nEpochs !== undefined) hp.n_epochs = flags.nEpochs as number; + if (flags.batchSize !== undefined) hp.batch_size = flags.batchSize as number; if (flags.learningRate !== undefined) hp.learning_rate = flags.learningRate as string; return hp; } diff --git a/packages/core/src/skills/agents.ts b/packages/core/src/skills/agents.ts index 1ddd369e..16b2e8d8 100644 --- a/packages/core/src/skills/agents.ts +++ b/packages/core/src/skills/agents.ts @@ -285,6 +285,21 @@ function isRecordedCopy(linkPath: string, recordedLinks: string[]): boolean { } } +/** + * Whether linkPath is a real directory containing a SKILL.md — indicating it is a + * skill artifact installed by another tool (e.g. `npx skills add`) or an older + * version predating bl's symlink management. These are safe to replace: they are + * not arbitrary user content but the same kind of artifact we manage. + */ +function isForeignSkillDir(linkPath: string): boolean { + try { + if (!lstatSync(linkPath).isDirectory()) return false; + return existsSync(join(linkPath, "SKILL.md")); + } catch { + return false; + } +} + export interface LinkResult { agent: string; path: string; @@ -326,6 +341,11 @@ export function linkSkillToAgents( // Copy-fallback artifact from a previous install → replace so updates // reach agents that have no symlink permission rmSync(linkPath, { recursive: true, force: true }); + } else if (isForeignSkillDir(linkPath)) { + // A real directory containing SKILL.md — a skill installed by another + // tool (e.g. `npx skills add`) or predating bl's symlink management. + // Replace with our symlink so future updates propagate automatically. + rmSync(linkPath, { recursive: true, force: true }); } else { results.push({ agent: agent.id, diff --git a/packages/core/src/types/api.ts b/packages/core/src/types/api.ts index fdc337ea..5d3692dd 100644 --- a/packages/core/src/types/api.ts +++ b/packages/core/src/types/api.ts @@ -206,6 +206,8 @@ export interface DashScopeVideoRequest { prompt: string; negative_prompt?: string; img_url?: string; + first_frame_url?: string; + last_frame_url?: string; media?: Array<{ type: "image" | "video" | "first_frame" | "last_frame" | "driving_audio" | "first_clip"; url: string; diff --git a/packages/core/tests/dataset-validate.test.ts b/packages/core/tests/dataset-validate.test.ts index b4b2b52b..b478cca6 100644 --- a/packages/core/tests/dataset-validate.test.ts +++ b/packages/core/tests/dataset-validate.test.ts @@ -56,8 +56,6 @@ describe("validateDataset — DPO schema", () => { }); test('schema "dpo" requires both chosen and rejected on every record', async () => { - // A record with neither chosen nor rejected is SFT-shaped; under --schema dpo - // it must be flagged as missing both preferences. const p = file("sft_under_dpo.jsonl", [SFT_OK]); const r = await validateDataset(p, { fullValidate: true, schema: "dpo" }); expect(r.valid).toBe(false); @@ -107,6 +105,43 @@ describe("validateDataset — DPO schema", () => { const r = await validateDataset(p, { fullValidate: true, schema: "dpo" }); expect(r.valid).toBe(true); }); + + test("DPO messages ending with assistant → DPO_LAST_MSG_NOT_USER error", async () => { + const p = file("dpo_last_asst.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"yo"}],"chosen":{"role":"assistant","content":"good"},"rejected":{"role":"assistant","content":"bad"}}', + ]); + const r = await validateDataset(p, { fullValidate: true, schema: "dpo" }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("DPO_LAST_MSG_NOT_USER"); + }); + + test("DPO with image content item → DPO_UNSUPPORTED_ELEMENT error", async () => { + const p = file("dpo_image.jsonl", [ + '{"messages":[{"role":"user","content":[{"text":"look"},{"image":"a.jpg"}]}],"chosen":{"role":"assistant","content":[{"text":"good"}]},"rejected":{"role":"assistant","content":[{"text":"bad"}]}}', + ]); + const r = await validateDataset(p, { fullValidate: true, schema: "dpo" }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("DPO_UNSUPPORTED_ELEMENT"); + }); + + test("DPO with tools / tool_calls → DPO_UNSUPPORTED_ELEMENT error", async () => { + const p = file("dpo_tools.jsonl", [ + '{"tools":[{"type":"function","function":{"name":"f","parameters":{}}}],"messages":[{"role":"user","content":"hi"}],"chosen":{"role":"assistant","content":"good"},"rejected":{"role":"assistant","content":"bad"}}', + ]); + const r = await validateDataset(p, { fullValidate: true, schema: "dpo" }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("DPO_UNSUPPORTED_ELEMENT"); + }); + + test("DPO chosen carrying an image item → DPO_UNSUPPORTED_ELEMENT error", async () => { + const p = file("dpo_chosen_image.jsonl", [ + '{"messages":[{"role":"user","content":"hi"}],"chosen":{"role":"assistant","content":[{"text":"good"},{"image":"x.png"}]},"rejected":{"role":"assistant","content":"bad"}}', + ]); + const r = await validateDataset(p, { fullValidate: true, schema: "dpo" }); + expect(r.valid).toBe(false); + const err = r.errors.find((e) => e.code === "DPO_UNSUPPORTED_ELEMENT"); + expect(err?.path).toContain("chosen"); + }); }); describe("validateDataset — CPT schema", () => { @@ -142,8 +177,6 @@ describe("validateDataset — CPT schema", () => { }); test("auto-detect routes a {text} record to CPT, not ChatML", async () => { - // A CPT record has no `messages`; under auto-detect it must NOT produce a - // ChatML MISSING_MESSAGES error — it should be validated as CPT and pass. const p = file("cpt_auto.jsonl", [CPT_OK]); const r = await validateDataset(p, { fullValidate: true }); expect(r.valid).toBe(true); @@ -151,8 +184,6 @@ describe("validateDataset — CPT schema", () => { }); test("SFT record with a stray text field still routes to ChatML", async () => { - // {messages, text} is ambiguous; CPT detect requires text AND no messages, - // so this falls through to ChatML and validates as SFT (text ignored). const p = file("mixed.jsonl", [ '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"yo"}],"text":"noise"}', ]); @@ -162,6 +193,400 @@ describe("validateDataset — CPT schema", () => { }); }); +describe("validateDataset — content array format", () => { + test("content as [{text}] array passes validation", async () => { + const p = file("content_arr.jsonl", [ + '{"messages":[{"role":"system","content":[{"text":"sys"}]},{"role":"user","content":[{"text":"hi"}]},{"role":"assistant","content":[{"text":"hello"}]}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).errors).toEqual([]); + }); + + test("content as plain string still passes (legacy format)", async () => { + const p = file("content_str.jsonl", [SFT_OK]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + }); + + test("content array with image item passes (VL multimodal)", async () => { + const p = file("content_img.jsonl", [ + '{"messages":[{"role":"user","content":[{"text":"describe"},{"image":"img1.jpg"}]},{"role":"assistant","content":[{"text":"a cat"}]}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + }); + + test("content array with video string item passes", async () => { + const p = file("content_vid.jsonl", [ + '{"messages":[{"role":"user","content":[{"text":"describe"},{"video":"vid1.mp4"}]},{"role":"assistant","content":[{"text":"a car"}]}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + }); + + test("content array with video frame list passes", async () => { + const p = file("content_frames.jsonl", [ + '{"messages":[{"role":"user","content":[{"text":"describe"},{"video":["0.jpg","1.jpg","2.jpg"]}]},{"role":"assistant","content":[{"text":"frames"}]}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + }); + + test("content array with invalid item (no text/image/video) → error", async () => { + const p = file("content_bad_item.jsonl", [ + '{"messages":[{"role":"user","content":[{"foo":"bar"}]},{"role":"assistant","content":"ok"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("CONTENT_ITEM_NO_KNOWN_FIELD"); + }); + + test("content as number → INVALID_CONTENT error", async () => { + const p = file("content_num.jsonl", [ + '{"messages":[{"role":"user","content":42},{"role":"assistant","content":"ok"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("INVALID_CONTENT"); + }); + + test("empty content array → EMPTY_CONTENT_ARRAY error", async () => { + const p = file("content_empty_arr.jsonl", [ + '{"messages":[{"role":"user","content":[]},{"role":"assistant","content":"ok"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("EMPTY_CONTENT_ARRAY"); + }); + + test("DPO with content array format passes", async () => { + const p = file("dpo_arr.jsonl", [ + '{"messages":[{"role":"user","content":[{"text":"hi"}]}],"chosen":{"role":"assistant","content":[{"text":"good"}]},"rejected":{"role":"assistant","content":[{"text":"bad"}]}}', + ]); + const r = await validateDataset(p, { fullValidate: true, schema: "dpo" }); + expect(r.valid).toBe(true); + }); +}); + +describe("validateDataset — tool calling (function calling)", () => { + const TOOL_OK = JSON.stringify({ + tools: [ + { + type: "function", + function: { + name: "get_weather", + description: "get weather", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }, + }, + }, + ], + messages: [ + { role: "user", content: [{ text: "weather in Beijing" }] }, + { + role: "assistant", + content: [{ text: "let me check" }], + tool_calls: [ + { + id: "call_1", + type: "function", + function: { name: "get_weather", arguments: '{"city":"Beijing"}' }, + }, + ], + }, + { role: "tool", tool_call_id: "call_1", content: [{ text: '{"weather":"sunny"}' }] }, + { role: "assistant", content: [{ text: "It is sunny." }] }, + ], + }); + + test("valid tool calling record passes", async () => { + const p = file("tool_ok.jsonl", [TOOL_OK]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).errors).toEqual([]); + }); + + test("tool role is accepted (no INVALID_ROLE)", async () => { + const p = file("tool_role.jsonl", [TOOL_OK]); + const r = await validateDataset(p, { fullValidate: true }); + expect(codes(r).errors).not.toContain("INVALID_ROLE"); + }); + + test("tool message without tool_call_id → TOOL_MISSING_CALL_ID", async () => { + const p = file("tool_no_id.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"","tool_calls":[{"id":"c1","type":"function","function":{"name":"f","arguments":"{}"}}]},{"role":"tool","content":"result"},{"role":"assistant","content":"done"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("TOOL_MISSING_CALL_ID"); + }); + + test("tool_call_id unmatched → TOOL_CALL_ID_UNMATCHED error", async () => { + const p = file("tool_unmatched.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"","tool_calls":[{"id":"c1","type":"function","function":{"name":"f","arguments":"{}"}}]},{"role":"tool","tool_call_id":"WRONG_ID","content":"result"},{"role":"assistant","content":"done"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("TOOL_CALL_ID_UNMATCHED"); + // The orphaned call side is advisory + expect(codes(r).warnings).toContain("TOOL_CALL_NO_RESPONSE"); + }); + + test("tool_calls without a tool response → TOOL_CALL_NO_RESPONSE warning", async () => { + const p = file("tool_no_resp.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"","tool_calls":[{"id":"c1","type":"function","function":{"name":"f","arguments":"{}"}}]},{"role":"assistant","content":"done"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(codes(r).warnings).toContain("TOOL_CALL_NO_RESPONSE"); + }); + + test("tool_calls with missing function name → TOOL_CALL_FN_NO_NAME", async () => { + const p = file("tool_no_fn_name.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"","tool_calls":[{"id":"c1","type":"function","function":{"arguments":"{}"}}]},{"role":"tool","tool_call_id":"c1","content":"r"},{"role":"assistant","content":"ok"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("TOOL_CALL_FN_NO_NAME"); + }); + + test("assistant with tool_calls but no content is valid", async () => { + const p = file("tool_no_content.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","tool_calls":[{"id":"c1","type":"function","function":{"name":"f","arguments":"{}"}}]},{"role":"tool","tool_call_id":"c1","content":"r"},{"role":"assistant","content":"ok"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).errors).not.toContain("MISSING_CONTENT"); + }); +}); + +describe("validateDataset — thinking tags", () => { + test("think tag in last assistant is valid", async () => { + const p = file("think_ok.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"\\nreasoning\\n\\n\\nanswer"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).warnings).not.toContain("THINK_TAG_NOT_LAST"); + }); + + test("think tag in non-last assistant → THINK_TAG_NOT_LAST warning", async () => { + const p = file("think_mid.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"\\nearly\\n\\n\\nmid"},{"role":"user","content":"more"},{"role":"assistant","content":"final"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).warnings).toContain("THINK_TAG_NOT_LAST"); + }); + + test("think tag in content array format detected", async () => { + const p = file("think_arr.jsonl", [ + '{"messages":[{"role":"user","content":[{"text":"hi"}]},{"role":"assistant","content":[{"text":"\\nreason\\n\\n\\nans"}]},{"role":"user","content":[{"text":"more"}]},{"role":"assistant","content":[{"text":"final"}]}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).warnings).toContain("THINK_TAG_NOT_LAST"); + }); + + test("official tool+thinking combo: think in non-last assistant WITH tool_calls is exempt", async () => { + // Mirrors the platform spec's 工具与思考组合 example: the assistant that + // issues tool_calls carries the block, the final assistant answers. + const p = file("think_tool_combo.jsonl", [ + JSON.stringify({ + tools: [ + { + type: "function", + function: { name: "get_weather", description: "d", parameters: { type: "object" } }, + }, + ], + messages: [ + { role: "user", content: [{ text: "weather in Beijing?" }] }, + { + role: "assistant", + content: [{ text: "\nneed the weather tool\n\n" }], + tool_calls: [ + { + id: "call_1", + type: "function", + function: { name: "get_weather", arguments: '{"city":"Beijing"}' }, + }, + ], + }, + { role: "tool", tool_call_id: "call_1", content: [{ text: '{"weather":"sunny"}' }] }, + { role: "assistant", content: [{ text: "It is sunny in Beijing." }] }, + ], + }), + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).warnings).not.toContain("THINK_TAG_NOT_LAST"); + }); +}); + +describe("validateDataset — OpenAI migration guards", () => { + test("message-level name field → UNSUPPORTED_FIELD_NAME error", async () => { + const p = file("openai_name.jsonl", [ + '{"messages":[{"role":"user","content":"hi","name":"alice"},{"role":"assistant","content":"hello"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("UNSUPPORTED_FIELD_NAME"); + }); + + test("message-level weight field → UNSUPPORTED_FIELD_WEIGHT error", async () => { + const p = file("openai_weight.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"hello","weight":0.5}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("UNSUPPORTED_FIELD_WEIGHT"); + }); + + test("record-level weight field → UNSUPPORTED_FIELD_WEIGHT error", async () => { + const p = file("record_weight.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"hello"}],"weight":1}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("UNSUPPORTED_FIELD_WEIGHT"); + }); +}); + +describe("validateDataset — loss_weight", () => { + test("valid loss_weight (0.5) passes", async () => { + const p = file("lw_ok.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"hello"}],"loss_weight":0.5}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).errors).not.toContain("INVALID_LOSS_WEIGHT"); + }); + + test("loss_weight out of range (1.5) → INVALID_LOSS_WEIGHT", async () => { + const p = file("lw_bad.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"hello"}],"loss_weight":1.5}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("INVALID_LOSS_WEIGHT"); + }); + + test("loss_weight negative → INVALID_LOSS_WEIGHT", async () => { + const p = file("lw_neg.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"hello"}],"loss_weight":-0.1}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("INVALID_LOSS_WEIGHT"); + }); + + test("loss_weight non-number → INVALID_LOSS_WEIGHT", async () => { + const p = file("lw_str.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"hello"}],"loss_weight":"high"}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("INVALID_LOSS_WEIGHT"); + }); + + test("loss_weight boundary values 0 and 1 pass", async () => { + const p = file("lw_boundary.jsonl", [ + '{"messages":[{"role":"user","content":"a"},{"role":"assistant","content":"b"}],"loss_weight":0}', + '{"messages":[{"role":"user","content":"c"},{"role":"assistant","content":"d"}],"loss_weight":1}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + }); + + test("message-level loss_weight on the LAST assistant passes without warning", async () => { + const p = file("lw_msg_last.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"hello","loss_weight":0.8}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).warnings).not.toContain("LOSS_WEIGHT_PLACEMENT"); + }); + + test("message-level loss_weight on a non-last assistant → LOSS_WEIGHT_PLACEMENT warning", async () => { + const p = file("lw_msg_mid.jsonl", [ + '{"messages":[{"role":"user","content":"a"},{"role":"assistant","content":"b","loss_weight":0.8},{"role":"user","content":"c"},{"role":"assistant","content":"d"}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(codes(r).warnings).toContain("LOSS_WEIGHT_PLACEMENT"); + }); + + test("message-level loss_weight out of range → INVALID_LOSS_WEIGHT", async () => { + const p = file("lw_msg_range.jsonl", [ + '{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"hello","loss_weight":2}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("INVALID_LOSS_WEIGHT"); + }); +}); + +describe("validateDataset — video content params", () => { + test("path-mode video with in-range fps and clip times passes", async () => { + const p = file("video_path_ok.jsonl", [ + '{"messages":[{"role":"user","content":[{"text":"desc"},{"video":"v.mp4","fps":3.0,"video_start":0.0,"video_end":3.0}]},{"role":"assistant","content":[{"text":"ok"}]}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).warnings).not.toContain("VIDEO_PARAM_MODE_MISMATCH"); + }); + + test("fps out of [0.1, 10] → INVALID_VIDEO_FPS error", async () => { + const p = file("video_fps_bad.jsonl", [ + '{"messages":[{"role":"user","content":[{"text":"desc"},{"video":"v.mp4","fps":30}]},{"role":"assistant","content":[{"text":"ok"}]}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(false); + expect(codes(r).errors).toContain("INVALID_VIDEO_FPS"); + }); + + test("sample_fps on path-mode video → VIDEO_PARAM_MODE_MISMATCH warning", async () => { + const p = file("video_mode_mix.jsonl", [ + '{"messages":[{"role":"user","content":[{"text":"desc"},{"video":"v.mp4","sample_fps":2.0}]},{"role":"assistant","content":[{"text":"ok"}]}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(codes(r).warnings).toContain("VIDEO_PARAM_MODE_MISMATCH"); + }); + + test("frame-list video with sample_fps passes; fps there is flagged", async () => { + const p = file("video_frames.jsonl", [ + '{"messages":[{"role":"user","content":[{"text":"desc"},{"video":["0.jpg","1.jpg"],"sample_fps":5.0}]},{"role":"assistant","content":[{"text":"ok"}]}]}', + '{"messages":[{"role":"user","content":[{"text":"desc"},{"video":["0.jpg"],"fps":2.0}]},{"role":"assistant","content":[{"text":"ok"}]}]}', + ]); + const r = await validateDataset(p, { fullValidate: true }); + expect(r.valid).toBe(true); + expect(codes(r).warnings).toContain("VIDEO_PARAM_MODE_MISMATCH"); + }); +}); + +describe("validateZipFilenames — macOS metadata entries", () => { + test("__MACOSX / .DS_Store / ._resource-fork entries are ignored", async () => { + const { validateZipFilenames } = await import("../src/dataset/validate/zip.ts"); + const issues = validateZipFilenames([ + "data.jsonl", + "image_1.jpg", + "__MACOSX/._image_1.jpg", + "__MACOSX/", + ".DS_Store", + "train/._clip.wav", + ]); + expect(issues).toEqual([]); + }); + + test("real charset violations are still reported", async () => { + const { validateZipFilenames } = await import("../src/dataset/validate/zip.ts"); + const issues = validateZipFilenames(["data.jsonl", "图片1.jpg"]); + expect(issues.map((issue) => issue.code)).toContain("INVALID_FILENAME_CHARSET"); + }); +}); + describe("parseDatasetSchemaFlag", () => { test("undefined / empty → undefined (auto)", () => { expect(parseDatasetSchemaFlag(undefined)).toBeUndefined(); diff --git a/packages/core/tests/skills-agents.test.ts b/packages/core/tests/skills-agents.test.ts index dc80ca31..7fa8125b 100644 --- a/packages/core/tests/skills-agents.test.ts +++ b/packages/core/tests/skills-agents.test.ts @@ -293,20 +293,51 @@ test("agents: Amp-style XDG config dir lights up the universal-xdg shared target }); }); -test("agents: recorded copy-fallback artifact is replaced; unrecorded dir stays skipped", async () => { +test("agents: foreign skill dir (contains SKILL.md) is replaced even without lock record", async () => { await inFakeHome(async (home) => { mkdirSync(join(home, ".claude"), { recursive: true }); const canonical = seedCanonicalSkill("demo"); const copyPath = join(home, ".claude", "skills", "demo"); - // Simulate a previous install that fell back to copy (no symlink permission, e.g. Windows) + // Simulate a skill installed by another tool (e.g. `npx skills add`) — a real + // directory containing SKILL.md, not recorded in our lock. mkdirSync(copyPath, { recursive: true }); - writeFileSync(join(copyPath, "SKILL.md"), "stale copy"); + writeFileSync(join(copyPath, "SKILL.md"), "stale copy from another tool"); + + // New behavior: contains SKILL.md → recognized as a skill artifact → replaced + const result = linkSkillToAgents("demo"); + expect(result[0]).toMatchObject({ agent: "claude-code", mode: "symlink" }); + expect(lstatSync(copyPath).isSymbolicLink()).toBe(true); + expect(readlinkSync(copyPath)).toBe(canonical); + }); +}); + +test("agents: foreign non-skill dir (no SKILL.md) stays skipped", async () => { + await inFakeHome(async (home) => { + mkdirSync(join(home, ".claude"), { recursive: true }); + seedCanonicalSkill("demo"); + const foreignPath = join(home, ".claude", "skills", "demo"); - // Without a lock record the dir is foreign → skipped, content untouched - const unrecorded = linkSkillToAgents("demo"); - expect(unrecorded[0].mode).toBe("skipped"); - expect(readFileSync(join(copyPath, "SKILL.md"), "utf-8")).toBe("stale copy"); + // A user's own directory that happens to share the skill name but has no SKILL.md + mkdirSync(foreignPath, { recursive: true }); + writeFileSync(join(foreignPath, "my-notes.txt"), "user content"); + + const result = linkSkillToAgents("demo"); + expect(result[0].mode).toBe("skipped"); + // User content untouched + expect(readFileSync(join(foreignPath, "my-notes.txt"), "utf-8")).toBe("user content"); + }); +}); + +test("agents: recorded copy-fallback artifact is replaced with symlink", async () => { + await inFakeHome(async (home) => { + mkdirSync(join(home, ".claude"), { recursive: true }); + const canonical = seedCanonicalSkill("demo"); + const copyPath = join(home, ".claude", "skills", "demo"); + + // Simulate a previous install that fell back to copy (no symlink permission) + mkdirSync(copyPath, { recursive: true }); + writeFileSync(join(copyPath, "SKILL.md"), "stale copy"); // With the recorded link the artifact is rebuilt and points at canonical again const recorded = linkSkillToAgents("demo", detectInstalledAgents(), [copyPath]); diff --git a/skills/bailian-finetune/SKILL.md b/skills/bailian-finetune/SKILL.md index 9f3180fe..d04ba77e 100644 --- a/skills/bailian-finetune/SKILL.md +++ b/skills/bailian-finetune/SKILL.md @@ -23,14 +23,14 @@ description: >- ``` 1. Validate data bl dataset validate --file train.jsonl [--schema chatml|dpo|cpt|tts|image] 2. Upload data bl dataset upload --file train.jsonl # returns a file-id -3. Create job bl finetune text|audio|image create --model --datasets +3. Create job bl finetune text|audio|image create --base-model --datasets 4. Watch progress bl finetune watch --job-id ft-xxx # or get / logs 5. Pick artifact bl finetune checkpoints --job-id ft-xxx 6. Export model bl finetune export --job-id ft-xxx --checkpoint ckpt-N --model-name my-model -7. Deploy service bl deploy text|audio|image create --model my-model --name my-svc +7. Deploy service bl deploy text|audio|image create --model-name my-model --display-name my-svc ``` -- Unsure which training methods a base model supports → `bl finetune capability --model ` or `--training-type sft|sft-lora|dpo|cpt`. +- Unsure which training methods a base model supports → `bl finetune capability --base-model ` or `--training-type sft|sft-lora|dpo|cpt`. - Text `--training-type` values: `sft` / `sft-lora` / `dpo` / `dpo-lora` / `cpt`. Audio bases include `cosyvoice-v3-flash`; image bases include `wan2.7-image-pro`. - Deployment plans: audio defaults to `--plan mu`; text/image default to `lora`. - Preview write operations (create / delete / cancel / scale) with `--dry-run` first, and confirm with the user before deleting a job or dataset. @@ -55,10 +55,10 @@ Flags, usage, and examples: see [`reference/`](reference/index.md) or `bl [--purpose ] [--schema ] [--no-validate] [--full-validate]` | +| Field | Value | +| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------- | +| **Name** | `dataset upload` | +| **Description** | Upload a dataset file (.jsonl or .zip) to Bailian | +| **Authentication** | API Key | +| **Usage** | `bl dataset upload --file [--purpose ] [--schema ] [--no-validate] [--full-validate]` | #### Flags -| Flag | Type | Required | Description | -| ------------------ | ------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--file ` | string | yes | Local dataset file (.jsonl or .zip; ≤300MB text, ≤1GB image) | -| `--purpose ` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") | -| `--schema ` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record. | -| `--no-validate` | switch | no | Skip the local JSONL pre-flight check (not recommended) | -| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) | -| `--api-key ` | string | no | API key | -| `--base-url ` | string | no | API base URL | +| Flag | Type | Required | Description | +| ------------------ | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `--file ` | string | yes | Local dataset file (.jsonl or .zip; ≤200MB SFT/DPO, ≤300MB CPT, ≤2GB media zip) | +| `--purpose ` | string | no | Dataset purpose tag (default: "fine-tune"; e.g. "evaluation") | +| `--schema ` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record. | +| `--no-validate` | switch | no | Skip the local JSONL pre-flight check (not recommended) | +| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) | +| `--api-key ` | string | no | API key | +| `--base-url ` | string | no | API base URL | #### Notes - Supports .jsonl (text) and .zip (audio/image archives with a data.jsonl -- manifest). Five record schemas are recognized: chatml = {messages:[...]} +- manifest). Six record schemas are recognized: chatml = {messages:[...]} - (SFT); dpo = {messages:[...], chosen, rejected}; cpt = {text:"..."} - (continual pre-training, raw text); tts = {wav_fn:"train/xxx.wav", - text:"..."} (audio fine-tuning); image = {img_path:"..."} (image -- generation). With no --schema, a record carrying wav_fn is validated as -- TTS, img_path as image, chosen/rejected as DPO, text (no messages) as CPT, -- otherwise ChatML. Upload cap: 300MB text, 1GB image. Upload uses the +- generation); video = {first_frame_path:...} (video generation). With no +- --schema, a record carrying wav_fn is validated as TTS, img_path as image, +- chosen/rejected as DPO, text (no messages) as CPT, otherwise ChatML. +- Upload cap: 200MB SFT/DPO text, 300MB CPT, 2GB media zip. Upload uses the - OpenAI-compatible /compatible-mode/v1/files endpoint so the purpose tag is - persisted (the DashScope-native /api/v1/files drops it). @@ -174,20 +175,20 @@ bl dataset upload --file train.jsonl --no-validate ### `bl dataset validate` -| Field | Value | -| ------------------ | ----------------------------------------------------------------------------------------------- | -| **Name** | `dataset validate` | -| **Description** | Locally validate a dataset file (.jsonl or .zip) without uploading | -| **Authentication** | No Auth | -| **Usage** | `bl dataset validate --file [--full-validate] [--schema ]` | +| Field | Value | +| ------------------ | ------------------------------------------------------------------------------------------------------ | +| **Name** | `dataset validate` | +| **Description** | Locally validate a dataset file (.jsonl or .zip) without uploading | +| **Authentication** | No Auth | +| **Usage** | `bl dataset validate --file [--full-validate] [--schema ]` | #### Flags -| Flag | Type | Required | Description | -| ----------------- | ------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--file ` | string | yes | Local dataset file (.jsonl or .zip) | -| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) | -| `--schema ` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), or "image" (image generation). Default auto-detects per record. | +| Flag | Type | Required | Description | +| ----------------- | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `--file ` | string | yes | Local dataset file (.jsonl or .zip) | +| `--full-validate` | switch | no | JSON.parse every line instead of sampling (slower) | +| `--schema ` | string | no | Record schema: "chatml" (SFT), "dpo" (chosen/rejected), "cpt" (raw text), "tts" (audio), "image" (image generation), or "video" (video generation). Default auto-detects per record. | #### Notes @@ -196,13 +197,15 @@ bl dataset upload --file train.jsonl --no-validate - Schemas: chatml = {messages:[...]} (SFT); dpo = {messages:[...], chosen, - rejected}; cpt = {text:"..."} (continual pre-training, raw text); - tts = {wav_fn:"train/xxx.wav", text:"..."} (audio fine-tuning); -- image = {img_path:"..."} (image generation). With no --schema, a record -- carrying wav_fn is validated as TTS, img_path as image, chosen/rejected -- as DPO, text (no messages) as CPT, otherwise ChatML. Pass --schema to -- require a specific shape on every record. ZIP archives (.zip) are -- validated structurally (data.jsonl present, media references resolve) in -- addition to per-record content checks. Use --full-validate to JSON.parse -- every line. +- image = {img_path:"..."} (image generation); +- video = {first_frame_path:"...", video_path:"..."} (video generation, +- i2v first-frame or kf2v first+last-frame with last_frame_path). With no +- --schema, a record carrying wav_fn is validated as TTS, img_path as image, +- first_frame_path/video_path as video, chosen/rejected as DPO, text (no +- messages) as CPT, otherwise ChatML. Pass --schema to require a specific +- shape on every record. ZIP archives (.zip) are validated structurally +- (data.jsonl present, media references resolve) in addition to per-record +- content checks. Use --full-validate to JSON.parse every line. #### Examples @@ -222,6 +225,10 @@ bl dataset validate --file cpt.jsonl --schema cpt bl dataset validate --file audio.zip --schema tts ``` +```bash +bl dataset validate --file wan-i2v-training-dataset.zip --schema video +``` + ```bash bl dataset validate --file eval.jsonl --full-validate ``` diff --git a/skills/bailian-finetune/reference/deploy.md b/skills/bailian-finetune/reference/deploy.md index 3e0e2c3f..65519097 100644 --- a/skills/bailian-finetune/reference/deploy.md +++ b/skills/bailian-finetune/reference/deploy.md @@ -7,44 +7,46 @@ Index: [index.md](index.md) ## Commands in this group -| Command | Authentication | Description | -| ------------------------ | -------------- | --------------------------------------------------------- | -| `bl deploy audio create` | API Key | Create an audio (TTS) model deployment | -| `bl deploy delete` | API Key | Delete a model deployment (must be STOPPED or FAILED) | -| `bl deploy get` | API Key | Get details of a single model deployment | -| `bl deploy image create` | API Key | Create an image generation model deployment | -| `bl deploy list` | API Key | List model deployments | -| `bl deploy models` | API Key | List models available for deployment | -| `bl deploy scale` | API Key | Scale a deployment's capacity | -| `bl deploy text create` | API Key | Create a text model deployment | -| `bl deploy update` | API Key | Update a deployment's rate limits (rpm_limit / tpm_limit) | +| Command | Authentication | Description | +| ------------------------ | -------------- | ------------------------------------------------------------- | +| `bl deploy audio create` | API Key | Create an audio (TTS) model deployment | +| `bl deploy delete` | API Key | Delete a model deployment (must be STOPPED or FAILED) | +| `bl deploy get` | API Key | Get details of a single model deployment | +| `bl deploy image create` | API Key | Create an image generation model deployment | +| `bl deploy list` | API Key | List model deployments | +| `bl deploy models` | API Key | List models available for deployment | +| `bl deploy pause` | Console | Pause a running model deployment (stops billing for mu/ptu) | +| `bl deploy resume` | Console | Resume a paused model deployment (brings service back online) | +| `bl deploy scale` | API Key | Scale a deployment's capacity | +| `bl deploy text create` | API Key | Create a text model deployment | +| `bl deploy update` | API Key | Update a deployment's rate limits (rpm_limit / tpm_limit) | ## Command details ### `bl deploy audio create` -| Field | Value | -| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Name** | `deploy audio create` | -| **Description** | Create an audio (TTS) model deployment | -| **Authentication** | API Key | -| **Usage** | `bl deploy audio create --model --name [--plan ] [--deploy-spec ] [--capacity ] [--billing-method ] [--input-tpm ] [--output-tpm ] [--thinking-output-tpm ]` | +| Field | Value | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Name** | `deploy audio create` | +| **Description** | Create an audio (TTS) model deployment | +| **Authentication** | API Key | +| **Usage** | `bl deploy audio create --model-name --display-name [--plan ] [--deploy-spec ] [--capacity ] [--billing-method ] [--input-tpm ] [--output-tpm ] [--thinking-output-tpm ]` | #### Flags -| Flag | Type | Required | Description | -| --------------------------- | ------ | -------- | ------------------------------------------------------------------------------- | -| `--model ` | string | yes | Model name (catalog model or fine-tuned output) (required) | -| `--name ` | string | yes | Console display name for the deployment (required) | -| `--plan ` | string | no | Billing plan: lora (default, Token-billed) \| ptu (Token-billed) \| mu | -| `--deploy-spec ` | string | no | Deploy spec (only used by plan=mu; auto-picked if omitted) | -| `--capacity ` | number | no | Resource units (plan=mu only; required by API; defaults to the template's unit) | -| `--billing-method ` | string | no | Billing method (plan=mu only; default "POST_PAY", the only supported value) | -| `--input-tpm ` | number | no | PTU max input tokens/min (required for plan=ptu) | -| `--output-tpm ` | number | no | PTU max output tokens/min (required for plan=ptu) | -| `--thinking-output-tpm ` | number | no | PTU max thinking-output tokens/min (optional, some models) | -| `--api-key ` | string | no | API key | -| `--base-url ` | string | no | API base URL | +| Flag | Type | Required | Description | +| ------------------------------- | ------ | -------- | ------------------------------------------------------------------------------- | +| `--model-name ` | string | yes | Model to deploy — fine-tuned output name or catalog model (required) | +| `--display-name ` | string | yes | Console display name for the deployment (required) | +| `--plan ` | string | no | Billing plan: lora (default, Token-billed) \| ptu (Token-billed) \| mu | +| `--deploy-spec ` | string | no | Deploy spec (only used by plan=mu; auto-picked if omitted) | +| `--capacity ` | number | no | Resource units (plan=mu only; required by API; defaults to the template's unit) | +| `--billing-method ` | string | no | Billing method (plan=mu only; default "POST_PAY", the only supported value) | +| `--input-tpm ` | number | no | PTU max input tokens/min (required for plan=ptu) | +| `--output-tpm ` | number | no | PTU max output tokens/min (required for plan=ptu) | +| `--thinking-output-tpm ` | number | no | PTU max thinking-output tokens/min (optional, some models) | +| `--api-key ` | string | no | API key | +| `--base-url ` | string | no | API base URL | #### Notes @@ -59,27 +61,24 @@ Index: [index.md](index.md) - Use `bl deploy models --source base` to inspect available templates. - After creation, status starts at PENDING and transitions to RUNNING. - Invoke the deployed model with: bl text chat --model -- WARNING: --model is overloaded across commands and refers to DIFFERENT -- values. `bl deploy create --model` takes the exported model_name -- (e.g. `qwen3-8b-ft-...`), but the create response also returns a -- `deployed_model` field (the deployment instance id, e.g. -- `qwen3-8b-5ecb5f068d79`). The inference call `bl text chat --model` must use -- the `deployed_model` from the create response — NOT the `model_name` you -- passed to `deploy create`. Do not reuse the value across the two -- commands. +- NOTE: --model-name is the model being deployed (e.g. `qwen3-8b-ft-...`). +- The create response also returns a `deployed_model` field — the deployment +- instance id (e.g. `qwen3-8b-5ecb5f068d79`). Use that id for inference +- (`bl text chat --model `) and lifecycle commands +- (`deploy get/scale/pause/resume/delete --deployed-model `). #### Examples ```bash -bl deploy audio create --model my-cosyvoice-ft --name my-tts +bl deploy audio create --model-name my-cosyvoice-ft --display-name my-tts ``` ```bash -bl deploy audio create --model my-cosyvoice-ft --name my-tts --deploy-spec dps-xxxx --capacity 1 +bl deploy audio create --model-name my-cosyvoice-ft --display-name my-tts --deploy-spec dps-xxxx --capacity 1 ``` ```bash -bl deploy audio create --model my-cosyvoice-ft --name my-tts --dry-run +bl deploy audio create --model-name my-cosyvoice-ft --display-name my-tts --dry-run ``` ### `bl deploy delete` @@ -139,28 +138,28 @@ bl deploy get --deployed-model qwen-plus-2025-12-01-b6d61c71 --output json ### `bl deploy image create` -| Field | Value | -| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Name** | `deploy image create` | -| **Description** | Create an image generation model deployment | -| **Authentication** | API Key | -| **Usage** | `bl deploy image create --model --name [--plan ] [--deploy-spec ] [--capacity ] [--billing-method ] [--input-tpm ] [--output-tpm ] [--thinking-output-tpm ]` | +| Field | Value | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Name** | `deploy image create` | +| **Description** | Create an image generation model deployment | +| **Authentication** | API Key | +| **Usage** | `bl deploy image create --model-name --display-name [--plan ] [--deploy-spec ] [--capacity ] [--billing-method ] [--input-tpm ] [--output-tpm ] [--thinking-output-tpm ]` | #### Flags -| Flag | Type | Required | Description | -| --------------------------- | ------ | -------- | ------------------------------------------------------------------------------- | -| `--model ` | string | yes | Model name (catalog model or fine-tuned output) (required) | -| `--name ` | string | yes | Console display name for the deployment (required) | -| `--plan ` | string | no | Billing plan: lora (default, Token-billed) \| ptu (Token-billed) \| mu | -| `--deploy-spec ` | string | no | Deploy spec (only used by plan=mu; auto-picked if omitted) | -| `--capacity ` | number | no | Resource units (plan=mu only; required by API; defaults to the template's unit) | -| `--billing-method ` | string | no | Billing method (plan=mu only; default "POST_PAY", the only supported value) | -| `--input-tpm ` | number | no | PTU max input tokens/min (required for plan=ptu) | -| `--output-tpm ` | number | no | PTU max output tokens/min (required for plan=ptu) | -| `--thinking-output-tpm ` | number | no | PTU max thinking-output tokens/min (optional, some models) | -| `--api-key ` | string | no | API key | -| `--base-url ` | string | no | API base URL | +| Flag | Type | Required | Description | +| ------------------------------- | ------ | -------- | ------------------------------------------------------------------------------- | +| `--model-name ` | string | yes | Model to deploy — fine-tuned output name or catalog model (required) | +| `--display-name ` | string | yes | Console display name for the deployment (required) | +| `--plan ` | string | no | Billing plan: lora (default, Token-billed) \| ptu (Token-billed) \| mu | +| `--deploy-spec ` | string | no | Deploy spec (only used by plan=mu; auto-picked if omitted) | +| `--capacity ` | number | no | Resource units (plan=mu only; required by API; defaults to the template's unit) | +| `--billing-method ` | string | no | Billing method (plan=mu only; default "POST_PAY", the only supported value) | +| `--input-tpm ` | number | no | PTU max input tokens/min (required for plan=ptu) | +| `--output-tpm ` | number | no | PTU max output tokens/min (required for plan=ptu) | +| `--thinking-output-tpm ` | number | no | PTU max thinking-output tokens/min (optional, some models) | +| `--api-key ` | string | no | API key | +| `--base-url ` | string | no | API base URL | #### Notes @@ -175,27 +174,24 @@ bl deploy get --deployed-model qwen-plus-2025-12-01-b6d61c71 --output json - Use `bl deploy models --source base` to inspect available templates. - After creation, status starts at PENDING and transitions to RUNNING. - Invoke the deployed model with: bl text chat --model -- WARNING: --model is overloaded across commands and refers to DIFFERENT -- values. `bl deploy create --model` takes the exported model_name -- (e.g. `qwen3-8b-ft-...`), but the create response also returns a -- `deployed_model` field (the deployment instance id, e.g. -- `qwen3-8b-5ecb5f068d79`). The inference call `bl text chat --model` must use -- the `deployed_model` from the create response — NOT the `model_name` you -- passed to `deploy create`. Do not reuse the value across the two -- commands. +- NOTE: --model-name is the model being deployed (e.g. `qwen3-8b-ft-...`). +- The create response also returns a `deployed_model` field — the deployment +- instance id (e.g. `qwen3-8b-5ecb5f068d79`). Use that id for inference +- (`bl text chat --model `) and lifecycle commands +- (`deploy get/scale/pause/resume/delete --deployed-model `). #### Examples ```bash -bl deploy image create --model my-wan-ft --name my-wan +bl deploy image create --model-name my-wan-ft --display-name my-wan ``` ```bash -bl deploy image create --model my-wan-ft --name my-wan-mu --plan mu +bl deploy image create --model-name my-wan-ft --display-name my-wan-mu --plan mu ``` ```bash -bl deploy image create --model my-wan-ft --name my-wan --dry-run +bl deploy image create --model-name my-wan-ft --display-name my-wan --dry-run ``` ### `bl deploy list` @@ -269,6 +265,84 @@ bl deploy models --source custom --page-size 50 bl deploy models --catalog-version v1.0 --output json ``` +### `bl deploy pause` + +| Field | Value | +| ------------------ | ----------------------------------------------------------- | +| **Name** | `deploy pause` | +| **Description** | Pause a running model deployment (stops billing for mu/ptu) | +| **Authentication** | Console | +| **Usage** | `bl deploy pause --deployed-model [--skip-precheck]` | + +#### Flags + +| Flag | Type | Required | Description | +| ------------------------------ | ------ | -------- | -------------------------------------------------------- | +| `--deployed-model ` | string | yes | Deployed model identifier (required) | +| `--skip-precheck` | switch | no | Skip the local RUNNING/PENDING status precheck | +| `--console-region ` | string | no | Console gateway region (e.g. cn-beijing, ap-southeast-1) | +| `--console-site ` | string | no | Console site: domestic, international | +| `--console-switch-agent ` | number | no | Switch agent UID for delegated access | +| `--workspace-id ` | string | no | Workspace ID (env: BAILIAN_WORKSPACE_ID) | + +#### Notes + +- While paused, billing ceases for mu/ptu plans. Use `deploy resume` to bring it back online or `deploy delete` to remove. +- Precheck verifies status is RUNNING/PENDING before issuing the pause; pass --skip-precheck to bypass. + +#### Examples + +```bash +bl deploy pause --deployed-model dep-... +``` + +```bash +bl deploy pause --deployed-model dep-... --skip-precheck +``` + +```bash +bl deploy pause --deployed-model dep-... --dry-run +``` + +### `bl deploy resume` + +| Field | Value | +| ------------------ | ------------------------------------------------------------- | +| **Name** | `deploy resume` | +| **Description** | Resume a paused model deployment (brings service back online) | +| **Authentication** | Console | +| **Usage** | `bl deploy resume --deployed-model [--skip-precheck]` | + +#### Flags + +| Flag | Type | Required | Description | +| ------------------------------ | ------ | -------- | -------------------------------------------------------- | +| `--deployed-model ` | string | yes | Deployed model identifier (required) | +| `--skip-precheck` | switch | no | Skip the local STOPPED status precheck | +| `--console-region ` | string | no | Console gateway region (e.g. cn-beijing, ap-southeast-1) | +| `--console-site ` | string | no | Console site: domestic, international | +| `--console-switch-agent ` | number | no | Switch agent UID for delegated access | +| `--workspace-id ` | string | no | Workspace ID (env: BAILIAN_WORKSPACE_ID) | + +#### Notes + +- Precheck verifies status is STOPPED before issuing the resume; pass --skip-precheck to bypass. +- For mu/ptu plans, billing resumes once the service is back online. + +#### Examples + +```bash +bl deploy resume --deployed-model dep-... +``` + +```bash +bl deploy resume --deployed-model dep-... --skip-precheck +``` + +```bash +bl deploy resume --deployed-model dep-... --dry-run +``` + ### `bl deploy scale` | Field | Value | @@ -301,28 +375,28 @@ bl deploy scale --deployed-model dep-... --capacity 2 ### `bl deploy text create` -| Field | Value | -| ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Name** | `deploy text create` | -| **Description** | Create a text model deployment | -| **Authentication** | API Key | -| **Usage** | `bl deploy text create --model --name [--plan ] [--deploy-spec ] [--capacity ] [--billing-method ] [--input-tpm ] [--output-tpm ] [--thinking-output-tpm ]` | +| Field | Value | +| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Name** | `deploy text create` | +| **Description** | Create a text model deployment | +| **Authentication** | API Key | +| **Usage** | `bl deploy text create --model-name --display-name [--plan ] [--deploy-spec ] [--capacity ] [--billing-method ] [--input-tpm ] [--output-tpm ] [--thinking-output-tpm ]` | #### Flags -| Flag | Type | Required | Description | -| --------------------------- | ------ | -------- | ------------------------------------------------------------------------------- | -| `--model ` | string | yes | Model name (catalog model or fine-tuned output) (required) | -| `--name ` | string | yes | Console display name for the deployment (required) | -| `--plan ` | string | no | Billing plan: lora (default, Token-billed) \| ptu (Token-billed) \| mu | -| `--deploy-spec ` | string | no | Deploy spec (only used by plan=mu; auto-picked if omitted) | -| `--capacity ` | number | no | Resource units (plan=mu only; required by API; defaults to the template's unit) | -| `--billing-method ` | string | no | Billing method (plan=mu only; default "POST_PAY", the only supported value) | -| `--input-tpm ` | number | no | PTU max input tokens/min (required for plan=ptu) | -| `--output-tpm ` | number | no | PTU max output tokens/min (required for plan=ptu) | -| `--thinking-output-tpm ` | number | no | PTU max thinking-output tokens/min (optional, some models) | -| `--api-key ` | string | no | API key | -| `--base-url ` | string | no | API base URL | +| Flag | Type | Required | Description | +| ------------------------------- | ------ | -------- | ------------------------------------------------------------------------------- | +| `--model-name ` | string | yes | Model to deploy — fine-tuned output name or catalog model (required) | +| `--display-name ` | string | yes | Console display name for the deployment (required) | +| `--plan ` | string | no | Billing plan: lora (default, Token-billed) \| ptu (Token-billed) \| mu | +| `--deploy-spec ` | string | no | Deploy spec (only used by plan=mu; auto-picked if omitted) | +| `--capacity ` | number | no | Resource units (plan=mu only; required by API; defaults to the template's unit) | +| `--billing-method ` | string | no | Billing method (plan=mu only; default "POST_PAY", the only supported value) | +| `--input-tpm ` | number | no | PTU max input tokens/min (required for plan=ptu) | +| `--output-tpm ` | number | no | PTU max output tokens/min (required for plan=ptu) | +| `--thinking-output-tpm ` | number | no | PTU max thinking-output tokens/min (optional, some models) | +| `--api-key ` | string | no | API key | +| `--base-url ` | string | no | API base URL | #### Notes @@ -337,31 +411,28 @@ bl deploy scale --deployed-model dep-... --capacity 2 - Use `bl deploy models --source base` to inspect available templates. - After creation, status starts at PENDING and transitions to RUNNING. - Invoke the deployed model with: bl text chat --model -- WARNING: --model is overloaded across commands and refers to DIFFERENT -- values. `bl deploy create --model` takes the exported model_name -- (e.g. `qwen3-8b-ft-...`), but the create response also returns a -- `deployed_model` field (the deployment instance id, e.g. -- `qwen3-8b-5ecb5f068d79`). The inference call `bl text chat --model` must use -- the `deployed_model` from the create response — NOT the `model_name` you -- passed to `deploy create`. Do not reuse the value across the two -- commands. +- NOTE: --model-name is the model being deployed (e.g. `qwen3-8b-ft-...`). +- The create response also returns a `deployed_model` field — the deployment +- instance id (e.g. `qwen3-8b-5ecb5f068d79`). Use that id for inference +- (`bl text chat --model `) and lifecycle commands +- (`deploy get/scale/pause/resume/delete --deployed-model `). #### Examples ```bash -bl deploy text create --model my-qwen-sft --name my-sft-test +bl deploy text create --model-name my-qwen-sft --display-name my-sft-test ``` ```bash -bl deploy text create --model qwen3.6-flash-2026-04-16 --name my-flash --plan ptu --input-tpm 10000 --output-tpm 1000 +bl deploy text create --model-name qwen3.6-flash-2026-04-16 --display-name my-flash --plan ptu --input-tpm 10000 --output-tpm 1000 ``` ```bash -bl deploy text create --model qwen3-8b --name my-qwen3-mu --plan mu +bl deploy text create --model-name qwen3-8b --display-name my-qwen3-mu --plan mu ``` ```bash -bl deploy text create --model qwen3-8b --name my-qwen3 --plan mu --deploy-spec MU1 --capacity 2 +bl deploy text create --model-name qwen3-8b --display-name my-qwen3 --plan mu --deploy-spec MU1 --capacity 2 ``` ### `bl deploy update` diff --git a/skills/bailian-finetune/reference/finetune.md b/skills/bailian-finetune/reference/finetune.md index f671b4a3..cb38d8ce 100644 --- a/skills/bailian-finetune/reference/finetune.md +++ b/skills/bailian-finetune/reference/finetune.md @@ -19,25 +19,27 @@ Index: [index.md](index.md) | `bl finetune image create` | API Key | Create an image generation model fine-tune job (sft-lora) | | `bl finetune list` | API Key | List fine-tune jobs | | `bl finetune logs` | API Key | Fetch training logs for a fine-tune job | +| `bl finetune price` | Console | Estimate the training cost for a fine-tune job (token billing) | | `bl finetune text create` | API Key | Create a text model fine-tune job (sft \| sft-lora \| dpo \| dpo-lora \| cpt) | +| `bl finetune video create` | API Key | Create a video generation model fine-tune job (Wan i2v/kf2v, efficient_sft) | | `bl finetune watch` | API Key | Probe a fine-tune job's status (default: single non-blocking fetch). Pass --follow to poll until terminal. | ## Command details ### `bl finetune audio create` -| Field | Value | -| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------- | -| **Name** | `finetune audio create` | -| **Description** | Create an audio TTS model fine-tune job (sft-lora) | -| **Authentication** | API Key | -| **Usage** | `bl finetune audio create --model --datasets [--validations ] [--model-name ] [--suffix ]` | +| Field | Value | +| ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | +| **Name** | `finetune audio create` | +| **Description** | Create an audio TTS model fine-tune job (sft-lora) | +| **Authentication** | API Key | +| **Usage** | `bl finetune audio create --base-model --datasets [--validations ] [--model-name ] [--suffix ]` | #### Flags | Flag | Type | Required | Description | | ---------------------------- | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `--model ` | string | yes | Base model to fine-tune | +| `--base-model ` | string | yes | Base model to fine-tune (e.g. qwen3-8b; not the output model name) | | `--datasets ` | string | yes | Comma-separated dataset file IDs or local paths (.jsonl for text, .zip for audio/image). Local paths are uploaded (validated) first, then their file-ids are used. | | `--validations ` | string | no | Comma-separated validation dataset file IDs or local paths (auto-uploaded like --datasets). | | `--model-name ` | string | no | Output model name (after training) | @@ -59,23 +61,23 @@ Index: [index.md](index.md) #### Examples ```bash -bl finetune audio create --model cosyvoice-v3-flash --datasets ./audio.zip +bl finetune audio create --base-model cosyvoice-v3-flash --datasets ./audio.zip ``` ```bash -bl finetune audio create --model cosyvoice-v3-flash --datasets file-xxx +bl finetune audio create --base-model cosyvoice-v3-flash --datasets file-xxx ``` ```bash -bl finetune audio create --model cosyvoice-v3-flash --datasets ./audio.zip --model-name my-tts +bl finetune audio create --base-model cosyvoice-v3-flash --datasets ./audio.zip --model-name my-tts ``` ```bash -bl finetune audio create --model cosyvoice-v3-flash --datasets file-xxx --output json +bl finetune audio create --base-model cosyvoice-v3-flash --datasets file-xxx --output json ``` ```bash -bl finetune audio create --model cosyvoice-v3-flash --datasets ./audio.zip --dry-run +bl finetune audio create --base-model cosyvoice-v3-flash --datasets ./audio.zip --dry-run ``` ### `bl finetune cancel` @@ -117,18 +119,18 @@ bl finetune cancel --job-id ft-xxx --dry-run | **Name** | `finetune capability` | | **Description** | Query fine-tune training capability — by model (which training types it supports) or by training type (which models support it) | | **Authentication** | No Auth | -| **Usage** | `bl finetune capability --model \| --training-type ` | +| **Usage** | `bl finetune capability --base-model \| --training-type ` | #### Flags | Flag | Type | Required | Description | | --------------------- | ------ | -------- | ------------------------------------------------------------------------------------- | -| `--model ` | string | no | List training types supported by this base model. | +| `--base-model ` | string | no | List training types supported by this base model. | | `--training-type ` | string | no | List models supporting this training type: sft \| sft-lora \| dpo \| dpo-lora \| cpt. | #### Notes -- Exactly one of --model / --training-type is required. +- Exactly one of --base-model / --training-type is required. - Training-type values use the `` / `-lora` convention: - sft | sft-lora | dpo | dpo-lora | cpt. (cpt has no -lora variant server-side.) - Queries listFoundationModels, a public API — no console login needed. @@ -136,7 +138,7 @@ bl finetune cancel --job-id ft-xxx --dry-run #### Examples ```bash -bl finetune capability --model qwen3-8b +bl finetune capability --base-model qwen3-8b ``` ```bash @@ -170,8 +172,8 @@ bl finetune capability --training-type sft --quiet #### Notes -- Use the returned `checkpoint` value with `finetune export` to publish -- a deployable model. +- `model_name` (shown for SUCCEEDED checkpoints) is the direct input for `deploy create --model-name`. +- Checkpoints expire ~15 days after creation; `expire_time` shows the deadline. Export or deploy before expiry. #### Examples @@ -275,18 +277,18 @@ bl finetune get --job-id ft-xxx --output json ### `bl finetune image create` -| Field | Value | -| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| **Name** | `finetune image create` | -| **Description** | Create an image generation model fine-tune job (sft-lora) | -| **Authentication** | API Key | -| **Usage** | `bl finetune image create --model --datasets [--validations ] [--model-name ] [--suffix ] [--generation-type ] [--learning-rate ]` | +| Field | Value | +| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Name** | `finetune image create` | +| **Description** | Create an image generation model fine-tune job (sft-lora) | +| **Authentication** | API Key | +| **Usage** | `bl finetune image create --base-model --datasets [--validations ] [--model-name ] [--suffix ] [--generation-type ] [--learning-rate ]` | #### Flags | Flag | Type | Required | Description | | ------------------------------ | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--model ` | string | yes | Base model to fine-tune | +| `--base-model ` | string | yes | Base model to fine-tune (e.g. qwen3-8b; not the output model name) | | `--datasets ` | string | yes | Comma-separated dataset file IDs or local paths (.jsonl for text, .zip for audio/image). Local paths are uploaded (validated) first, then their file-ids are used. | | `--validations ` | string | no | Comma-separated validation dataset file IDs or local paths (auto-uploaded like --datasets). | | `--model-name ` | string | no | Output model name (after training) | @@ -312,47 +314,48 @@ bl finetune get --job-id ft-xxx --output json #### Examples ```bash -bl finetune image create --model wan2.7-image-pro --datasets ./images.zip +bl finetune image create --base-model wan2.7-image-pro --datasets ./images.zip ``` ```bash -bl finetune image create --model wan2.7-image-pro --datasets file-xxx +bl finetune image create --base-model wan2.7-image-pro --datasets file-xxx ``` ```bash -bl finetune image create --model wan2.7-image-pro --datasets file-xxx --generation-type i2i +bl finetune image create --base-model wan2.7-image-pro --datasets file-xxx --generation-type i2i ``` ```bash -bl finetune image create --model wan2.7-image-pro --datasets ./images.zip --model-name my-wan +bl finetune image create --base-model wan2.7-image-pro --datasets ./images.zip --model-name my-wan ``` ```bash -bl finetune image create --model wan2.7-image-pro --datasets file-xxx --output json +bl finetune image create --base-model wan2.7-image-pro --datasets file-xxx --output json ``` ```bash -bl finetune image create --model wan2.7-image-pro --datasets ./images.zip --dry-run +bl finetune image create --base-model wan2.7-image-pro --datasets ./images.zip --dry-run ``` ### `bl finetune list` -| Field | Value | -| ------------------ | ---------------------------------------------------------------- | -| **Name** | `finetune list` | -| **Description** | List fine-tune jobs | -| **Authentication** | API Key | -| **Usage** | `bl finetune list [--page ] [--page-size ] [--status ]` | +| Field | Value | +| ------------------ | --------------------------------------------------------------------------------------- | +| **Name** | `finetune list` | +| **Description** | List fine-tune jobs | +| **Authentication** | API Key | +| **Usage** | `bl finetune list [--page ] [--page-size ] [--status ] [--base-model ]` | #### Flags -| Flag | Type | Required | Description | -| ------------------ | ------ | -------- | -------------------------------------------------------------------- | -| `--page ` | number | no | Page number (default: 1) | -| `--page-size ` | number | no | Results per page (default: 10, max 100) | -| `--status ` | string | no | Filter by status (PENDING / RUNNING / SUCCEEDED / FAILED / CANCELED) | -| `--api-key ` | string | no | API key | -| `--base-url ` | string | no | API base URL | +| Flag | Type | Required | Description | +| ---------------------- | ------ | -------- | -------------------------------------------------------------------- | +| `--page ` | number | no | Page number (default: 1) | +| `--page-size ` | number | no | Results per page (default: 10, max 100) | +| `--status ` | string | no | Filter by status (PENDING / RUNNING / SUCCEEDED / FAILED / CANCELED) | +| `--base-model ` | string | no | Filter by base model ID (server-side) | +| `--api-key ` | string | no | API key | +| `--base-url ` | string | no | API base URL | #### Examples @@ -365,7 +368,11 @@ bl finetune list --status RUNNING ``` ```bash -bl finetune list --page-size 20 --output json +bl finetune list --base-model qwen3-8b +``` + +```bash +bl finetune list --page-size 20 ``` ### `bl finetune logs` @@ -415,20 +422,62 @@ bl finetune logs --job-id ft-xxx --tail 20 bl finetune logs --job-id ft-xxx --search checkpoint --tail 5 ``` +### `bl finetune price` + +| Field | Value | +| ------------------ | --------------------------------------------------------------------------------------------------- | +| **Name** | `finetune price` | +| **Description** | Estimate the training cost for a fine-tune job (token billing) | +| **Authentication** | Console | +| **Usage** | `bl finetune price --base-model --datasets [--training-type ] [--n-epochs ]` | + +#### Flags + +| Flag | Type | Required | Description | +| ------------------------------ | ------ | -------- | ------------------------------------------------------------------ | +| `--base-model ` | string | yes | Base model to fine-tune (e.g. qwen3-8b; not the output model name) | +| `--datasets ` | string | yes | Training dataset file IDs, comma-separated (required) | +| `--training-type ` | string | no | Training type: sft \| dpo \| cpt (default: sft) | +| `--n-epochs ` | number | no | Number of training epochs (default: 3) | +| `--console-region ` | string | no | Console gateway region (e.g. cn-beijing, ap-southeast-1) | +| `--console-site ` | string | no | Console site: domestic, international | +| `--console-switch-agent ` | number | no | Switch agent UID for delegated access | +| `--workspace-id ` | string | no | Workspace ID (env: BAILIAN_WORKSPACE_ID) | + +#### Notes + +- Estimate only — the server computes token usage from the datasets; final cost is subject to the bill. +- Covers token billing for sft / dpo / cpt. Training-unit (MTU) billing is not supported by this command. +- Hyper-parameters other than --n-epochs are fixed at representative defaults for estimation. + +#### Examples + +```bash +bl finetune price --base-model qwen3-8b --datasets file-ft-xxx +``` + +```bash +bl finetune price --base-model qwen3-8b --datasets file-ft-xxx,file-ft-yyy --n-epochs 2 +``` + +```bash +bl finetune price --base-model qwen3-8b --datasets file-ft-xxx --training-type cpt +``` + ### `bl finetune text create` -| Field | Value | -| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Name** | `finetune text create` | -| **Description** | Create a text model fine-tune job (sft \| sft-lora \| dpo \| dpo-lora \| cpt) | -| **Authentication** | API Key | -| **Usage** | `bl finetune text create --model --datasets [--validations ] [--model-name ] [--suffix ] [--n-epochs ] [--batch-size ] [--learning-rate ] [--max-length ] [--training-type ]` | +| Field | Value | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Name** | `finetune text create` | +| **Description** | Create a text model fine-tune job (sft \| sft-lora \| dpo \| dpo-lora \| cpt) | +| **Authentication** | API Key | +| **Usage** | `bl finetune text create --base-model --datasets [--validations ] [--model-name ] [--suffix ] [--n-epochs ] [--batch-size ] [--learning-rate ] [--max-length ] [--training-type ]` | #### Flags | Flag | Type | Required | Description | | ---------------------------- | ------ | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--model ` | string | yes | Base model to fine-tune | +| `--base-model ` | string | yes | Base model to fine-tune (e.g. qwen3-8b; not the output model name) | | `--datasets ` | string | yes | Comma-separated dataset file IDs or local paths (.jsonl for text, .zip for audio/image). Local paths are uploaded (validated) first, then their file-ids are used. | | `--validations ` | string | no | Comma-separated validation dataset file IDs or local paths (auto-uploaded like --datasets). | | `--model-name ` | string | no | Output model name (after training) | @@ -464,35 +513,90 @@ bl finetune logs --job-id ft-xxx --search checkpoint --tail 5 #### Examples ```bash -bl finetune text create --model qwen3-8b --datasets file-xxx +bl finetune text create --base-model qwen3-8b --datasets file-xxx ``` ```bash -bl finetune text create --model qwen3-8b --datasets ./train.jsonl +bl finetune text create --base-model qwen3-8b --datasets ./train.jsonl ``` ```bash -bl finetune text create --model qwen3-8b --datasets ./train.jsonl --validations ./eval.jsonl +bl finetune text create --base-model qwen3-8b --datasets ./train.jsonl --validations ./eval.jsonl ``` ```bash -bl finetune text create --model qwen3-8b --datasets file-aaa,./extra.jsonl +bl finetune text create --base-model qwen3-8b --datasets file-aaa,./extra.jsonl ``` ```bash -bl finetune text create --model qwen3-8b --datasets ./train.jsonl --training-type sft +bl finetune text create --base-model qwen3-8b --datasets ./train.jsonl --training-type sft +``` + +```bash +bl finetune text create --base-model qwen3-8b --datasets file-xxx --learning-rate "1.6e-5" --n-epochs 4 +``` + +```bash +bl finetune text create --base-model qwen3-8b --datasets file-xxx --output json +``` + +```bash +bl finetune text create --base-model qwen3-8b --datasets file-xxx --dry-run +``` + +### `bl finetune video create` + +| Field | Value | +| ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Name** | `finetune video create` | +| **Description** | Create a video generation model fine-tune job (Wan i2v/kf2v, efficient_sft) | +| **Authentication** | API Key | +| **Usage** | `bl finetune video create --base-model --datasets [--validations ] [--model-name ] [--suffix ] [--n-epochs ] [--batch-size ] [--learning-rate ]` | + +#### Flags + +| Flag | Type | Required | Description | +| ---------------------------- | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `--base-model ` | string | yes | Base model to fine-tune (e.g. qwen3-8b; not the output model name) | +| `--datasets ` | string | yes | Comma-separated dataset file IDs or local paths (.jsonl for text, .zip for audio/image). Local paths are uploaded (validated) first, then their file-ids are used. | +| `--validations ` | string | no | Comma-separated validation dataset file IDs or local paths (auto-uploaded like --datasets). | +| `--model-name ` | string | no | Output model name (after training) | +| `--suffix ` | string | no | Output suffix appended by the platform (finetuned_output_suffix) | +| `--n-epochs ` | number | no | Training epochs (default: 50) | +| `--batch-size ` | number | no | Batch size (default: model-specific, 1 for wan2.7, 4 for wan2.5/2.2) | +| `--learning-rate ` | string | no | Learning rate as a string to preserve precision (default: "2e-5") | +| `--api-key ` | string | no | API key | +| `--base-url ` | string | no | API base URL | + +#### Notes + +- Creating a job uploads any local datasets and consumes training quota. +- Use --dry-run to preview the request body without submitting. +- --datasets / --validations accept either file-ids (from `dataset upload`) +- or local paths. Local paths are validated and uploaded first, then their +- file-ids are submitted — a one-step upload-and-train. +- Video generation training (Wan i2v/kf2v) runs efficient_sft with model- +- specific defaults: wan2.7 (batch_size=1, max_pixels=102400), wan2.5/2.2 +- (batch_size=4, max_pixels per model). Override with --batch-size/--n-epochs. +- Datasets are .zip archives with data.jsonl + frame images + videos. +- Recommended: ≥10 training samples, 20-100 for stable results. + +#### Examples + +```bash +bl finetune video create --base-model wan2.7-i2v --datasets file-xxx ``` ```bash -bl finetune text create --model qwen3-8b --datasets file-xxx --learning-rate "1.6e-5" --n-epochs 4 +bl finetune video create --base-model wan2.7-i2v --datasets ./i2v-data.zip ``` ```bash -bl finetune text create --model qwen3-8b --datasets file-xxx --output json +bl finetune video create --base-model wan2.2-kf2v-flash --datasets file-xxx --n-epochs 100 ``` ```bash -bl finetune text create --model qwen3-8b --datasets file-xxx --dry-run +bl finetune video create --base-model wan2.7-i2v --datasets file-xxx --dry-run ``` ### `bl finetune watch` diff --git a/skills/bailian-finetune/reference/index.md b/skills/bailian-finetune/reference/index.md index 9ec562e3..3dff1281 100644 --- a/skills/bailian-finetune/reference/index.md +++ b/skills/bailian-finetune/reference/index.md @@ -22,6 +22,8 @@ Use this index for the skill-scoped quick index and global flags. | `bl deploy image create` | API Key | Create an image generation model deployment | [deploy.md](deploy.md) | | `bl deploy list` | API Key | List model deployments | [deploy.md](deploy.md) | | `bl deploy models` | API Key | List models available for deployment | [deploy.md](deploy.md) | +| `bl deploy pause` | Console | Pause a running model deployment (stops billing for mu/ptu) | [deploy.md](deploy.md) | +| `bl deploy resume` | Console | Resume a paused model deployment (brings service back online) | [deploy.md](deploy.md) | | `bl deploy scale` | API Key | Scale a deployment's capacity | [deploy.md](deploy.md) | | `bl deploy text create` | API Key | Create a text model deployment | [deploy.md](deploy.md) | | `bl deploy update` | API Key | Update a deployment's rate limits (rpm_limit / tpm_limit) | [deploy.md](deploy.md) | @@ -35,16 +37,18 @@ Use this index for the skill-scoped quick index and global flags. | `bl finetune image create` | API Key | Create an image generation model fine-tune job (sft-lora) | [finetune.md](finetune.md) | | `bl finetune list` | API Key | List fine-tune jobs | [finetune.md](finetune.md) | | `bl finetune logs` | API Key | Fetch training logs for a fine-tune job | [finetune.md](finetune.md) | +| `bl finetune price` | Console | Estimate the training cost for a fine-tune job (token billing) | [finetune.md](finetune.md) | | `bl finetune text create` | API Key | Create a text model fine-tune job (sft \| sft-lora \| dpo \| dpo-lora \| cpt) | [finetune.md](finetune.md) | +| `bl finetune video create` | API Key | Create a video generation model fine-tune job (Wan i2v/kf2v, efficient_sft) | [finetune.md](finetune.md) | | `bl finetune watch` | API Key | Probe a fine-tune job's status (default: single non-blocking fetch). Pass --follow to poll until terminal. | [finetune.md](finetune.md) | ## By group -| Group | Commands | Reference | -| ---------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -------------------------- | -| `dataset` | `delete`, `get`, `list`, `upload`, `validate` | [dataset.md](dataset.md) | -| `deploy` | `audio create`, `delete`, `get`, `image create`, `list`, `models`, `scale`, `text create`, `update` | [deploy.md](deploy.md) | -| `finetune` | `audio create`, `cancel`, `capability`, `checkpoints`, `delete`, `export`, `get`, `image create`, `list`, `logs`, `text create`, `watch` | [finetune.md](finetune.md) | +| Group | Commands | Reference | +| ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------- | +| `dataset` | `delete`, `get`, `list`, `upload`, `validate` | [dataset.md](dataset.md) | +| `deploy` | `audio create`, `delete`, `get`, `image create`, `list`, `models`, `pause`, `resume`, `scale`, `text create`, `update` | [deploy.md](deploy.md) | +| `finetune` | `audio create`, `cancel`, `capability`, `checkpoints`, `delete`, `export`, `get`, `image create`, `list`, `logs`, `price`, `text create`, `video create`, `watch` | [finetune.md](finetune.md) | ## Global flags diff --git a/skills/bailian-gen/reference/video.md b/skills/bailian-gen/reference/video.md index 85d412e5..52788733 100644 --- a/skills/bailian-gen/reference/video.md +++ b/skills/bailian-gen/reference/video.md @@ -111,6 +111,7 @@ bl video edit --video https://example.com/input.mp4 --prompt "Put clothes on the | `--model ` | string | no | Model ID (default: happyhorse-1.1-t2v, or happyhorse-1.1-i2v with --image) | | `--prompt ` | string | yes | Video description | | `--image ` | string | no | Input image URL for image-to-video generation | +| `--last-frame ` | string | no | Last frame image URL (with --image, enables kf2v first+last frame mode) | | `--negative-prompt ` | string | no | Negative prompt to exclude unwanted content | | `--resolution ` | string | no | Resolution: 720P or 1080P (default: 1080P) | | `--ratio ` | string | no | Aspect ratio (e.g. 16:9, 9:16, 1:1) |