fix(skills): read the site's compact catalog and fetch only the needed Skills (0.5.2)

CHANGELOG.md

@@ -1,5 +1,12 @@
 # lovstudio
 
+## 0.5.2
+
+### Patch Changes
+
+- Fix `skills add` stopping at "读取 Lovstudio Skills 目录失败 … aborted due to timeout" on slow cross-border links. The catalog is now read from lovstudio.ai's `/api/skills/catalog` (about 20 KB with only the install fields) instead of the GitHub API copy of `skills.yaml` (about 740 KB). If the site cannot answer, the CLI falls back to the GitHub API (using `GITHUB_TOKEN` / `GH_TOKEN` when set, sent only to api.github.com), then raw.githubusercontent.com, then jsDelivr. A source that answers with something other than a catalog no longer passes as an empty one, and nothing is installed when no source returns the catalog.
+- Free Skills, including a paid Skill's free dependencies, are fetched with a partial sparse clone of only their `skills/<name>` directories instead of a full clone of the catalog repository (over 100 MB, which slow links could not finish within the installer's 300 s clone limit), then installed from that local copy. Each git step is retried up to three times, since flaky links drop some of the connections. Like paid Skills, these installs are updated by running `skills add` again rather than by `npx skills update`.
+
 ## 0.5.1
 
 ### Patch Changes

README.md

@@ -126,7 +126,8 @@ automatically via their declared `install` command.
 
 Under the hood:
 - `account connect` → browser device confirmation on `lovstudio.ai`; the shared session is stored owner-only under `~/.lovstudio/` for local Agent tools
-- `skills add <name>` → resolves the product slug in `skills.yaml`, then passes its exact `runtime_name` to `npx -y skills@latest add lovstudio/skills --skill <runtime_name>` (vercel-labs/skills)
+- `skills add <name>` → resolves the product slug in the catalog, fetches free Skills' `skills/<name>` directories with a partial sparse clone of `lovstudio/skills` (paid Skills download from lovstudio.ai), then passes their exact `runtime_name` to `npx -y skills@latest add <local copy> --skill <runtime_name>` (vercel-labs/skills). Reinstall with the same command to update
+- The catalog is read from `https://lovstudio.ai/api/skills/catalog` first, then from the GitHub API (authenticated with `GITHUB_TOKEN` / `GH_TOKEN` when set; anonymous calls share 60 requests per hour per IP), raw.githubusercontent.com and jsDelivr. `LOVSTUDIO_SKILLS_CATALOG_URL` replaces the whole chain. If no source answers, nothing is installed
 - `skills add skills` → resolves the unified catalog and passes every free Skill's exact `runtime_name`
 - `license *` → `uvx lovstudio-skill-helper *` (pinned version)
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "lovstudio",
-  "version": "0.5.1",
+  "version": "0.5.2",
   "description": "Lovstudio CLI \u2014 install free Skills directly and access paid Skills with VIP Membership.",
   "type": "module",
   "packageManager": "npm@11.17.0",

src/commands/skills/index.mjs

@@ -1,5 +1,5 @@
 import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
-import { existsSync } from "node:fs";
+import { existsSync, rmSync } from "node:fs";
 import { homedir, tmpdir } from "node:os";
 import { join } from "node:path";
 import { parse as parseYaml } from "yaml";
@@ -14,9 +14,23 @@ const SKILLS_NPX_SPEC = "skills@latest";
 const SKILL_PREFIX = "lov-";
 const LEGACY_SKILL_PREFIX = "lovstudio-";
 const ALL_SKILLS_NAMES = new Set(["*", "all", "skills", GALLERY]);
-const CATALOG_URL = process.env.LOVSTUDIO_SKILLS_CATALOG_URL ||
-  `https://api.github.com/repos/${GALLERY}/contents/skills.yaml?ref=main`;
+const GIT_ATTEMPTS = 3;
 const WEB_URL = (process.env.LOVSTUDIO_WEB_URL || "https://lovstudio.ai").replace(/\/$/, "");
+const CATALOG_SOURCES = process.env.LOVSTUDIO_SKILLS_CATALOG_URL
+  ? [process.env.LOVSTUDIO_SKILLS_CATALOG_URL]
+  : [
+      // About 20 KB with only the install fields, from the site that also
+      // serves entitlements and paid downloads. The GitHub API copy of
+      // skills.yaml is about 740 KB, which slow cross-border links cannot
+      // download within the timeout.
+      `${WEB_URL}/api/skills/catalog`,
+      // The API reflects a push immediately but allows only 60 anonymous
+      // requests per hour per IP, which a shared NAT can exhaust. The CDN
+      // copies lag by minutes (raw) to hours (jsDelivr) and are not rate limited.
+      `https://api.github.com/repos/${GALLERY}/contents/skills.yaml?ref=main`,
+      `https://raw.githubusercontent.com/${GALLERY}/main/skills.yaml`,
+      `https://cdn.jsdelivr.net/gh/${GALLERY}@main/skills.yaml`,
+    ];
 
 function isAllSkillsName(name) {
   return ALL_SKILLS_NAMES.has(String(name).trim().toLowerCase());
@@ -57,27 +71,49 @@ async function requireAccountToken() {
   }
 }
 
-async function loadCatalog() {
-  const response = await hfetch(CATALOG_URL, {
-    headers: { accept: "application/vnd.github+json, text/yaml, text/plain" },
+function catalogRequestHeaders(url) {
+  // The raw media type makes the API return the file itself instead of base64
+  // JSON, which GitHub leaves empty once the file passes 1 MB.
+  const headers = { accept: "application/vnd.github.raw+json, application/json, text/yaml, text/plain" };
+  const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
+  // Only GitHub's API receives the token; mirrors and custom URLs never see it.
+  if (token && new URL(url).hostname === "api.github.com") {
+    headers.authorization = `Bearer ${token}`;
+  }
+  return headers;
+}
+
+async function fetchCatalogSource(url) {
+  const response = await hfetch(url, {
+    headers: catalogRequestHeaders(url),
     signal: AbortSignal.timeout(15_000),
   });
-  if (!response.ok) {
-    throw new Error(`catalog request failed: HTTP ${response.status}`);
-  }
-  const contentType = response.headers.get("content-type") || "";
-  let text;
-  if (contentType.includes("application/json")) {
-    const payload = await response.json();
-    if (typeof payload?.content !== "string") {
-      throw new Error("GitHub API catalog response has no encoded content");
+  if (!response.ok) throw new Error(`HTTP ${response.status}`);
+  // The site answers JSON, which parses as YAML too.
+  const data = parseYaml(await response.text());
+  // A blocked or hijacked mirror can answer 200 with an HTML page; that must
+  // not pass as an empty catalog.
+  if (!Array.isArray(data?.skills)) throw new Error("response is not a skills catalog");
+  return data.skills.filter((skill) => !skill.test);
+}
+
+// Every source describes the same catalog, so the first one that answers
+// wins. Callers still refuse to install when all of them fail.
+export async function loadCatalog(sources = CATALOG_SOURCES) {
+  const failures = [];
+  for (const url of sources) {
+    try {
+      const skills = await fetchCatalogSource(url);
+      if (failures.length) {
+        console.error(`Lovstudio Skills 目录改用 ${new URL(url).host} 读取（${failures.join("; ")}）。`);
+      }
+      return skills;
+    } catch (error) {
+      const cause = error?.cause?.code ? ` (${error.cause.code})` : "";
+      failures.push(`${new URL(url).host} ${error instanceof Error ? error.message : String(error)}${cause}`);
     }
-    text = Buffer.from(payload.content, "base64").toString("utf8");
-  } else {
-    text = await response.text();
   }
-  const data = parseYaml(text) || {};
-  return Array.isArray(data.skills) ? data.skills.filter((skill) => !skill.test) : [];
+  throw new Error(`catalog request failed: ${failures.join("; ")}`);
 }
 
 export function canonicalSkillName(name) {
@@ -294,6 +330,36 @@ async function downloadPaidSkill(skill, token) {
   }
 }
 
+// Free Skills are mirrored at skills/<name> in the public catalog repository,
+// which also carries every other Skill and its case media (over 100 MB, past
+// the installer's 300 s clone limit on slow links). A partial sparse clone
+// fetches only the selected directories. As with paid Skills, the copy is
+// installed from a local path, so `npx skills update` leaves it alone.
+export function checkoutFreeSkills(source, skills, workDir) {
+  const repoDir = join(workDir, "catalog");
+  const steps = [
+    ["clone", "--quiet", "--depth", "1", "--filter=blob:none", "--sparse", source, repoDir],
+    ["-C", repoDir, "sparse-checkout", "set", ...skills.map((skill) => `skills/${skill.name}`)],
+  ];
+  for (const args of steps) {
+    // Each step opens its own GitHub connection (blobs are fetched on
+    // checkout), and flaky links drop some of them. Both steps can be rerun;
+    // a clone whose checkout failed leaves its directory behind.
+    for (let attempt = 1; ; attempt += 1) {
+      if (args[0] === "clone") rmSync(repoDir, { recursive: true, force: true });
+      const result = runCapture("git", args, {
+        env: { ...process.env, GIT_TERMINAL_PROMPT: "0" },
+        timeout: 300_000,
+      });
+      if (result.status === 0) break;
+      const message = result.error?.message || result.stderr.trim() || `git exited ${result.status}`;
+      if (attempt === GIT_ATTEMPTS) throw new Error(message);
+      console.error(`GitHub 连接失败，正在重试（${attempt + 1}/${GIT_ATTEMPTS}）：${message.split("\n").at(-1)}`);
+    }
+  }
+  return repoDir;
+}
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Frontmatter / dependency preflight
 // ─────────────────────────────────────────────────────────────────────────────
@@ -468,6 +534,17 @@ async function addAction(rawArgs) {
       workDir = download.workDir;
       const version = download.version ? ` v${download.version}` : "";
       console.log(`✓ 已下载「${plan.skills[0].name}」${version}（${download.ref}）`);
+    } else if (plan.skills.length) {
+      workDir = await mkdtemp(join(tmpdir(), "lovstudio-skills-"));
+      const names = plan.skills.map((skill) => skill.name).join("」「");
+      try {
+        source = checkoutFreeSkills(plan.source, plan.skills, workDir);
+      } catch (error) {
+        await rm(workDir, { recursive: true, force: true });
+        console.error(`获取「${names}」失败：${error instanceof Error ? error.message : String(error)}`);
+        process.exit(1);
+      }
+      console.log(`✓ 已获取「${names}」`);
     }
     const skillArgs = [
       "-y", SKILLS_NPX_SPEC, "add", source,

test/skills-command.test.mjs

@@ -1,19 +1,171 @@
 import assert from "node:assert/strict";
-import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
+import { spawnSync } from "node:child_process";
+import { randomBytes } from "node:crypto";
+import { existsSync } from "node:fs";
+import { mkdir, mkdtemp, readdir, rm, stat, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
+import { pathToFileURL } from "node:url";
 import test from "node:test";
 
 import {
   catalogSkillDependencyClosure,
   catalogSkillInstallPlans,
   catalogSkillSelector,
   canonicalSkillName,
+  checkoutFreeSkills,
   findCatalogSkill,
+  loadCatalog,
   locateExtractedSkill,
   skillSelector,
 } from "../src/commands/skills/index.mjs";
 
+const SITE_CATALOG = "https://lovstudio.ai/api/skills/catalog";
+const API_CATALOG = "https://api.github.com/repos/lovstudio/skills/contents/skills.yaml?ref=main";
+const RAW_CATALOG = "https://raw.githubusercontent.com/lovstudio/skills/main/skills.yaml";
+const CDN_CATALOG = "https://cdn.jsdelivr.net/gh/lovstudio/skills@main/skills.yaml";
+const usesDefaultCatalog = !process.env.LOVSTUDIO_SKILLS_CATALOG_URL && !process.env.LOVSTUDIO_WEB_URL;
+
+function withEnv(t, name, value) {
+  const previous = process.env[name];
+  process.env[name] = value;
+  t.after(() => {
+    if (previous === undefined) delete process.env[name];
+    else process.env[name] = previous;
+  });
+}
+
+test("loadCatalog reads the site's compact catalog before GitHub", { skip: !usesDefaultCatalog }, async (t) => {
+  const requests = [];
+  t.mock.method(globalThis, "fetch", async (url) => {
+    requests.push(url);
+    return Response.json({
+      skills: [{ name: "koubo-media-creator", runtime_name: "lov-koubo-media-creator", paid: true, depends_on: ["branding-consistency"] }],
+    });
+  });
+
+  const catalog = await loadCatalog();
+
+  assert.deepEqual(requests, [SITE_CATALOG]);
+  assert.deepEqual(catalog, [
+    { name: "koubo-media-creator", runtime_name: "lov-koubo-media-creator", paid: true, depends_on: ["branding-consistency"] },
+  ]);
+});
+
+test("loadCatalog falls back to the GitHub API when the site is unavailable", { skip: !usesDefaultCatalog }, async (t) => {
+  t.mock.method(console, "error", () => {});
+  const requests = [];
+  t.mock.method(globalThis, "fetch", async (url) => {
+    requests.push(url);
+    if (url === SITE_CATALOG) return Response.json({ error: "catalog_unavailable" }, { status: 503 });
+    return new Response("skills:\n- name: branding-consistency\n  paid: false\n");
+  });
+
+  const catalog = await loadCatalog();
+
+  assert.deepEqual(catalog.map((skill) => skill.name), ["branding-consistency"]);
+  assert.deepEqual(requests, [SITE_CATALOG, API_CATALOG]);
+});
+
+test("loadCatalog falls back to the raw mirror when the GitHub API answers 403", async (t) => {
+  withEnv(t, "GITHUB_TOKEN", "test-token");
+  t.mock.method(console, "error", () => {});
+  const requests = [];
+  t.mock.method(globalThis, "fetch", async (url, init) => {
+    requests.push({ url, authorization: init.headers.authorization });
+    if (url === API_CATALOG) {
+      return new Response('{"message":"API rate limit exceeded"}', {
+        status: 403,
+        headers: { "content-type": "application/json" },
+      });
+    }
+    return new Response("version: 1\nskills:\n- name: wxmp-cli\n  paid: false\n- name: smoke\n  test: true\n", {
+      headers: { "content-type": "text/plain; charset=utf-8" },
+    });
+  });
+
+  const catalog = await loadCatalog([API_CATALOG, RAW_CATALOG, CDN_CATALOG]);
+
+  assert.deepEqual(catalog.map((skill) => skill.name), ["wxmp-cli"]);
+  assert.deepEqual(requests, [
+    { url: API_CATALOG, authorization: "Bearer test-token" },
+    { url: RAW_CATALOG, authorization: undefined },
+  ]);
+});
+
+test("loadCatalog fails instead of returning an empty catalog when every source fails", async (t) => {
+  t.mock.method(globalThis, "fetch", async (url) => {
+    if (url === SITE_CATALOG) throw new DOMException("The operation was aborted due to timeout", "TimeoutError");
+    if (url === API_CATALOG) return new Response("", { status: 429 });
+    if (url === RAW_CATALOG) return new Response("<html>blocked</html>", { headers: { "content-type": "text/html" } });
+    throw new TypeError("fetch failed", { cause: Object.assign(new Error("getaddrinfo"), { code: "ENOTFOUND" }) });
+  });
+
+  await assert.rejects(
+    loadCatalog([SITE_CATALOG, API_CATALOG, RAW_CATALOG, CDN_CATALOG]),
+    {
+      message: "catalog request failed: lovstudio.ai The operation was aborted due to timeout; " +
+        "api.github.com HTTP 429; " +
+        "raw.githubusercontent.com response is not a skills catalog; " +
+        "cdn.jsdelivr.net fetch failed (ENOTFOUND)",
+    },
+  );
+});
+
+async function directoryBytes(dir) {
+  let total = 0;
+  for (const entry of await readdir(dir, { withFileTypes: true })) {
+    const path = join(dir, entry.name);
+    total += entry.isDirectory() ? await directoryBytes(path) : (await stat(path)).size;
+  }
+  return total;
+}
+
+test("checkoutFreeSkills fetches only the selected Skill directories", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "lovstudio-catalog-repo-"));
+  t.after(() => rm(root, { recursive: true, force: true }));
+  const origin = join(root, "origin");
+  for (const name of ["branding-consistency", "human-writing", "heavy"]) {
+    await mkdir(join(origin, "skills", name), { recursive: true });
+    await writeFile(join(origin, "skills", name, "SKILL.md"), `---\nname: lov-${name}\n---\n`);
+  }
+  // Random bytes do not compress, so fetching this blob would show in .git.
+  await writeFile(join(origin, "skills", "heavy", "sample.bin"), randomBytes(2 * 1024 * 1024));
+  const git = (...args) => {
+    const result = spawnSync("git", args, { cwd: origin, encoding: "utf8" });
+    assert.equal(result.status, 0, result.stderr);
+  };
+  git("init", "--quiet");
+  git("config", "uploadpack.allowFilter", "true");
+  git("config", "uploadpack.allowAnySHA1InWant", "true");
+  git("add", ".");
+  git("-c", "user.name=fixture", "-c", "user.email=fixture@example.com", "commit", "--quiet", "-m", "catalog");
+  const workDir = join(root, "work");
+  await mkdir(workDir);
+
+  const repoDir = checkoutFreeSkills(
+    pathToFileURL(origin).href,
+    [{ name: "branding-consistency" }, { name: "human-writing" }],
+    workDir,
+  );
+
+  assert.ok(existsSync(join(repoDir, "skills", "branding-consistency", "SKILL.md")));
+  assert.ok(existsSync(join(repoDir, "skills", "human-writing", "SKILL.md")));
+  assert.equal(existsSync(join(repoDir, "skills", "heavy")), false);
+  assert.ok(await directoryBytes(join(repoDir, ".git")) < 1024 * 1024, "unselected blobs must not be downloaded");
+});
+
+test("checkoutFreeSkills retries git, then reports its error", async (t) => {
+  const retries = t.mock.method(console, "error", () => {});
+  const workDir = await mkdtemp(join(tmpdir(), "lovstudio-catalog-missing-"));
+  t.after(() => rm(workDir, { recursive: true, force: true }));
+  assert.throws(
+    () => checkoutFreeSkills(pathToFileURL(join(workDir, "missing")).href, [{ name: "x" }], workDir),
+    /missing/,
+  );
+  assert.equal(retries.mock.callCount(), 2);
+});
+
 test("catalogSkillDependencyClosure installs transitive Skill dependencies first", () => {
   const catalog = [
     { name: "branding-consistency", runtime_name: "lov-branding-consistency" },