diff --git a/docs/cli-reference.md b/docs/cli-reference.md index ae0b840..38c3337 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -224,7 +224,7 @@ registry release, a skill added this way is **not scanned, signed or pinned in | `owner/repo` | A GitHub repository (default branch) | | `owner/repo/skill` | One skill of it, by name or directory name (same as `--skill skill`) | | `github:owner/repo[/path][@ref]` | A directory of the repository, at a branch or tag | -| `https://github.com/owner/repo[/tree/[/]]` | Read as `github:` | +| `https://github.com/owner/repo[/tree/[/]]` | Read as `github:`; a branch name with `/` is found by asking the remote which leading segments name a branch or tag (longest wins) | | `https://host/repo.git` | Any https git repository (default branch) | | `./dir`, `../dir`, an absolute path | A local directory | @@ -232,8 +232,12 @@ Only `https` is accepted. `http`, `ssh`, `git@host:` (scp style), `file://`, `ext::` and other `::` transport helpers, a URL with a query or fragment, and anything starting with `-` are refused as `usage_error`; git is also run with every other protocol disabled. A path or ref with `..`, `.`, empty, `\` or `:` -segments is refused. `@publisher/package`, `kit:` and `collection:` belong to -`skillx install`; `@agentkit/*` is reserved. +segments is refused. A URL that holds an `@` but does not parse, or whose path +holds one (`https://user:pa/ss@host/...`), is refused without being echoed, +as is any refused source holding an `@`: percent-encode `/`, `?`, `#` and `@` in a user name or password. +`@publisher/package`, `kit:` and `collection:` belong to `skillx install`; +`@agentkit/*` is reserved. A ref is a branch or tag name; commit SHAs are not +accepted. **Options:** | Flag | Description | Default | @@ -253,8 +257,10 @@ is not front matter, and tags or duplicate keys reject the file, so nothing in a source ever runs. When the source (or `github:` path) is itself a skill, it is the only one; otherwise every skill up to 6 directories deep counts, outside `.git`, other VCS and cache directories and `node_modules`, and a -skill's own subdirectories are its content. Skills that cannot be installed are -listed under `skipped` with the reason. +skill's own subdirectories are its content. Sizes are read before content: more +than 1,000 `SKILL.md` files under the search path is a `usage_error` (point at a +smaller path), and a `SKILL.md` over 1 MiB is skipped unread. Skills that cannot +be installed are listed under `skipped` with the reason. - **Names.** `name` is the installed directory name and is never rewritten: it must be a valid alias (up to 64 lowercase letters, digits, `.`, `_`, `-`, @@ -267,14 +273,28 @@ listed under `skipped` with the reason. limits (4,096 files, 32 MiB per file, 256 MiB in all) refuses the skill as `incompatible`. Links are never followed or copied, neither while searching nor while reading. -- **Git.** The clone is shallow, single-branch and without a checkout, into a - temporary directory removed afterwards: files are read from git objects, so - no repository file is written, and no hook, filter or LFS download runs. git - runs without a shell, with `--` before the URL and all credential prompts - off (`core.askPass=`, `GIT_TERMINAL_PROMPT=0`, `GCM_INTERACTIVE=never`), so - a private repository works only with credentials git already has (a +- **Git.** The clone is shallow, single-branch, without a checkout and asks + for no blobs (`--filter=blob:none`; a server without partial clone sends + them all): the listing comes from tree objects and only the files discovery + and the chosen skills need are fetched, so a skill in a large monorepo does + not download the rest. A fetched file arrives whole (git cannot report a + size first), so download time is bounded by the 120 s limit per git call; + sizes are then read before content and content is read in bounded batches. + No repository file is written, and no hook, filter or LFS download runs. + The temporary directory is removed afterwards, also on Ctrl+C, SIGTERM or + SIGHUP, including while git is running (exit `130`, `143`, `129`). git runs + without a shell, with `--` before the URL and all credential prompts off + (`core.askPass=`, `GIT_TERMINAL_PROMPT=0`, `GCM_INTERACTIVE=never`), so a + private repository works only with credentials git already has (a credential helper, or a token in the URL). -- **Credentials.** A token in the URL is used to clone and nothing else: it is +- **Credentials.** A token in the URL is used to clone and nothing else. It is + removed from the URL git sees and passed as an `Authorization` header through + `GIT_CONFIG_*` environment entries (git 2.31 or newer), so it never appears + in git's argument list, the clone's `.git/config` or a credential helper. + git would send that header on to wherever a redirect leads, so redirects are + not followed while a token is in use, even if your git config follows them + for that URL. A server that redirects (for example GitLab without `.git`) + fails with `usage_error`: use the repository's final URL. The token is removed from the recorded source and masked (`https://***@host`) in every message and envelope. diff --git a/packages/cli/src/commands/add.ts b/packages/cli/src/commands/add.ts index e5a339e..f4851e7 100644 --- a/packages/cli/src/commands/add.ts +++ b/packages/cli/src/commands/add.ts @@ -2,9 +2,10 @@ import { join } from 'node:path'; import { Command } from 'commander'; import chalk from 'chalk'; import { applyAdd, planAdd } from '../lib/add/add-install.js'; -import { parseAddSource, redactCredentials, searchPathOf } from '../lib/add/add-source.js'; +import { parseAddSource, redactCredentials, resolveTreeRef, searchPathOf, treeRefCandidates } from '../lib/add/add-source.js'; import type { AddSource } from '../lib/add/add-source.js'; import { openGitSource } from '../lib/add/git-source.js'; +import { remoteOf, remoteRefNames } from '../lib/add/git-run.js'; import { openLocalSource } from '../lib/add/local-source.js'; import { discoverSkills, prepareSkill } from '../lib/add/skill-discovery.js'; import type { SourceSnapshot } from '../lib/add/skill-discovery.js'; @@ -36,6 +37,13 @@ export function openSource(source: AddSource): SourceSnapshot { return openGitSource(source.cloneUrl, source.label, source.kind === 'github' ? source.ref : undefined); } +/** A `/tree//` URL with several segments, split where the remote has a branch or tag. */ +export function settleSource(source: AddSource, transport = 'https'): AddSource { + const candidates = treeRefCandidates(source); + if (source.kind !== 'github' || candidates.length === 0) return source; + return resolveTreeRef(source, remoteRefNames(remoteOf(source.cloneUrl), source.label, candidates, transport)); +} + /** Where a skill came from: the source plus the skill's directory in it, never credentials. */ export function originOf(source: AddSource, dir: string): string { if (source.kind === 'local') return dir === '' ? source.dir : join(source.dir, ...dir.split('/')); @@ -44,10 +52,11 @@ export function originOf(source: AddSource, dir: string): string { } export async function runAdd(input: string, options: AddOptions): Promise { - const source = parseAddSource(input); - // Resolve the runtime before any clone, so a bad --runtime fails fast. + const parsed = parseAddSource(input); + // Resolve the runtime before any network call, so a bad --runtime fails fast. const ctx = options.list ? null : contextFrom(options); if (ctx !== null) assertCanPrompt(options); + const source = settleSource(parsed); if (!options.json && source.kind !== 'local') console.error(chalk.dim(`cloning ${source.label} ...`)); const snapshot = openSource(source); try { diff --git a/packages/cli/src/lib/add/add-source.test.ts b/packages/cli/src/lib/add/add-source.test.ts index 1c82989..060b504 100644 --- a/packages/cli/src/lib/add/add-source.test.ts +++ b/packages/cli/src/lib/add/add-source.test.ts @@ -1,7 +1,7 @@ import { resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; import { CliError } from '../output.js'; -import { parseAddSource, redactCredentials } from './add-source.js'; +import { parseAddSource, redactCredentials, resolveTreeRef, treeRefCandidates } from './add-source.js'; import { redactError } from '../../commands/add.js'; const refusal = (input: string) => { @@ -105,4 +105,32 @@ describe('credentials in URLs', () => { expect(out.code).toBe('network_error'); expect((redactError(new Error(`boom https://${token}@h`)) as Error).message).toBe('boom https://***@h'); }); + + it('refuses a URL whose password holds a slash without echoing any of it', () => { + for (const input of ['https://user:pa/ss@example.com/r.git', 'https://user:12/ss@example.com/r.git', 'https://tok/en@example.com/r.git']) { + const error = refusal(input); + expect(error.code).toBe('usage_error'); + expect(error.message).not.toMatch(/pa\/ss|12\/ss|tok\/en|example/); + } + expect(redactCredentials('clone https://user:pa/ss@example.com/r.git failed')).toBe('clone https://***@example.com/r.git failed'); + expect(redactCredentials('https://u:a?b#c@h/x')).toBe('https://***@h/x'); + }); + + it('keeps every segment after /tree/ until the remote says which ones name the ref', () => { + const source = parseAddSource('https://github.com/acme/skills/tree/feature/x/skills/a'); + expect(source).toMatchObject({ ref: 'feature', path: 'x/skills/a', tree: ['feature', 'x', 'skills', 'a'] }); + expect(treeRefCandidates(source)).toEqual(['feature/x/skills/a', 'feature/x/skills', 'feature/x', 'feature']); + expect(resolveTreeRef(source, new Set(['feature/x', 'feature']))).toEqual({ + kind: 'github', owner: 'acme', repo: 'skills', ref: 'feature/x', path: 'skills/a', cloneUrl: 'https://github.com/acme/skills.git', label: 'github:acme/skills/skills/a@feature/x', + }); + expect(resolveTreeRef(source, new Set(['feature']))).toMatchObject({ ref: 'feature', path: 'x/skills/a' }); + const unknown = resolveTreeRef(source, new Set()); + expect(unknown).toMatchObject({ ref: 'feature', path: 'x/skills/a' }); + expect(unknown).not.toHaveProperty('tree'); + // A whole URL naming only a ref, or a ref and one directory, still splits per segment. + expect(parseAddSource('https://github.com/acme/skills/tree/main')).not.toHaveProperty('tree'); + expect(treeRefCandidates(parseAddSource('https://github.com/acme/skills/tree/main/skills'))).toEqual(['main/skills', 'main']); + // An encoded slash stays inside its segment. + expect(resolveTreeRef(parseAddSource('https://github.com/acme/skills/tree/rel%2F1/a/b'), new Set(['rel/1']))).toMatchObject({ ref: 'rel/1', path: 'a/b' }); + }); }); diff --git a/packages/cli/src/lib/add/add-source.ts b/packages/cli/src/lib/add/add-source.ts index 8fc28f9..637116d 100644 --- a/packages/cli/src/lib/add/add-source.ts +++ b/packages/cli/src/lib/add/add-source.ts @@ -16,10 +16,24 @@ import { usageError } from '../install/install-errors.js'; * * Credentials in a URL may be used to clone but never leave this module in a * label: `label` strips them, and `redactCredentials` masks them in any text. + * A URL that holds an `@` but does not parse, or whose path holds one, is + * refused without being echoed, since where its credentials end cannot be + * known. */ export type AddSource = - | { kind: 'github'; owner: string; repo: string; path: string; ref?: string; skill?: string; cloneUrl: string; label: string } + | { + kind: 'github'; + owner: string; + repo: string; + path: string; + ref?: string; + skill?: string; + /** Segments after `/tree/` when there are several: which ones name the ref is settled against the remote (`resolveTreeRef`). */ + tree?: string[]; + cloneUrl: string; + label: string; + } | { kind: 'git'; cloneUrl: string; label: string } | { kind: 'local'; dir: string; label: string }; @@ -33,14 +47,19 @@ const REF = /^(?!-)(?!.*\.\.)(?!.*\/\/)(?!.*@\{)(?!.*\.lock$)(?!.*\/$)[A-Za-z0-9 const USAGE = 'owner/repo[/skill], github:owner/repo[/path][@ref], an https git URL or a local directory'; /** - * Masks the userinfo of every `scheme://user:secret@host` in `text`. Quotes - * end a match, so the result of redacting serialized JSON is still JSON. + * Masks the userinfo of every `scheme://user:secret@host` in `text`, up to + * the last `@` of the URL, so a secret holding `/`, `?` or `#` is masked whole + * (at the cost of masking a path `@` too). Whitespace and quotes end a match, + * so the result of redacting serialized JSON is still JSON. */ export function redactCredentials(text: string): string { - return text.replace(/([a-z][a-z0-9+.-]*:\/\/)[^\s/?#@"'<>]*(?:@[^\s/?#@"'<>]*)*@/gi, '$1***@'); + return text.replace(/([a-z][a-z0-9+.-]*:\/\/)[^\s"'<>]*@/gi, '$1***@'); } -const refuse = (input: string, why: string) => usageError(`refusing source '${redactCredentials(input)}': ${why}`); +const UNSHOWN_URL = + 'refusing the source: not a valid git URL (not shown, as it may hold credentials; percent-encode `/`, `?`, `#` and `@` in a user name or password)'; +/** A refusal echoes the input only when it holds no `@`, since where a credential in it ends cannot be known for sure. */ +const refuse = (input: string, why: string) => usageError(input.includes('@') ? `refusing the source: ${why}` : `refusing source '${input}': ${why}`); function checkRepo(repo: string, input: string): void { if (repo === '.' || repo === '..') throw refuse(input, 'the repository name is not a name'); @@ -61,10 +80,38 @@ function checkRef(ref: string, input: string): string { return ref; } -function github(input: string, owner: string, repo: string, path: string, ref: string | undefined, skill?: string): AddSource { +const githubLabel = (owner: string, repo: string, path: string, ref?: string) => `github:${owner}/${repo}${path ? `/${path}` : ''}${ref ? `@${ref}` : ''}`; + +type GithubSource = Extract; + +function github(input: string, owner: string, repo: string, path: string, ref: string | undefined, skill?: string): GithubSource { checkRepo(repo, input); - const label = `github:${owner}/${repo}${path ? `/${path}` : ''}${ref ? `@${ref}` : ''}`; - return { kind: 'github', owner, repo, path, ...(ref ? { ref } : {}), ...(skill ? { skill } : {}), cloneUrl: `https://github.com/${owner}/${repo}.git`, label }; + return { kind: 'github', owner, repo, path, ...(ref ? { ref } : {}), ...(skill ? { skill } : {}), cloneUrl: `https://github.com/${owner}/${repo}.git`, label: githubLabel(owner, repo, path, ref) }; +} + +/** How a `/tree/a/b/c` URL may split into a ref and a path, longest ref first; refs that are not valid names are left out. */ +function treeSplits(tree: readonly string[]): Array<{ ref: string; path: string }> { + return tree + .map((_, i) => ({ ref: tree.slice(0, tree.length - i).join('/'), path: tree.slice(tree.length - i).join('/') })) + .filter(({ ref }) => REF.test(ref)); +} + +/** The ref names a `/tree/` URL may mean, to look up on the remote. */ +export const treeRefCandidates = (source: AddSource): string[] => (source.kind === 'github' && source.tree ? treeSplits(source.tree).map((s) => s.ref) : []); + +/** + * Settles a `/tree//` URL against the refs the remote has: a branch + * name may hold `/`, so the longest leading segments that name a branch or tag + * are the ref and the rest the path. With no match the first segment stays the + * ref, and the clone reports it missing. + */ +export function resolveTreeRef(source: AddSource, remoteRefs: ReadonlySet): AddSource { + if (source.kind !== 'github' || !source.tree) return source; + const { tree, ...rest } = source; + const split = treeSplits(tree).find((s) => remoteRefs.has(s.ref)); + if (split === undefined) return rest; + const path = checkRepoPath(split.path, source.label); + return { ...rest, ref: split.ref, path, label: githubLabel(source.owner, source.repo, path, split.ref) }; } function fromUrl(input: string): AddSource { @@ -74,8 +121,11 @@ function fromUrl(input: string): AddSource { try { url = new URL(input); } catch { + if (input.includes('@')) throw usageError(UNSHOWN_URL); throw refuse(input, 'not a valid URL'); } + // `https://user:12/ss@host` parses as host `user`: an `@` past the host means a secret may sit in the path. + if (url.pathname.includes('@')) throw usageError(UNSHOWN_URL); if (url.protocol !== 'https:') throw refuse(input, `only https git URLs are accepted, not ${url.protocol.replace(/:$/, '')}`); if (url.hostname === '') throw refuse(input, 'the URL has no host'); if (url.search !== '' || url.hash !== '') throw refuse(input, 'a git URL takes no query or fragment'); @@ -90,7 +140,9 @@ function fromUrl(input: string): AddSource { throw refuse(input, 'the URL path is not valid percent-encoding'); } }; - return github(input, gh[1]!, gh[2]!, checkRepoPath(decode(gh[4] ?? ''), input), gh[3] ? checkRef(decode(gh[3]), input) : undefined); + const source = github(input, gh[1]!, gh[2]!, checkRepoPath(decode(gh[4] ?? ''), input), gh[3] ? checkRef(decode(gh[3]), input) : undefined); + const tree = gh[3] && gh[4] ? [gh[3], ...(gh[4].split('/').filter(Boolean))].map(decode) : []; + return tree.length > 1 ? { ...source, tree } : source; } const bare = new URL(url.href); bare.username = ''; @@ -122,7 +174,7 @@ export function parseAddSource(input: string, cwd: string = process.cwd()): AddS if (raw.includes('://') || raw.includes('::') || /^[^/:]+@[^/:]+:/.test(raw)) return fromUrl(raw); const m = SHORTHAND.exec(raw); if (m) return github(raw, m[1]!, m[2]!, '', undefined, m[3]); - throw usageError(`'${redactCredentials(raw)}' is not a source: ${USAGE}`); + throw usageError(`${raw.includes('@') ? 'the source' : `'${raw}'`} is not a source: ${USAGE}`); } /** Where discovery starts inside the source: the `github:` path, else the root. */ diff --git a/packages/cli/src/lib/add/git-credentials.test.ts b/packages/cli/src/lib/add/git-credentials.test.ts new file mode 100644 index 0000000..f7e03c9 --- /dev/null +++ b/packages/cli/src/lib/add/git-credentials.test.ts @@ -0,0 +1,123 @@ +import { execFileSync, spawn } from 'node:child_process'; +import type { ChildProcess } from 'node:child_process'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { CliError } from '../output.js'; +import { openGitSource } from './git-source.js'; + +/** + * A URL credential travels as an `Authorization` header, and git would send + * it to wherever a redirect leads. Two servers run in a child process (git + * runs synchronously, so they cannot share this event loop): `origin` sends + * every request on to `elsewhere`, a working smart-HTTP server on another + * host, and both log the Authorization header they receive. Only the `http` transport, which tests alone may allow, can + * reach them. + */ + +const SERVERS = ` +const http = require('node:http'); +const fs = require('node:fs'); +const { spawn } = require('node:child_process'); +const [log, projectRoot] = process.argv.slice(1); +const record = (name, req) => fs.appendFileSync(log, JSON.stringify({ name, method: req.method, auth: req.headers.authorization ?? null }) + '\\n'); +// A real smart-HTTP server (git http-backend as CGI), so git goes on to POST after the redirect. +const elsewhere = http.createServer((req, res) => { + record('elsewhere', req); + const [path, query = ''] = req.url.split('?'); + const cgi = spawn('git', ['http-backend'], { env: { ...process.env, GIT_PROJECT_ROOT: projectRoot, GIT_HTTP_EXPORT_ALL: '1', PATH_INFO: path, QUERY_STRING: query, REQUEST_METHOD: req.method, CONTENT_TYPE: req.headers['content-type'] ?? '', GIT_PROTOCOL: req.headers['git-protocol'] ?? '' } }); + req.pipe(cgi.stdin); + const chunks = []; + cgi.stdout.on('data', (c) => chunks.push(c)); + cgi.on('close', () => { + const out = Buffer.concat(chunks); + const split = out.indexOf('\\r\\n\\r\\n'); + const headers = out.subarray(0, split).toString('latin1').split('\\r\\n'); + let status = 200; + for (const h of headers) { + const [k, ...v] = h.split(':'); + if (k.toLowerCase() === 'status') status = Number(v.join(':').trim().split(' ')[0]); + else res.setHeader(k, v.join(':').trim()); + } + res.statusCode = status; + res.end(out.subarray(split + 4)); + }); +}); +elsewhere.listen(0, '127.0.0.2', () => { + const to = elsewhere.address().port; + const origin = http.createServer((req, res) => { + record('origin', req); + res.writeHead(302, { location: 'http://127.0.0.2:' + to + req.url }); + res.end(); + }); + origin.listen(0, '127.0.0.1', () => console.log(String(origin.address().port))); +}); +`; + +let dir: string; +let log: string; +let servers: ChildProcess; +let originPort: string; + +beforeAll(async () => { + dir = mkdtempSync(join(tmpdir(), 'skillx-add-cred-')); + log = join(dir, 'requests.jsonl'); + const work = join(dir, 'work'); + mkdirSync(work); + const run = (cwd: string, args: string[]) => execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', '-c', 'commit.gpgsign=false', ...args], { cwd, stdio: 'ignore' }); + run(work, ['init', '-q', '-b', 'main']); + writeFileSync(join(work, 'SKILL.md'), '---\nname: demo\ndescription: Demo\n---\n'); + run(work, ['add', '-A']); + run(work, ['commit', '-q', '-m', 'skill']); + run(dir, ['clone', '-q', '--bare', work, join(dir, 'repo.git')]); + servers = spawn(process.execPath, ['-e', SERVERS, log, dir], { stdio: ['ignore', 'pipe', 'inherit'] }); + originPort = await new Promise((resolve) => servers.stdout!.once('data', (chunk: Buffer) => resolve(chunk.toString('utf8').trim()))); +}); + +afterAll(() => { + servers.kill(); + rmSync(dir, { recursive: true, force: true }); +}); + +describe('URL credentials', () => { + it('reach the origin of the URL and never a host it redirects to', () => { + const token = 'SECRET-TOKEN-1'; + let failure: unknown; + try { + openGitSource(`http://user:${token}@127.0.0.1:${originPort}/repo.git`, 'test-repo', undefined, 'http').dispose(); + } catch (error) { + failure = error; + } + const requests = readFileSync(log, 'utf8').trim().split('\n').map((line) => JSON.parse(line) as { name: string; auth: string | null }); + const basic = `Basic ${Buffer.from(`user:${token}`).toString('base64')}`; + expect(requests.filter((r) => r.name === 'origin').map((r) => r.auth)).toContain(basic); + expect(requests.filter((r) => r.name === 'elsewhere' && r.auth !== null)).toEqual([]); + // With redirects off the clone stops at the 302, and its message carries no credential. + expect(failure).toBeInstanceOf(CliError); + expect((failure as CliError).code).toBe('usage_error'); + expect((failure as CliError).extra.hint).toMatch(/final URL/); + expect(JSON.stringify({ m: (failure as CliError).message, e: (failure as CliError).extra })).not.toContain(token); + }, 60_000); + + it("stay off a redirect when the user's git config follows redirects for that URL", () => { + const token = 'SECRET-TOKEN-2'; + const url = `http://127.0.0.1:${originPort}/repo.git`; + const global = join(dir, 'gitconfig'); + writeFileSync(global, `[http "${url}"]\n\tfollowRedirects = true\n[http "http://127.0.0.1:${originPort}/"]\n\tfollowRedirects = true\n`); + const saved = process.env.GIT_CONFIG_GLOBAL; + process.env.GIT_CONFIG_GLOBAL = global; + writeFileSync(log, ''); + try { + openGitSource(`http://user:${token}@127.0.0.1:${originPort}/repo.git`, 'test-repo', undefined, 'http').dispose(); + } catch { + // The clone stops at the redirect; only the requests matter here. + } finally { + if (saved === undefined) delete process.env.GIT_CONFIG_GLOBAL; + else process.env.GIT_CONFIG_GLOBAL = saved; + } + const requests = readFileSync(log, 'utf8').trim().split('\n').map((line) => JSON.parse(line) as { name: string; auth: string | null }); + expect(requests.some((r) => r.name === 'origin')).toBe(true); + expect(requests.filter((r) => r.name === 'elsewhere' && r.auth !== null)).toEqual([]); + }, 60_000); +}); diff --git a/packages/cli/src/lib/add/git-run.ts b/packages/cli/src/lib/add/git-run.ts new file mode 100644 index 0000000..9d8c8c5 --- /dev/null +++ b/packages/cli/src/lib/add/git-run.ts @@ -0,0 +1,142 @@ +import { spawnSync } from 'node:child_process'; +import { InstallError } from '../install/install-errors.js'; +import { redactCredentials } from './add-source.js'; +import { exitOnSignal, isCleanupSignal } from './scratch-dirs.js'; + +/** + * Runs git for `skillx add`: argument arrays (no shell), every credential + * prompt off (`core.askPass=` plus an empty `GIT_ASKPASS`, + * `GIT_TERMINAL_PROMPT=0`, `GCM_INTERACTIVE=never`), only one transport + * allowed (`https` outside tests) so `ext::`, `file://` and `ssh` are refused + * by git itself too, also on a redirect, a bounded output and a timeout. + * Variables that point git at another repository (`GIT_DIR`, + * `GIT_WORK_TREE`, ...) are never inherited. Every message is redacted. + * + * A URL's credentials never reach git's argument list or `.git/config`: + * `remoteOf` strips them and hands them over as an `Authorization` header in + * `GIT_CONFIG_*` environment entries (`http./.extraHeader`), so no + * credential helper is asked to store them either. git matches that key once, + * against the URL it was given, and then sends the header to wherever a + * redirect leads, so redirects are turned off whenever a header is passed. + * + * git runs synchronously, so Ctrl+C reaches git (same process group) before + * Node can handle it: a git call killed by SIGINT, SIGTERM or SIGHUP exits + * the CLI as that signal would, after removing every temporary clone. + */ + +export const GIT_TIMEOUT_MS = 120_000; + +export type ConfigPair = readonly [key: string, value: string]; + +/** Variables that redirect git to another repository, index or object store. */ +const REPO_ENV = new Set(['GIT_DIR', 'GIT_WORK_TREE', 'GIT_INDEX_FILE', 'GIT_OBJECT_DIRECTORY', 'GIT_ALTERNATE_OBJECT_DIRECTORIES', 'GIT_COMMON_DIR', 'GIT_NAMESPACE', 'GIT_CEILING_DIRECTORIES']); + +export function gitEnv(base: NodeJS.ProcessEnv = process.env, config: readonly ConfigPair[] = []): NodeJS.ProcessEnv { + // Compared in upper case: Windows environment names are case-insensitive. + const kept = Object.fromEntries(Object.entries(base).filter(([key]) => !REPO_ENV.has(key.toUpperCase()))); + const env: NodeJS.ProcessEnv = { ...kept, GIT_TERMINAL_PROMPT: '0', GCM_INTERACTIVE: 'never', GIT_ASKPASS: '', SSH_ASKPASS: '', GIT_LFS_SKIP_SMUDGE: '1' }; + if (config.length === 0) return env; + const start = Number(base.GIT_CONFIG_COUNT) > 0 ? Number(base.GIT_CONFIG_COUNT) : 0; + config.forEach(([key, value], i) => { + env[`GIT_CONFIG_KEY_${start + i}`] = key; + env[`GIT_CONFIG_VALUE_${start + i}`] = value; + }); + env.GIT_CONFIG_COUNT = String(start + config.length); + return env; +} + +/** Options that come before every git command reaching a remote. */ +export const guardArgs = (transport: string): string[] => ['-c', 'core.askPass=', '-c', 'protocol.allow=never', '-c', `protocol.${transport}.allow=always`]; + +export interface Remote { + /** The URL git sees, without credentials. */ + url: string; + /** Config entries passed through the environment (the credential header, if any). */ + config: ConfigPair[]; +} + +const decode = (part: string) => { + try { + return decodeURIComponent(part); + } catch { + return part; + } +}; + +export function remoteOf(url: string): Remote { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return { url, config: [] }; + } + if (parsed.username === '' && parsed.password === '') return { url, config: [] }; + const basic = Buffer.from(`${decode(parsed.username)}:${decode(parsed.password)}`, 'utf8').toString('base64'); + parsed.username = ''; + parsed.password = ''; + return { + url: parsed.href, + config: [ + [`http.${parsed.origin}/.extraHeader`, `Authorization: Basic ${basic}`], + // Scoped to the whole URL: a user's URL-scoped `followRedirects` would beat a shorter key. + [`http.${parsed.href}.followRedirects`, 'false'], + ], + }; +} + +export interface GitRun { + input?: string; + config?: readonly ConfigPair[]; + maxBuffer: number; + /** Extra environment, e.g. `GIT_NO_LAZY_FETCH` for local object reads. */ + env?: NodeJS.ProcessEnv; +} + +export function git(args: string[], what: string, run: GitRun): Buffer { + const env = gitEnv({ ...process.env, ...run.env }, run.config); + const r = spawnSync('git', args, { env, input: run.input, maxBuffer: run.maxBuffer, timeout: GIT_TIMEOUT_MS, windowsHide: true }); + if (r.error) { + const code = (r.error as NodeJS.ErrnoException).code; + if (code === 'ENOENT') throw new InstallError('unexpected_error', 'git is not installed or not on PATH'); + if (code === 'ETIMEDOUT') throw new InstallError('network_error', `${what} timed out after ${GIT_TIMEOUT_MS / 1000}s`); + if (code === 'ENOBUFS') throw new InstallError('incompatible', `${what} printed more than ${run.maxBuffer} bytes; point at a smaller repository or path`, { code: 'tree.limit_exceeded' }); + throw new InstallError('unexpected_error', `${what} failed: ${redactCredentials(r.error.message)}`); + } + if (r.signal !== null && isCleanupSignal(r.signal)) exitOnSignal(r.signal); + if (r.status !== 0) { + const stderr = redactCredentials(r.stderr.toString('utf8')).trim(); + const line = stderr.split('\n').filter((l) => !/^\s*$/.test(l)).pop() ?? `exit ${r.status}`; + if (carriesCredential(run.config) && /returned error: 30[1278]/.test(stderr)) { + const hint = "the server redirects, and redirects are not followed while the URL holds credentials; use the repository's final URL (often ending in .git)"; + throw new InstallError('usage_error', `${what} failed: ${line}`, undefined, hint); + } + throw new InstallError(classify(stderr), `${what} failed: ${line}`); + } + return r.stdout; +} + +const carriesCredential = (config: readonly ConfigPair[] = []) => config.some(([key]) => key.endsWith('.extraHeader')); + +function classify(stderr: string): 'auth_error' | 'not_found' | 'usage_error' | 'network_error' { + if (/authentication failed|could not read (username|password)|terminal prompts disabled|returned error: 40[13]/i.test(stderr)) return 'auth_error'; + if (/repository .*not found|not found|remote branch .* not found|returned error: 404/i.test(stderr)) return 'not_found'; + if (/transport .* not allowed|unsupported protocol/i.test(stderr)) return 'usage_error'; + return 'network_error'; +} + +/** Which of `names` the remote has as a branch or tag, asked with exact ref patterns. */ +export function remoteRefNames(remote: Remote, label: string, names: readonly string[], transport = 'https'): Set { + if (names.length === 0) return new Set(); + const patterns = names.flatMap((n) => [`refs/heads/${n}`, `refs/tags/${n}`]); + const out = git([...guardArgs(transport), 'ls-remote', '--heads', '--tags', '--', remote.url, ...patterns], `looking up refs of ${label}`, { + config: remote.config, + maxBuffer: 16 * 1024 * 1024, + }); + const found = new Set(); + for (const line of out.toString('utf8').split('\n')) { + const ref = line.split('\t')[1]?.replace(/\^\{\}$/, ''); + const name = ref?.replace(/^refs\/(?:heads|tags)\//, ''); + if (name !== undefined && name !== ref && names.includes(name)) found.add(name); + } + return found; +} diff --git a/packages/cli/src/lib/add/git-source.test.ts b/packages/cli/src/lib/add/git-source.test.ts index 37f5ebd..be81e1a 100644 --- a/packages/cli/src/lib/add/git-source.test.ts +++ b/packages/cli/src/lib/add/git-source.test.ts @@ -1,11 +1,13 @@ import { execFileSync } from 'node:child_process'; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { pathToFileURL } from 'node:url'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { CliError } from '../output.js'; -import { cloneArgs, gitEnv, openGitSource } from './git-source.js'; +import { cloneArgs, openGitSource } from './git-source.js'; +import { gitEnv, remoteOf, remoteRefNames } from './git-run.js'; +import { openScratchDirs } from './scratch-dirs.js'; import { discoverSkills, prepareSkill } from './skill-discovery.js'; /** @@ -16,6 +18,7 @@ import { discoverSkills, prepareSkill } from './skill-discovery.js'; let root: string; let repoUrl: string; +let partialUrl: string; function git(cwd: string, args: string[], input?: string): string { return execFileSync('git', ['-c', 'core.autocrlf=false', '-c', 'commit.gpgsign=false', '-c', 'user.name=Test', '-c', 'user.email=test@example.com', ...args], { cwd, input, encoding: 'utf8' }).trim(); @@ -39,6 +42,23 @@ beforeAll(() => { git(repo, ['update-index', '--add', '--cacheinfo', `120000,${oid},skills/leaky/leak`]); git(repo, ['commit', '-q', '-m', 'skills']); repoUrl = pathToFileURL(repo).href; + // A second repository whose server allows partial clones, with a large blob outside the skills and a branch with a slash. + const partial = join(root, 'partial'); + mkdirSync(partial); + git(partial, ['init', '-q', '-b', 'main']); + git(partial, ['config', 'uploadpack.allowFilter', 'true']); + const writeIn = (rel: string, text: string) => { + mkdirSync(dirname(join(partial, rel)), { recursive: true }); + writeFileSync(join(partial, rel), text); + }; + writeIn('skills/demo/SKILL.md', '---\nname: demo\ndescription: Demo skill\n---\n'); + writeIn('skills/demo/notes.md', 'notes\n'); + writeIn('skills/huge/SKILL.md', `---\nname: huge\ndescription: Huge\n---\n${'x'.repeat(1024 * 1024)}`); + writeIn('assets/big.bin', 'y'.repeat(3 * 1024 * 1024)); + git(partial, ['add', '-A']); + git(partial, ['commit', '-q', '-m', 'skills']); + git(partial, ['branch', 'feature/x']); + partialUrl = pathToFileURL(partial).href; }, 60_000); afterAll(() => rmSync(root, { recursive: true, force: true })); @@ -56,6 +76,7 @@ function failure(run: () => unknown): CliError { describe('git sources', () => { it('clones with no prompt, no checkout and the URL after --', () => { const args = cloneArgs('https://host/x.git', '/tmp/dest', 'main'); + expect(args).toContain('--filter=blob:none'); expect(args.slice(0, 6)).toEqual(['-c', 'core.askPass=', '-c', 'protocol.allow=never', '-c', 'protocol.https.allow=always']); expect(args).toEqual(expect.arrayContaining(['--depth', '1', '--no-checkout', '--branch', 'main'])); expect(args.slice(-3)).toEqual(['--', 'https://host/x.git', '/tmp/dest']); @@ -128,4 +149,61 @@ describe('git sources', () => { expect(refused.code).toBe('network_error'); expect(JSON.stringify({ m: refused.message, e: refused.extra })).not.toContain(token); }, 60_000); + + it('clones without blobs where the server allows it and fetches only what it reads', () => { + const source = openGitSource(partialUrl, 'partial', undefined, 'file'); + try { + const clone = join(openScratchDirs().at(-1)!, 'repo'); + const present = (rev: string) => { + try { + execFileSync('git', ['-C', clone, 'cat-file', '-e', rev], { env: { ...process.env, GIT_NO_LAZY_FETCH: '1' }, stdio: 'ignore' }); + return true; + } catch { + return false; + } + }; + expect(execFileSync('git', ['-C', clone, 'config', '--get', 'remote.origin.promisor'], { encoding: 'utf8' }).trim()).toBe('true'); + expect(execFileSync('git', ['-C', clone, 'config', '--get', 'remote.origin.url'], { encoding: 'utf8' }).trim()).toBe(partialUrl); + const { skills, skipped } = discoverSkills(source, 'skills'); + expect(skills.map((s) => s.name)).toEqual(['demo']); + expect(skipped).toEqual([{ dir: 'skills/huge', reason: 'SKILL.md is larger than 1048576 bytes' }]); + expect(prepareSkill(source, skills[0]!).files.map((f) => f.path)).toEqual(['SKILL.md', 'notes.md']); + // The over-size SKILL.md is sized from its object header and never read; the blob outside every skill is never fetched. + expect(present('HEAD:skills/demo/notes.md')).toBe(true); + expect(present('HEAD:assets/big.bin')).toBe(false); + } finally { + source.dispose(); + } + expect(openScratchDirs()).toEqual([]); + }); + + it('finds which /tree/ segments name a branch on the remote', () => { + const found = remoteRefNames(remoteOf(partialUrl), 'partial', ['feature/x/skills', 'feature/x', 'feature'], 'file'); + expect([...found]).toEqual(['feature/x']); + }); + + it('keeps URL credentials out of the clone URL and git config, as a header with redirects off', () => { + const remote = remoteOf('https://user:pa%2Fss@git.example.com:8443/team/x.git'); + expect(remote.url).toBe('https://git.example.com:8443/team/x.git'); + expect(remote.config).toEqual([ + ['http.https://git.example.com:8443/.extraHeader', `Authorization: Basic ${Buffer.from('user:pa/ss').toString('base64')}`], + ['http.https://git.example.com:8443/team/x.git.followRedirects', 'false'], + ]); + expect(remoteOf('https://host/x.git')).toEqual({ url: 'https://host/x.git', config: [] }); + // Entries already in the environment are kept; ours are appended after them. + expect(gitEnv({ GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'a.b', GIT_CONFIG_VALUE_0: 'c' }, remote.config)).toMatchObject({ + GIT_CONFIG_COUNT: '3', GIT_CONFIG_KEY_0: 'a.b', GIT_CONFIG_KEY_1: 'http.https://git.example.com:8443/.extraHeader', + }); + expect(cloneArgs(remote.url, '/tmp/dest').join(' ')).not.toContain('pa'); + }); + + it('removes the clone on dispose and drops its signal handlers', () => { + const before = process.listenerCount('SIGINT'); + const source = openGitSource(repoUrl, 'test-repo', 'main', 'file'); + const [dir] = openScratchDirs(); + expect(process.listenerCount('SIGINT')).toBe(before + 1); + source.dispose(); + expect(existsSync(dir!)).toBe(false); + expect(process.listenerCount('SIGINT')).toBe(before); + }); }); diff --git a/packages/cli/src/lib/add/git-source.ts b/packages/cli/src/lib/add/git-source.ts index 4f10ee2..142902f 100644 --- a/packages/cli/src/lib/add/git-source.ts +++ b/packages/cli/src/lib/add/git-source.ts @@ -1,85 +1,51 @@ -import { spawnSync } from 'node:child_process'; -import { mkdtempSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { LIMITS } from '@skillx/skill-tree'; import { InstallError } from '../install/install-errors.js'; -import { redactCredentials } from './add-source.js'; +import { git, guardArgs, remoteOf } from './git-run.js'; +import { scratchDir } from './scratch-dirs.js'; import type { ListedEntry, SourceSnapshot } from './skill-discovery.js'; import { isDiscoverable } from './skill-discovery.js'; /** * A git source, cloned into a private temporary directory and read from git - * objects only. The clone is shallow, single-branch and has no checkout, so - * no file of the repository is ever written to disk, no checkout filter, LFS - * smudge or hook runs, and a symlink stays a `120000` tree entry that the - * tree rules refuse. git runs with argument arrays (no shell), `--` before the - * URL, every credential prompt off (`core.askPass=` plus an empty - * `GIT_ASKPASS`, `GIT_TERMINAL_PROMPT=0`, `GCM_INTERACTIVE=never`), and only - * the https transport allowed, so `ext::`, `file://` and `ssh` are refused by - * git itself too, also on a redirect. Variables that point git at another - * repository (`GIT_DIR`, `GIT_WORK_TREE`, ...) are never inherited, and every - * read names the clone's `--git-dir`. Every message is redacted. + * objects only. The clone is shallow, single-branch, has no checkout and asks + * for no blobs (`--filter=blob:none`; a server without partial clone sends + * them all): the listing comes from tree objects, and only the blobs + * discovery or a chosen skill needs are fetched, in one batch each (a no-op + * for blobs already there). A fetched blob is downloaded whole, since git + * offers no way to learn a size first, so download time is bounded by the + * per-call timeout; memory is bounded by reading sizes from the local object + * headers before any content, and content in batches. No file of the + * repository is written to disk, no checkout filter, LFS smudge or hook runs, + * and a symlink stays a `120000` tree entry that the tree rules refuse. How + * git is run, and how URL credentials are kept off disk, is in `git-run.ts`. + * Every local call names the clone's `--git-dir`, never a repository found + * from the directory or environment. */ -const CLONE_TIMEOUT_MS = 120_000; -const MAX_OUTPUT = 1024 * 1024 * 1024; - -/** Variables that redirect git to another repository, index or object store. */ -const REPO_ENV = new Set(['GIT_DIR', 'GIT_WORK_TREE', 'GIT_INDEX_FILE', 'GIT_OBJECT_DIRECTORY', 'GIT_ALTERNATE_OBJECT_DIRECTORIES', 'GIT_COMMON_DIR', 'GIT_NAMESPACE', 'GIT_CEILING_DIRECTORIES']); - -export function gitEnv(base: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv { - // Compared in upper case: Windows environment names are case-insensitive. - const env = Object.fromEntries(Object.entries(base).filter(([key]) => !REPO_ENV.has(key.toUpperCase()))); - return { ...env, GIT_TERMINAL_PROMPT: '0', GCM_INTERACTIVE: 'never', GIT_ASKPASS: '', SSH_ASKPASS: '', GIT_LFS_SKIP_SMUDGE: '1' }; -} - -/** Pins a read to the clone itself, never a repository found from the directory or environment. */ -const inClone = (repo: string) => ['-C', repo, `--git-dir=${join(repo, '.git')}`]; +const MAX_LISTING = 256 * 1024 * 1024; +const READ_BATCH = 64 * 1024 * 1024; /** The clone command line. `transport` is `https` outside tests. */ export function cloneArgs(url: string, dest: string, ref?: string, transport = 'https'): string[] { return [ - '-c', 'core.askPass=', - '-c', 'protocol.allow=never', - '-c', `protocol.${transport}.allow=always`, - 'clone', '--depth', '1', '--no-checkout', '--single-branch', '--no-tags', + ...guardArgs(transport), + 'clone', '--depth', '1', '--no-checkout', '--single-branch', '--no-tags', '--filter=blob:none', ...(ref === undefined ? [] : ['--branch', ref]), '--', url, dest, ]; } -function git(args: string[], what: string, input?: string): Buffer { - const r = spawnSync('git', args, { env: gitEnv(), input, maxBuffer: MAX_OUTPUT, timeout: CLONE_TIMEOUT_MS, windowsHide: true }); - if (r.error) { - const code = (r.error as NodeJS.ErrnoException).code; - if (code === 'ENOENT') throw new InstallError('unexpected_error', 'git is not installed or not on PATH'); - if (code === 'ETIMEDOUT') throw new InstallError('network_error', `${what} timed out after ${CLONE_TIMEOUT_MS / 1000}s`); - throw new InstallError('unexpected_error', `${what} failed: ${redactCredentials(r.error.message)}`); - } - if (r.status !== 0) { - const stderr = redactCredentials(r.stderr.toString('utf8')).trim(); - const line = stderr.split('\n').filter((l) => !/^\s*$/.test(l)).pop() ?? `exit ${r.status}`; - throw new InstallError(classify(stderr), `${what} failed: ${line}`); - } - return r.stdout; -} - -function classify(stderr: string): 'auth_error' | 'not_found' | 'usage_error' | 'network_error' { - if (/authentication failed|could not read (username|password)|terminal prompts disabled|returned error: 40[13]/i.test(stderr)) return 'auth_error'; - if (/repository .*not found|not found|remote branch .* not found|returned error: 404/i.test(stderr)) return 'not_found'; - if (/transport .* not allowed|unsupported protocol/i.test(stderr)) return 'usage_error'; - return 'network_error'; -} +type GitEntry = Omit & { oid: string }; -/** `ls-tree -r -l -z` records: every blob and submodule of the commit, with its mode and size. */ -function listTree(dir: string): Array { - const out = git([...inClone(dir), 'ls-tree', '-r', '-l', '-z', '--full-tree', 'HEAD'], 'git ls-tree'); +/** `ls-tree -r -z` records: every blob and submodule of the commit, with its mode. No size, so no blob is fetched. */ +function parseTree(out: Buffer): GitEntry[] { const decoder = new TextDecoder('utf-8', { fatal: true }); - const entries: Array = []; + const entries: GitEntry[] = []; for (const record of out.toString('latin1').split('\0')) { if (record === '') continue; const tab = record.indexOf('\t'); - const m = /^(\d{6}) \w+ ([0-9a-f]{40,64}) +(\S+)$/.exec(record.slice(0, tab)); + const m = /^(\d{6}) \w+ ([0-9a-f]{40,64})$/.exec(record.slice(0, tab)); if (tab < 0 || !m) throw new InstallError('unexpected_error', 'git ls-tree printed an unexpected record'); let path: string; try { @@ -87,17 +53,15 @@ function listTree(dir: string): Array { } catch { throw new InstallError('incompatible', 'a path in the repository is not valid UTF-8'); } - const [mode, oid, size] = [m[1]!, m[2]!, m[3]!]; + const [mode, oid] = [m[1]!, m[2]!]; const kind = mode === '120000' ? 'symlink' : mode === '160000' ? 'special' : 'regular'; - entries.push({ path, kind, mode: mode === '100664' ? '100644' : mode, size: size === '-' ? 0 : Number(size), oid }); + entries.push({ path, kind, mode: mode === '100664' ? '100644' : mode, oid }); } return entries; } /** Splits `git cat-file --batch` output into blob contents, in request order. */ -function readBlobs(dir: string, oids: readonly string[]): Uint8Array[] { - if (oids.length === 0) return []; - const out = git([...inClone(dir), 'cat-file', '--batch'], 'git cat-file', `${oids.join('\n')}\n`); +function splitBlobs(out: Buffer, oids: readonly string[]): Uint8Array[] { const blobs: Uint8Array[] = []; let at = 0; for (const oid of oids) { @@ -111,37 +75,97 @@ function readBlobs(dir: string, oids: readonly string[]): Uint8Array[] { return blobs; } -/** Clones `url` and returns a snapshot of its HEAD; the clone is removed by `dispose`. */ +/** Pins a call to the clone itself, never a repository found from the directory or environment. */ +const inClone = (repo: string) => ['-C', repo, `--git-dir=${join(repo, '.git')}`]; + +/** Clones `url` and returns a snapshot of its HEAD; the clone is removed by `dispose`, or when the process exits or is interrupted. */ export function openGitSource(url: string, label: string, ref?: string, transport = 'https'): SourceSnapshot { - const dir = mkdtempSync(join(tmpdir(), 'skillx-add-')); - const dispose = () => rmSync(dir, { recursive: true, force: true }); + const remote = remoteOf(url); + const scratch = scratchDir('skillx-add-'); + const repo = join(scratch.dir, 'repo'); + // Missing blobs are fetched explicitly, in one batch; git 2.44+ also refuses to fetch them lazily here. + const local = (args: string[], what: string, maxBuffer: number, input?: string) => + git([...guardArgs(transport), ...inClone(repo), ...args], what, { input, maxBuffer, config: remote.config, env: { GIT_NO_LAZY_FETCH: '1' } }); try { - git(cloneArgs(url, join(dir, 'repo'), ref, transport), `cloning ${label}`); - const repo = join(dir, 'repo'); + git(cloneArgs(remote.url, repo, ref, transport), `cloning ${label}`, { config: remote.config, maxBuffer: 1024 * 1024 }); let commit: string; try { - commit = git([...inClone(repo), 'rev-parse', '--verify', 'HEAD^{commit}'], 'git rev-parse').toString('utf8').trim(); + commit = local(['rev-parse', '--verify', 'HEAD^{commit}'], 'git rev-parse', 4096).toString('utf8').trim(); } catch { throw new InstallError('not_found', `${label} has no commit to read`); } - const entries = listTree(repo); + const entries = parseTree(local(['ls-tree', '-r', '-z', '--full-tree', 'HEAD'], 'git ls-tree', MAX_LISTING)); const byPath = new Map(entries.map((e) => [e.path, e])); + const fetched = new Set(); + const sizes = new Map(); + + const fetchBlobs = (oids: readonly string[]) => { + const want = [...new Set(oids)].filter((oid) => !fetched.has(oid) && !sizes.has(oid)); + if (want.length === 0) return; + git( + [...guardArgs(transport), ...inClone(repo), '-c', 'fetch.negotiationAlgorithm=noop', 'fetch', '-q', '--no-tags', '--no-write-fetch-head', '--recurse-submodules=no', '--filter=blob:none', '--stdin', 'origin'], + `fetching files of ${label}`, + { input: `${want.join('\n')}\n`, config: remote.config, maxBuffer: 1024 * 1024 }, + ); + want.forEach((oid) => fetched.add(oid)); + }; + const sizeOf = (list: readonly GitEntry[]): number[] => { + const blobs = list.filter((e) => e.kind !== 'special'); + fetchBlobs(blobs.map((e) => e.oid)); + const unknown = [...new Set(blobs.map((e) => e.oid).filter((oid) => !sizes.has(oid)))]; + if (unknown.length > 0) { + const out = local(['cat-file', '--batch-check'], 'git cat-file', unknown.length * 160 + 1024, `${unknown.join('\n')}\n`).toString('utf8'); + out.split('\n').filter(Boolean).forEach((line, i) => { + const [id, type, size] = line.split(' '); + if (id !== unknown[i] || type !== 'blob') throw new InstallError('unexpected_error', `git could not read object ${unknown[i]}`); + sizes.set(id, Number(size)); + }); + } + return list.map((e) => (e.kind === 'special' ? 0 : sizes.get(e.oid)!)); + }; + const readBlobs = (list: readonly GitEntry[]): Uint8Array[] => { + const lengths = sizeOf(list); + const blobs: Uint8Array[] = []; + for (let i = 0; i < list.length; ) { + let end = i + 1; + let bytes = lengths[i]!; + while (end < list.length && bytes + lengths[end]! <= READ_BATCH) bytes += lengths[end++]!; + const oids = list.slice(i, end).map((e) => e.oid); + blobs.push(...splitBlobs(local(['cat-file', '--batch'], 'git cat-file', bytes + oids.length * 160 + 1024, `${oids.join('\n')}\n`), oids)); + i = end; + } + return blobs; + }; const under = (d: string) => (p: string) => d === '' || p.startsWith(`${d}/`); + const entry = (p: string): GitEntry => { + const found = byPath.get(p); + if (!found) throw new InstallError('unexpected_error', `${p} is not a file of ${label}`); + return found; + }; + return { label, commit, skillFiles: (searchPath) => entries.filter((e) => e.kind === 'regular' && under(searchPath)(e.path) && (e.path === 'SKILL.md' || e.path.endsWith('/SKILL.md')) && isDiscoverable(e.path, searchPath)).map((e) => e.path), + sizes: (paths) => sizeOf(paths.map(entry)), hasDir: (path) => entries.some((e) => e.path.startsWith(`${path}/`)), - entriesUnder: (d) => entries.filter((e) => under(d)(e.path)).map(({ path, kind, mode, size }) => ({ path: d === '' ? path : path.slice(d.length + 1), kind, mode, size })), + entriesUnder: (d) => { + const list = entries.filter((e) => under(d)(e.path)); + if (list.length > LIMITS.maxEntries) { + throw new InstallError('incompatible', `${d || label} has more than ${LIMITS.maxEntries} entries`, { code: 'tree.limit_exceeded', dir: d }); + } + const lengths = sizeOf(list); + return list.map(({ path, kind, mode }, i) => ({ path: d === '' ? path : path.slice(d.length + 1), kind, mode, size: lengths[i]! })); + }, read: (paths) => { - const blobs = readBlobs(repo, paths.map((p) => byPath.get(p)!.oid)); + const blobs = readBlobs(paths.map(entry)); return new Map(paths.map((p, i) => [p, blobs[i]!])); }, - dispose, + dispose: scratch.remove, }; } catch (error) { - dispose(); + scratch.remove(); throw error; } } diff --git a/packages/cli/src/lib/add/local-source.ts b/packages/cli/src/lib/add/local-source.ts index e29ae72..786de1b 100644 --- a/packages/cli/src/lib/add/local-source.ts +++ b/packages/cli/src/lib/add/local-source.ts @@ -68,6 +68,7 @@ export function openLocalSource(dir: string): SourceSnapshot { ) .filter((e) => e.kind === 'regular' && (e.path === 'SKILL.md' || e.path.endsWith('/SKILL.md'))) .map((e) => e.path), + sizes: (paths) => paths.map((path) => lstatSync(containedPath(root, path)).size), hasDir: (path) => linkFreeParents(root, `${path}/x`), entriesUnder: (d) => walk(root, d, () => true, LIMITS.maxEntries, () => diff --git a/packages/cli/src/lib/add/scratch-dirs.ts b/packages/cli/src/lib/add/scratch-dirs.ts new file mode 100644 index 0000000..60755bf --- /dev/null +++ b/packages/cli/src/lib/add/scratch-dirs.ts @@ -0,0 +1,60 @@ +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +/** + * Private temporary directories that never outlive the process. A clone may + * hold a credential-free remote but still a whole repository, so each + * directory is removed by its own `remove`, on a normal or `process.exit` + * exit, and on SIGINT, SIGTERM or SIGHUP while one is open (the handlers are + * installed only then, so Ctrl+C keeps its default behaviour otherwise). + */ + +const live = new Set(); +const SIGNALS = { SIGINT: 130, SIGTERM: 143, SIGHUP: 129 } as const; +type Signal = keyof typeof SIGNALS; + +function removeAll(): void { + for (const dir of live) rmSync(dir, { recursive: true, force: true }); + live.clear(); +} + +export const isCleanupSignal = (signal: string): signal is Signal => Object.hasOwn(SIGNALS, signal); + +/** Removes every open directory and exits with the shell's code for `signal` (128 + its number). */ +export function exitOnSignal(signal: Signal): never { + removeAll(); + process.exit(SIGNALS[signal]); +} + +const onSignal = (signal: Signal) => exitOnSignal(signal); + +function hook(on: boolean): void { + for (const signal of Object.keys(SIGNALS) as Signal[]) { + if (on) process.on(signal, onSignal); + else process.off(signal, onSignal); + } +} + +let exitHooked = false; + +/** Creates a directory under the system temp directory; `remove` deletes it (idempotent). */ +export function scratchDir(prefix: string): { dir: string; remove: () => void } { + const dir = mkdtempSync(join(tmpdir(), prefix)); + if (!exitHooked) { + process.on('exit', removeAll); + exitHooked = true; + } + if (live.size === 0) hook(true); + live.add(dir); + return { + dir, + remove: () => { + rmSync(dir, { recursive: true, force: true }); + if (live.delete(dir) && live.size === 0) hook(false); + }, + }; +} + +/** Directories still open; for tests. */ +export const openScratchDirs = (): string[] => [...live]; diff --git a/packages/cli/src/lib/add/skill-discovery.ts b/packages/cli/src/lib/add/skill-discovery.ts index 4bd25fa..a3ba06e 100644 --- a/packages/cli/src/lib/add/skill-discovery.ts +++ b/packages/cli/src/lib/add/skill-discovery.ts @@ -2,7 +2,7 @@ import { LIMITS, buildTree, isNoisePath, isSkillTreeError } from '@skillx/skill- import type { EntryKind, TreeEntry } from '@skillx/skill-tree'; import type { VerifiedTree } from '../install/fetch-verify.js'; import { sha256Hex } from '../install/fetch-verify.js'; -import { InstallError } from '../install/install-errors.js'; +import { InstallError, usageError } from '../install/install-errors.js'; import { readFrontmatter } from './skill-frontmatter.js'; /** @@ -13,7 +13,10 @@ import { readFrontmatter } from './skill-frontmatter.js'; * is a directory holding a regular `SKILL.md`; when the search path itself is * one, it is the only skill. Otherwise every such directory up to * `MAX_DEPTH` below it counts, outside noise directories and `node_modules`, - * and a skill's own subdirectories are its content, not more skills. + * and a skill's own subdirectories are its content, not more skills. Sizes + * come before bytes: more than `MAX_SKILL_FILES` candidates refuse the search + * path, and a `SKILL.md` over `MAX_SKILL_MD_SIZE` is skipped without its + * content being read into memory. * * `prepareSkill` builds the tree with the registry's `agentkit.skill-tree/v1` * rules: a symlink, junction, hardlink, submodule or other special entry, @@ -36,6 +39,8 @@ export interface SourceSnapshot { commit?: string; /** Regular `SKILL.md` files under `searchPath` that discovery may consider, relative to the source root. */ skillFiles(searchPath: string): string[]; + /** Sizes of regular files, read without their content. */ + sizes(paths: readonly string[]): number[]; /** True when `path` names a directory of the source. */ hasDir(path: string): boolean; /** Every entry under directory `dir` ('' for the root), relative to it. */ @@ -58,6 +63,11 @@ export interface SkippedSkill { } export const MAX_DEPTH = 6; +/** Discovery refuses a search path holding more `SKILL.md` files than this, before reading any of them. */ +export const MAX_SKILL_FILES = 1000; +/** A larger `SKILL.md` is skipped without its content being read. */ +export const MAX_SKILL_MD_SIZE = 1024 * 1024; +const READ_CHUNK = 64; const SKIPPED_DIRS = new Set(['node_modules']); /** True when discovery may look at this `SKILL.md` path (depth, noise and skipped directories). */ @@ -75,18 +85,32 @@ export function discoverSkills(source: SourceSnapshot, searchPath: string): { sk throw new InstallError('not_found', `${searchPath} is not a directory in ${source.label}`, { path: searchPath }); } const files = source.skillFiles(searchPath); + if (files.length > MAX_SKILL_FILES) { + throw usageError(`more than ${MAX_SKILL_FILES} SKILL.md files under ${searchPath || 'the root'} of ${source.label}; point at a smaller directory`, { path: searchPath, count: files.length }); + } const root = join(searchPath, 'SKILL.md'); const dirs = files.includes(root) ? [searchPath] : files.map(parentOf).sort(); // A skill's subdirectories are its content: drop any skill nested inside another one. const outer = dirs.filter((d) => !dirs.some((o) => o !== d && (o === '' || d.startsWith(`${o}/`)))); - const bytes = source.read(outer.map((d) => join(d, 'SKILL.md'))); + const sizes = source.sizes(outer.map((d) => join(d, 'SKILL.md'))); const skills: FoundSkill[] = []; const skipped: SkippedSkill[] = []; - for (const dir of outer) { - const fm = readFrontmatter(bytes.get(join(dir, 'SKILL.md'))!); - if (fm.ok) skills.push({ name: fm.name, description: fm.description, dir }); - else skipped.push({ dir, reason: fm.reason }); + const readable = outer.filter((dir, i) => { + if (sizes[i]! <= MAX_SKILL_MD_SIZE) return true; + skipped.push({ dir, reason: `SKILL.md is larger than ${MAX_SKILL_MD_SIZE} bytes` }); + return false; + }); + // Read in chunks and keep only the front matter, so memory stays bounded by the chunk. + for (let i = 0; i < readable.length; i += READ_CHUNK) { + const chunk = readable.slice(i, i + READ_CHUNK); + const bytes = source.read(chunk.map((d) => join(d, 'SKILL.md'))); + for (const dir of chunk) { + const fm = readFrontmatter(bytes.get(join(dir, 'SKILL.md'))!); + if (fm.ok) skills.push({ name: fm.name, description: fm.description, dir }); + else skipped.push({ dir, reason: fm.reason }); + } } + skipped.sort((a, b) => (a.dir < b.dir ? -1 : a.dir > b.dir ? 1 : 0)); return { skills, skipped }; } diff --git a/packages/cli/src/lib/add/skill-safety.test.ts b/packages/cli/src/lib/add/skill-safety.test.ts index 815d489..b7993df 100644 --- a/packages/cli/src/lib/add/skill-safety.test.ts +++ b/packages/cli/src/lib/add/skill-safety.test.ts @@ -4,7 +4,7 @@ import { dirname, join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { CliError } from '../output.js'; import { openLocalSource } from './local-source.js'; -import { discoverSkills, prepareSkill } from './skill-discovery.js'; +import { MAX_SKILL_FILES, MAX_SKILL_MD_SIZE, discoverSkills, prepareSkill } from './skill-discovery.js'; import { readFrontmatter } from './skill-frontmatter.js'; const enc = (text: string) => new TextEncoder().encode(text); @@ -165,3 +165,21 @@ describe('a chosen skill is refused, never copied, when it holds links', () => { expect(refusedWith(() => openLocalSource(file)).code).toBe('usage_error'); }); }); + +describe('discovery bounds', () => { + it('skips a SKILL.md over the size cap without reading it', () => { + write('a/SKILL.md', skillMd('a')); + write('b/SKILL.md', skillMd('b', `pad: ${'x'.repeat(MAX_SKILL_MD_SIZE)}\n`)); + const { skills, skipped } = discoverSkills(openLocalSource(root), ''); + expect(skills.map((s) => s.name)).toEqual(['a']); + expect(skipped).toEqual([{ dir: 'b', reason: `SKILL.md is larger than ${MAX_SKILL_MD_SIZE} bytes` }]); + }); + + it('refuses a search path with too many SKILL.md files before reading one', () => { + for (let i = 0; i <= MAX_SKILL_FILES; i++) write(`s${i}/SKILL.md`, 'not front matter'); + const error = refusedWith(() => discoverSkills(openLocalSource(root), '')); + expect(error.code).toBe('usage_error'); + expect(error.message).toContain(`more than ${MAX_SKILL_FILES} SKILL.md files`); + }); +}); +