diff --git a/.github/workflows/cli-build.yml b/.github/workflows/cli-build.yml index afd8c9ef9..b6fad80ad 100644 --- a/.github/workflows/cli-build.yml +++ b/.github/workflows/cli-build.yml @@ -352,7 +352,7 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - name: Generate CLI docs - run: pnpm --filter sentry run generate:schema && pnpm --filter sentry run generate:docs + run: pnpm --filter sentry run generate:docs - name: Set docs release run: | release=$(node -p 'require("./packages/cli/package.json").version') diff --git a/apps/cli-docs/package.json b/apps/cli-docs/package.json index 5d715a96f..14b94bd55 100644 --- a/apps/cli-docs/package.json +++ b/apps/cli-docs/package.json @@ -6,7 +6,7 @@ "dependencies": ["sentry"] }, "scripts": { - "ci:prepare": "pnpm --filter sentry run generate:schema && pnpm --filter sentry run generate:docs", + "ci:prepare": "pnpm --filter sentry run generate:docs", "dev": "astro dev", "build": "astro build", "test": "node --test test/*.test.mjs", diff --git a/package.json b/package.json index 0a4993af9..9b679d6cb 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "build": "dotenv -e .env -e .env.local -- pnpm -r --filter '!sentry' --filter '!sentry-cli-docs' --if-present run build", "build:cli": "dotenv -e .env -e .env.local -- pnpm --filter sentry run build && pnpm --filter sentry-cli-docs run build", "check:generated": "pnpm --filter @sentry/mcp-core generate-definitions && git diff --exit-code -- packages/mcp-core/src/toolDefinitions.json packages/mcp-core/src/skillDefinitions.json plugins/sentry-mcp/agents/sentry-mcp.md plugins/sentry-mcp-experimental/agents/sentry-mcp.md", - "deploy:docs": "pnpm --filter sentry run generate:schema && pnpm --filter sentry run generate:docs && pnpm --filter sentry-cli-docs run build && pnpm --filter sentry-cli-docs run deploy", + "deploy:docs": "pnpm --filter sentry run generate:docs && pnpm --filter sentry-cli-docs run build && pnpm --filter sentry-cli-docs run deploy", "eval": "pnpm --filter '@sentry/mcp-server-evals...' --if-present run build && dotenv -e .env -e .env.local -- pnpm --filter @sentry/mcp-server-evals run eval", "eval:ci": "pnpm --filter '@sentry/mcp-server-evals...' --if-present run build && CI=true dotenv -e .env -e .env.local -- pnpm --stream -r run eval:ci", "format": "oxfmt --write", diff --git a/packages/cli/AGENTS.md b/packages/cli/AGENTS.md index 925f67aaf..afe0aad7e 100644 --- a/packages/cli/AGENTS.md +++ b/packages/cli/AGENTS.md @@ -820,7 +820,10 @@ Use `"date"` for timestamp-based sort (not `"time"`). Export sort types from the ### Generated Docs & Skills -All command docs and skill files are generated via `pnpm run generate:docs` (which runs `generate:banner`, `generate:completions`, `generate:command-docs`, `generate:skill`, then `generate:docs-sections`). This runs automatically as part of `dev`, `build`, `typecheck`, and `test` scripts. +All command docs and skill files are generated via `pnpm run generate:docs`. This runs automatically as part of `dev`, `build`, `typecheck`, and `test` scripts. + +- **One process**: the `generate:*` scripts (except `generate:grammar`, a plain Node script) call `script/generate.ts`, which runs the requested steps in one process so the route tree loads once. Phase 1 writes what the tree imports (API schema, grammars, output-schema field data, banner), runs those steps concurrently, and must not load the tree; its steps must not read each other's outputs. Phase 2 stubs the generated files the tree imports (`script/lib/route-tree-stubs.ts`), then runs each step's exported `emit()`. Scripts that load the tree without generating, such as `check-fragments`, leave those stubs behind, so `ensure-generated.mjs` treats a file that still holds its stub (`script/lib/route-tree-stub-files.mjs`) as missing. Phase-2 modules may import the tree statically, because the runner imports them only after the stubs exist. A new generator exports `emit()` and gets a step in `script/generate.ts`, with no stub code of its own. A phase-2 step may also declare `prepare`, which the runner calls as the run begins: `command-api` uses it to parse `src/` in a worker thread (`script/lib/command-api-analysis.ts`) during phase 1 and the tree load. The worker reads the API schema once the `schema` step finishes, and `command-api` runs last because its `emit` waits for that worker. In `package.json`, call the runner (`tsx script/generate.ts `) rather than chaining `pnpm run generate:*`. `docs` names every step but `sdk`, including `schema`, so callers need not run `generate:schema` first. +- **Skipped steps**: a phase-1 step with `files` in `script/generate.ts` runs only when the content hash of its inputs or outputs differs from its entry in `src/generated/.stamps.json` (`script/lib/step-stamp.ts`). When such a step starts reading a new file or package, add it to that step's inputs. `output-schemas` and the phase-2 steps have no stamp: their inputs are the source they import, which is no short list. - **Command docs** (`apps/cli-docs/src/content/docs/commands/*.md`) and the configuration page (`apps/cli-docs/src/content/docs/configuration.md`) are **gitignored** and generated from CLI metadata + hand-written fragments in `apps/cli-docs/src/fragments/` (paths live in `script/paths.ts`). - **Skill files** (`plugins/sentry-cli/skills/sentry-cli/`) are **committed** (consumed by external plugin systems) and auto-committed by CI when stale. diff --git a/packages/cli/package.json b/packages/cli/package.json index 7ada146a2..5e78f3f56 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -56,30 +56,31 @@ "scripts": { "tsx": "tsx --env-file-if-exists=.env.local --import ./script/require-shim.mjs", "cli": "node script/ensure-generated.mjs && tsx --env-file-if-exists=.env.local --import ./script/require-shim.mjs --import ./script/strip-arg-separator.mjs src/bin.ts", - "dev": "pnpm run generate:schema && pnpm run generate:docs && pnpm run generate:sdk && tsx --import ./script/require-shim.mjs --import ./script/strip-arg-separator.mjs src/bin.ts", - "build": "pnpm --filter @sentry/mcp-core run build && pnpm run generate:schema && pnpm run generate:docs && pnpm run generate:sdk && pnpm tsx script/build.ts --single", - "build:all": "pnpm --filter @sentry/mcp-core run build && pnpm run generate:schema && pnpm run generate:docs && pnpm run generate:sdk && pnpm tsx script/build.ts", - "bundle": "pnpm --filter @sentry/mcp-core run build && pnpm run generate:schema && pnpm run generate:docs && pnpm run generate:sdk && pnpm tsx script/bundle.ts", - "typecheck": "pnpm --filter @sentry/mcp-core run build && pnpm run generate:docs && pnpm run generate:sdk && tsc --noEmit", + "dev": "tsx script/generate.ts docs sdk && tsx --import ./script/require-shim.mjs --import ./script/strip-arg-separator.mjs src/bin.ts", + "build": "pnpm --filter @sentry/mcp-core run build && tsx script/generate.ts docs sdk && pnpm tsx script/build.ts --single", + "build:all": "pnpm --filter @sentry/mcp-core run build && tsx script/generate.ts docs sdk && pnpm tsx script/build.ts", + "bundle": "pnpm --filter @sentry/mcp-core run build && tsx script/generate.ts docs sdk && pnpm tsx script/bundle.ts", + "typecheck": "pnpm --filter @sentry/mcp-core run build && tsx script/generate.ts docs sdk && tsc --noEmit", "lint": "pnpm -w exec oxlint --report-unused-disable-directives packages/cli && pnpm -w exec oxfmt --check packages/cli", "lint:fix": "pnpm -w exec oxlint --fix packages/cli && pnpm -w exec oxfmt --write packages/cli", "test": "pnpm run test:unit", - "test:unit": "pnpm run generate:docs && pnpm run generate:sdk && vitest run test/lib test/commands test/types test/script --coverage", - "test:changed": "pnpm run generate:docs && pnpm run generate:sdk && vitest run --changed", + "test:unit": "tsx script/generate.ts docs sdk && vitest run test/lib test/commands test/types test/script --coverage", + "test:changed": "tsx script/generate.ts docs sdk && vitest run --changed", "test:e2e": "pnpm tsx script/prepare-e2e-bundle.ts && vitest run test/e2e", "test:init-eval": "vitest run test/init-eval --testTimeout 600000", - "generate:sdk": "pnpm tsx script/generate-sdk.ts", - "generate:skill": "pnpm tsx script/generate-skill.ts", - "generate:banner": "pnpm tsx script/generate-banner-image.ts", - "generate:docs": "node script/ensure-generated.mjs schema && pnpm run generate:grammar && pnpm run generate:output-schemas && pnpm run generate:command-api && pnpm run generate:banner && pnpm run generate:completions && pnpm run generate:top-level-routes && pnpm run generate:command-docs && pnpm run generate:docs-sections && pnpm run generate:skill", + "generate": "tsx script/generate.ts docs sdk", + "generate:sdk": "tsx script/generate.ts sdk", + "generate:skill": "tsx script/generate.ts skill", + "generate:banner": "tsx script/generate.ts banner", + "generate:docs": "tsx script/generate.ts docs", "generate:grammar": "node ../../scripts/generate-parsers.mjs .", - "generate:output-schemas": "pnpm tsx script/generate-output-schemas.ts", - "generate:completions": "pnpm tsx script/generate-completion-targets.ts", - "generate:top-level-routes": "pnpm tsx script/generate-top-level-routes.ts", - "generate:docs-sections": "pnpm tsx script/generate-docs-sections.ts", - "generate:schema": "pnpm tsx script/generate-api-schema.ts", - "generate:command-api": "pnpm tsx script/generate-command-api.ts", - "generate:command-docs": "pnpm tsx script/generate-command-docs.ts", + "generate:output-schemas": "tsx script/generate.ts output-schemas", + "generate:completions": "tsx script/generate.ts completions", + "generate:top-level-routes": "tsx script/generate.ts top-level-routes", + "generate:docs-sections": "tsx script/generate.ts docs-sections", + "generate:schema": "tsx script/generate.ts schema", + "generate:command-api": "tsx script/generate.ts command-api", + "generate:command-docs": "tsx script/generate.ts command-docs", "eval:skill": "pnpm tsx script/eval-skill.ts", "bench": "pnpm tsx script/bench.ts", "bench:save": "pnpm tsx script/bench.ts --save-baseline", @@ -89,7 +90,7 @@ "check:deps": "pnpm tsx script/check-no-deps.ts", "check:errors": "pnpm tsx script/check-error-patterns.ts", "check:patches": "pnpm tsx script/check-patches.ts", - "check:docs-sections": "pnpm tsx script/generate-docs-sections.ts --check", + "check:docs-sections": "tsx script/generate.ts docs-sections --check", "check:env-coverage": "pnpm tsx script/check-env-coverage.ts", "check:stale-refs": "pnpm tsx script/check-stale-references.ts", "check:doc-examples": "pnpm tsx script/check-doc-examples.ts", diff --git a/packages/cli/script/check-doc-examples.ts b/packages/cli/script/check-doc-examples.ts index ad6689cbd..648048435 100644 --- a/packages/cli/script/check-doc-examples.ts +++ b/packages/cli/script/check-doc-examples.ts @@ -27,8 +27,8 @@ * 1 - At least one example names a missing command, flag, or ambiguous target */ -import { mkdirSync, readdirSync } from "node:fs"; -import { access, readFile, writeFile } from "node:fs/promises"; +import { readdirSync } from "node:fs"; +import { readFile } from "node:fs/promises"; import { join } from "node:path"; import { exampleSteps } from "../src/lib/command-example.js"; import type { @@ -37,6 +37,7 @@ import type { RouteMap, RouteTarget, } from "../src/lib/introspect.js"; +import { writeRouteTreeStubs } from "./lib/route-tree-stubs.js"; import { blankCodeBlocks, markdownCodeBlocks } from "./markdown.js"; import { DOCS_CONTENT, DOCS_FRAGMENTS } from "./paths.js"; import { @@ -46,19 +47,7 @@ import { } from "./sdk-run-calls.js"; import { shellCommands } from "./shell-words.js"; -// Bootstrap: skill-content stub (same pattern as generate-docs-sections.ts) -const SKILL_CONTENT_PATH = "src/generated/skill-content.ts"; -const skillContentExists = await access(SKILL_CONTENT_PATH).then( - () => true, - () => false, -); -if (!skillContentExists) { - mkdirSync("src/generated", { recursive: true }); - await writeFile( - SKILL_CONTENT_PATH, - "export const SKILL_FILES: [string, string][] = [];\n", - ); -} +await writeRouteTreeStubs(); const { routes } = await import("../src/app.js"); const { isCommand, isRouteMap } = await import("../src/lib/introspect.js"); diff --git a/packages/cli/script/check-fragments.ts b/packages/cli/script/check-fragments.ts index 00b18ade8..fe97a9d60 100644 --- a/packages/cli/script/check-fragments.ts +++ b/packages/cli/script/check-fragments.ts @@ -20,26 +20,15 @@ * tsx script/check-fragments.ts --strict # Errors for missing subcommands */ -import { mkdirSync, readdirSync } from "node:fs"; -import { access, readFile, writeFile } from "node:fs/promises"; +import { readdirSync } from "node:fs"; +import { access, readFile } from "node:fs/promises"; import type { RouteInfo, RouteMap } from "../src/lib/introspect.js"; import { findFragmentCommandIssues } from "./fragment-commands.js"; +import { writeRouteTreeStubs } from "./lib/route-tree-stubs.js"; import { markdownHeadings } from "./markdown.js"; import { DOCS_FRAGMENTS } from "./paths.js"; -// Ensure skill-content stub exists (see generate-command-docs.ts for rationale) -const SKILL_CONTENT_PATH = "src/generated/skill-content.ts"; -const skillContentExists = await access(SKILL_CONTENT_PATH).then( - () => true, - () => false, -); -if (!skillContentExists) { - mkdirSync("src/generated", { recursive: true }); - await writeFile( - SKILL_CONTENT_PATH, - "export const SKILL_FILES: [string, string][] = [];\n", - ); -} +await writeRouteTreeStubs(); const { routes } = await import("../src/app.js"); const { extractAllRoutes } = await import("../src/lib/introspect.js"); diff --git a/packages/cli/script/ensure-generated.mjs b/packages/cli/script/ensure-generated.mjs index 238d133ba..a6345a015 100644 --- a/packages/cli/script/ensure-generated.mjs +++ b/packages/cli/script/ensure-generated.mjs @@ -17,7 +17,8 @@ */ import { execSync } from "node:child_process"; -import { existsSync } from "node:fs"; +import { existsSync, readFileSync, statSync } from "node:fs"; +import { ROUTE_TREE_STUBS } from "./lib/route-tree-stub-files.mjs"; /** Produced by `generate:schema`; imported at runtime by `src/`. */ const SCHEMA_OUTPUTS = [ @@ -40,6 +41,26 @@ const DOCS_OUTPUTS = [ "src/generated/top-level-routes.json", ]; +const STUBS = new Map(ROUTE_TREE_STUBS); + +/** + * Whether `path` holds generated output. A file that still holds its + * route-tree stub does not: scripts that load the tree without generating, + * such as `check-fragments`, write stubs and leave them behind. + */ +function isGenerated(path) { + if (!existsSync(path)) { + return false; + } + const stub = STUBS.get(path); + // Compare sizes first so generated files are not read. + return ( + stub === undefined || + statSync(path).size !== Buffer.byteLength(stub) || + readFileSync(path, "utf8") !== stub + ); +} + function run(script) { console.error(`Missing generated files; running \`pnpm run ${script}\`...`); // Send generator output to stderr: `pnpm run cli mcp` serves JSON-RPC on @@ -59,6 +80,6 @@ if (!SCHEMA_OUTPUTS.every(existsSync)) { if (wants("grammar") && !GRAMMAR_OUTPUTS.every(existsSync)) { run("generate:grammar"); } -if (wants("docs") && !DOCS_OUTPUTS.every(existsSync)) { +if (wants("docs") && !DOCS_OUTPUTS.every(isGenerated)) { run("generate:docs"); } diff --git a/packages/cli/script/generate-api-schema.ts b/packages/cli/script/generate-api-schema.ts index b746eb772..9fb87d615 100644 --- a/packages/cli/script/generate-api-schema.ts +++ b/packages/cli/script/generate-api-schema.ts @@ -1,8 +1,7 @@ -#!/usr/bin/env tsx /** * Generate the CLI discovery index from the installed SDK's operation catalog. * - * Usage: tsx script/generate-api-schema.ts + * Usage: tsx script/generate.ts schema * Outputs: src/generated/api-schema.json and src/generated/idempotent-operations.json */ @@ -14,22 +13,26 @@ import { projectApiOperations } from "./lib/api-schema.js"; const OUTPUT_PATH = "src/generated/api-schema.json"; const IDEMPOTENT_OUTPUT_PATH = "src/generated/idempotent-operations.json"; -const endpoints = projectApiOperations(operations); -await mkdir(dirname(OUTPUT_PATH), { recursive: true }); -await writeFile(OUTPUT_PATH, JSON.stringify(endpoints, null, 2)); +/** Write the endpoint index and the idempotent write operations. */ +export async function emit(): Promise { + const endpoints = projectApiOperations(operations); -console.log( - `Generated ${OUTPUT_PATH} (${endpoints.length} endpoints, ${Math.round(JSON.stringify(endpoints).length / 1024)}KB)`, -); + await mkdir(dirname(OUTPUT_PATH), { recursive: true }); + await writeFile(OUTPUT_PATH, JSON.stringify(endpoints, null, 2)); -const idempotentOperations = selectIdempotentWrites(operations); + console.log( + `Generated ${OUTPUT_PATH} (${endpoints.length} endpoints, ${Math.round(JSON.stringify(endpoints).length / 1024)}KB)`, + ); -await writeFile( - IDEMPOTENT_OUTPUT_PATH, - JSON.stringify(idempotentOperations, null, 2), -); + const idempotentOperations = selectIdempotentWrites(operations); -console.log( - `Generated ${IDEMPOTENT_OUTPUT_PATH} (${idempotentOperations.length} operations)`, -); + await writeFile( + IDEMPOTENT_OUTPUT_PATH, + JSON.stringify(idempotentOperations, null, 2), + ); + + console.log( + `Generated ${IDEMPOTENT_OUTPUT_PATH} (${idempotentOperations.length} operations)`, + ); +} diff --git a/packages/cli/script/generate-banner-image.ts b/packages/cli/script/generate-banner-image.ts index 235b4cf4b..43979afc9 100644 --- a/packages/cli/script/generate-banner-image.ts +++ b/packages/cli/script/generate-banner-image.ts @@ -1,4 +1,3 @@ -#!/usr/bin/env tsx /** * Generate the Graphics Banner * @@ -16,7 +15,7 @@ * Output is a committed module (like `src/generated/skill-content.ts`) so the * runtime never needs to read the asset from disk. * - * Usage: tsx script/generate-banner-image.ts + * Usage: tsx script/generate.ts banner * Output: src/generated/banner-image.ts */ @@ -119,7 +118,8 @@ function encodePngBase64(mask: PNG): string { return PNG.sync.write(out, { deflateLevel: 9 }).toString("base64"); } -async function main(): Promise { +/** Write `src/generated/banner-image.ts` from the banner mask. */ +export async function emit(): Promise { const png = PNG.sync.read(readFileSync(MASK_PATH)); const pngBase64 = encodePngBase64(png); @@ -149,8 +149,3 @@ export const BANNER_IMAGE = { `Wrote ${OUT_PATH} (${png.width}x${png.height}, png ${pngBase64.length} base64 bytes)\n`, ); } - -main().catch((err) => { - process.stderr.write(`generate-banner-image failed: ${String(err)}\n`); - process.exit(1); -}); diff --git a/packages/cli/script/generate-command-api.ts b/packages/cli/script/generate-command-api.ts index e6cf730dd..c6c0e8f03 100644 --- a/packages/cli/script/generate-command-api.ts +++ b/packages/cli/script/generate-command-api.ts @@ -1,138 +1,101 @@ -#!/usr/bin/env tsx /** Generate command/API relationships from source calls and the public route tree. */ -import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import { writeFile } from "node:fs/promises"; import { resolve } from "node:path"; import { pathToFileURL } from "node:url"; -import { glob } from "tinyglobby"; -import type { ApiEndpoint, ApiReference } from "../src/lib/api-schema.js"; -import type { RouteMap, RouteTarget } from "../src/lib/introspect.js"; -import { analyzeSourceApiUsage } from "./lib/source-api-usage.js"; +import { routes } from "../src/app.js"; +import type { ApiReference } from "../src/lib/api-schema.js"; +import { + isCommand, + publicTargetPaths, + type RouteMap, + type RouteTarget, +} from "../src/lib/introspect.js"; +import { commandApiAnalysis } from "./lib/command-api-analysis.js"; const OUTPUT_PATH = "src/generated/command-api.json"; -// The route tree imports these generated files. Bootstrap before importing it; -// generate:top-level-routes and generate:skill replace their stubs later in -// the same generate:docs pipeline. -await mkdir("src/generated", { recursive: true }); -for (const [path, stub] of [ - [OUTPUT_PATH, "{}\n"], - ["src/generated/top-level-routes.json", "{}\n"], - [ - "src/generated/skill-content.ts", - "export const SKILL_FILES: [string, string][] = [];\n", - ], -] as const) { - if ( - !(await access(path).then( - () => true, - () => false, - )) - ) { - await writeFile(path, stub); - } -} - -const files = await glob(["src/**/*.{ts,tsx}"], { - ignore: ["src/generated/**"], -}); -const sources = new Map( - await Promise.all( - files - .sort() - .map(async (file) => [file, await readFile(file, "utf8")] as const), - ), -); -const endpoints = JSON.parse( - await readFile("src/generated/api-schema.json", "utf8"), -) as ApiEndpoint[]; -const analysis = analyzeSourceApiUsage( - sources, - endpoints - .filter((entry) => entry.fn) - .map((entry) => ({ - functionName: entry.fn, - method: entry.method, - path: entry.path, - })), -); +/** Write which API operations each public command calls. */ +export async function emit(): Promise { + const analysis = await commandApiAnalysis(); -const { routes } = await import("../src/app.js"); -const { isCommand, publicTargetPaths } = - await import("../src/lib/introspect.js"); -const paths = publicTargetPaths(routes as unknown as RouteMap); -const matched = new Set(); -const references: Record = {}; -const unresolved = new Map; reason: string }>(); + const paths = publicTargetPaths(routes as unknown as RouteMap); + const matched = new Set(); + const references: Record = {}; + const unresolved = new Map< + string, + { commands: Set; reason: string } + >(); -for (const { file, name } of analysis.commands) { - const module = await import(pathToFileURL(resolve(file)).href); - const target = module[name] as RouteTarget | undefined; - const path = target && paths.get(target); - if (!target || !path || !isCommand(target)) continue; - matched.add(target); - const command = `sentry ${path.join(" ")}`; - const usage = analysis.exports.get(file)?.get(name); - if (!usage) continue; - const entries = (references[command] ??= []); - for (const operation of usage.operations) { - const context = Object.entries(operation.query ?? {}) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([key, value]) => `${key}=${value}`) - .join(", "); - const reference: ApiReference = { - fn: operation.functionName, - ...(context - ? { - note: `Known query context: ${context}. The command may add filters or requests.`, - } - : {}), - }; - if ( - !entries.some( - (entry) => entry.fn === reference.fn && entry.note === reference.note, + for (const { file, name } of analysis.commands) { + const module = await import(pathToFileURL(resolve(file)).href); + const target = module[name] as RouteTarget | undefined; + const path = target && paths.get(target); + if (!target || !path || !isCommand(target)) continue; + matched.add(target); + const command = `sentry ${path.join(" ")}`; + const usage = analysis.exports.get(file)?.get(name); + if (!usage) continue; + const entries = (references[command] ??= []); + for (const operation of usage.operations) { + const context = Object.entries(operation.query ?? {}) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([key, value]) => `${key}=${value}`) + .join(", "); + const reference: ApiReference = { + fn: operation.functionName, + ...(context + ? { + note: `Known query context: ${context}. The command may add filters or requests.`, + } + : {}), + }; + if ( + !entries.some( + (entry) => entry.fn === reference.fn && entry.note === reference.note, + ) ) - ) - entries.push(reference); - } - for (const request of usage.unresolvedRequests) { - const key = `${request.file}:${request.line}`; - const diagnostic = unresolved.get(key) ?? { - commands: new Set(), - reason: request.reason, - }; - diagnostic.commands.add(command); - unresolved.set(key, diagnostic); + entries.push(reference); + } + for (const request of usage.unresolvedRequests) { + const key = `${request.file}:${request.line}`; + const diagnostic = unresolved.get(key) ?? { + commands: new Set(), + reason: request.reason, + }; + diagnostic.commands.add(command); + unresolved.set(key, diagnostic); + } } -} -for (const [target, path] of paths) { - if (isCommand(target) && !matched.has(target)) { + for (const [target, path] of paths) { + if (isCommand(target) && !matched.has(target)) { + console.warn( + `API discovery: no source command export found for sentry ${path.join(" ")}`, + ); + } + } + for (const [location, { commands, reason }] of unresolved) { console.warn( - `API discovery: no source command export found for sentry ${path.join(" ")}`, + `API discovery: ${location}: ${reason} (${[...commands].sort().join(", ")})`, ); } -} -for (const [location, { commands, reason }] of unresolved) { - console.warn( - `API discovery: ${location}: ${reason} (${[...commands].sort().join(", ")})`, + + const catalog = Object.fromEntries( + Object.entries(references) + .filter(([, entries]) => entries.length) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([command, entries]) => [ + command, + entries.sort( + (a, b) => + a.fn.localeCompare(b.fn) || + (a.note ?? "").localeCompare(b.note ?? ""), + ), + ]), + ); + await writeFile(OUTPUT_PATH, `${JSON.stringify(catalog, null, 2)}\n`); + console.log( + `Generated ${OUTPUT_PATH} (${Object.keys(catalog).length} commands; ${unresolved.size} unresolved requests)`, ); } - -const catalog = Object.fromEntries( - Object.entries(references) - .filter(([, entries]) => entries.length) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([command, entries]) => [ - command, - entries.sort( - (a, b) => - a.fn.localeCompare(b.fn) || - (a.note ?? "").localeCompare(b.note ?? ""), - ), - ]), -); -await writeFile(OUTPUT_PATH, `${JSON.stringify(catalog, null, 2)}\n`); -console.log( - `Generated ${OUTPUT_PATH} (${Object.keys(catalog).length} commands; ${unresolved.size} unresolved requests)`, -); diff --git a/packages/cli/script/generate-command-docs.ts b/packages/cli/script/generate-command-docs.ts index 36e735a5a..6827a57d0 100644 --- a/packages/cli/script/generate-command-docs.ts +++ b/packages/cli/script/generate-command-docs.ts @@ -1,4 +1,3 @@ -#!/usr/bin/env tsx /** * Generate Command Reference & Configuration Documentation * @@ -15,7 +14,7 @@ * source of truth for custom content. * * Usage: - * tsx script/generate-command-docs.ts + * tsx script/generate.ts command-docs * * Output: * docs/src/content/docs/commands/{route}.md (one per visible route) @@ -24,7 +23,16 @@ */ import { mkdirSync, rmSync } from "node:fs"; -import { access, readFile, writeFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; +import { routes } from "../src/app.js"; +import { ENV_VAR_REGISTRY, type EnvVarEntry } from "../src/lib/env-registry.js"; +import { + type CommandInfo, + extractAllRoutes, + type FlagInfo, + type RouteInfo, + type RouteMap, +} from "../src/lib/introspect.js"; import { formatCommandArguments, formatCommandExamples, @@ -33,36 +41,6 @@ import { import { stripAllowCommandMarkers } from "./fragment-commands.js"; import { DOCS_CONTENT, DOCS_FRAGMENTS } from "./paths.js"; -// Ensure src/generated/skill-content.ts exists before importing the route tree. -// The route tree transitively imports agent-skills.ts → skill-content.ts, which -// is generated by generate:skill. On a fresh checkout (or CI without prior runs), -// the file won't exist yet. We create a minimal stub so the import resolves; -// generate:skill will overwrite it with real content afterward. -// Must run before the dynamic import below (static imports hoist above top-level await). -const SKILL_CONTENT_PATH = "src/generated/skill-content.ts"; -const SKILL_CONTENT_STUB = - "export const SKILL_FILES: [string, string][] = [];\n"; -const skillContentExists = await access(SKILL_CONTENT_PATH).then( - () => true, - () => false, -); -if (!skillContentExists) { - mkdirSync("src/generated", { recursive: true }); - await writeFile(SKILL_CONTENT_PATH, SKILL_CONTENT_STUB); -} - -import type { EnvVarEntry } from "../src/lib/env-registry.js"; -import type { - CommandInfo, - FlagInfo, - RouteInfo, - RouteMap, -} from "../src/lib/introspect.js"; - -const { routes } = await import("../src/app.js"); -const { extractAllRoutes } = await import("../src/lib/introspect.js"); -const { ENV_VAR_REGISTRY } = await import("../src/lib/env-registry.js"); - const DOCS_DIR = `${DOCS_CONTENT}/commands`; const DOCS_CONTENT_DIR = DOCS_CONTENT; const FRAGMENTS_DIR = `${DOCS_FRAGMENTS}/commands`; @@ -385,72 +363,75 @@ async function readTopLevelFragment(fragmentName: string): Promise { // Main // --------------------------------------------------------------------------- -const routeMap = routes as unknown as RouteMap; -const routeInfos = extractAllRoutes(routeMap).filter( - (r) => !SKIP_ROUTES.has(r.name), -); +/** Write the command reference pages, their index, and the configuration page. */ +export async function emit(): Promise { + const routeMap = routes as unknown as RouteMap; + const routeInfos = extractAllRoutes(routeMap).filter( + (r) => !SKIP_ROUTES.has(r.name), + ); -const generatedFiles: string[] = []; + const generatedFiles: string[] = []; -// Ensure output directory exists (it's gitignored, may be absent on fresh checkout) -mkdirSync(DOCS_DIR, { recursive: true }); + // Ensure output directory exists (it's gitignored, may be absent on fresh checkout) + mkdirSync(DOCS_DIR, { recursive: true }); -// Clean up legacy cli/ subdirectory if it exists (we now use cli.md) -try { - rmSync(`${DOCS_DIR}/cli`, { recursive: true, force: true }); -} catch { - // Directory may not exist -} + // Clean up legacy cli/ subdirectory if it exists (we now use cli.md) + try { + rmSync(`${DOCS_DIR}/cli`, { recursive: true, force: true }); + } catch { + // Directory may not exist + } -// Generate command doc pages -for (const route of routeInfos) { - const filePath = `${DOCS_DIR}/${route.name}.md`; - const pageContent = generatePage(route); - const customContent = await readCustomContent(route.name); - const fullContent = customContent - ? pageContent + customContent - : `${pageContent}\n`; - - await writeFile(filePath, fullContent); - generatedFiles.push(filePath); -} + // Generate command doc pages + for (const route of routeInfos) { + const filePath = `${DOCS_DIR}/${route.name}.md`; + const pageContent = generatePage(route); + const customContent = await readCustomContent(route.name); + const fullContent = customContent + ? pageContent + customContent + : `${pageContent}\n`; + + await writeFile(filePath, fullContent); + generatedFiles.push(filePath); + } -// Update commands/index.md -const indexCustomContent = await readCustomContent("index"); -const indexLines: string[] = []; -indexLines.push("---"); -indexLines.push("title: Commands"); -indexLines.push("description: Available commands in the Sentry CLI"); -indexLines.push("---"); -indexLines.push(""); -indexLines.push( - "The Sentry CLI provides commands for interacting with various Sentry resources.", -); -indexLines.push(""); -indexLines.push("## Available Commands"); -indexLines.push(""); -indexLines.push(generateCommandsTable(routeInfos)); -indexLines.push(""); -indexLines.push(GENERATED_END_MARKER); - -const indexContent = indexCustomContent - ? indexLines.join("\n") + indexCustomContent - : `${indexLines.join("\n")}\n`; - -await writeFile(INDEX_PATH, indexContent); + // Update commands/index.md + const indexCustomContent = await readCustomContent("index"); + const indexLines: string[] = []; + indexLines.push("---"); + indexLines.push("title: Commands"); + indexLines.push("description: Available commands in the Sentry CLI"); + indexLines.push("---"); + indexLines.push(""); + indexLines.push( + "The Sentry CLI provides commands for interacting with various Sentry resources.", + ); + indexLines.push(""); + indexLines.push("## Available Commands"); + indexLines.push(""); + indexLines.push(generateCommandsTable(routeInfos)); + indexLines.push(""); + indexLines.push(GENERATED_END_MARKER); -// --------------------------------------------------------------------------- -// Generate configuration.md (env var reference + fragment) -// --------------------------------------------------------------------------- + const indexContent = indexCustomContent + ? indexLines.join("\n") + indexCustomContent + : `${indexLines.join("\n")}\n`; + + await writeFile(INDEX_PATH, indexContent); + + // --------------------------------------------------------------------------- + // Generate configuration.md (env var reference + fragment) + // --------------------------------------------------------------------------- -const configGenerated = generateConfigurationPage(ENV_VAR_REGISTRY); -const configFragment = await readTopLevelFragment("configuration"); -const configContent = configFragment - ? configGenerated + configFragment - : `${configGenerated}\n`; + const configGenerated = generateConfigurationPage(ENV_VAR_REGISTRY); + const configFragment = await readTopLevelFragment("configuration"); + const configContent = configFragment + ? configGenerated + configFragment + : `${configGenerated}\n`; -await writeFile(CONFIG_PATH, configContent); + await writeFile(CONFIG_PATH, configContent); -console.log( - `Generated ${generatedFiles.length} command doc pages + ${INDEX_PATH} + ${CONFIG_PATH}`, -); + console.log( + `Generated ${generatedFiles.length} command doc pages + ${INDEX_PATH} + ${CONFIG_PATH}`, + ); +} diff --git a/packages/cli/script/generate-completion-targets.ts b/packages/cli/script/generate-completion-targets.ts index 8512fe98e..714ed4e30 100644 --- a/packages/cli/script/generate-completion-targets.ts +++ b/packages/cli/script/generate-completion-targets.ts @@ -1,4 +1,3 @@ -#!/usr/bin/env tsx /** * Generate Shell-Completion Target Sets * @@ -16,50 +15,38 @@ * The output is gitignored and regenerated by `generate:docs`, which runs as * part of `dev`, `build`, `bundle`, `typecheck`, and `test`. * - * Usage: tsx script/generate-completion-targets.ts + * Usage: tsx script/generate.ts completions * Output: src/generated/completion-targets.json */ import { mkdirSync } from "node:fs"; -import { access, writeFile } from "node:fs/promises"; - -// Bootstrap: skill-content stub (same pattern as generate-command-docs.ts). -// The route tree transitively imports skill-content.ts, which may not exist -// yet on a fresh checkout. Must run before the dynamic import below. -const SKILL_CONTENT_PATH = "src/generated/skill-content.ts"; -const SKILL_CONTENT_STUB = - "export const SKILL_FILES: [string, string][] = [];\n"; -const skillContentExists = await access(SKILL_CONTENT_PATH).then( - () => true, - () => false, -); -if (!skillContentExists) { - mkdirSync("src/generated", { recursive: true }); - await writeFile(SKILL_CONTENT_PATH, SKILL_CONTENT_STUB); -} - -import type { RouteMap } from "../src/lib/introspect.js"; - -const { routes } = await import("../src/app.js"); -const { extractCommandCompletions, extractRouteGroups } = - await import("../src/lib/introspect.js"); -const { COMPLETION_VALUES } = await import("../src/lib/completion-target.js"); -const { getApiCompletionPaths } = await import("../src/lib/api-schema.js"); +import { writeFile } from "node:fs/promises"; +import { routes } from "../src/app.js"; +import { getApiCompletionPaths } from "../src/lib/api-schema.js"; +import { COMPLETION_VALUES } from "../src/lib/completion-target.js"; +import { + extractCommandCompletions, + extractRouteGroups, + type RouteMap, +} from "../src/lib/introspect.js"; const OUTPUT_PATH = "src/generated/completion-targets.json"; -const routeMap = routes as unknown as RouteMap; -const commands = extractCommandCompletions(routeMap); -const groups = extractRouteGroups(routeMap); +/** Write the completion kinds, route groups, and completer values. */ +export async function emit(): Promise { + const routeMap = routes as unknown as RouteMap; + const commands = extractCommandCompletions(routeMap); + const groups = extractRouteGroups(routeMap); -// Plain JSON (like api-schema.json) so the output is produced by a real -// serializer rather than hand-assembled source text. -mkdirSync("src/generated", { recursive: true }); -await writeFile( - OUTPUT_PATH, - `${JSON.stringify({ commands, groups, values: { ...COMPLETION_VALUES, apiPaths: getApiCompletionPaths() } }, null, 2)}\n`, - "utf8", -); -process.stdout.write( - `Wrote ${OUTPUT_PATH} (${Object.keys(commands).length} commands)\n`, -); + // Plain JSON (like api-schema.json) so the output is produced by a real + // serializer rather than hand-assembled source text. + mkdirSync("src/generated", { recursive: true }); + await writeFile( + OUTPUT_PATH, + `${JSON.stringify({ commands, groups, values: { ...COMPLETION_VALUES, apiPaths: getApiCompletionPaths() } }, null, 2)}\n`, + "utf8", + ); + process.stdout.write( + `Wrote ${OUTPUT_PATH} (${Object.keys(commands).length} commands)\n`, + ); +} diff --git a/packages/cli/script/generate-docs-sections.ts b/packages/cli/script/generate-docs-sections.ts index 0bcca0e6c..3db221fad 100644 --- a/packages/cli/script/generate-docs-sections.ts +++ b/packages/cli/script/generate-docs-sections.ts @@ -1,4 +1,3 @@ -#!/usr/bin/env tsx /** * Generate Documentation Sections (Marker-Based) * @@ -26,45 +25,35 @@ * this script edits committed files in-place between marker pairs. * * Usage: - * tsx script/generate-docs-sections.ts # Generate (write) - * tsx script/generate-docs-sections.ts --check # Dry-run, exit 1 if stale + * tsx script/generate.ts docs-sections # Generate (write) + * tsx script/generate.ts docs-sections --check # Dry-run, exit 1 if stale */ -import { existsSync, mkdirSync } from "node:fs"; -import { access, readFile, writeFile } from "node:fs/promises"; +import { existsSync } from "node:fs"; +import { readFile, writeFile } from "node:fs/promises"; import peggy from "peggy"; import { DOCS_CONTENT, DOCS_FRAGMENTS } from "./paths.js"; import { shellCommands } from "./shell-words.js"; -// Bootstrap: skill-content stub (same pattern as generate-command-docs.ts) -const SKILL_CONTENT_PATH = "src/generated/skill-content.ts"; -const SKILL_CONTENT_STUB = - "export const SKILL_FILES: [string, string][] = [];\n"; -const skillContentExists = await access(SKILL_CONTENT_PATH).then( - () => true, - () => false, -); -if (!skillContentExists) { - mkdirSync("src/generated", { recursive: true }); - await writeFile(SKILL_CONTENT_PATH, SKILL_CONTENT_STUB); -} - -import type { EnvVarEntry } from "../src/lib/env-registry.js"; +import { routes } from "../src/app.js"; +import { parse as parseIssueSyntax } from "../src/generated/issue-argument-grammar.js"; +import { buildOrgAwareAliases } from "../src/lib/alias.js"; +import { parseSelector } from "../src/lib/arg-parsing.js"; import { exampleSteps } from "../src/lib/command-example.js"; -import type { Command, RouteInfo, RouteMap } from "../src/lib/introspect.js"; - -const { routes } = await import("../src/app.js"); -const { extractAllRoutes, isCommand, isRouteMap } = - await import("../src/lib/introspect.js"); -const { OAUTH_SCOPES } = await import("../src/lib/oauth.js"); -const { ENV_VAR_REGISTRY } = await import("../src/lib/env-registry.js"); -const { AGENT_DISPLAY_NAMES } = await import("../src/lib/detect-agent.js"); -const { TEXT_EXTENSIONS } = await import("../src/lib/scan/constants.js"); -const { buildOrgAwareAliases } = await import("../src/lib/alias.js"); -const { parseIssueArgument } = await import("../src/lib/issue-arguments.js"); -const { parseSelector } = await import("../src/lib/arg-parsing.js"); -const { parse: parseIssueSyntax } = - await import("../src/generated/issue-argument-grammar.js"); +import { AGENT_DISPLAY_NAMES } from "../src/lib/detect-agent.js"; +import { ENV_VAR_REGISTRY, type EnvVarEntry } from "../src/lib/env-registry.js"; +import { + type Command, + extractAllRoutes, + isCommand, + isRouteMap, + type RouteInfo, + type RouteMap, +} from "../src/lib/introspect.js"; +import { parseIssueArgument } from "../src/lib/issue-arguments.js"; +import { OAUTH_SCOPES } from "../src/lib/oauth.js"; +import { TEXT_EXTENSIONS } from "../src/lib/scan/constants.js"; + const pkg = JSON.parse(await readFile("package.json", "utf-8")); const TARGET_GRAMMAR = "grammar/org-project-target.pegjs"; const targetGrammar = await readFile(TARGET_GRAMMAR, "utf-8"); @@ -72,7 +61,7 @@ const ISSUE_GRAMMAR = "grammar/issue-argument.pegjs"; const issueGrammar = await readFile(ISSUE_GRAMMAR, "utf-8"); const workspace = JSON.parse(await readFile("../../package.json", "utf-8")); -const isCheck = process.argv.includes("--check"); +const routeMap = routes as unknown as RouteMap; // --------------------------------------------------------------------------- // Marker Replacement @@ -978,193 +967,199 @@ type SectionDef = { // Main // --------------------------------------------------------------------------- -const routeMap = routes as unknown as RouteMap; -const routeInfos = extractAllRoutes(routeMap); - -const sections: SectionDef[] = [ - // -- Existing sections -- - { - filePath: `${DOCS_CONTENT}/contributing.md`, - sectionName: "project-structure", - generate: () => generateProjectStructure(routeInfos), - }, - { - filePath: "DEVELOPMENT.md", - sectionName: "oauth-scopes", - generate: () => generateScopesBulletList(OAUTH_SCOPES), - }, - { - filePath: `${DOCS_CONTENT}/self-hosted.md`, - sectionName: "oauth-scopes", - generate: () => generateScopesInline(OAUTH_SCOPES), - }, - // -- Prerequisites (version numbers from package.json) -- - { - filePath: "README.md", - sectionName: "dev-prereq", - generate: generateDevPrereq, - }, - { - filePath: "README.md", - sectionName: "library-prereq", - generate: generateLibraryPrereq, - }, - { - filePath: `${DOCS_CONTENT}/contributing.md`, - sectionName: "dev-prereq", - generate: generateDevPrereqContributing, - }, - { - filePath: "DEVELOPMENT.md", - sectionName: "dev-prereq", - generate: generateDevPrereqDevelopment, - }, - { - filePath: "DEVELOPMENT.md", - sectionName: "build-toolchain", - generate: generateBuildToolchain, - }, - { - filePath: `${DOCS_CONTENT}/contributing.md`, - sectionName: "build-commands", - generate: generateBuildCommands, - }, - // -- Dev scripts (README.md) -- - { - filePath: "README.md", - sectionName: "dev-scripts", - generate: generateDevScripts, - }, - // -- Env var tables -- - { - filePath: "DEVELOPMENT.md", - sectionName: "dev-env-vars", - generate: generateDevEnvVarsTable, - }, - { - filePath: `${DOCS_CONTENT}/self-hosted.md`, - sectionName: "self-hosted-env-vars", - generate: generateSelfHostedEnvVarsTable, - }, - // -- Platform support (getting-started.mdx) -- - { - filePath: `${DOCS_CONTENT}/getting-started.mdx`, - sectionName: "platform-support", - generate: generatePlatformSupport, - markerStyle: "mdx", - }, - // -- Lists that mirror code -- - { - filePath: `${DOCS_CONTENT}/agent-guidance.md`, - sectionName: "destructive-commands", - generate: () => generateDestructiveCommands(routeInfos), - }, - { - filePath: `${DOCS_CONTENT}/agentic-usage.md`, - sectionName: "detected-agents", - generate: generateDetectedAgents, - }, - { - filePath: `${DOCS_FRAGMENTS}/commands/index.md`, - sectionName: "target-syntax", - generate: generateTargetSyntax, - }, - { - filePath: "CONTRIBUTING.md", - sectionName: "target-syntax", - generate: generateTargetSyntax, - }, - { - filePath: "plugins/README.md", - sectionName: "skill-layout", - generate: generateSkillLayout, - }, - { - filePath: `${DOCS_CONTENT}/features.md`, - sectionName: "dsn-extensions", - generate: generateDsnExtensions, - }, - { - filePath: `${DOCS_CONTENT}/features.md`, - sectionName: "alias-unique-prefix", - generate: () => - generateAliasTable( - "Project Slug", - inOrg("acme", ["frontend", "functions", "backend"]), - projectSlug, - ), - }, - { - filePath: `${DOCS_CONTENT}/features.md`, - sectionName: "alias-common-prefix", - generate: () => - generateAliasTable( - "Project Slug", - inOrg("acme", [ - "spotlight-electron", - "spotlight-website", - "spotlight-backend", - ]), - projectSlug, - ), - }, - { - filePath: `${DOCS_CONTENT}/features.md`, - sectionName: "alias-cross-org", - generate: () => - generateAliasTable( - "Org/Project", - [ - { org: "acme", project: "api" }, - { org: "globex", project: "api" }, - ], - orgProject, - ), - }, - { - filePath: `${DOCS_CONTENT}/features.md`, - sectionName: "issue-id-formats", - generate: generateIssueIdFormats, - }, - { - filePath: `${DOCS_CONTENT}/library-usage.md`, - sectionName: "sdk-run-examples", - generate: generateSdkRunExamples, - }, -]; +/** + * Regenerate every marker section. With `check`, write nothing and set a + * failing exit code when any section is stale. + */ +export async function emit({ check }: { check: boolean }): Promise { + const routeInfos = extractAllRoutes(routeMap); + + const sections: SectionDef[] = [ + // -- Existing sections -- + { + filePath: `${DOCS_CONTENT}/contributing.md`, + sectionName: "project-structure", + generate: () => generateProjectStructure(routeInfos), + }, + { + filePath: "DEVELOPMENT.md", + sectionName: "oauth-scopes", + generate: () => generateScopesBulletList(OAUTH_SCOPES), + }, + { + filePath: `${DOCS_CONTENT}/self-hosted.md`, + sectionName: "oauth-scopes", + generate: () => generateScopesInline(OAUTH_SCOPES), + }, + // -- Prerequisites (version numbers from package.json) -- + { + filePath: "README.md", + sectionName: "dev-prereq", + generate: generateDevPrereq, + }, + { + filePath: "README.md", + sectionName: "library-prereq", + generate: generateLibraryPrereq, + }, + { + filePath: `${DOCS_CONTENT}/contributing.md`, + sectionName: "dev-prereq", + generate: generateDevPrereqContributing, + }, + { + filePath: "DEVELOPMENT.md", + sectionName: "dev-prereq", + generate: generateDevPrereqDevelopment, + }, + { + filePath: "DEVELOPMENT.md", + sectionName: "build-toolchain", + generate: generateBuildToolchain, + }, + { + filePath: `${DOCS_CONTENT}/contributing.md`, + sectionName: "build-commands", + generate: generateBuildCommands, + }, + // -- Dev scripts (README.md) -- + { + filePath: "README.md", + sectionName: "dev-scripts", + generate: generateDevScripts, + }, + // -- Env var tables -- + { + filePath: "DEVELOPMENT.md", + sectionName: "dev-env-vars", + generate: generateDevEnvVarsTable, + }, + { + filePath: `${DOCS_CONTENT}/self-hosted.md`, + sectionName: "self-hosted-env-vars", + generate: generateSelfHostedEnvVarsTable, + }, + // -- Platform support (getting-started.mdx) -- + { + filePath: `${DOCS_CONTENT}/getting-started.mdx`, + sectionName: "platform-support", + generate: generatePlatformSupport, + markerStyle: "mdx", + }, + // -- Lists that mirror code -- + { + filePath: `${DOCS_CONTENT}/agent-guidance.md`, + sectionName: "destructive-commands", + generate: () => generateDestructiveCommands(routeInfos), + }, + { + filePath: `${DOCS_CONTENT}/agentic-usage.md`, + sectionName: "detected-agents", + generate: generateDetectedAgents, + }, + { + filePath: `${DOCS_FRAGMENTS}/commands/index.md`, + sectionName: "target-syntax", + generate: generateTargetSyntax, + }, + { + filePath: "CONTRIBUTING.md", + sectionName: "target-syntax", + generate: generateTargetSyntax, + }, + { + filePath: "plugins/README.md", + sectionName: "skill-layout", + generate: generateSkillLayout, + }, + { + filePath: `${DOCS_CONTENT}/features.md`, + sectionName: "dsn-extensions", + generate: generateDsnExtensions, + }, + { + filePath: `${DOCS_CONTENT}/features.md`, + sectionName: "alias-unique-prefix", + generate: () => + generateAliasTable( + "Project Slug", + inOrg("acme", ["frontend", "functions", "backend"]), + projectSlug, + ), + }, + { + filePath: `${DOCS_CONTENT}/features.md`, + sectionName: "alias-common-prefix", + generate: () => + generateAliasTable( + "Project Slug", + inOrg("acme", [ + "spotlight-electron", + "spotlight-website", + "spotlight-backend", + ]), + projectSlug, + ), + }, + { + filePath: `${DOCS_CONTENT}/features.md`, + sectionName: "alias-cross-org", + generate: () => + generateAliasTable( + "Org/Project", + [ + { org: "acme", project: "api" }, + { org: "globex", project: "api" }, + ], + orgProject, + ), + }, + { + filePath: `${DOCS_CONTENT}/features.md`, + sectionName: "issue-id-formats", + generate: generateIssueIdFormats, + }, + { + filePath: `${DOCS_CONTENT}/library-usage.md`, + sectionName: "sdk-run-examples", + generate: generateSdkRunExamples, + }, + ]; -let staleCount = 0; + let staleCount = 0; -for (const section of sections) { - const original = await readFile(section.filePath, "utf-8"); - const generated = section.generate(); - const updated = replaceMarkerSection( - original, - section.sectionName, - generated, - section.markerStyle ?? "html", - ); + for (const section of sections) { + const original = await readFile(section.filePath, "utf-8"); + const generated = section.generate(); + const updated = replaceMarkerSection( + original, + section.sectionName, + generated, + section.markerStyle ?? "html", + ); - if (updated !== original) { - if (isCheck) { - console.error(`STALE: ${section.filePath} [${section.sectionName}]`); - staleCount += 1; + if (updated !== original) { + if (check) { + console.error(`STALE: ${section.filePath} [${section.sectionName}]`); + staleCount += 1; + } else { + await writeFile(section.filePath, updated); + console.log(`Updated: ${section.filePath} [${section.sectionName}]`); + } } else { - await writeFile(section.filePath, updated); - console.log(`Updated: ${section.filePath} [${section.sectionName}]`); + console.log(`Up to date: ${section.filePath} [${section.sectionName}]`); } - } else { - console.log(`Up to date: ${section.filePath} [${section.sectionName}]`); } -} -if (isCheck && staleCount > 0) { - console.error( - `\n${staleCount} section(s) are stale. Run: pnpm run generate:docs`, - ); - process.exit(1); -} + if (check && staleCount > 0) { + console.error( + `\n${staleCount} section(s) are stale. Run: pnpm run generate:docs`, + ); + process.exitCode = 1; + return; + } -if (!isCheck) { - console.log("All docs sections generated."); + if (!check) { + console.log("All docs sections generated."); + } } diff --git a/packages/cli/script/generate-output-schemas.ts b/packages/cli/script/generate-output-schemas.ts index 126b5fb04..4c66cd88d 100644 --- a/packages/cli/script/generate-output-schemas.ts +++ b/packages/cli/script/generate-output-schemas.ts @@ -1,4 +1,3 @@ -#!/usr/bin/env tsx /** * Generate Output Schema Fields * @@ -21,7 +20,7 @@ * The output is gitignored and regenerated by `generate:docs`, which runs as * part of `dev`, `build`, `bundle`, `typecheck`, and `test`. * - * Usage: tsx script/generate-output-schemas.ts + * Usage: tsx script/generate.ts output-schemas * Output: src/generated/output-schemas/*.ts */ @@ -75,28 +74,31 @@ function renderModule( return `${lines.join("\n")}\n`; } -const files = (await readdir(SOURCE_DIR)) - .filter((file) => file.endsWith(".ts")) - .sort(); +/** Write the field data of every output schema. */ +export async function emit(): Promise { + const files = (await readdir(SOURCE_DIR)) + .filter((file) => file.endsWith(".ts")) + .sort(); -await rm(OUTPUT_DIR, { recursive: true, force: true }); -await mkdir(OUTPUT_DIR, { recursive: true }); + await rm(OUTPUT_DIR, { recursive: true, force: true }); + await mkdir(OUTPUT_DIR, { recursive: true }); -let count = 0; -for (const file of files) { - const name = basename(file, ".ts"); - const exports: Record = await import( - pathToFileURL(resolve(SOURCE_DIR, file)).href - ); - const schemas = Object.entries(exports).filter( - (entry): entry is [string, GenericSchema] => - entry[0].endsWith("Schema") && isSchema(entry[1]), - ); - if (schemas.length === 0) { - continue; + let count = 0; + for (const file of files) { + const name = basename(file, ".ts"); + const exports: Record = await import( + pathToFileURL(resolve(SOURCE_DIR, file)).href + ); + const schemas = Object.entries(exports).filter( + (entry): entry is [string, GenericSchema] => + entry[0].endsWith("Schema") && isSchema(entry[1]), + ); + if (schemas.length === 0) { + continue; + } + count += schemas.length; + await writeFile(`${OUTPUT_DIR}/${name}.ts`, renderModule(name, schemas)); } - count += schemas.length; - await writeFile(`${OUTPUT_DIR}/${name}.ts`, renderModule(name, schemas)); -} -console.log(`Generated ${OUTPUT_DIR}/ (${count} output schemas)`); + console.log(`Generated ${OUTPUT_DIR}/ (${count} output schemas)`); +} diff --git a/packages/cli/script/generate-sdk.ts b/packages/cli/script/generate-sdk.ts index c91c1bc6a..910d2cad3 100644 --- a/packages/cli/script/generate-sdk.ts +++ b/packages/cli/script/generate-sdk.ts @@ -1,4 +1,3 @@ -#!/usr/bin/env tsx /** * Generate typed SDK methods from the Stricli command tree. * @@ -14,7 +13,7 @@ * * Zero manual config — all commands are auto-discovered. * - * Run: tsx script/generate-sdk.ts + * Run: tsx script/generate.ts sdk */ import { writeFile } from "node:fs/promises"; @@ -754,94 +753,97 @@ function createSdkMethodsFunction(root: NamespaceNode): ts.FunctionDeclaration { // Main // --------------------------------------------------------------------------- -const allCommands = discoverCommands(routes as unknown as RouteMap, []); -console.log(`Discovered ${allCommands.length} commands`); +/** Write `src/sdk.generated.ts` and `src/sdk.generated.d.cts`. */ +export async function emit(): Promise { + const allCommands = discoverCommands(routes as unknown as RouteMap, []); + console.log(`Discovered ${allCommands.length} commands`); -const paramTypes: ts.Statement[] = []; -const returnTypes: ts.Statement[] = []; -const sdkTree = createNamespaceNode(); + const paramTypes: ts.Statement[] = []; + const returnTypes: ts.Statement[] = []; + const sdkTree = createNamespaceNode(); -for (const { path, command } of allCommands) { - const flags = extractSdkFlags(command); - const positional = derivePositional(command); + for (const { path, command } of allCommands) { + const flags = extractSdkFlags(command); + const positional = derivePositional(command); - const returnType = buildReturnType(command, `${buildTypeName(path)}Result`); - if (returnType) { - returnTypes.push(returnType); - } + const returnType = buildReturnType(command, `${buildTypeName(path)}Result`); + if (returnType) { + returnTypes.push(returnType); + } - const params = buildParamsType( - `${buildTypeName(path)}Params`, - positional, - flags, - ); - if (params) { - paramTypes.push(params); - } - - insertMethod(sdkTree, { - path, - brief: command.brief || path.join(" "), - positional, - flags, - streamingFlags: flags.filter((flag) => STREAMING_FLAGS.has(flag.name)), - paramsTypeName: params?.name.text, - paramsRequired: flags.some((flag) => !flag.optional), - resultTypeName: returnType?.name.text, - }); -} + const params = buildParamsType( + `${buildTypeName(path)}Params`, + positional, + flags, + ); + if (params) { + paramTypes.push(params); + } -const sdkType = exportedObjectType(SDK_TYPE_NAME, [ - ...namespaceType(sdkTree).members, -]); + insertMethod(sdkTree, { + path, + brief: command.brief || path.join(" "), + positional, + flags, + streamingFlags: flags.filter((flag) => STREAMING_FLAGS.has(flag.name)), + paramsTypeName: params?.name.text, + paramsRequired: flags.some((flag) => !flag.optional), + resultTypeName: returnType?.name.text, + }); + } -/** Sections shared verbatim by the implementation file and the published declarations. */ -const sharedTypes = [ - "// --- Return types (derived from __jsonFields) ---", - "", - returnTypes.length > 0 - ? printNodes(returnTypes) - : "// No commands have registered schemas yet.", - "", - "// --- Parameter types ---", - "", - paramTypes.length > 0 - ? printNodes(paramTypes) - : "// No commands have parameters.", - "", - "// --- SDK type ---", - "", - printNodes([sdkType]), - "", -]; - -const header = [ - "// Auto-generated by script/generate-sdk.ts — DO NOT EDIT", - "// Run `pnpm run generate:sdk` to regenerate.", - "", -]; - -const output = [ - ...header, - 'import type { buildInvoker } from "./lib/sdk-invoke.js";', - 'import { positionals } from "./lib/sdk-positionals.js";', - "", - ...sharedTypes, - "// --- SDK factory ---", - "", - "/** Invoke function type from sdk-invoke.ts */", - "type Invoke = ReturnType;", - "", - printNodes([createSdkMethodsFunction(sdkTree)]), - "", -].join("\n"); - -const outPath = "./src/sdk.generated.ts"; -await writeFile(outPath, output); -console.log(`Generated ${outPath}`); - -// Standalone type declarations (.d.cts) for the npm bundle: only the shared type -// exports — no runtime imports or implementations. -const dtsPath = "./src/sdk.generated.d.cts"; -await writeFile(dtsPath, [...header, ...sharedTypes].join("\n")); -console.log(`Generated ${dtsPath}`); + const sdkType = exportedObjectType(SDK_TYPE_NAME, [ + ...namespaceType(sdkTree).members, + ]); + + /** Sections shared verbatim by the implementation file and the published declarations. */ + const sharedTypes = [ + "// --- Return types (derived from __jsonFields) ---", + "", + returnTypes.length > 0 + ? printNodes(returnTypes) + : "// No commands have registered schemas yet.", + "", + "// --- Parameter types ---", + "", + paramTypes.length > 0 + ? printNodes(paramTypes) + : "// No commands have parameters.", + "", + "// --- SDK type ---", + "", + printNodes([sdkType]), + "", + ]; + + const header = [ + "// Auto-generated by script/generate-sdk.ts — DO NOT EDIT", + "// Run `pnpm run generate:sdk` to regenerate.", + "", + ]; + + const output = [ + ...header, + 'import type { buildInvoker } from "./lib/sdk-invoke.js";', + 'import { positionals } from "./lib/sdk-positionals.js";', + "", + ...sharedTypes, + "// --- SDK factory ---", + "", + "/** Invoke function type from sdk-invoke.ts */", + "type Invoke = ReturnType;", + "", + printNodes([createSdkMethodsFunction(sdkTree)]), + "", + ].join("\n"); + + const outPath = "./src/sdk.generated.ts"; + await writeFile(outPath, output); + console.log(`Generated ${outPath}`); + + // Standalone type declarations (.d.cts) for the npm bundle: only the shared type + // exports — no runtime imports or implementations. + const dtsPath = "./src/sdk.generated.d.cts"; + await writeFile(dtsPath, [...header, ...sharedTypes].join("\n")); + console.log(`Generated ${dtsPath}`); +} diff --git a/packages/cli/script/generate-skill.ts b/packages/cli/script/generate-skill.ts index e077a2069..0fb14ea03 100644 --- a/packages/cli/script/generate-skill.ts +++ b/packages/cli/script/generate-skill.ts @@ -1,4 +1,3 @@ -#!/usr/bin/env tsx /** * Generate Skill Files from Stricli Command Metadata and Docs * @@ -22,8 +21,16 @@ * docs/public/.well-known/skills/index.json */ -import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, rmSync } from "node:fs"; import { access, readFile, writeFile } from "node:fs/promises"; +import { routes } from "../src/app.js"; +import { + type CommandInfo, + extractAllRoutes, + type FlagInfo, + type RouteInfo, + type RouteMap, +} from "../src/lib/introspect.js"; import { formatCommandArguments, formatCommandExamples, @@ -36,32 +43,7 @@ import { } from "./markdown.js"; import { DOCS_CONTENT, DOCS_PUBLIC } from "./paths.js"; -// Bootstrap: ensure the generated skill-content module exists before -// importing the route tree (app.ts → agent-skills.ts → skill-content.ts). -// On a fresh checkout the file won't exist, causing a module resolution -// error. Write a minimal stub so the import resolves; the real content -// is generated at the end of this script. -// NOTE: This must run before the dynamic import() below because static -// imports are hoisted by the runtime before any code executes. const SKILL_CONTENT_PATH = "src/generated/skill-content.ts"; -if (!existsSync(SKILL_CONTENT_PATH)) { - mkdirSync("src/generated", { recursive: true }); - writeFileSync( - SKILL_CONTENT_PATH, - "export const SKILL_FILES: ReadonlyMap = new Map();\n", - ); -} - -const { routes } = await import("../src/app.js"); - -import type { - CommandInfo, - FlagInfo, - RouteInfo, - RouteMap, -} from "../src/lib/introspect.js"; -import { extractAllRoutes } from "../src/lib/introspect.js"; - const SKILL_DIR = "plugins/sentry-cli/skills/sentry-cli"; const OUTPUT_PATH = `${SKILL_DIR}/SKILL.md`; const PUBLIC_SKILL_DIR = `${DOCS_PUBLIC}/.well-known/skills/sentry-cli`; @@ -690,48 +672,50 @@ function generateIndexJson(generatedFiles: GeneratedFiles): string { // Main // --------------------------------------------------------------------------- -const files = await generateAllSkillFiles(routes as unknown as RouteMap); - -// The public references are a symlink to these generated plugin references. -rmSync(`${SKILL_DIR}/references`, { recursive: true, force: true }); -for (const [relativePath, content] of files) { - const fullPath = `${SKILL_DIR}/${relativePath}`; - const dir = fullPath.substring(0, fullPath.lastIndexOf("/")); - mkdirSync(dir, { recursive: true }); - await writeFile(fullPath, content); -} +/** Write the skill, its references, the public skill index, and `skill-content.ts`. */ +export async function emit(): Promise { + const files = await generateAllSkillFiles(routes as unknown as RouteMap); + + // The public references are a symlink to these generated plugin references. + rmSync(`${SKILL_DIR}/references`, { recursive: true, force: true }); + for (const [relativePath, content] of files) { + const fullPath = `${SKILL_DIR}/${relativePath}`; + const dir = fullPath.substring(0, fullPath.lastIndexOf("/")); + mkdirSync(dir, { recursive: true }); + await writeFile(fullPath, content); + } -const publicSkill = files.get("SKILL.md"); -if (publicSkill === undefined) { - throw new Error("Generated skill is missing SKILL.md"); -} -mkdirSync(PUBLIC_SKILL_DIR, { recursive: true }); -await writeFile(`${PUBLIC_SKILL_DIR}/SKILL.md`, publicSkill); - -// Write index.json -const indexJson = generateIndexJson(files); -const indexJsonDir = INDEX_JSON_PATH.substring( - 0, - INDEX_JSON_PATH.lastIndexOf("/"), -); -mkdirSync(indexJsonDir, { recursive: true }); -await writeFile(INDEX_JSON_PATH, indexJson); - -// Generate src/generated/skill-content.ts with inlined file contents. -// This embeds all skill files into the binary at build time so that -// agent-skills.ts can install them without any network fetching. -const skillEntries = [...files.entries()] - .sort(([a], [b]) => a.localeCompare(b)) - .map(([path, content]) => { - const escaped = content - .replaceAll("\\", "\\\\") - .replaceAll("`", "\\`") - .replaceAll("${", "\\${"); - return ` ["${path}", \`${escaped}\`],`; - }) - .join("\n"); - -const skillContentModule = `/** + const publicSkill = files.get("SKILL.md"); + if (publicSkill === undefined) { + throw new Error("Generated skill is missing SKILL.md"); + } + mkdirSync(PUBLIC_SKILL_DIR, { recursive: true }); + await writeFile(`${PUBLIC_SKILL_DIR}/SKILL.md`, publicSkill); + + // Write index.json + const indexJson = generateIndexJson(files); + const indexJsonDir = INDEX_JSON_PATH.substring( + 0, + INDEX_JSON_PATH.lastIndexOf("/"), + ); + mkdirSync(indexJsonDir, { recursive: true }); + await writeFile(INDEX_JSON_PATH, indexJson); + + // Generate src/generated/skill-content.ts with inlined file contents. + // This embeds all skill files into the binary at build time so that + // agent-skills.ts can install them without any network fetching. + const skillEntries = [...files.entries()] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([path, content]) => { + const escaped = content + .replaceAll("\\", "\\\\") + .replaceAll("`", "\\`") + .replaceAll("${", "\\${"); + return ` ["${path}", \`${escaped}\`],`; + }) + .join("\n"); + + const skillContentModule = `/** * Embedded skill file contents for agent skill installation. * Auto-generated by script/generate-skill.ts — do not edit manually. */ @@ -742,12 +726,13 @@ ${skillEntries} ]); `; -await writeFile(SKILL_CONTENT_PATH, skillContentModule); + await writeFile(SKILL_CONTENT_PATH, skillContentModule); -// Report what was generated -const refCount = [...files.keys()].filter((k) => - k.startsWith("references/"), -).length; -console.log( - `Generated ${OUTPUT_PATH} + ${refCount} reference files + ${INDEX_JSON_PATH} + ${SKILL_CONTENT_PATH}`, -); + // Report what was generated + const refCount = [...files.keys()].filter((k) => + k.startsWith("references/"), + ).length; + console.log( + `Generated ${OUTPUT_PATH} + ${refCount} reference files + ${INDEX_JSON_PATH} + ${SKILL_CONTENT_PATH}`, + ); +} diff --git a/packages/cli/script/generate-top-level-routes.ts b/packages/cli/script/generate-top-level-routes.ts index c2285f04f..201a1daeb 100644 --- a/packages/cli/script/generate-top-level-routes.ts +++ b/packages/cli/script/generate-top-level-routes.ts @@ -1,4 +1,3 @@ -#!/usr/bin/env tsx /** * Generate What the Top-Level Help Shows for Each Route * @@ -12,39 +11,25 @@ * The output is gitignored and regenerated by `generate:docs`, which runs as * part of `dev`, `build`, `bundle`, `typecheck`, and `test`. * - * Usage: tsx script/generate-top-level-routes.ts + * Usage: tsx script/generate.ts top-level-routes * Output: src/generated/top-level-routes.json */ import { mkdirSync } from "node:fs"; -import { access, writeFile } from "node:fs/promises"; - -// Bootstrap: skill-content stub (same pattern as -// generate-completion-targets.ts). The route tree transitively imports -// skill-content.ts, which may not exist yet on a fresh checkout. Must run -// before the dynamic import below. -const SKILL_CONTENT_PATH = "src/generated/skill-content.ts"; -const SKILL_CONTENT_STUB = - "export const SKILL_FILES: [string, string][] = [];\n"; -const skillContentExists = await access(SKILL_CONTENT_PATH).then( - () => true, - () => false, -); -if (!skillContentExists) { - mkdirSync("src/generated", { recursive: true }); - await writeFile(SKILL_CONTENT_PATH, SKILL_CONTENT_STUB); -} - -const { app, routes } = await import("../src/app.js"); -const { describeTopLevelRoutes } = await import("./lib/top-level-routes.js"); +import { writeFile } from "node:fs/promises"; +import { app, routes } from "../src/app.js"; +import { describeTopLevelRoutes } from "./lib/top-level-routes.js"; const OUTPUT_PATH = "src/generated/top-level-routes.json"; -const described = describeTopLevelRoutes(routes, app.config.documentation); +/** Write what the top-level help shows for each visible route. */ +export async function emit(): Promise { + const described = describeTopLevelRoutes(routes, app.config.documentation); -mkdirSync("src/generated", { recursive: true }); -await writeFile(OUTPUT_PATH, `${JSON.stringify(described, null, 2)}\n`); + mkdirSync("src/generated", { recursive: true }); + await writeFile(OUTPUT_PATH, `${JSON.stringify(described, null, 2)}\n`); -console.log( - `Generated ${OUTPUT_PATH} (${Object.keys(described).length} routes)`, -); + console.log( + `Generated ${OUTPUT_PATH} (${Object.keys(described).length} routes)`, + ); +} diff --git a/packages/cli/script/generate.ts b/packages/cli/script/generate.ts new file mode 100644 index 000000000..1cbb7014d --- /dev/null +++ b/packages/cli/script/generate.ts @@ -0,0 +1,244 @@ +#!/usr/bin/env tsx +/** + * Run the CLI's Code Generators in One Process + * + * Most generators read the route tree, and loading it through tsx is most of + * what each of them costs. Run as separate `pnpm tsx` processes, they loaded + * the tree once each and paid pnpm's startup every time. This runner loads it + * once and runs every requested step in two phases: + * + * 1. Steps the route tree depends on: the API schema, the grammar parsers, + * the output-schema field data that commands import, and the banner. + * They are independent of each other and run concurrently; the grammar + * compiles in a child process. Nothing in this phase may load the route + * tree. A step that lists its `files` is skipped when their content + * matches its last run. + * 2. Steps that read the route tree. The runner writes stubs for the + * generated files the tree imports but these steps produce + * (`lib/route-tree-stubs.ts`), loads the tree, then runs each step in + * the order below, whatever order they are named in. Step modules import + * the tree statically, so they are imported only here. + * + * Work a phase-2 step can do without the tree starts with the run, in its + * `prepare`: `command-api` parses `src/` in a worker thread while phase 1 + * runs and the tree loads. + * + * Like `pnpm tsx`, the runner loads `.env.local` when it exists and installs + * `require-shim.mjs` before any step. + * + * Usage: + * tsx script/generate.ts docs # what generate:docs writes + * tsx script/generate.ts docs sdk # everything, as `build` runs it + * tsx script/generate.ts skill # one step + * tsx script/generate.ts docs-sections --check # exit 1 if a section is stale + */ + +import "./require-shim.mjs"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { readdirSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { commandApiAnalysis } from "./lib/command-api-analysis.js"; +import { writeRouteTreeStubs } from "./lib/route-tree-stubs.js"; +import { + packageManifest, + runUnlessUnchanged, + type StepFiles, +} from "./lib/step-stamp.js"; + +try { + process.loadEnvFile(".env.local"); +} catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } +} + +type Options = { check: boolean }; + +type Step = { + name: string; + /** Import the step's module. Route-tree steps import the tree statically. */ + load: () => Promise<{ emit: (options: Options) => Promise }>; + /** + * Phase 1 only: what the step reads and writes, so the runner can skip it + * when neither changed (`lib/step-stamp.ts`). Evaluated before `load`, so + * a skipped step's dependencies are never imported. + */ + files?: () => StepFiles; + /** + * Phase 2 only: start work that needs no route tree, such as a worker + * thread, as the run begins, so it overlaps phase 1 and the tree load. + * `after(name)` settles when that phase-1 step does (at once if it was + * not requested). `emit` awaits the work. + */ + prepare?: (after: (name: string) => Promise) => void; +}; + +const GRAMMAR_SCRIPT = "../../scripts/generate-parsers.mjs"; + +/** Compile the grammars; a plain Node script shared with other packages. */ +async function generateGrammar(): Promise { + const child = spawn(process.execPath, [GRAMMAR_SCRIPT, "."], { + stdio: "inherit", + }); + const [code, signal] = (await once(child, "exit")) as [ + number | null, + NodeJS.Signals | null, + ]; + if (code !== 0) { + throw new Error(`${GRAMMAR_SCRIPT} exited with ${signal ?? code}`); + } +} + +/** The API schema: the SDK's operation catalog, projected. */ +function schemaFiles(): StepFiles { + return { + inputs: [ + fileURLToPath(import.meta.resolve("@sentry/api/operations")), + "script/generate-api-schema.ts", + "script/lib/api-schema.ts", + "../toolkit-core/src/api-client/idempotent-operations.ts", + "../toolkit-core/src/api-client/retry.ts", + ], + outputs: [ + "src/generated/api-schema.json", + "src/generated/idempotent-operations.json", + ], + }; +} + +/** Each `grammar/.pegjs` and its `.d.ts` become `-grammar.*`. */ +function grammarFiles(): StepFiles { + const sources = readdirSync("grammar").sort(); + const names = sources + .filter((file) => file.endsWith(".pegjs")) + .map((file) => file.slice(0, -".pegjs".length)); + return { + inputs: [ + ...sources.map((file) => `grammar/${file}`), + GRAMMAR_SCRIPT, + packageManifest("peggy", GRAMMAR_SCRIPT), + packageManifest("typescript", GRAMMAR_SCRIPT), + ], + outputs: names.flatMap((name) => [ + `src/generated/${name}-grammar.js`, + `src/generated/${name}-grammar.d.ts`, + ]), + }; +} + +function bannerFiles(): StepFiles { + const script = "script/generate-banner-image.ts"; + return { + inputs: [ + "assets/banner-mask.png", + script, + packageManifest("pngjs", script), + ], + outputs: ["src/generated/banner-image.ts"], + }; +} + +/** + * Phase 1: steps whose outputs the route tree imports. `output-schemas` + * has no `files`: it loads `src/output-schemas/` and everything that + * imports, which is no short list. + */ +const PRE_TREE_STEPS: readonly Step[] = [ + { + name: "schema", + load: () => import("./generate-api-schema.js"), + files: schemaFiles, + }, + { + name: "grammar", + load: async () => ({ emit: generateGrammar }), + files: grammarFiles, + }, + { + name: "output-schemas", + load: () => import("./generate-output-schemas.js"), + }, + { + name: "banner", + load: () => import("./generate-banner-image.js"), + files: bannerFiles, + }, +]; + +/** Phase 2: steps that read the route tree. */ +const TREE_STEPS: readonly Step[] = [ + { + name: "completions", + load: () => import("./generate-completion-targets.js"), + }, + { + name: "top-level-routes", + load: () => import("./generate-top-level-routes.js"), + }, + { name: "command-docs", load: () => import("./generate-command-docs.js") }, + { name: "docs-sections", load: () => import("./generate-docs-sections.js") }, + { name: "skill", load: () => import("./generate-skill.js") }, + { name: "sdk", load: () => import("./generate-sdk.js") }, + // Last: its emit waits for the worker `prepare` starts, and no step reads + // `command-api.json` back (only `sentry schema` does, at runtime). + { + name: "command-api", + load: () => import("./generate-command-api.js"), + prepare: (after) => { + void commandApiAnalysis(after("schema")); + }, + }, +]; + +const STEP_NAMES = [...PRE_TREE_STEPS, ...TREE_STEPS].map((step) => step.name); +/** `docs`: everything but the SDK, which `generate:sdk` writes. */ +const DOCS_STEPS = STEP_NAMES.filter((name) => name !== "sdk"); + +const args = process.argv.slice(2); +const options: Options = { check: args.includes("--check") }; +const requested = new Set(); +for (const arg of args.filter((a) => a !== "--check")) { + for (const name of arg === "docs" ? DOCS_STEPS : [arg]) { + if (!STEP_NAMES.includes(name)) { + throw new Error( + `Unknown generator step "${arg}". Steps: docs, ${STEP_NAMES.join(", ")}`, + ); + } + requested.add(name); + } +} + +const preTreeSteps = PRE_TREE_STEPS.filter((step) => requested.has(step.name)); +const treeSteps = TREE_STEPS.filter((step) => requested.has(step.name)); + +async function runPreTreeStep(step: Step): Promise { + const emit = async () => { + await (await step.load()).emit(options); + }; + if (step.files) { + await runUnlessUnchanged(step.name, step.files(), emit); + } else { + await emit(); + } +} + +const preTreeRuns = new Map( + preTreeSteps.map((step) => [step.name, runPreTreeStep(step)] as const), +); +for (const step of treeSteps) { + step.prepare?.((name) => preTreeRuns.get(name) ?? Promise.resolve()); +} +await Promise.all(preTreeRuns.values()); + +if (treeSteps.length > 0) { + await writeRouteTreeStubs(); + await import("../src/app.js"); + for (const step of treeSteps) { + await (await step.load()).emit(options); + if (process.exitCode) { + break; + } + } +} diff --git a/packages/cli/script/lib/command-api-analysis-worker.ts b/packages/cli/script/lib/command-api-analysis-worker.ts new file mode 100644 index 000000000..f59b5c770 --- /dev/null +++ b/packages/cli/script/lib/command-api-analysis-worker.ts @@ -0,0 +1,42 @@ +/** + * Worker Thread for the `command-api` Source Analysis + * + * Started by `command-api-analysis.ts`. Loads the analyzer and reads every + * source file at once, then waits for the main thread's message that the + * API schema is written, reads it, analyzes which operations each export + * reaches, and posts the result (plain objects and Maps) back. + */ + +import { readFile } from "node:fs/promises"; +import { parentPort } from "node:worker_threads"; +import { glob } from "tinyglobby"; +import type { ApiEndpoint } from "../../src/lib/api-schema.js"; +import { analyzeSourceApiUsage } from "./source-api-usage.js"; + +const files = await glob(["src/**/*.{ts,tsx}"], { + ignore: ["src/generated/**"], +}); +const sources = new Map( + await Promise.all( + files + .sort() + .map(async (file) => [file, await readFile(file, "utf8")] as const), + ), +); +await new Promise((resolve) => parentPort?.once("message", resolve)); +const endpoints = JSON.parse( + await readFile("src/generated/api-schema.json", "utf8"), +) as ApiEndpoint[]; + +parentPort?.postMessage( + analyzeSourceApiUsage( + sources, + endpoints + .filter((entry) => entry.fn) + .map((entry) => ({ + functionName: entry.fn, + method: entry.method, + path: entry.path, + })), + ), +); diff --git a/packages/cli/script/lib/command-api-analysis.ts b/packages/cli/script/lib/command-api-analysis.ts new file mode 100644 index 000000000..1abce7534 --- /dev/null +++ b/packages/cli/script/lib/command-api-analysis.ts @@ -0,0 +1,53 @@ +/** + * Start the `command-api` Source Analysis in a Worker + * + * Finding the API operations each command reaches parses all of `src/` with + * the TypeScript compiler. It is the slowest work in `generate` and needs + * none of the route tree, so `script/generate.ts` starts it in a worker + * thread (`command-api-analysis-worker.ts`) as the run begins, so it + * overlaps phase 1 and the tree load. `generate-command-api.ts` then awaits + * the result and maps it onto the tree. + * + * This module imports the analyzer only as a type, so starting the worker + * does not load the compiler into the main thread. + */ + +import { Worker } from "node:worker_threads"; +import type { analyzeSourceApiUsage } from "./source-api-usage.js"; + +export type CommandApiAnalysis = ReturnType; + +let analysis: Promise | undefined; + +/** + * The source analysis, started in a worker on the first call. The worker + * reads `src/generated/api-schema.json` once `schemaReady` resolves; later + * calls share the first one's result. + */ +export function commandApiAnalysis( + schemaReady: Promise = Promise.resolve(), +): Promise { + if (!analysis) { + analysis = new Promise((resolve, reject) => { + const worker = new Worker( + new URL("./command-api-analysis-worker.ts", import.meta.url), + ); + worker.once("message", resolve); + worker.once("error", reject); + worker.once("exit", (code) => { + reject(new Error(`command-api analysis worker exited with ${code}`)); + }); + schemaReady.then( + () => worker.postMessage("schema-ready"), + (error: unknown) => { + reject(error); + void worker.terminate(); + }, + ); + }); + // Started early and awaited later: a failure before then must not + // count as unhandled. Whoever awaits it still sees the rejection. + analysis.catch(() => undefined); + } + return analysis; +} diff --git a/packages/cli/script/lib/route-tree-stub-files.mjs b/packages/cli/script/lib/route-tree-stub-files.mjs new file mode 100644 index 000000000..f5264c714 --- /dev/null +++ b/packages/cli/script/lib/route-tree-stub-files.mjs @@ -0,0 +1,18 @@ +/** + * The Stub Contents of the Generated Files the Route Tree Imports + * + * `route-tree-stubs.ts` writes these before a script loads the tree. They + * live in plain JavaScript so `ensure-generated.mjs`, which runs without + * tsx, can tell a stub left behind by a tree-loading script, such as + * `check-fragments`, from generated output, and regenerate it. + */ + +/** @type {ReadonlyArray} */ +export const ROUTE_TREE_STUBS = [ + ["src/generated/command-api.json", "{}\n"], + ["src/generated/top-level-routes.json", "{}\n"], + [ + "src/generated/skill-content.ts", + "export const SKILL_FILES: ReadonlyMap = new Map();\n", + ], +]; diff --git a/packages/cli/script/lib/route-tree-stubs.ts b/packages/cli/script/lib/route-tree-stubs.ts new file mode 100644 index 000000000..aacbcb7ab --- /dev/null +++ b/packages/cli/script/lib/route-tree-stubs.ts @@ -0,0 +1,38 @@ +/** + * Stubs for the Generated Files the Route Tree Imports + * + * `src/app.ts` imports these files, and the generators that read the route + * tree are what write them, so a fresh checkout has none of them when the + * tree first loads. No generator reads them back through the tree, so a stub + * that resolves is enough; the generator that owns each file replaces it + * later in the same `script/generate.ts` run. + * + * Scripts that load the route tree call {@link writeRouteTreeStubs} before + * the first `import("../src/app.js")`. Static imports run before any code, + * so the tree has to be loaded with a dynamic import after this. + * + * A script that only checks the tree, such as `check-fragments`, leaves its + * stubs behind. `ensure-generated.mjs` treats a file that still holds its + * stub (`route-tree-stub-files.mjs`) as missing, so the CLI never runs on + * one. + */ + +import { mkdir, writeFile } from "node:fs/promises"; +import { ROUTE_TREE_STUBS } from "./route-tree-stub-files.mjs"; + +/** Write a stub for each route-tree input that does not exist yet. */ +export async function writeRouteTreeStubs(): Promise { + await mkdir("src/generated", { recursive: true }); + await Promise.all( + ROUTE_TREE_STUBS.map(async ([path, stub]) => { + try { + // `wx` fails with EEXIST rather than replace a generated file. + await writeFile(path, stub, { flag: "wx" }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") { + throw error; + } + } + }), + ); +} diff --git a/packages/cli/script/lib/step-stamp.ts b/packages/cli/script/lib/step-stamp.ts new file mode 100644 index 000000000..dfab48567 --- /dev/null +++ b/packages/cli/script/lib/step-stamp.ts @@ -0,0 +1,116 @@ +/** + * Skip a Generator Step When Nothing It Reads Has Changed + * + * A step that declares its inputs and outputs runs only when the content + * hash of either differs from the stamp its last run left in + * `src/generated/.stamps.json`, one entry per step. Hashes, not mtimes: a checkout or a CI cache + * restore resets mtimes without changing content. The output hash catches + * outputs that something else rewrote since, such as `generate:grammar` + * run on its own or another branch's pipeline, and missing outputs never + * match, so `git clean -fdx src/generated` regenerates everything. + * + * Inputs are listed by hand. A step only qualifies when that list is short + * and stable: its own script, the source it reads, and the packages it + * reads through (their `package.json`, which names the installed version). + */ + +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { dirname, resolve } from "node:path"; + +const STAMPS_PATH = "src/generated/.stamps.json"; + +/** The files a step reads and the files it writes. */ +export type StepFiles = { + inputs: readonly string[]; + outputs: readonly string[]; +}; + +type Stamp = { inputs: string; outputs: string }; + +/** Each step's stamp, by step name. */ +type Stamps = Partial>; + +/** The last pending write to the stamp file. */ +let stampWrites: Promise = Promise.resolve(); + +/** The contents of `path`, or `undefined` if it does not exist. */ +async function readIfExists(path: string): Promise { + try { + return await readFile(path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return; + } + throw error; + } +} + +/** Hash the paths and contents of `paths`, or `undefined` if one is missing. */ +async function digest(paths: readonly string[]): Promise { + const contents = await Promise.all(paths.map(readIfExists)); + const hash = createHash("sha256"); + for (const [index, content] of contents.entries()) { + if (content === undefined) { + return; + } + hash.update(`${paths[index]}\0`).update(content).update("\0"); + } + return hash.digest("hex"); +} + +async function readStamps(): Promise { + const content = await readIfExists(STAMPS_PATH); + try { + return content + ? (JSON.parse(new TextDecoder().decode(content)) as Stamps) + : {}; + } catch { + // A corrupt stamp file only means every stamped step runs again. + return {}; + } +} + +/** + * The `package.json` of package `name` as installed for `from`. As an + * input it stands for the package: it names the installed version. + */ +export function packageManifest(name: string, from: string): string { + return createRequire(resolve(from)).resolve(`${name}/package.json`); +} + +/** Run `step` unless its inputs and outputs match the last run's stamp. */ +export async function runUnlessUnchanged( + name: string, + files: StepFiles, + step: () => Promise, +): Promise { + const inputs = await digest(files.inputs); + const last = (await readStamps())[name]; + if ( + inputs !== undefined && + last?.inputs === inputs && + last.outputs === (await digest(files.outputs)) + ) { + console.log(`Skipped ${name}: inputs and outputs unchanged`); + return; + } + + await step(); + + const outputs = await digest(files.outputs); + const stamp = + inputs === undefined || outputs === undefined + ? undefined + : { inputs, outputs }; + // Steps run concurrently and share one file: chain the read-modify-write + // so no step's update overwrites another's. + stampWrites = stampWrites.then(async () => { + const stamps = await readStamps(); + stamps[name] = stamp; + await mkdir(dirname(STAMPS_PATH), { recursive: true }); + await writeFile(STAMPS_PATH, `${JSON.stringify(stamps, null, 2)}\n`); + }); + await stampWrites; +} diff --git a/packages/cli/test/script/step-stamp.test.ts b/packages/cli/test/script/step-stamp.test.ts new file mode 100644 index 000000000..aefffcc77 --- /dev/null +++ b/packages/cli/test/script/step-stamp.test.ts @@ -0,0 +1,80 @@ +import { mkdtempSync, rmSync, utimesSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { runUnlessUnchanged } from "../../script/lib/step-stamp.js"; + +const files = { inputs: ["input.txt"], outputs: ["output.txt"] }; + +describe("runUnlessUnchanged", () => { + const originalCwd = process.cwd(); + let dir: string; + let runs: number; + + /** A step that copies its input to its output, counting its runs. */ + async function run(): Promise { + await runUnlessUnchanged("copy", files, async () => { + runs += 1; + writeFileSync("output.txt", `copy of ${runs}`); + }); + } + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "sentry-step-stamp-")); + process.chdir(dir); + writeFileSync("input.txt", "v1"); + runs = 0; + }); + + afterEach(() => { + process.chdir(originalCwd); + rmSync(dir, { recursive: true, force: true }); + }); + + test("skips a step whose inputs and outputs are unchanged", async () => { + await run(); + await run(); + expect(runs).toBe(1); + }); + + test("ignores modification times", async () => { + await run(); + const later = new Date(Date.now() + 60_000); + utimesSync("input.txt", later, later); + utimesSync("output.txt", later, later); + await run(); + expect(runs).toBe(1); + }); + + test("reruns when an input changes", async () => { + await run(); + writeFileSync("input.txt", "v2"); + await run(); + expect(runs).toBe(2); + }); + + test("reruns when an output was rewritten or removed", async () => { + await run(); + writeFileSync("output.txt", "edited"); + await run(); + expect(runs).toBe(2); + + rmSync("output.txt"); + await run(); + expect(runs).toBe(3); + }); + + test("reruns when the stamps are cleaned", async () => { + await run(); + rmSync("src/generated", { recursive: true }); + await run(); + expect(runs).toBe(2); + }); + + test("always runs when an input is missing", async () => { + rmSync("input.txt"); + await run(); + await run(); + expect(runs).toBe(2); + }); +}); diff --git a/packages/mcp-server/package.json b/packages/mcp-server/package.json index 584cdc4a1..88e39f171 100644 --- a/packages/mcp-server/package.json +++ b/packages/mcp-server/package.json @@ -40,7 +40,7 @@ "dist/sentry-mcp.cjs" ], "scripts": { - "bundle": "pnpm --filter @sentry/mcp-core run build && pnpm --filter sentry run generate:schema && pnpm --filter sentry run generate:docs && pnpm --filter sentry run generate:sdk && tsx script/bundle.ts", + "bundle": "pnpm --filter @sentry/mcp-core run build && pnpm --filter sentry run generate && tsx script/bundle.ts", "test:bundle": "node --test test/bundle.test.mjs" }, "devDependencies": { diff --git a/scripts/ci-projects.test.mjs b/scripts/ci-projects.test.mjs index cc1396274..9be16701c 100644 --- a/scripts/ci-projects.test.mjs +++ b/scripts/ci-projects.test.mjs @@ -40,7 +40,6 @@ describe("workspace CI selection", () => { const docsScripts = JSON.parse( readFileSync(resolve(root, "apps/cli-docs/package.json"), "utf8"), ).scripts; - assert.match(docsScripts[docs.prepare], /sentry run generate:schema/); assert.match(docsScripts[docs.prepare], /sentry run generate:docs/); const cli = matrix.include.find(({ name }) => name === "sentry"); assert.equal(cli.policy, "ci:policy");