diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 462f0e44e9fef..280b9e7379a3b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -102,6 +102,7 @@ The repository uses a Go workspace with modules in `tsc/` and `tools/`. npx hereby build # Build the native compiler into built/local/tsc npx hereby test # Run compiler and language-service Go tests npx hereby test:all # Also run benchmarks, tools, and API tests +npx hereby test:codegen # Run codegen tests without regenerating source outputs npx hereby lint # Run custom golangci-lint for both Go modules npx hereby generate:go # Run the generators also exposed through go generate npx hereby generate # Regenerate all sources, assets, localization, and vendored files @@ -117,6 +118,13 @@ to bypass incremental caches. `generate` includes every generator and fetches the pinned LSP model when its local cache is missing or stale. Use `generate:go` for the narrower scope of `go generate ./...` in the `tsc` module. +`npx hereby build` runs `generate` before compiling; a separate generation step +is not needed. The API and package build tasks share this prerequisite, and +`build:watch` runs the full incremental generation suite before every rebuild. +Each watch pass clears cached input fingerprints and runs generation in-process. +Restart the watcher after editing generator implementations. Generator-owned outputs are +excluded from watch events to avoid regeneration loops. + Package-specific commands: ```bash @@ -142,19 +150,8 @@ go -C ./tsc test -run='TestLocal/' ./internal/testrunner Run: ```bash -npx hereby generate -npx hereby build -npx hereby test -npx hereby test:all -npx hereby lint -npx hereby format -npx hereby check:format -npm run -w @typescript/typescript build -npm run -w @typescript/typescript test -npm run -w native-preview build -go -C ./tsc mod tidy -diff -go -C ./tools mod tidy -diff -go work sync +npx hereby validate --all +npx hereby tidy git diff --exit-code ``` diff --git a/Herebyfile.mjs b/Herebyfile.mjs index ad1fb24d43cfa..117920769adb4 100644 --- a/Herebyfile.mjs +++ b/Herebyfile.mjs @@ -2,7 +2,7 @@ import AdmZip from "adm-zip"; import chokidar from "chokidar"; -import { task } from "hereby"; +import { task as herebyTask } from "hereby"; import assert from "node:assert"; import crypto from "node:crypto"; import fs from "node:fs"; @@ -18,8 +18,18 @@ import * as tar from "tar"; import { xSync } from "tinyexec"; import { enableFileFingerprintCache, + invalidateFileFingerprints, run, } from "./tools/scripts/gen/utils.mts"; +import { + ensureModel as ensureLSPModel, + outputs as lspModelOutputs, +} from "./tools/scripts/lsp/fetchModel.mts"; +import { outputs as encoderOutputs } from "./tools/scripts/tsc/generate-encoder.ts"; +import { outputs as enumOutputs } from "./tools/scripts/tsc/generate-enums.ts"; +import { outputs as goASTOutputs } from "./tools/scripts/tsc/generate-go-ast.ts"; +import { outputs as tsASTOutputs } from "./tools/scripts/tsc/generate-ts-ast.ts"; +import generateUnicodeData, { outputs as unicodeOutputs } from "./tsc/internal/stringutil/_scripts/generate-unicode-data.mts"; enableFileFingerprintCache(); @@ -30,6 +40,17 @@ if (process.platform === "win32") { const __filename = url.fileURLToPath(new URL(import.meta.url)); const __dirname = path.dirname(__filename); +/** @typedef {import("hereby").TaskOptions & { outputs?: readonly string[] }} TaskOptions */ +/** @type {TaskOptions[]} */ +const taskDescriptors = []; + +/** @param {TaskOptions} options */ +function task(options) { + const result = herebyTask(options); + taskDescriptors.push(options); + return result; +} + const isCI = !!process.env.CI || !!process.env.TF_BUILD; const stableThreeComponentVersionPatternSource = String.raw`^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$`; const stableThreeComponentVersionPattern = new RegExp(stableThreeComponentVersionPatternSource); @@ -219,7 +240,6 @@ async function runWithConcurrencyLimit(tasks, concurrency) { const builtLocal = "./built/local"; const libsDir = "./tsc/internal/bundled/libs"; -const libsRegexp = /(?:^|[\\/])internal[\\/]bundled[\\/]libs[\\/]/; /** * @param {string} out @@ -230,7 +250,7 @@ async function generateLibs(out) { const libs = await fs.promises.readdir(libsDir); await Promise.all(libs.map(async lib => { - fs.promises.copyFile(path.join(libsDir, lib), path.join(out, lib)); + await fs.promises.copyFile(path.join(libsDir, lib), path.join(out, lib)); })); } @@ -271,71 +291,6 @@ function buildTsc(opts) { }); } -export const tscBuild = task({ - name: "tsc:build", - description: "Builds the tsc binary.", - run: async () => { - await buildTsc({ extraFlags: options.release ? getReleaseBuildFlags() : [] }); - }, -}); - -export const tsgo = task({ - name: "tsgo", - dependencies: [lib, tscBuild], -}); - -export const local = task({ - name: "local", - dependencies: [tsgo], -}); - -export const build = task({ - name: "build", - dependencies: [local], -}); - -export const buildWatch = task({ - name: "build:watch", - description: "Builds the tsc binary and watches for changes.", - run: async () => { - await watchDebounced("build:watch", async (paths, abortSignal) => { - let libsChanged = false; - let goChanged = false; - - if (paths) { - for (const p of paths) { - if (libsRegexp.test(p)) { - libsChanged = true; - } - else if (p.endsWith(".go")) { - goChanged = true; - } - if (libsChanged && goChanged) { - break; - } - } - } - else { - libsChanged = true; - goChanged = true; - } - - if (libsChanged) { - console.log("Generating libs..."); - await generateLibs(builtLocal); - } - - if (goChanged) { - console.log("Building tsgo..."); - await buildTsc({ abortSignal }); - } - }, { - paths: ["tsc/cmd", "tsc/internal"], - ignored: path => /[\\/]testdata[\\/]/.test(path), - }); - }, -}); - export const cleanBuilt = task({ name: "clean:built", hiddenFromTaskList: true, @@ -409,8 +364,9 @@ async function runGoGenerator(name, { file, ...spec }) { /** * @param {string} name * @param {GoGenerator[] | (() => Promise)} generators + * @param {readonly string[]} [extraOutputs] */ -function goGenerateTask(name, generators) { +function goGenerateTask(name, generators, extraOutputs = []) { const run = typeof generators === "function" ? generators : async () => { for (const generator of generators) await runGoGenerator(name, generator); }; @@ -418,6 +374,10 @@ function goGenerateTask(name, generators) { return task({ name, description: `Generates ${name.slice("generate:".length)} files. Pass --force to regenerate unchanged files.`, + outputs: [ + ...(typeof generators === "function" ? [] : generators.flatMap(generator => generator.outputs.map(output => path.resolve(__dirname, generator.cwd ?? path.dirname(generator.file), output)))), + ...extraOutputs, + ], run, }); } @@ -460,11 +420,13 @@ function moqGenerator(file, type, output, { source = ".", packageName, inputs = }; } +const astStringerGenerator = stringerGenerator("tsc/internal/ast/kind_generated.go", "Kind", "kind_stringer_generated.go"); + async function runGenerateASTStringer() { - await runGoGenerator("generate:ast-stringer", stringerGenerator("tsc/internal/ast/kind_generated.go", "Kind", "kind_stringer_generated.go")); + await runGoGenerator("generate:ast-stringer", astStringerGenerator); } -export const generateASTStringer = goGenerateTask("generate:ast-stringer", runGenerateASTStringer); +export const generateASTStringer = goGenerateTask("generate:ast-stringer", [astStringerGenerator]); export const generateBundled = goGenerateTask("generate:bundled", [{ file: "tsc/internal/bundled/bundled.go", @@ -492,6 +454,19 @@ goGenerateActions.push(runGenerateCompilerOptions); export const generateCompilerOptions = task({ name: "generate:compileroptions", description: "Generates compiler options and their dependent API files. Pass --force to regenerate unchanged files.", + outputs: [ + "packages/typescript/src/api/compilerOptions.generated.ts", + "packages/typescript/src/api/prepareCompilerOptions.generated.ts", + "packages/typescript/schemas/tsconfig.schema.json", + "packages/typescript/schemas/jsconfig.schema.json", + "tsc/internal/core/options_generated.go", + "tsc/internal/core/modulekind_stringer_generated.go", + "tsc/internal/core/scripttarget_stringer_generated.go", + "tsc/internal/transpile/options_generated.go", + "tsc/internal/tsoptions/declarations_generated.go", + "tsc/internal/tsoptions/options_generated.go", + "tsc/internal/testrunner/options_generated.go", + ], run: async () => { await runGenerateCompilerOptions(); await runGenerateEnums(); @@ -516,7 +491,7 @@ export const generateDiagnostics = goGenerateTask("generate:diagnostics", [ file: "tsc/internal/diagnostics/diagnostics.go", inputs: ["generate.go", "diagnosticMessages.json", "../../../tools/LocProject.json", "../{collections,json}/*.go", "loc/*.generated.json"], exclude: ["**/*_test.go"], - outputs: ["diagnostics_generated.go", "diagnosticMessages.generated.json", "loc_generated.go", "loc/*.json.gz"], + outputs: ["diagnostics_generated.go", "loc_generated.go", "diagnosticMessages.generated.json", "loc/*.json.gz"], commands: [ [ "go", @@ -537,7 +512,7 @@ export const generateDiagnostics = goGenerateTask("generate:diagnostics", [ ], }, stringerGenerator("tsc/internal/diagnostics/diagnostics.go", "Category", "stringer_generated.go"), -]); +], ["tsc/internal/diagnostics/loc"]); export const generateAutoImport = goGenerateTask("generate:autoimport", [ stringerGenerator("tsc/internal/ls/autoimport/export.go", "ExportSyntax", "export_stringer_generated.go"), @@ -571,16 +546,18 @@ export const generateVFSMatch = goGenerateTask("generate:vfsmatch", [ ]); export const generateUnicode = goGenerateTask("generate:unicode", async () => { - const { default: generate } = await import("./tsc/internal/stringutil/_scripts/generate-unicode-data.mts"); - await generate(!!options.force); -}); + await generateUnicodeData(!!options.force); +}, unicodeOutputs.map(output => path.join("tsc/internal/stringutil", output))); + +const extensionOutputs = ["packages/vscode-typescript/l10n/bundle.l10n.json"]; +const extensionTestOutputs = ["packages/vscode-typescript/l10n/bundle.l10n.qps-ploc.json", "packages/vscode-typescript/package.nls.qps-ploc.json"]; async function runGenerateExtension() { const { default: cache } = await import("./tools/scripts/gen/cache.mts"); await cache({ cwd: __dirname, inputs: [__filename, "tools/scripts/gen/extensionLocalization.mts", "packages/vscode-typescript/package.json", "packages/vscode-typescript/src/**/*"], - outputs: ["packages/vscode-typescript/l10n/bundle.l10n.json"], + outputs: extensionOutputs, commands: [["npm", "run", "-w", "native-preview", "generateLocBundle"]], envInputs: [], force: !!options.force, @@ -590,6 +567,7 @@ async function runGenerateExtension() { export const generateExtension = task({ name: "generate:extension", description: "Generates files in the extension. Pass --force to regenerate unchanged files.", + outputs: extensionOutputs, run: runGenerateExtension, }); @@ -598,7 +576,7 @@ async function runGenerateExtensionTest() { await cache({ cwd: __dirname, inputs: [__filename, "tools/scripts/gen/extensionLocalization.mts", "packages/vscode-typescript/package.json", "packages/vscode-typescript/l10n/bundle.l10n.json", "packages/vscode-typescript/package.nls.json"], - outputs: ["packages/vscode-typescript/l10n/bundle.l10n.qps-ploc.json", "packages/vscode-typescript/package.nls.qps-ploc.json"], + outputs: extensionTestOutputs, commands: [["npm", "run", "-w", "native-preview", "generateLocTest"]], envInputs: [], force: !!options.force, @@ -609,20 +587,17 @@ export const generateExtensionTest = task({ name: "generate:extension-test", description: "Generates pseudo-localized extension resources. Pass --force to regenerate unchanged files.", dependencies: [generateExtension], + outputs: extensionTestOutputs, run: runGenerateExtensionTest, }); +const lspOutputs = ["tsc/internal/lsp/lsproto/lsp_generated.go", "packages/typescript/src/vscode/protocol.generated.ts"]; + async function runGenerateLSP() { const { GeneratedFile } = await import("./tools/scripts/gen/generatedFile.mts"); const directory = path.join(__dirname, "tools/scripts/lsp"); - const modelFiles = ["metaModel.json", "metaModelSchema.mts"].map(file => new GeneratedFile(path.join(directory, file), [path.join(directory, "fetchModel.mts"), path.join(__dirname, "package-lock.json")])); - if (!modelFiles.every(file => file.isCurrent(!!options.force))) { - for (const file of modelFiles) file.invalidate(); - const { default: fetchModel } = await import("./tools/scripts/lsp/fetchModel.mts"); - await fetchModel(); - for (const file of modelFiles) file.markCurrent(); - } - const output = new GeneratedFile(path.join(__dirname, "tsc/internal/lsp/lsproto/lsp_generated.go"), [ + const modelFiles = await ensureLSPModel(!!options.force); + const output = new GeneratedFile(path.join(__dirname, lspOutputs[0]), [ __filename, path.join(directory, "generate.mts"), ...modelFiles.map(file => file.fileName), @@ -636,7 +611,7 @@ async function runGenerateLSP() { else { console.log("LSP bindings are up to date."); } - const typeScriptOutput = new GeneratedFile(path.join(__dirname, "packages/typescript/src/vscode/protocol.generated.ts"), [ + const typeScriptOutput = new GeneratedFile(path.join(__dirname, lspOutputs[1]), [ __filename, path.join(directory, "generate.mts"), path.join(directory, "generateTypeScript.mts"), @@ -655,6 +630,7 @@ async function runGenerateLSP() { export const generateLSP = task({ name: "generate:lsp", description: "Generates Go LSP bindings and extension API types from the pinned protocol model. Pass --force to regenerate unchanged files.", + outputs: [...lspOutputs, ...lspModelOutputs.map(output => path.join("tools/scripts/lsp", output))], run: runGenerateLSP, }); @@ -666,6 +642,7 @@ async function runGenerateEnums() { export const generateEnums = task({ name: "generate:enums", description: "Generates TypeScript enums from metadata and Go source. Pass --force to regenerate unchanged files.", + outputs: enumOutputs, run: runGenerateEnums, }); @@ -678,6 +655,7 @@ async function runGenerateAST() { export const generateAST = task({ name: "generate:ast", description: "Generates AST, kind stringer, and encoder files from ast.json. Pass --force to regenerate unchanged files.", + outputs: [...goASTOutputs, ...tsASTOutputs, ...encoderOutputs], run: runGenerateAST, }); @@ -689,9 +667,16 @@ async function runGenerateSync() { export const generateSync = task({ name: "generate:sync", description: "Generates synchronous and generator APIs and tests. Pass --force to regenerate unchanged files.", + outputs: ["packages/typescript/src/api/sync", "packages/typescript/test/sync", "packages/typescript/test/generators/api.bench.ts"], run: runGenerateSync, }); +const apiOutputs = ["packages/typescript/src/api/proto.generated.ts"]; +const preferenceOutputs = [ + "tsc/internal/ls/lsutil/userpreferences_generated.go", + "packages/typescript/src/api/userPreferences.generated.ts", +]; + async function runGeneratePreferences() { await runGoGenerator("generate:preferences", { file: "tsc/internal/ls/lsutil/userpreferences.go", @@ -699,10 +684,7 @@ async function runGeneratePreferences() { inputs: ["tools/userPreferences.schema.json", "tools/gen-preferences/*.go"], exclude: ["**/*_test.go"], envInputs: [], - outputs: [ - "tsc/internal/ls/lsutil/userpreferences_generated.go", - "packages/typescript/src/api/userPreferences.generated.ts", - ], + outputs: preferenceOutputs, commands: [ ["go", "-C", "./tools", "run", "./gen-preferences", "./userPreferences.schema.json", "../tsc/internal/ls/lsutil/userpreferences_generated.go", "../packages/typescript/src/api/userPreferences.generated.ts"], ["dprint", "fmt", "tsc/internal/ls/lsutil/userpreferences_generated.go", "packages/typescript/src/api/userPreferences.generated.ts"], @@ -710,7 +692,7 @@ async function runGeneratePreferences() { }); } -export const generatePreferences = goGenerateTask("generate:preferences", runGeneratePreferences); +export const generatePreferences = goGenerateTask("generate:preferences", runGeneratePreferences, preferenceOutputs); async function runGenerateAPI() { await runGeneratePreferences(); @@ -728,7 +710,7 @@ async function runGenerateAPI() { ], exclude: ["**/*_test.go"], envInputs: [], - outputs: ["packages/typescript/src/api/proto.generated.ts"], + outputs: apiOutputs, commands: [ ["go", "-C", "./tools", "run", "./gen-proto", "../tsc/internal/api/proto.go", "../packages/typescript/src/api/proto.generated.ts"], ["dprint", "fmt", "packages/typescript/src/api/proto.generated.ts"], @@ -739,6 +721,7 @@ async function runGenerateAPI() { export const generateAPI = task({ name: "generate:api", description: "Generates API files. Pass --force to regenerate unchanged files.", + outputs: apiOutputs, run: runGenerateAPI, }); @@ -773,20 +756,108 @@ async function runGenerateVendor() { export const generateVendor = task({ name: "generate:vendor", description: "Updates the vendored copy of vscode-jsonrpc from node_modules. Pass --force to regenerate unchanged files.", + outputs: [vendorJsonrpcDir], run: runGenerateVendor, }); -const generateCompiler = task({ - name: "generate:compiler", - hiddenFromTaskList: true, - dependencies: [generateAST, generateLSP], - run: runGenerateGo, -}); +async function runGenerate() { + await runWithConcurrencyLimit([ + async () => { + await runWithConcurrencyLimit([runGenerateAST, runGenerateLSP], 2); + await runGenerateGo(); + }, + runGenerateSync, + async () => { + await runGenerateExtension(); + await runGenerateExtensionTest(); + }, + runGenerateVendor, + ], 4); +} export const generate = task({ name: "generate", description: "Runs all code generation, including AST, LSP, APIs, extension localization, and vendored dependencies.", - dependencies: [generateCompiler, generateSync, generateExtensionTest, generateVendor], + run: runGenerate, +}); + +export const tscBuild = task({ + name: "tsc:build", + description: "Generates sources and builds the tsc binary.", + dependencies: [generate], + run: async () => { + await buildTsc({ extraFlags: options.release ? getReleaseBuildFlags() : [] }); + }, +}); + +export const tsgo = task({ + name: "tsgo", + dependencies: [lib, tscBuild], +}); + +export const local = task({ + name: "local", + dependencies: [tsgo], +}); + +export const build = task({ + name: "build", + dependencies: [local], +}); + +function getGeneratedOutputPaths() { + const outputs = taskDescriptors.flatMap(descriptor => descriptor.outputs ?? []); + return new Set([...outputs, ...fs.globSync(outputs, { cwd: __dirname })].map(output => path.resolve(__dirname, output))); +} + +export const buildWatch = task({ + name: "build:watch", + description: "Watches sources and generator inputs, regenerating and rebuilding the tsc binary on changes.", + run: async () => { + const generatedOutputs = getGeneratedOutputPaths(); + const generatorDirectories = [ + "tsc/internal/stringutil/_scripts", + ].map(directory => path.resolve(directory)); + await watchDebounced("build:watch", async (_paths, abortSignal) => { + abortSignal.throwIfAborted(); + invalidateFileFingerprints(); + await runGenerate(); + abortSignal.throwIfAborted(); + await generateLibs(builtLocal); + abortSignal.throwIfAborted(); + console.log("Building tsgo..."); + await buildTsc({ abortSignal }); + }, { + paths: [ + "tsc/cmd", + "tsc/internal", + "tools/scripts/tsc/ast.json", + "tools/gen-proto", + "tools/gen-preferences", + "tools/userPreferences.schema.json", + "packages/typescript/src/api/async", + "packages/typescript/test/async", + "packages/vscode-typescript/src", + "packages/vscode-typescript/package.json", + "packages/vscode-typescript/package.nls.json", + "node_modules/vscode-jsonrpc/package.json", + "package-lock.json", + ".dprint.jsonc", + "go.work", + "go.work.sum", + "tsc/go.mod", + "tsc/go.sum", + "tools/go.mod", + "tools/go.sum", + ], + ignored: file => { + const absolutePath = path.resolve(file); + return generatedOutputs.has(absolutePath) + || generatorDirectories.some(directory => absolutePath === directory || absolutePath.startsWith(directory + path.sep)) + || path.normalize(file).split(path.sep).includes("testdata"); + }, + }); + }, }); const coverageDir = path.join(__dirname, "coverage"); @@ -1023,9 +1094,8 @@ export const testTools = task({ export const testCodegen = task({ name: "test:codegen", - description: "Runs incremental codegen tests.", + description: "Runs codegen tests without regenerating sources.", run: async () => { - await run("go", ["-C", "tsc", "mod", "download"]); await run("node", ["--test", "--test-concurrency=1", "./tools/scripts/gen/*.test.mts", "./tools/scripts/tsc/*.test.ts"]); }, }); @@ -1033,6 +1103,7 @@ export const testCodegen = task({ export const buildAPI = task({ name: "build:api", description: "Builds @typescript/typescript JS API.", + dependencies: [generate], run: async () => { await run("npm", ["run", "-w", "@typescript/typescript", "build"]); }, @@ -1049,15 +1120,14 @@ export const checkAPIDeclarations = task({ run: runCheckAPIDeclarations, }); -async function runBuildAPITests(generateSources = true) { - if (generateSources) await runGenerateSync(); +async function runBuildAPITests() { await run("npm", ["run", "-w", "@typescript/typescript", "build:test"]); } export const buildAPITests = task({ name: "build:api:test", description: "Builds the @typescript/typescript JS API tests.", - dependencies: [generateEnums, generateAPI], + dependencies: [generate], run: runBuildAPITests, }); @@ -1215,12 +1285,9 @@ async function runFormat() { export const validate = task({ name: "validate", - description: "Generates, builds, tests, lints, and formats the repo. Pass --api to include API tests, or --all to include all code generation and ancillary repository tests Benchmarks are separate: test:benchmarks and test:benchmarks:api.", - dependencies: [options.all ? generate : generateGo], + description: "Generates, builds, tests, lints, and formats the repo. Pass --api to include API tests, or --all to include API and ancillary repository tests. Benchmarks are separate: test:benchmarks and test:benchmarks:api.", + dependencies: [build], run: async () => { - await generateLibs(builtLocal); - await buildTsc({ extraFlags: options.release ? getReleaseBuildFlags() : [] }); - /** @type {{ name: string; error: unknown }[]} */ const failures = []; /** @param {string} name @param {() => Promise} action */ @@ -1238,8 +1305,7 @@ export const validate = task({ await runValidation("test:extension", runTestExtension); await runValidation("check:api-declarations", runCheckAPIDeclarations); if (options.api || options.all) { - if (!options.all) await runGenerateEnums(); - await runBuildAPITests(!options.all); + await runBuildAPITests(); await runValidation("test:api", runTestAPI); } if (options.all) { @@ -2267,6 +2333,7 @@ function stripConditionsFromValue(value) { export const buildNativePreviewPackages = task({ name: "typescript:build", description: "Builds TypeScript npm packages for the current platform. Pass --respectGoEnv to preserve caller-provided Go build settings.", + dependencies: usePublishedPlatformPackagesForVsix ? undefined : [generate], run: runBuildNativePreviewPackages, }); @@ -3129,6 +3196,7 @@ export const vscodeTypescriptRelease = task({ export const nativePreviewRelease = task({ name: "typescript:release", hiddenFromTaskList: true, + dependencies: [generate], run: async () => { if (releaseVscodeTypescript) { throw new Error("typescript:release cannot be used with --vscodeTypescriptRelease; use vscode-typescript:release"); diff --git a/tools/scripts/gen/cache.mts b/tools/scripts/gen/cache.mts index 99b9e6e3efb1f..1b74eade7bfab 100644 --- a/tools/scripts/gen/cache.mts +++ b/tools/scripts/gen/cache.mts @@ -8,6 +8,7 @@ import { goInputs, repoRoot, run, + withFileFingerprintWrites, } from "./utils.mts"; export interface CacheOptions { @@ -21,6 +22,10 @@ export interface CacheOptions { force?: boolean; } +export interface CacheHost { + run(command: string, args: readonly string[], options: { cwd: string; env: NodeJS.ProcessEnv; }): Promise; +} + function expand(patterns: string[], cwd: string, exclude: string[] = []) { return [...new Set(fs.globSync(patterns, { cwd, exclude }).map(file => path.resolve(cwd, file)))].sort(); } @@ -34,7 +39,7 @@ export default async function cache({ envInputs = ["GOOS", "GOARCH", "GOFLAGS", "GOTOOLCHAIN", "GOEXPERIMENT", "CGO_ENABLED", "GOWORK"], env, force = false, -}: CacheOptions): Promise { +}: CacheOptions, host: CacheHost = { run }): Promise { if (!outputs.length || !commands.length || commands.some(command => !command.length)) { throw new Error("Cached generation requires outputs and nonempty commands."); } @@ -91,9 +96,11 @@ export default async function cache({ return true; } for (const file of previous) file.invalidate(); - for (const [command, ...args] of commands) { - await run(command, args, { cwd, env: environment }); - } + await withFileFingerprintWrites(async () => { + for (const [command, ...args] of commands) { + await host.run(command, args, { cwd, env: environment }); + } + }); if (!complete()) throw new Error(`Generation did not produce all declared outputs: ${outputs.join(", ")}`); if (snapshotInputs(true).hash === before.hash) { for (const file of artifacts(outputFiles())) file.markCurrent(); diff --git a/tools/scripts/gen/cache.test.mts b/tools/scripts/gen/cache.test.mts index 4dd87ed158687..fa38763801129 100644 --- a/tools/scripts/gen/cache.test.mts +++ b/tools/scripts/gen/cache.test.mts @@ -1,59 +1,121 @@ import assert from "node:assert/strict"; -import * as fs from "node:fs"; +import fs from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; import os from "node:os"; import path from "node:path"; import { test } from "node:test"; -import cache from "./cache.mts"; +import cache, { type CacheHost } from "./cache.mts"; +import { GeneratedFile } from "./generatedFile.mts"; +import { enableFileFingerprintCache } from "./utils.mts"; + +test("command cache snapshots and generated files share invocation fingerprints", async context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-fingerprint-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const input = path.join(directory, "input.txt"); + fs.writeFileSync(input, "input"); + const host: CacheHost = { + async run() { + fs.writeFileSync(path.join(directory, "output.txt"), "generated"); + }, + }; + const options = { + cwd: directory, + inputs: ["input.txt"], + outputs: ["output.txt"], + commands: [["generate"]], + }; + assert.equal(await cache(options, host), false); + context.after(enableFileFingerprintCache()); + const reads = context.mock.method(fs, "readFileSync"); + syncBuiltinESMExports(); + context.after(() => { + reads.mock.restore(); + syncBuiltinESMExports(); + }); + assert.equal(await cache(options, host), true); + assert.equal(await cache(options, host), true); + new GeneratedFile(path.join(directory, "other.txt"), [input]); + assert.equal(reads.mock.calls.filter(call => call.arguments[0] === input).length, 1); +}); test("cache commands use and track an explicit environment without changing the parent", async context => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-cache-env-")); context.after(() => fs.rmSync(directory, { recursive: true, force: true })); const parentValue = process.env.GOFILE; + let calls = 0; + const host: CacheHost = { + async run(command, args, { cwd, env }) { + calls++; + assert.equal(command, "generate"); + assert.deepEqual(args, []); + fs.writeFileSync(path.join(cwd, "output.json"), JSON.stringify({ file: env.GOFILE, pkg: env.GOPACKAGE, cwd })); + }, + }; const options = { cwd: directory, inputs: [], outputs: ["output.json"], envInputs: ["GOFILE", "GOPACKAGE"], env: { GOFILE: "source.go", GOPACKAGE: "example" }, - commands: [[process.execPath, "-e", "require('node:fs').writeFileSync('output.json', JSON.stringify({ file: process.env.GOFILE, pkg: process.env.GOPACKAGE, cwd: process.cwd() }))"]], + commands: [["generate"]], }; - assert.equal(await cache(options), false); + assert.equal(await cache(options, host), false); assert.deepEqual(JSON.parse(fs.readFileSync(path.join(directory, "output.json"), "utf8")), { file: "source.go", pkg: "example", cwd: fs.realpathSync(directory), }); - assert.equal(await cache(options), true); - assert.equal(await cache({ ...options, env: { ...options.env, GOFILE: "other.go" } }), false); + assert.equal(await cache(options, host), true); + assert.equal(calls, 1); + assert.equal(await cache({ ...options, env: { ...options.env, GOFILE: "other.go" } }, host), false); + assert.equal(calls, 2); assert.equal(process.env.GOFILE, parentValue); }); test("cache commands preserve the working directory and arguments", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_cache probe-")); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-cache probe-")); context.after(() => fs.rmSync(directory, { recursive: true, force: true, maxRetries: 10 })); fs.writeFileSync(path.join(directory, "input.txt"), "input"); - fs.writeFileSync(path.join(directory, "generate.cjs"), "const fs = require('node:fs'); fs.writeFileSync('output.txt', fs.readFileSync('input.txt') + JSON.stringify(process.argv.slice(2)));\n"); - fs.writeFileSync(path.join(directory, "finish.cjs"), "require('node:fs').appendFileSync('output.txt', ' finished');\n"); + const calls: string[] = []; + const host: CacheHost = { + async run(command, args, { cwd }) { + calls.push(command); + assert.equal(cwd, directory); + if (command === "generate") { + assert.deepEqual(args, ["--help", "--force", "space value"]); + fs.writeFileSync(path.join(cwd, "output.txt"), fs.readFileSync(path.join(cwd, "input.txt")) + JSON.stringify(args)); + } + else { + assert.deepEqual(args, []); + if (command === "fail") throw new Error("Command failed"); + fs.appendFileSync(path.join(cwd, "output.txt"), " finished"); + } + }, + }; const generate = (force = false, failure = false) => cache({ cwd: directory, - inputs: ["input.txt", "*.cjs"], + inputs: ["input.txt"], outputs: ["output.txt"], commands: [ - [process.execPath, "generate.cjs", "--help", "--force", "space value"], - [process.execPath, failure ? "missing.cjs" : "finish.cjs"], + ["generate", "--help", "--force", "space value"], + [failure ? "fail" : "finish"], + ["last"], ], force, - }); + }, host); const output = path.join(directory, "output.txt"); assert.equal(await generate(), false); - assert.equal(fs.readFileSync(output, "utf8"), 'input["--help","--force","space value"] finished'); - const timestamp = fs.statSync(output).mtimeMs; + assert.equal(fs.readFileSync(output, "utf8"), 'input["--help","--force","space value"] finished finished'); + assert.deepEqual(calls, ["generate", "finish", "last"]); + calls.length = 0; assert.equal(await generate(), true); - assert.equal(fs.statSync(output).mtimeMs, timestamp); + assert.deepEqual(calls, []); assert.equal(await generate(true), false); - await assert.rejects(generate(false, true)); + assert.deepEqual(calls, ["generate", "finish", "last"]); + calls.length = 0; + await assert.rejects(generate(false, true), /Command failed/); + assert.deepEqual(calls, ["generate", "fail"]); assert.equal(await generate(), false); assert.equal(await generate(), true); }); @@ -64,26 +126,38 @@ test("cache commands track inputs, outputs, arguments, and failures", async cont const input = path.join(directory, "input.txt"); const output = path.join(directory, "output.txt"); fs.writeFileSync(input, "first"); - fs.writeFileSync(path.join(directory, "generate.cjs"), "const fs = require('node:fs'); fs.writeFileSync('output.txt', fs.readFileSync('input.txt'));\n"); - fs.writeFileSync(path.join(directory, "finish.cjs"), "const fs = require('node:fs'); fs.appendFileSync('output.txt', process.argv[2]); fs.writeFileSync('extra.txt', 'extra');\n"); + const host: CacheHost = { + async run(command, args, { cwd }) { + if (command === "generate") fs.writeFileSync(output, fs.readFileSync(input)); + else if (command === "fail") throw new Error("Command failed"); + else { + fs.appendFileSync(output, args[0]); + fs.writeFileSync(path.join(cwd, "extra.txt"), "extra"); + } + }, + }; const generate = (suffix = " done", force = false, failure = false, env: NodeJS.ProcessEnv = {}) => cache({ cwd: directory, - inputs: ["input*.txt", "*.cjs"], + inputs: ["input*.txt"], + exclude: ["input-ignored.txt"], outputs: ["output.txt", "extra*.txt"], envInputs: ["TSGO_CACHE_PROBE"], env, commands: [ - [process.execPath, "generate.cjs"], - [process.execPath, failure ? "missing.cjs" : "finish.cjs", suffix], + ["generate"], + [failure ? "fail" : "finish", suffix], ], force, - }); + }, host); await generate(); assert.equal(fs.readFileSync(output, "utf8"), "first done"); const timestamp = fs.statSync(output).mtimeMs; assert.equal(await generate(), true); assert.equal(fs.statSync(output).mtimeMs, timestamp); + fs.writeFileSync(path.join(directory, "input-ignored.txt"), "ignored"); + fs.writeFileSync(path.join(directory, "unrelated.txt"), "unrelated"); + assert.equal(await generate(), true); fs.writeFileSync(input, "second"); await generate(); assert.equal(fs.readFileSync(output, "utf8"), "second done"); @@ -113,40 +187,77 @@ test("cache commands track inputs, outputs, arguments, and failures", async cont assert.equal(await generate("--force"), true); }); +test("failed commands invalidate metadata even when outputs are unchanged", async context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-cache-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + let fail = false; + let calls = 0; + const host: CacheHost = { + async run() { + calls++; + fs.writeFileSync(path.join(directory, "output.txt"), "generated"); + if (fail) throw new Error("Command failed"); + }, + }; + const options = { cwd: directory, inputs: [], outputs: ["output.txt"], commands: [["generate"]] }; + assert.equal(await cache(options, host), false); + fail = true; + await assert.rejects(cache({ ...options, force: true }, host), /Command failed/); + fail = false; + assert.equal(await cache(options, host), false); + assert.equal(await cache(options, host), true); + assert.equal(calls, 3); +}); + test("cache commands reject missing inputs, outputs, and empty commands", async context => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-cache-")); context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + let calls = 0; + const host: CacheHost = { + async run() { + calls++; + fs.writeFileSync(path.join(directory, "output.txt"), "generated"); + }, + }; const options = { cwd: directory, inputs: ["input.txt"], outputs: ["output.txt"], - commands: [[process.execPath, "-e", "require('node:fs').writeFileSync('output.txt', 'generated')"]], + commands: [["generate"]], }; - await assert.rejects(cache(options), /No inputs matched/); + await assert.rejects(cache(options, host), /No inputs matched/); assert.equal(fs.existsSync(path.join(directory, "output.txt")), false); fs.writeFileSync(path.join(directory, "input.txt"), "input"); - await assert.rejects(cache({ ...options, inputs: ["missing*.txt"] }), /No inputs matched/); - await assert.rejects(cache({ ...options, commands: [] }), /nonempty commands/); - await assert.rejects(cache({ ...options, commands: [[]] }), /nonempty commands/); - await assert.rejects(cache({ ...options, outputs: [] }), /requires outputs/); - await assert.rejects(cache({ ...options, outputs: ["output.txt", "missing.txt"] }), /did not produce all declared outputs/); - assert.equal(await cache(options), false); - assert.equal(await cache(options), true); + await assert.rejects(cache({ ...options, inputs: ["missing*.txt"] }, host), /No inputs matched/); + await assert.rejects(cache({ ...options, commands: [] }, host), /nonempty commands/); + await assert.rejects(cache({ ...options, commands: [[]] }, host), /nonempty commands/); + await assert.rejects(cache({ ...options, outputs: [] }, host), /requires outputs/); + assert.equal(calls, 0); + await assert.rejects(cache({ ...options, outputs: ["output.txt", "missing.txt"] }, host), /did not produce all declared outputs/); + assert.equal(calls, 1); + assert.equal(await cache(options, host), false); + assert.equal(await cache(options, host), true); }); test("cache commands do not cache inputs changed during execution", async context => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-cache-")); context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - for (const change of ["fs.writeFileSync('input.txt', 'changed')", "fs.writeFileSync('input-added.txt', 'added')"]) { + for (const changedFile of ["input.txt", "input-added.txt"]) { fs.writeFileSync(path.join(directory, "input.txt"), "initial"); + const host: CacheHost = { + async run() { + fs.writeFileSync(path.join(directory, "output.txt"), "generated"); + fs.writeFileSync(path.join(directory, changedFile), "changed"); + }, + }; const options = { cwd: directory, inputs: ["input*.txt"], outputs: ["output.txt"], - commands: [[process.execPath, "-e", `const fs = require('node:fs'); fs.writeFileSync('output.txt', 'generated'); ${change};`]], + commands: [["generate"]], }; - assert.equal(await cache(options), false); - assert.equal(await cache(options), false); - assert.equal(await cache(options), true); + assert.equal(await cache(options, host), false); + assert.equal(await cache(options, host), false); + assert.equal(await cache(options, host), true); } }); diff --git a/tools/scripts/gen/generatedFile.test.mts b/tools/scripts/gen/generatedFile.test.mts index d063b8d098b22..612e327cb92f1 100644 --- a/tools/scripts/gen/generatedFile.test.mts +++ b/tools/scripts/gen/generatedFile.test.mts @@ -5,11 +5,6 @@ import { syncBuiltinESMExports } from "node:module"; import os from "node:os"; import path from "node:path"; import { test } from "node:test"; -import { stripVTControlCharacters } from "node:util"; -import { runInNewContext } from "node:vm"; -import { x } from "tinyexec"; -import ts from "typescript"; -import cache from "./cache.mts"; import { defaultCacheDirectory, GeneratedFile, @@ -18,70 +13,29 @@ import { enableFileFingerprintCache, getFileFingerprint, repoRoot, - run, + withFileFingerprintWrites, } from "./utils.mts"; -test("validate generates before building and selects the generation scope", async () => { - const fileName = path.resolve(import.meta.dirname, "../../../Herebyfile.mjs"); - const source = ts.createSourceFile(fileName, fs.readFileSync(fileName, "utf8"), ts.ScriptTarget.Latest, true, ts.ScriptKind.JS); - const declaration = source.statements.filter(ts.isVariableStatement) - .flatMap(statement => statement.declarationList.declarations) - .find(declaration => declaration.name.getText(source) === "validate"); - assert.ok(declaration?.initializer); - for (const options of [{}, { api: true }, { all: true }]) { - const calls: string[] = []; - const action = (name: string) => async () => { - calls.push(name); - }; - const generate = { run: action("generate") }; - const generateGo = { run: action("generate:go") }; - const build = { run: action("build") }; - const validation = runInNewContext(declaration.initializer.getText(source), { - task: (spec: unknown) => spec, - options, - generate, - generateGo, - build, - builtLocal: "./built/local", - generateLibs: action("lib"), - buildTsc: action("build"), - getReleaseBuildFlags: () => [], - runGenerateGo: action("generate:go"), - runGenerateEnums: action("generate:enums"), - runGenerateAPI: action("generate:api"), - runGenerateExtension: action("generate:extension"), - runGenerateVendor: action("generate:vendor"), - runBuildAPITests: async (generate = true) => { - if (generate) calls.push("generate:sync"); - calls.push("build:api:test"); - }, - runTests: action("test:tsc"), - runTestExtension: action("test:extension"), - runCheckAPIDeclarations: action("check:api-declarations"), - runTestAPI: action("test:api"), - runTestBenchmarks: action("test:benchmarks"), - runTestTools: action("test:tools"), - run: async (command: string, args: string[]) => { - assert.equal(command, "node"); - assert.deepEqual(Array.from(args), ["--test", "./tools/scripts/tsc/options.test.ts"]); - calls.push("test:options"); - }, - runSmokeTest: action("test:smoke"), - runLint: action("lint"), - runFormat: action("format"), - }) as { dependencies: { run: () => Promise; }[]; run: () => Promise; }; - await Promise.all(validation.dependencies.map(dependency => dependency.run())); - await validation.run(); - assert.equal(calls[0], "all" in options ? "generate" : "generate:go"); - assert.ok(calls.indexOf("build") > 0); - assert.ok(calls.indexOf("build") < calls.indexOf("test:tsc")); - assert.ok(calls.indexOf("check:api-declarations") > calls.indexOf("test:extension")); - assert.equal(calls.includes("test:api"), "api" in options || "all" in options); - assert.equal(calls.includes("test:tools"), "all" in options); - assert.equal(calls.includes("test:options"), "all" in options); - if ("all" in options) { - assert.deepEqual(calls.filter(name => name.startsWith("generate")), ["generate"]); - } +test("Go AST generator validates bases after schema reloads", async context => { + const { default: generateGoAST } = await import("../tsc/generate-go-ast.ts"); + const schema = await import("../tsc/schema.ts"); + context.mock.method(GeneratedFile.prototype, "isCurrent", () => true); + context.after(() => schema.reloadSchema()); + generateGoAST(); + + for ( + const { name, extendsKeys, error } of [ + { name: "duplicate bases", extendsKeys: ["NodeBase", "ExpressionBase"], error: /declares duplicate embedded bases/ }, + { name: "missing NodeBase", extendsKeys: [], error: /does not embed NodeBase at offset zero/ }, + ] + ) { + await context.test(name, context => { + schema.reloadSchema(); + const node = [...schema.api.nodes()][0]; + assert.ok(node); + context.mock.getter(node, "extendsKeys", () => extendsKeys); + assert.throws(() => generateGoAST(), error); + }); } }); @@ -170,7 +124,7 @@ test("generated files share input fingerprints and refresh changed inputs", cont assert.equal(new GeneratedFile(output, [input], cache).isCurrent(), true); }); -test("invocation fingerprints reuse metadata and invalidate writes and commands", async context => { +test("invocation fingerprints reuse metadata and invalidate writes", async context => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-fingerprint-")); context.after(() => fs.rmSync(directory, { recursive: true, force: true })); context.after(enableFileFingerprintCache()); @@ -206,21 +160,26 @@ test("invocation fingerprints reuse metadata and invalidate writes and commands" assert.equal(new GeneratedFile(output, [input], metadata).isCurrent(), false); for (const failure of [false, true]) { const before = getFileFingerprint(input); - const content = failure ? "failed command" : "successful command"; - const command = run(process.execPath, ["-e", `require('node:fs').writeFileSync(process.argv[1], ${JSON.stringify(content)}); process.exit(${failure ? 1 : 0});`, input]); - if (failure) await assert.rejects(command); - else await command; + const content = failure ? "failed write" : "successful write"; + const writing = withFileFingerprintWrites(() => { + fs.writeFileSync(input, content); + if (failure) throw new Error("Write failed"); + }); + if (failure) await assert.rejects(writing, /Write failed/); + else await writing; assert.notEqual(getFileFingerprint(input), before); assert.equal(getFileFingerprint(input), createHash("sha256").update(content).digest("hex")); } - const running = run(process.execPath, ["-e", "process.exit(0)"]); + const finished = Promise.withResolvers(); + const writing = withFileFingerprintWrites(() => finished.promise); try { const before = getFileFingerprint(input); - fs.writeFileSync(input, "changed while a command is running"); + fs.writeFileSync(input, "changed while a writer is active"); assert.notEqual(getFileFingerprint(input), before); } finally { - await running; + finished.resolve(); + await writing; } }); @@ -256,31 +215,6 @@ test("fingerprint cache scopes can overlap", context => { } }); -test("command cache snapshots and generated files share invocation fingerprints", async context => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-fingerprint-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - const input = path.join(directory, "input.txt"); - fs.writeFileSync(input, "input"); - const options = { - cwd: directory, - inputs: ["input.txt"], - outputs: ["output.txt"], - commands: [[process.execPath, "-e", "require('node:fs').writeFileSync('output.txt', 'generated')"]], - }; - assert.equal(await cache(options), false); - context.after(enableFileFingerprintCache()); - const reads = context.mock.method(fs, "readFileSync"); - syncBuiltinESMExports(); - context.after(() => { - reads.mock.restore(); - syncBuiltinESMExports(); - }); - assert.equal(await cache(options), true); - assert.equal(await cache(options), true); - new GeneratedFile(path.join(directory, "other.txt"), [input]); - assert.equal(reads.mock.calls.filter(call => call.arguments[0] === input).length, 1); -}); - test("changing inputs during generation does not mark stale output current", context => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-codegen-")); context.after(() => fs.rmSync(directory, { recursive: true, force: true })); @@ -351,578 +285,3 @@ test("generated files track each output independently", context => { assert.equal(files[0].isCurrent(), true); assert.equal(files[1].isCurrent(), false); }); - -test("generate:go runs Go generators directly and shares caches with Go fallback", async () => { - const root = path.resolve(import.meta.dirname, "../../.."); - const generate = () => x("npx", ["hereby", "generate:go"], { throwOnError: true, nodeOptions: { cwd: root } }); - const first = await generate(); - assert.doesNotMatch(first.stdout, /\$ go generate|npm run --silent cache|\$ node .*generate-unicode-data/); - const files = fs.globSync(["tsc/internal/**/*generated.go", "packages/typescript/src/api/*.generated.ts", "packages/typescript/src/enums/*.ts", "packages/typescript/schemas/*.schema.json"], { cwd: root }); - const timestamps = files.map(file => fs.statSync(path.join(root, file)).mtimeMs); - const current = await generate(); - assert.doesNotMatch(current.stdout, /Generated codegen outputs|Generated Unicode tables/); - assert.match(current.stdout, /Unicode tables are up to date/); - assert.deepEqual(files.map(file => fs.statSync(path.join(root, file)).mtimeMs), timestamps); - const fallback = await x("go", ["-C", "./tsc", "generate", "./internal/diagnostics"], { throwOnError: true, nodeOptions: { cwd: root } }); - assert.equal(fallback.stdout.match(/codegen outputs are already up to date/g)?.length, 2); - const nested = await x("npx", ["hereby", "generate:compileroptions"], { throwOnError: true, nodeOptions: { cwd: path.join(root, "tsc/internal/core") } }); - assert.equal(nested.stdout.match(/codegen outputs are already up to date/g)?.length, 4); - assert.match(nested.stdout, /Enums are up to date/); -}); - -test("generate:go generates new option diagnostics before Go-dependent generators", async () => { - const root = path.resolve(import.meta.dirname, "../../.."); - const generate = async () => { - const result = await x("npx", ["hereby", "generate:go"], { nodeOptions: { cwd: root } }); - assert.equal(result.exitCode, 0, result.stdout + result.stderr); - return result; - }; - await generate(); - const metadataFile = path.join(root, "tools/scripts/tsc/options.ts"); - const diagnosticsFile = path.join(root, "tsc/internal/diagnostics/diagnosticMessages.json"); - const protocolFile = path.join(root, "packages/typescript/src/api/proto.generated.ts"); - const originalMetadata = fs.readFileSync(metadataFile, "utf8"); - const originalDiagnostics = fs.readFileSync(diagnosticsFile, "utf8"); - const originalProtocol = fs.readFileSync(protocolFile, "utf8"); - const messages: Record = JSON.parse(originalDiagnostics); - const message = "Codegen ordering probe"; - assert.equal(messages[message], undefined); - messages[message] = { category: "Message", code: Math.max(...Object.values(messages).map(message => message.code)) + 1 }; - const metadata = originalMetadata.replace( - 'diagnostic("Print all of the files read during the compilation.")', - `diagnostic("${message}")`, - ); - assert.notEqual(metadata, originalMetadata); - try { - fs.writeFileSync(diagnosticsFile, JSON.stringify(messages, null, 4) + "\n"); - fs.writeFileSync(metadataFile, metadata); - // Exercise protocol generation too, rather than accepting its cached output. - fs.writeFileSync(protocolFile, originalProtocol + "\n"); - const { stdout } = await generate(); - assert.match(stdout, /All generated values match Go/); - assert.match(stdout, /\$ go -C \.\/tools run \.\/gen-proto/); - assert.equal(fs.readFileSync(protocolFile, "utf8"), originalProtocol); - assert.match(fs.readFileSync(path.join(root, "tsc/internal/tsoptions/declarations_generated.go"), "utf8"), /diagnostics\.Codegen_ordering_probe/); - assert.match(fs.readFileSync(path.join(root, "tsc/internal/diagnostics/diagnostics_generated.go"), "utf8"), /Codegen_ordering_probe/); - } - finally { - fs.writeFileSync(metadataFile, originalMetadata); - fs.writeFileSync(diagnosticsFile, originalDiagnostics); - await generate(); - } -}); - -test("generate includes standalone generators without Go traversal", async () => { - const root = path.resolve(import.meta.dirname, "../../.."); - const { stdout } = await x("npx", ["hereby", "generate"], { throwOnError: true, nodeOptions: { cwd: root } }); - const log = stripVTControlCharacters(stdout); - const eventIndex = (event: string) => { - const index = log.indexOf(event); - assert.ok(index >= 0, `Missing task event: ${event}`); - return index; - }; - const compilerStart = eventIndex("Starting generate:compiler"); - assert.ok(eventIndex("Finished generate:ast ") < compilerStart); - assert.ok(eventIndex("Finished generate:lsp ") < compilerStart); - assert.ok(eventIndex("Starting generate:sync") < compilerStart); - assert.ok(eventIndex("Starting generate:vendor") < compilerStart); - assert.ok(eventIndex("Finished generate:extension ") < eventIndex("Starting generate:extension-test")); - assert.doesNotMatch(stdout, /\$ go generate|npm run --silent cache/); - assert.match(stdout, /Unicode tables/); - assert.match(stdout, /Enums are up to date|All generated values match Go/); - assert.match(stdout, /Sync API is up to date|Formatting\.\.\./); - assert.match(stdout, /LSP bindings are up to date|Successfully generated .*lsp_generated\.go/); - assert.match(stdout, /generateLocBundle/); - assert.match(stdout, /generateLocTest/); - assert.deepEqual( - fs.readFileSync(path.join(root, "packages/typescript/vendor/vscode-jsonrpc/package.json")), - fs.readFileSync(path.join(root, "node_modules/vscode-jsonrpc/package.json")), - ); - const lspOutputs = ["tsc/internal/lsp/lsproto/lsp_generated.go", "packages/typescript/src/vscode/protocol.generated.ts"].map(file => path.join(root, file)); - const timestamps = lspOutputs.map(file => fs.statSync(file).mtimeMs); - const current = await x("npx", ["hereby", "generate:lsp"], { throwOnError: true, nodeOptions: { cwd: root } }); - assert.match(current.stdout, /LSP bindings are up to date/); - assert.doesNotMatch(current.stdout, /Using vscode-languageclient/); - assert.deepEqual(lspOutputs.map(file => fs.statSync(file).mtimeMs), timestamps); -}); - -test("localization and vendoring preserve current outputs", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const generate = (force = false) => x("npx", ["hereby", "generate:extension-test", "generate:vendor", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const outputs = fs.globSync([ - "packages/vscode-typescript/l10n/*.json", - "packages/vscode-typescript/package.nls.qps-ploc.json", - "packages/typescript/vendor/vscode-jsonrpc/**/*", - ], { cwd: root }).map(file => path.join(root, file)).filter(file => fs.statSync(file).isFile()); - const contents = outputs.map(file => fs.readFileSync(file)); - context.after(() => { - for (const [index, file] of outputs.entries()) fs.writeFileSync(file, contents[index]); - }); - const oldTime = new Date("2000-01-01T00:00:00Z"); - for (const file of outputs) fs.utimesSync(file, oldTime, oldTime); - const timestamps = outputs.map(file => fs.statSync(file).mtimeMs); - await generate(); - assert.deepEqual(outputs.map(file => fs.statSync(file).mtimeMs), timestamps); - await generate(true); - for (const file of outputs) assert.notEqual(fs.statSync(file).mtimeMs, oldTime.getTime()); - for (const [index, file] of outputs.entries()) { - if (index % 2) fs.rmSync(file); - else fs.writeFileSync(file, "modified"); - } - await generate(); - assert.deepEqual(outputs.map(file => fs.readFileSync(file)), contents); -}); - -test("localization tracks source membership and package strings", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const extension = path.join(root, "packages/vscode-typescript"); - const source = path.join(extension, "src/cache-probe.ts"); - const packageStrings = path.join(extension, "package.nls.json"); - const bundle = path.join(extension, "l10n/bundle.l10n.json"); - const pseudoBundle = path.join(extension, "l10n/bundle.l10n.qps-ploc.json"); - const pseudoPackage = path.join(extension, "package.nls.qps-ploc.json"); - const originals = [packageStrings, bundle, pseudoBundle, pseudoPackage].map(file => [file, fs.readFileSync(file)] as const); - assert.equal(fs.existsSync(source), false); - context.after(() => { - fs.rmSync(source, { force: true }); - for (const [file, content] of originals) fs.writeFileSync(file, content); - }); - const generate = () => x("npx", ["hereby", "generate:extension-test"], { throwOnError: true, nodeOptions: { cwd: root } }); - for (const message of ["Codegen cache probe", "Updated codegen cache probe"]) { - fs.writeFileSync(source, `import * as vscode from "vscode";\nexport const message = vscode.l10n.t(${JSON.stringify(message)});\n`); - await generate(); - assert.ok(Object.hasOwn(JSON.parse(fs.readFileSync(bundle, "utf8")), message)); - assert.ok(Object.hasOwn(JSON.parse(fs.readFileSync(pseudoBundle, "utf8")), message)); - } - fs.rmSync(source); - await generate(); - assert.deepEqual(fs.readFileSync(bundle), originals[1][1]); - assert.deepEqual(fs.readFileSync(pseudoBundle), originals[2][1]); - const strings = JSON.parse(fs.readFileSync(packageStrings, "utf8")); - strings["cache.probe"] = "Package cache probe"; - fs.writeFileSync(packageStrings, JSON.stringify(strings)); - await generate(); - assert.ok(Object.hasOwn(JSON.parse(fs.readFileSync(pseudoPackage, "utf8")), "cache.probe")); -}); - -test("vendoring tracks package versions and supports force", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const source = path.join(root, "node_modules/vscode-jsonrpc/lib/cache-probe.txt"); - const destination = path.join(root, "packages/typescript/vendor/vscode-jsonrpc/lib/cache-probe.txt"); - const sourceManifest = path.join(root, "node_modules/vscode-jsonrpc/package.json"); - const destinationManifest = path.join(root, "packages/typescript/vendor/vscode-jsonrpc/package.json"); - const originalSourceManifest = fs.readFileSync(sourceManifest); - const originalDestinationManifest = fs.readFileSync(destinationManifest); - assert.equal(fs.existsSync(source), false); - assert.equal(fs.existsSync(destination), false); - context.after(() => { - fs.rmSync(source, { force: true }); - fs.rmSync(destination, { force: true }); - fs.writeFileSync(sourceManifest, originalSourceManifest); - fs.writeFileSync(destinationManifest, originalDestinationManifest); - }); - const generate = (force = false) => x("npx", ["hereby", "generate:vendor", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - fs.writeFileSync(source, "first"); - await generate(); - assert.equal(fs.existsSync(destination), false); - const manifest = JSON.parse(originalSourceManifest.toString()); - manifest.version += "-cache-probe"; - fs.writeFileSync(sourceManifest, JSON.stringify(manifest)); - await generate(); - assert.equal(fs.readFileSync(destination, "utf8"), "first"); - assert.equal(JSON.parse(fs.readFileSync(destinationManifest, "utf8")).version, manifest.version); - fs.writeFileSync(source, "changed"); - await generate(); - assert.equal(fs.readFileSync(destination, "utf8"), "first"); - await generate(true); - assert.equal(fs.readFileSync(destination, "utf8"), "changed"); - fs.rmSync(source); - await generate(); - assert.equal(fs.readFileSync(destination, "utf8"), "changed"); - fs.rmSync(destinationManifest); - await generate(); - assert.equal(fs.existsSync(destination), false); - assert.equal(JSON.parse(fs.readFileSync(destinationManifest, "utf8")).version, manifest.version); -}); - -test("bundled generation skips unchanged library outputs", async () => { - const root = path.resolve(import.meta.dirname, "../../.."); - const generate = () => x("go", ["-C", "./tsc", "generate", "./internal/bundled"], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const files = ["libs_generated.go", "embed_generated.go"].map(file => path.join(root, "tsc/internal/bundled", file)); - const timestamps = files.map(file => fs.statSync(file).mtimeMs); - assert.match((await generate()).stdout, /codegen outputs are already up to date/); - assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); -}); - -test("bundled generation tracks libraries, validates inputs, and supports force", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_bundled-probe-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true, maxRetries: 10 })); - const notice = path.join(directory, "CopyrightNotice.txt"); - const libs = path.join(directory, "libs"); - fs.mkdirSync(libs); - fs.writeFileSync(notice, "Fixture notice\n\n"); - const library = path.join(libs, "lib.first.d.ts"); - const content = "Fixture notice\n\n\ndeclare const first: string;\n"; - fs.writeFileSync(library, content); - const options = { - cwd: directory, - inputs: [path.join(root, "tsc/internal/bundled/generate.go"), "CopyrightNotice.txt", "libs/*"], - outputs: ["libs_generated.go", "embed_generated.go"], - commands: [["go", "run", path.join(root, "tsc/internal/bundled/generate.go")]], - }; - const generate = (force = false) => cache({ ...options, force }); - await generate(); - const files = ["libs_generated.go", "embed_generated.go"].map(file => path.join(directory, file)); - const originals = files.map(file => fs.readFileSync(file)); - assert.equal(await generate(), true); - const added = path.join(libs, "lib.second.d.ts"); - fs.writeFileSync(added, content); - assert.equal(await generate(), false); - for (const file of files) assert.match(fs.readFileSync(file, "utf8"), /lib\.second\.d\.ts/); - fs.rmSync(added); - assert.equal(await generate(), false); - for (const [index, file] of files.entries()) { - assert.deepEqual(fs.readFileSync(file), originals[index]); - fs.writeFileSync(file, "modified"); - assert.equal(await generate(), false); - assert.deepEqual(fs.readFileSync(file), originals[index]); - fs.rmSync(file, { recursive: true, maxRetries: 10 }); - assert.equal(await generate(), false); - assert.deepEqual(fs.readFileSync(file), originals[index]); - } - fs.writeFileSync(library, content.replace("string", "number")); - assert.equal(await generate(), false); - fs.writeFileSync(library, content.replaceAll("\n", "\r\n")); - await assert.rejects(generate()); - fs.writeFileSync(library, content); - assert.equal(await generate(), false); - fs.mkdirSync(path.join(libs, "unexpected")); - await assert.rejects(generate()); - fs.rmdirSync(path.join(libs, "unexpected")); - assert.equal(await generate(), false); - fs.writeFileSync(notice, "Changed fixture notice\n\n"); - await assert.rejects(generate()); - fs.writeFileSync(notice, "Fixture notice\n\n"); - assert.equal(await generate(), false); - assert.equal(await generate(true), false); - assert.equal(await generate(), true); -}); - -test("stringer generation tracks declared inputs, options, output, and force", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_stringer-probe-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - const source = path.join(directory, "kind.go"); - const extra = path.join(directory, "extra.go"); - const output = path.join(directory, "kind_stringer_generated.go"); - fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); - const generate = (trimPrefix = false, force = false) => - cache({ - cwd: directory, - inputs: ["kind.go"], - outputs: ["kind_stringer_generated.go"], - env: { GOFILE: "kind.go" }, - force, - commands: [ - ["go", "tool", "golang.org/x/tools/cmd/stringer", "-type=Kind", "-output=kind_stringer_generated.go", ...(trimPrefix ? ["-trimprefix=Kind"] : [])], - ["dprint", "fmt", "kind_stringer_generated.go"], - ], - }); - await generate(); - const timestamp = fs.statSync(output).mtimeMs; - assert.equal(await generate(), true); - assert.equal(fs.statSync(output).mtimeMs, timestamp); - fs.writeFileSync(extra, "package probe\n\nconst Unrelated = 1\n"); - assert.equal(await generate(), true); - fs.appendFileSync(source, "\nconst KindSecond Kind = 1\n"); - assert.equal(await generate(), false); - assert.match(fs.readFileSync(output, "utf8"), /KindSecond/); - fs.rmSync(extra); - assert.equal(await generate(), true); - fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); - assert.equal(await generate(), false); - assert.doesNotMatch(fs.readFileSync(output, "utf8"), /KindSecond/); - assert.equal(await generate(true), false); - assert.match(fs.readFileSync(output, "utf8"), /_Kind_name = "First"/); - assert.equal(await generate(true), true); - fs.appendFileSync(output, "\n"); - assert.equal(await generate(true), false); - fs.rmSync(output); - assert.equal(await generate(true), false); - assert.equal(await generate(true, true), false); - assert.equal(await generate(true), true); - fs.appendFileSync(source, "\nconst KindInvalid Kind = missing\n"); - await assert.rejects(generate(true)); - fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); - assert.equal(await generate(true), false); - assert.equal(await generate(true), true); -}); - -test("moq generation tracks interface inputs, output, and force", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_moq-probe-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - const source = path.join(directory, "client.go"); - const base = path.join(directory, "base.go"); - fs.mkdirSync(path.join(directory, "mock")); - const output = path.join(directory, "mock/mock_generated.go"); - fs.writeFileSync(base, "package probe\n\ntype Base interface { Read() string }\n"); - fs.writeFileSync(source, "package probe\n\ntype Client interface { Base }\n"); - const options = { - cwd: directory, - inputs: ["client.go", "base.go"], - outputs: ["mock/mock_generated.go"], - commands: [ - ["go", "tool", "github.com/matryer/moq", "-stub", "-fmt=goimports", "-pkg=mock", "-out=mock/mock_generated.go", ".", "Client"], - ["dprint", "fmt", "mock/mock_generated.go"], - ], - }; - const generate = (force = false) => cache({ ...options, force }); - await generate(); - const timestamp = fs.statSync(output).mtimeMs; - assert.equal(await generate(), true); - assert.equal(fs.statSync(output).mtimeMs, timestamp); - fs.writeFileSync(base, "package probe\n\ntype Base interface { Read() string; Close() error }\n"); - assert.equal(await generate(), false); - assert.match(fs.readFileSync(output, "utf8"), /CloseFunc/); - fs.rmSync(output); - assert.equal(await generate(), false); - assert.equal(await generate(true), false); - assert.equal(await generate(), true); -}); - -test("diagnostic generation tracks Go and locale outputs and supports force", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_diagnostics-probe-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - const output = path.join(directory, "diagnostics_generated.go"); - const source = path.join(directory, "diagnosticMessages.generated.json"); - const localized = path.join(directory, "loc_generated.go"); - const locDirectory = path.join(directory, "loc"); - fs.mkdirSync(locDirectory); - const handbacks = fs.globSync("loc/*.generated.json", { cwd: path.join(root, "tsc/internal/diagnostics") }); - for (const handback of handbacks) { - fs.copyFileSync(path.join(root, "tsc/internal/diagnostics", handback), path.join(locDirectory, path.basename(handback))); - } - const project = path.join(directory, "tools/LocProject.json"); - fs.mkdirSync(path.dirname(project)); - const projectData = JSON.parse(fs.readFileSync(path.join(root, "tools/LocProject.json"), "utf8")); - projectData.Projects[0].LocItems = [projectData.Projects[0].LocItems[0]]; - projectData.Projects[0].LocItems[0].SourceFile = "diagnosticMessages.generated.json"; - fs.writeFileSync(project, JSON.stringify(projectData)); - const options = { - cwd: path.join(root, "tsc/internal/diagnostics"), - inputs: ["generate.go", "diagnosticMessages.json", project, "../{collections,json}/*.go", path.join(locDirectory, "*.generated.json")], - exclude: ["**/*_test.go"], - outputs: [output, source, localized, path.join(locDirectory, "*.json.gz")], - commands: [ - ["go", "run", "generate.go", "-diagnostics", output, "-loc", localized, "-locdir", locDirectory, "-locproject", project, "-locsource", source], - ["dprint", "fmt", output, localized], - ], - }; - const generate = (force = false) => cache({ ...options, force }); - await generate(); - const locales = fs.globSync("loc/*.json.gz", { cwd: directory }).map(file => path.join(directory, file)); - assert.ok(locales.length > 0); - const files = [output, source, localized, ...locales]; - const timestamps = files.map(file => fs.statSync(file).mtimeMs); - assert.equal(await generate(), true); - assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); - fs.appendFileSync(project, "\n"); - assert.equal(await generate(), false); - fs.appendFileSync(path.join(locDirectory, path.basename(handbacks[0])), "\n"); - assert.equal(await generate(), false); - const archive = fs.readFileSync(locales[0]); - fs.rmSync(locales[0]); - assert.equal(await generate(), false); - assert.deepEqual(fs.readFileSync(locales[0]), archive); - fs.writeFileSync(locales[0], "modified"); - assert.equal(await generate(), false); - assert.deepEqual(fs.readFileSync(locales[0]), archive); - const unexpected = path.join(directory, "loc/unexpected.json.gz"); - fs.writeFileSync(unexpected, "unexpected"); - assert.equal(await generate(), false); - assert.equal(fs.existsSync(unexpected), false); - for (const locale of locales) fs.rmSync(locale); - assert.equal(await generate(), false); - assert.deepEqual(fs.readFileSync(locales[0]), archive); - fs.rmSync(localized); - assert.equal(await generate(), false); - assert.equal(await generate(true), false); - assert.equal(await generate(), true); -}); - -test("Unicode generation skips unchanged tables and formatting", async () => { - const root = path.resolve(import.meta.dirname, "../../.."); - const generate = () => x("go", ["-C", "./tsc", "generate", "./internal/stringutil"], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const files = ["js_case_generated.go", "identifier_parts_generated.go"].map(file => path.join(root, "tsc/internal/stringutil", file)); - const timestamps = files.map(file => fs.statSync(file).mtimeMs); - const current = await generate(); - assert.match(current.stdout, /Unicode tables are up to date\./); - assert.doesNotMatch(current.stdout, /Formatted/); - assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); -}); - -test("Unicode generation repairs outputs and supports force", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_unicode-probe-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - const command = [path.join(root, "tsc/internal/stringutil/_scripts/generate-unicode-data.mts"), "--outDir", directory]; - const generate = (...args: string[]) => x(process.execPath, [...command, ...args], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const files = ["js_case_generated.go", "identifier_parts_generated.go"].map(file => path.join(directory, file)); - const originals = files.map(file => fs.readFileSync(file)); - const timestamps = files.map(file => fs.statSync(file).mtimeMs); - assert.match((await generate()).stdout, /Unicode tables are up to date\./); - assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); - for (const [index, file] of files.entries()) { - assert.deepEqual(originals[index], fs.readFileSync(path.join(root, "tsc/internal/stringutil", path.basename(file)))); - fs.writeFileSync(file, "modified"); - assert.match((await generate()).stdout, /Generated Unicode tables\./); - assert.deepEqual(fs.readFileSync(file), originals[index]); - fs.rmSync(file); - assert.match((await generate()).stdout, /Generated Unicode tables\./); - assert.deepEqual(fs.readFileSync(file), originals[index]); - } - const cacheFile = path.join(defaultCacheDirectory, createHash("sha256").update(files[0]).digest("hex") + ".json"); - context.after(() => fs.rmSync(cacheFile, { force: true })); - fs.rmSync(cacheFile); - assert.match((await generate()).stdout, /Generated Unicode tables\./); - assert.match((await generate("--force")).stdout, /Generated Unicode tables\./); - assert.match((await generate()).stdout, /Unicode tables are up to date\./); - const forced = await x(process.execPath, command, { - throwOnError: true, - nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, - }); - assert.match(forced.stdout, /Generated Unicode tables\./); - const overridden = await x(process.execPath, [...command, "--no-force"], { - throwOnError: true, - nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, - }); - assert.match(overridden.stdout, /Unicode tables are up to date\./); -}); - -test("enum generation skips unchanged outputs and Go verification", async () => { - const root = path.resolve(import.meta.dirname, "../../.."); - const generate = (force = false) => x("npx", ["hereby", "generate:enums", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const files = [ - ...fs.globSync("packages/typescript/src/enums/*.ts", { cwd: root }), - "tsc/internal/api/enum_values_generated.go", - ].map(file => path.join(root, file)); - const timestamps = files.map(file => fs.statSync(file).mtimeMs); - const { stdout } = await generate(); - assert.match(stdout, /Enums are up to date\./); - assert.doesNotMatch(stdout, /Getting values from go/); - assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); - const forced = await generate(true); - assert.match(forced.stdout, /All generated values match Go\./); - assert.match((await generate()).stdout, /Enums are up to date\./); - const verifier = path.join(root, "tsc/internal/api/enum_values_generated.go"); - const cacheFile = path.join(defaultCacheDirectory, createHash("sha256").update(verifier).digest("hex") + ".json"); - fs.rmSync(cacheFile); - const regenerated = await generate(); - assert.match(regenerated.stdout, /All generated values match Go\./); - assert.match((await generate()).stdout, /Enums are up to date\./); - - const astSchema = path.join(root, "tools/scripts/tsc/ast.json"); - const originalSchema = fs.readFileSync(astSchema, "utf8"); - const goKinds = path.join(root, "tsc/internal/ast/kind_generated.go"); - const goKindsTimestamp = fs.statSync(goKinds).mtimeMs; - try { - fs.writeFileSync(astSchema, originalSchema + "\n"); - assert.match((await generate()).stdout, /All generated values match Go\./); - assert.equal(fs.statSync(goKinds).mtimeMs, goKindsTimestamp); - assert.match((await generate()).stdout, /Enums are up to date\./); - } - finally { - fs.writeFileSync(astSchema, originalSchema); - await generate(); - } -}); - -test("AST generation forwards force to schema generators and the kind stringer", async () => { - const root = path.resolve(import.meta.dirname, "../../.."); - const generate = (force = false) => x("npx", ["hereby", "generate:ast", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const output = path.join(root, "tsc/internal/ast/kind_stringer_generated.go"); - const oldTime = new Date("2000-01-01T00:00:00Z"); - fs.utimesSync(output, oldTime, oldTime); - const forced = await generate(true); - assert.match(forced.stdout, /Wrote .*encoder_generated\.go/); - assert.match(forced.stdout, /Wrote .*ast_generated\.go/); - assert.match(forced.stdout, /Generated .*ast\.generated\.ts/); - assert.notEqual(fs.statSync(output).mtimeMs, oldTime.getTime()); - const timestamp = fs.statSync(output).mtimeMs; - assert.doesNotMatch(forced.stdout, /\$ node .*tools\/scripts\/tsc\/generate\.ts/); - const current = await generate(); - assert.doesNotMatch(current.stdout, /(?:Wrote|Generated) /); - assert.match(current.stdout, /codegen outputs are already up to date/); - assert.equal(fs.statSync(output).mtimeMs, timestamp); -}); - -test("package generation forwards force to AST, encoder, and sync generators", async () => { - const root = path.resolve(import.meta.dirname, "../../.."); - const generate = (force = false) => x("npm", ["run", "-w", "@typescript/typescript", "generate", ...(force ? ["--", "--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const forced = await generate(true); - assert.match(forced.stdout, /Generated .*ast\.generated\.ts/); - assert.match(forced.stdout, /Wrote .*encoder_generated\.go/); - assert.match(forced.stdout, /Formatting\.\.\./); - const current = await generate(); - assert.doesNotMatch(current.stdout, /(?:Wrote|Generated) |Formatting\.\.\./); - assert.match(current.stdout, /Sync API is up to date\./); -}); - -test("API protocol generation caches formatted output and supports force", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const output = path.join(root, "packages/typescript/src/api/proto.generated.ts"); - const compilerOptions = path.join(root, "packages/typescript/src/api/compilerOptions.generated.ts"); - const compilerOptionsTimestamp = fs.statSync(compilerOptions).mtimeMs; - const generate = (force = false) => x("npx", ["hereby", "generate:api", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const timestamp = fs.statSync(output).mtimeMs; - const current = await generate(); - assert.match(current.stdout, /codegen outputs are already up to date/); - assert.doesNotMatch(current.stdout, /\$ node .*cache\.mts/); - assert.equal(fs.statSync(output).mtimeMs, timestamp); - const unrelated = path.join(root, "tsc/internal/parser/codegen_cache_probe.go"); - assert.equal(fs.existsSync(unrelated), false); - context.after(() => fs.rmSync(unrelated, { force: true })); - fs.writeFileSync(unrelated, "package parser\n"); - assert.match((await generate()).stdout, /codegen outputs are already up to date/); - fs.rmSync(unrelated); - const schema = path.join(root, "tsc/internal/api/requestfilesystem/codegen_cache_probe.go"); - assert.equal(fs.existsSync(schema), false); - const original = fs.readFileSync(output, "utf8"); - context.after(async () => { - if (fs.existsSync(schema)) { - fs.rmSync(schema); - await generate(true); - } - }); - fs.writeFileSync(schema, 'package requestfilesystem\n\nconst KindCodegenCacheProbe Kind = "codegen-cache-probe"\n'); - await generate(); - assert.match(fs.readFileSync(output, "utf8"), /"codegen-cache-probe"/); - fs.rmSync(schema); - await generate(); - assert.equal(fs.readFileSync(output, "utf8"), original); - const oldTime = new Date("2000-01-01T00:00:00Z"); - fs.utimesSync(output, oldTime, oldTime); - await generate(true); - assert.notEqual(fs.statSync(output).mtimeMs, oldTime.getTime()); - assert.equal(fs.statSync(compilerOptions).mtimeMs, compilerOptionsTimestamp, "Protocol generation must not regenerate compiler option definitions"); - assert.match((await generate()).stdout, /codegen outputs are already up to date/); - fs.utimesSync(output, oldTime, oldTime); - const nested = await x("go", ["-C", "./tsc", "generate", "./internal/api"], { - throwOnError: true, - nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, - }); - assert.notEqual(fs.statSync(output).mtimeMs, oldTime.getTime()); - assert.match((await generate()).stdout, /codegen outputs are already up to date/); -}); diff --git a/tools/scripts/gen/lspTypeScript.test.mts b/tools/scripts/gen/lspTypeScript.test.mts index 888120494c75a..b0d407cb03955 100644 --- a/tools/scripts/gen/lspTypeScript.test.mts +++ b/tools/scripts/gen/lspTypeScript.test.mts @@ -4,13 +4,17 @@ import fs from "node:fs"; import path from "node:path"; import { test } from "node:test"; import ts from "typescript"; -import { getTypeScriptModel } from "../lsp/generate.mts"; -import { getMiddlewareMethods } from "../lsp/generateTypeScript.mts"; +import { ensureModel } from "../lsp/fetchModel.mts"; import type { MetaModel, Type, } from "../lsp/metaModelSchema.mts"; -import { generateTypeScript } from "../lsp/typeScript.mts"; + +await ensureModel(); + +const { getTypeScriptModel } = await import("../lsp/generate.mts"); +const { getMiddlewareMethods } = await import("../lsp/generateTypeScript.mts"); +const { generateTypeScript } = await import("../lsp/typeScript.mts"); function fixture(): MetaModel { return { @@ -131,7 +135,7 @@ test("committed TypeScript protocol types are reproducible from the shared model const outputPath = path.join(root, "packages/typescript/src/vscode/protocol.generated.ts"); const methods = getMiddlewareMethods(fs.readFileSync(path.join(root, "packages/vscode-typescript/src/lspMiddleware.ts"), "utf8")); const output = generateTypeScript(getTypeScriptModel(), methods); - const formatted = execFileSync(path.join(root, "node_modules/.bin/dprint"), ["fmt", "--stdin", outputPath], { input: output, encoding: "utf8" }); + const formatted = execFileSync(process.execPath, [path.join(root, "node_modules/dprint/bin.cjs"), "fmt", "--stdin", outputPath], { input: output, encoding: "utf8" }); assert.equal(formatted, fs.readFileSync(outputPath, "utf8")); const file = ts.createSourceFile("generated.ts", output, ts.ScriptTarget.Latest, true); assert.ok(file.statements.every(statement => ts.isInterfaceDeclaration(statement) || ts.isTypeAliasDeclaration(statement))); diff --git a/tools/scripts/gen/utils.mts b/tools/scripts/gen/utils.mts index fc8e830103d62..bae735bc518a5 100644 --- a/tools/scripts/gen/utils.mts +++ b/tools/scripts/gen/utils.mts @@ -42,6 +42,16 @@ function beginFileFingerprintWrite(): () => void { }; } +export async function withFileFingerprintWrites(action: () => T | PromiseLike): Promise { + const finishWrite = beginFileFingerprintWrite(); + try { + return await action(); + } + finally { + finishWrite(); + } +} + export function getFileFingerprint(file: string, fresh = false): string { file = path.resolve(file); fresh ||= activeFingerprintWriters > 0; @@ -76,9 +86,8 @@ export function formatCommandArg(arg: string) { export async function run(command: string, args: readonly string[] = [], options: RunOptions = {}) { console.log("$ " + [command, ...args].map(formatCommandArg).join(" ")); - const finishWrite = beginFileFingerprintWrite(); - try { - return await x(command, args, { + return withFileFingerprintWrites(() => + x(command, args, { throwOnError: true, ...(options.signal ? { signal: options.signal } : {}), nodeOptions: { @@ -86,11 +95,8 @@ export async function run(command: string, args: readonly string[] = [], options env: options.env ? { ...process.env, ...options.env } : undefined, stdio: options.captureOutput ? "pipe" : "inherit", }, - }); - } - finally { - finishWrite(); - } + }) + ); } export function globInputs(patterns: string[], exclude: string[] = []): string[] { @@ -115,13 +121,7 @@ export function parseGeneratorArgs } export async function formatFiles(files: string[]): Promise { - const finishWrite = beginFileFingerprintWrite(); - try { - await x("dprint", ["fmt", ...files], { throwOnError: true, nodeOptions: { cwd: repoRoot, stdio: "inherit" } }); - } - finally { - finishWrite(); - } + await withFileFingerprintWrites(() => x("dprint", ["fmt", ...files], { throwOnError: true, nodeOptions: { cwd: repoRoot, stdio: "inherit" } })); } export function formatFilesSync(files: string[]): void { diff --git a/tools/scripts/lsp/fetchModel.mts b/tools/scripts/lsp/fetchModel.mts index e24a717d8ba9d..25a945a535cd1 100644 --- a/tools/scripts/lsp/fetchModel.mts +++ b/tools/scripts/lsp/fetchModel.mts @@ -5,15 +5,27 @@ import fs from "node:fs"; import path from "node:path"; import url from "node:url"; +import { GeneratedFile } from "../gen/generatedFile.mts"; const __filename = url.fileURLToPath(new URL(import.meta.url)); const __dirname = path.dirname(__filename); -const metaModelPath = path.join(__dirname, "metaModel.json"); -const metaModelSchemaPath = path.join(__dirname, "metaModelSchema.mts"); +export const outputs = ["metaModel.json", "metaModelSchema.mts"] as const; +const metaModelPath = path.join(__dirname, outputs[0]); +const metaModelSchemaPath = path.join(__dirname, outputs[1]); +const lockfilePath = path.resolve(__dirname, "../../../package-lock.json"); + +export async function ensureModel(force = false): Promise { + const files = outputs.map(output => new GeneratedFile(path.join(__dirname, output), [__filename, lockfilePath])); + if (!files.every(file => file.isCurrent(force))) { + for (const file of files) file.invalidate(); + await fetchModel(); + for (const file of files) file.markCurrent(); + } + return files; +} export default async function fetchModel() { - const lockfilePath = path.resolve(__dirname, "../../../package-lock.json"); const lockfile = JSON.parse(fs.readFileSync(lockfilePath, "utf-8")); const clientVersion: string = lockfile.packages["node_modules/vscode-languageclient"].version; diff --git a/tools/scripts/tsc/generate-encoder.ts b/tools/scripts/tsc/generate-encoder.ts index 1b106bb8ff555..791bbbbe238d8 100644 --- a/tools/scripts/tsc/generate-encoder.ts +++ b/tools/scripts/tsc/generate-encoder.ts @@ -2059,35 +2059,43 @@ function writeAndFormat(filePath: string, generate: () => string, force: boolean console.log(`Wrote ${filePath}`); } +export const outputs = [ + "tsc/internal/api/encoder/encoder_generated.go", + "tsc/internal/api/encoder/decoder_generated.go", + "packages/typescript/src/api/node/protocol.generated.ts", + "packages/typescript/src/api/node/encoder.generated.ts", + "packages/typescript/src/api/node/node.generated.ts", +] as const; + export default function main(force = false) { console.log("Generating encoder/decoder code..."); writeAndFormat( - path.join(ROOT, "tsc/internal/api/encoder/encoder_generated.go"), + path.join(ROOT, outputs[0]), generateGoEncoder, force, ); writeAndFormat( - path.join(ROOT, "tsc/internal/api/encoder/decoder_generated.go"), + path.join(ROOT, outputs[1]), generateGoDecoder, force, ); writeAndFormat( - path.join(ROOT, "packages/typescript/src/api/node/protocol.generated.ts"), + path.join(ROOT, outputs[2]), generateTSProtocol, force, ); writeAndFormat( - path.join(ROOT, "packages/typescript/src/api/node/encoder.generated.ts"), + path.join(ROOT, outputs[3]), generateTSEncoder, force, ); writeAndFormat( - path.join(ROOT, "packages/typescript/src/api/node/node.generated.ts"), + path.join(ROOT, outputs[4]), generateTSNodeGenerated, force, ); diff --git a/tools/scripts/tsc/generate-enums.ts b/tools/scripts/tsc/generate-enums.ts index 757f598fa88c6..a31503f932445 100644 --- a/tools/scripts/tsc/generate-enums.ts +++ b/tools/scripts/tsc/generate-enums.ts @@ -373,6 +373,14 @@ export function generateEnum(def: EnumDef): { members: EnumMember[]; code: strin const enumValuesGeneratedGoPath = path.join(ROOT, "tsc/internal/api/enum_values_generated.go"); +export const outputs = [ + enumValuesGeneratedGoPath, + ...enumDefs.flatMap(def => { + const camelName = def.name.charAt(0).toLowerCase() + def.name.slice(1); + return [path.join(ROOT, def.outDir, `${camelName}.enum.ts`), path.join(ROOT, def.outDir, `${camelName}.ts`)]; + }), +]; + interface GeneratedEnum { def: EnumDef; code: string; diff --git a/tools/scripts/tsc/generate-go-ast.ts b/tools/scripts/tsc/generate-go-ast.ts index db4d110de8f8a..279813aaa2e63 100644 --- a/tools/scripts/tsc/generate-go-ast.ts +++ b/tools/scripts/tsc/generate-go-ast.ts @@ -198,9 +198,6 @@ function verifyNodeBaseAtOffsetZero(): void { } } -verifyNoDuplicateBases(); -verifyNodeBaseAtOffsetZero(); - // A base generated without fields or embeddings is a struct{} marker. Go pads a struct whose last field is // zero-size (so the field's address stays in bounds), so markers are embedded first, never last; being // zero-size, they don't move NodeBase off offset zero. @@ -1483,13 +1480,18 @@ function writeAndFormat(filePath: string, generateContent: () => string, force: console.log(`Wrote ${filePath}`); } +export const outputs = ["tsc/internal/ast/ast_generated.go", "tsc/internal/ast/kind_generated.go"] as const; + export default function main(force = false) { + verifyNoDuplicateBases(); + verifyNodeBaseAtOffsetZero(); + console.log("Generating Go AST code..."); - const outPath = path.join(ROOT, "tsc/internal/ast/ast_generated.go"); + const outPath = path.join(ROOT, outputs[0]); writeAndFormat(outPath, () => generate() + "\n", force); - const kindOutPath = path.join(ROOT, "tsc/internal/ast/kind_generated.go"); + const kindOutPath = path.join(ROOT, outputs[1]); writeAndFormat(kindOutPath, () => generateKind() + "\n", force); } diff --git a/tools/scripts/tsc/generate-ts-ast.ts b/tools/scripts/tsc/generate-ts-ast.ts index 4414e14148881..f17f2931d8c11 100644 --- a/tools/scripts/tsc/generate-ts-ast.ts +++ b/tools/scripts/tsc/generate-ts-ast.ts @@ -42,7 +42,7 @@ function isInheritedFromTsBase(member: MemberInfo): boolean { // NodeBase fields (e.g. Flags) are class-level properties on NodeObject, // not data members stored in _data. -const nodeBaseFieldNames = new Set( +let nodeBaseFieldNames = new Set( api.bases().find(b => b.name === "NodeBase")?.fields.filter(f => !f.noTS).map(f => f.name) ?? [], ); @@ -108,7 +108,7 @@ function syntaxKindChecksForNode(node: NodeType): string[] { // Bases for ast.generated.ts // ──────────────────────────────────────────────────────────────────────────── -const goOnlyBases = new Set( +let goOnlyBases = new Set( api.bases().filter(base => base.isGoOnly()).map(base => base.key), ); @@ -217,7 +217,7 @@ function computeTsVariants(): TsVariant[] { return result; } -const tsVariants = computeTsVariants(); +let tsVariants = computeTsVariants(); function visitorNodes(): NodeType[] { return api.nodes().filter(node => !isVariantNode(node)); @@ -1823,13 +1823,23 @@ function writeAndFormat(filePath: string, generate: () => string, force: boolean console.log(`Generated ${filePath}`); } +export const outputs = [ + "packages/typescript/src/ast/factory.generated.ts", + "packages/typescript/src/ast/is.generated.ts", + "packages/typescript/src/ast/ast.generated.ts", + "packages/typescript/src/ast/visitor.generated.ts", +] as const; + export default function main(force = false) { console.log("Generating TS AST code..."); - - const factoryPath = path.join(ROOT, "packages/typescript/src/ast/factory.generated.ts"); - const isGenPath = path.join(ROOT, "packages/typescript/src/ast/is.generated.ts"); - const astGenPath = path.join(ROOT, "packages/typescript/src/ast/ast.generated.ts"); - const visitorPath = path.join(ROOT, "packages/typescript/src/ast/visitor.generated.ts"); + nodeBaseFieldNames = new Set(api.bases().find(base => base.name === "NodeBase")?.fields.filter(field => !field.noTS).map(field => field.name) ?? []); + goOnlyBases = new Set(api.bases().filter(base => base.isGoOnly()).map(base => base.key)); + tsVariants = computeTsVariants(); + + const factoryPath = path.join(ROOT, outputs[0]); + const isGenPath = path.join(ROOT, outputs[1]); + const astGenPath = path.join(ROOT, outputs[2]); + const visitorPath = path.join(ROOT, outputs[3]); writeAndFormat(astGenPath, generateAstGenerated, force); writeAndFormat(factoryPath, generateFactory, force); diff --git a/tools/scripts/tsc/generate.ts b/tools/scripts/tsc/generate.ts index 65f2a3997980a..434db91c38ca1 100644 --- a/tools/scripts/tsc/generate.ts +++ b/tools/scripts/tsc/generate.ts @@ -3,8 +3,10 @@ import { parseGeneratorArgs } from "../gen/utils.mts"; import generateEncoder from "./generate-encoder.ts"; import generateGoAST from "./generate-go-ast.ts"; import generateTSAST from "./generate-ts-ast.ts"; +import { reloadSchema } from "./schema.ts"; export default function generate(force = false) { + reloadSchema(); generateEncoder(force); generateGoAST(force); generateTSAST(force); diff --git a/tools/scripts/tsc/schema.ts b/tools/scripts/tsc/schema.ts index 6d2985d433435..1dcaf10796f02 100644 --- a/tools/scripts/tsc/schema.ts +++ b/tools/scripts/tsc/schema.ts @@ -1261,7 +1261,16 @@ export class SchemaAPI { } } -export const api = new SchemaAPI(JSON.parse( - fs.readFileSync(path.join(ROOT, "tools/scripts/tsc/ast.json"), "utf-8"), -) as Schema); -api.validate(); +function readSchema(): SchemaAPI { + const schema = new SchemaAPI(JSON.parse( + fs.readFileSync(path.join(ROOT, "tools/scripts/tsc/ast.json"), "utf-8"), + ) as Schema); + schema.validate(); + return schema; +} + +export let api = readSchema(); + +export function reloadSchema(): void { + api = readSchema(); +} diff --git a/tsc/internal/stringutil/_scripts/generate-unicode-data.mts b/tsc/internal/stringutil/_scripts/generate-unicode-data.mts index e24c8a5d7152b..504db9bcc6ac5 100644 --- a/tsc/internal/stringutil/_scripts/generate-unicode-data.mts +++ b/tsc/internal/stringutil/_scripts/generate-unicode-data.mts @@ -231,14 +231,16 @@ ${renderRangeTable("unicodeESNextIdentifierPart", partTable)} `; } +export const outputs = ["js_case_generated.go", "identifier_parts_generated.go"] as const; + export default async function generateUnicode(force = false, outDir = path.resolve(scriptDir, "..")) { outDir = path.resolve(outDir); const inputs = [ import.meta.filename, fileURLToPath(import.meta.resolve(`${PACKAGE}/package.json`)), ]; - const caseOutput = new GeneratedFile(path.join(outDir, "js_case_generated.go"), inputs); - const identifierOutput = new GeneratedFile(path.join(outDir, "identifier_parts_generated.go"), inputs); + const caseOutput = new GeneratedFile(path.join(outDir, outputs[0]), inputs); + const identifierOutput = new GeneratedFile(path.join(outDir, outputs[1]), inputs); const generatedFiles = [caseOutput, identifierOutput]; if (generatedFiles.every(file => file.isCurrent(force))) { console.log("Unicode tables are up to date.");