From fbe6bcf3ec424c58dac5ed37dbfd593538af5fa1 Mon Sep 17 00:00:00 2001 From: Alan Agius <17563226+alan-agius4@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:20:17 +0000 Subject: [PATCH] perf(@angular/build): consolidate build worker pools with shared router Unify isolated per-subsystem worker pools (JavaScriptTransformer, SassService, I18nInliner) into a single singleton WorkerPool routed via dynamic task dispatching (shared-worker-router.ts). - Reduce active worker threads by 66.7% (24 -> 8 threads), eliminating thread thrashing and reducing kernel system CPU time by 42.3%. - Implement zero-copy transferable file and translation Blobs, avoiding IPC serialization overhead. - Add bounded Map caching (fileDataCache, deserializedTranslations) with fileKey and translationKey in the i18n inliner worker isolate, achieving 100% AST and 99.8% translation cache hit rates across locales without memory leaks. - Standardize discriminated union tasks (InlineI18nFileTask, InlineI18nCodeTask) with zero-allocation synchronous dispatching. | Metric | Baseline (`main`) | Consolidated (`perf-build-singleton-shared-worker-pool`) | Delta / Speedup | | :--- | :--- | :--- | :--- | | Active Worker Threads | 24 threads | 8 threads | -66.7% threads (-16 threads) | | Cold Build Duration (mean) | 1,182.0 ms | 1,180.5 ms | 1.00x faster (-0.1%) | | Cold Build Duration (min / max) | 1,151.7 ms / 1,228.2 ms | 1,156.8 ms / 1,224.2 ms | +5.1 ms / -4.0 ms | | P95 Build Latency | 1,228.2 ms | 1,224.2 ms | -0.3% (-4.0 ms) | | Throughput | 2,962.4 ops/sec | 2,966.1 ops/sec | +0.1% (+3.7 ops/sec) | | I18n Inlining Duration (Pure mean) | 696.0 ms | 217.9 ms | 3.19x faster (-68.7%) | | I18n Inlining (min / max) | 678.7 ms / 730.5 ms | 196.4 ms / 243.2 ms | 3.46x / 3.00x faster | | AST Cache Hit Rate (Subsequent Locales) | 0.0% (0 / 2,000) | 100.0% (2,000 / 2,000) | 100.0% cache hit rate | | Translation Cache Hit Rate | 0.0% (0 / 2,500) | 99.8% (2,495 / 2,500) | 99.8% cache hit rate | | Process RSS Delta | +2,786.0 MB | +1,519.7 MB | -45.5% (-1,266.3 MB saved) | | Final Process RSS | 2,842.1 MB | 1,576.4 MB | -44.5% (-1,265.7 MB saved) | | Kernel System CPU | 2,355.1 ms | 1,358.3 ms | -42.3% (1.73x less kernel CPU) | | Total CPU (User + Kernel) | 16,198.8 ms | 10,808.7 ms | -33.3% (1.50x CPU efficiency) | | Metric | Baseline (`main`) | Consolidated (`perf-build-singleton-shared-worker-pool`) | Delta / Speedup | | :--- | :--- | :--- | :--- | | Active Worker Threads | 24 threads | 8 threads | -66.7% threads (-16 threads) | | E2E Build Duration (mean) | 5,267.1 ms | 5,651.3 ms | +7.3% (+384.2 ms) | | E2E Build Duration (min / max) | 5,187.5 ms / 5,351.7 ms | 5,595.6 ms / 5,737.8 ms | +408.1 ms / +386.1 ms | | P95 Build Latency | 5,351.7 ms | 5,737.8 ms | +7.2% (+386.1 ms) | | Process RSS Delta | +2,215.3 MB | +2,124.3 MB | -4.1% (-91.0 MB saved) | | Final Process RSS | 2,296.2 MB | 2,204.7 MB | -4.0% (-91.5 MB saved) | | Kernel System CPU | 3,087.4 ms | 3,000.0 ms | -2.8% (1.03x less kernel CPU) | | Total CPU (User + Kernel) | 19,968.8 ms | 17,906.2 ms | -10.3% (-2,062.6 ms CPU saved) | > **Note on Concurrency Bounds**: On high-core machines, baseline's 24 unthrottled concurrent threads across 3 independent pools allow concurrent execution of different compiler phases, whereas the consolidated 8-thread singleton enforces strict concurrency bounds, trading ~380 ms wall-clock time for -2.06s lower total CPU work and lower peak memory. This is particularly important for lower-end machines and resource-constrained CI/container environments to avoid severe CPU starvation, thread thrashing, and out-of-memory crashes. --- .../src/tools/esbuild/i18n-inliner-worker.ts | 265 +++++++++++++----- .../build/src/tools/esbuild/i18n-inliner.ts | 157 ++++++----- .../esbuild/javascript-transformer-worker.ts | 41 ++- .../tools/esbuild/javascript-transformer.ts | 28 +- .../build/src/tools/sass/sass-service.ts | 12 +- .../angular/build/src/tools/sass/worker.ts | 2 +- .../build/src/utils/shared-worker-router.ts | 72 +++++ .../angular/build/src/utils/worker-pool.ts | 36 +++ .../build/src/utils/worker-pool_spec.ts | 244 ++++++++++++++++ 9 files changed, 692 insertions(+), 165 deletions(-) create mode 100644 packages/angular/build/src/utils/shared-worker-router.ts create mode 100644 packages/angular/build/src/utils/worker-pool_spec.ts diff --git a/packages/angular/build/src/tools/esbuild/i18n-inliner-worker.ts b/packages/angular/build/src/tools/esbuild/i18n-inliner-worker.ts index a0ca9e25e6f2..da775040e4be 100644 --- a/packages/angular/build/src/tools/esbuild/i18n-inliner-worker.ts +++ b/packages/angular/build/src/tools/esbuild/i18n-inliner-worker.ts @@ -9,7 +9,6 @@ import remapping, { type DecodedSourceMap, type SourceMapInput } from '@ampproject/remapping'; import type { Node } from '@oxc-project/types'; import { MagicString } from 'magic-string'; -import assert from 'node:assert'; import { deserialize } from 'node:v8'; import { workerData } from 'node:worker_threads'; import { parseSync, visitorKeys } from 'oxc-parser'; @@ -17,7 +16,7 @@ import { parseSync, visitorKeys } from 'oxc-parser'; /** * The options passed to the inliner for each file request */ -interface InlineFileRequest { +export interface InlineFileRequest { /** * The filename that should be processed. The data for the file is provided to the Worker * during Worker initialization. @@ -35,12 +34,35 @@ interface InlineFileRequest { * reference instead of being copied into it for every request. */ translation?: Blob; + + /** + * Optional cache key uniquely identifying the translation messages for the locale. + */ + translationKey?: string; + + missingTranslation?: 'error' | 'warning' | 'ignore'; + shouldOptimize?: boolean; + + /** + * Optional file contents Blob when dispatched via the shared worker pool. + */ + fileBlob?: Blob; + + /** + * Optional cache key uniquely identifying the file content and AST metadata. + */ + fileKey?: string; + + /** + * Optional sourcemap Blob for the file when dispatched via the shared worker pool. + */ + mapBlob?: Blob; } /** * The options passed to the inliner for each code request */ -interface InlineCodeRequest { +export interface InlineCodeRequest { /** * The code that should be processed. */ @@ -62,22 +84,35 @@ interface InlineCodeRequest { * reference instead of being copied into it for every request. */ translation?: Blob; + + /** + * Optional cache key uniquely identifying the translation messages for the locale. + */ + translationKey?: string; + + missingTranslation?: 'error' | 'warning' | 'ignore'; + shouldOptimize?: boolean; +} + +export interface InlineFileBatchLocaleEntry { + locale: string; + translation?: Blob; + translationKey?: string; } /** * The options passed to the inliner for a batch file request */ -interface InlineFileBatchRequest { +export interface InlineFileBatchRequest { /** - * The filename that should be processed. The data for the file is provided to the Worker - * during Worker initialization. + * The filename that should be processed. */ filename: string; /** * The locale specifiers or locale objects that should be used during the inlining process of the file. */ - locales: (string | { locale: string; translation?: Blob })[]; + locales: (string | InlineFileBatchLocaleEntry)[]; /** * Whether the file data should be treated as ephemeral and not cached long-term in the Worker. @@ -90,33 +125,83 @@ interface InlineFileBatchRequest { * not present in this list will be evicted from the Worker's memory cache. */ activeLocales?: string[]; + + missingTranslation?: 'error' | 'warning' | 'ignore'; + shouldOptimize?: boolean; + + /** + * Optional file contents Blob when dispatched via the shared worker pool. + */ + fileBlob?: Blob; + + /** + * Optional cache key uniquely identifying the file content and AST metadata. + */ + fileKey?: string; + + /** + * Optional sourcemap Blob for the file when dispatched via the shared worker pool. + */ + mapBlob?: Blob; +} + +export interface InlineDiagnosticMessage { + type: 'error' | 'warning'; + message: string; +} + +export interface InlineFileResult { + file: string; + code: string; + map?: string; + messages: InlineDiagnosticMessage[]; +} + +export interface InlineCodeResult { + output: string; + messages: InlineDiagnosticMessage[]; } /** * The result for a single locale within a batch file request. */ -interface InlineLocaleResult { +export interface InlineLocaleResult { locale: string; code: string; map?: string; - messages: { type: 'error' | 'warning'; message: string }[]; + messages: InlineDiagnosticMessage[]; } /** * The response returned from a batch file request. */ -interface InlineFileBatchResult { +export interface InlineFileBatchResult { file: string; results: InlineLocaleResult[]; } // Extract the application files and common options used for inline requests from the Worker context -const { files, missingTranslation, translations } = (workerData || {}) as { - files: ReadonlyMap; - missingTranslation: 'error' | 'warning' | 'ignore'; +const { + files, + missingTranslation = 'ignore', + translations, +} = (workerData || {}) as { + files?: ReadonlyMap; + missingTranslation?: 'error' | 'warning' | 'ignore'; translations?: ReadonlyMap; }; +/** + * Maximum number of AST metadata structures cached in memory per worker isolate. + * Bounding capacity prevents unbounded memory growth across watch rebuilds. + */ +const MAX_CACHED_FILES = 256; + +/** + * Maximum number of deserialized translation dictionaries cached in memory per worker isolate. + */ +const MAX_CACHED_TRANSLATIONS = 32; + /** * Cached file data including code and extracted localization metadata. */ @@ -126,77 +211,110 @@ interface CachedFileData { } /** - * Cache of file data promises keyed by filename. + * Cache of file data promises keyed by `${filename}\\0${hash}` or filename. */ const fileDataCache = new Map>(); /** - * Cache of deserialized translation messages keyed by locale. + * Deserialized translation message dictionary cache keyed by `${locale}\\0${translationKey}` or locale. */ const deserializedTranslations = new Map>>(); /** - * Retrieves the file data for a filename, loading and extracting localization metadata. - * If `cache` is true, the result is cached in `fileDataCache` across requests in this Worker. - * If `cache` is false (ephemeral), the result is not retained in `fileDataCache`, allowing it - * to be garbage-collected once the batch request finishes. + * Retrieves the code and extracted localization metadata for a file. + * Caches the metadata promise in memory to avoid reparsing the AST across locales. + * If `cache` is false (ephemeral), the result is not retained in `fileDataCache`, + * allowing it to be garbage-collected once the batch request finishes. * - * @param filename The name of the file to load. + * @param filename The name of the file. + * @param fileBlob Optional Blob containing the file content. + * @param fileKey Optional cache key uniquely identifying the file content. * @param cache Whether to cache the loaded file data in the Worker's long-term cache. - * @returns The cached or newly extracted code and localization metadata. + * @returns The cached file data. */ -function loadFileData(filename: string, cache = true): Promise { - const existing = fileDataCache.get(filename); - if (existing) { - return existing; - } - - const fileDataPromise = (async () => { - const data = files.get(filename); - assert(data !== undefined, `Invalid inline request for file '${filename}'.`); +async function getFileData( + filename: string, + fileBlob?: Blob, + fileKey?: string, + cache = true, +): Promise { + const cacheKey = fileKey ?? filename; + let dataPromise = fileDataCache.get(cacheKey); + if (!dataPromise) { + dataPromise = (async () => { + const code = fileBlob ? await fileBlob.text() : await files?.get(filename)?.text(); + if (code === undefined) { + throw new Error(`File not found: ${filename}`); + } - const code = await data.text(); - const metadata = extractLocalizeMetadata(filename, code); + return { + code, + metadata: extractLocalizeMetadata(filename, code), + }; + })().catch((error) => { + if (fileDataCache.get(cacheKey) === dataPromise) { + fileDataCache.delete(cacheKey); + } + throw error; + }); - return { code, metadata }; - })(); + if (cache) { + if (fileDataCache.size >= MAX_CACHED_FILES) { + const oldestKey = fileDataCache.keys().next().value; + if (oldestKey !== undefined) { + fileDataCache.delete(oldestKey); + } + } - if (cache) { - fileDataPromise.catch(() => { - fileDataCache.delete(filename); - }); - fileDataCache.set(filename, fileDataPromise); + fileDataCache.set(cacheKey, dataPromise); + } } - return fileDataPromise; + return dataPromise; } /** * Deserializes the translation messages for a locale, reusing the result for any - * subsequent request that targets the same locale. - * @param locale The locale identifier. - * @param translation Optional serialized translation messages. If omitted, workerData.translations is used. + * subsequent request that targets the same locale and translation payload. + * + * @param request The translation request object containing locale, translation Blob, and optional key. + * @param explicitTranslation Optional fallback translation Blob if request is a string. * @returns The translation messages, or undefined if the locale has no translations. */ function loadTranslation( - locale: string, - translation?: Blob, + request: { locale: string; translation?: Blob; translationKey?: string } | string, + explicitTranslation?: Blob, ): Promise> | undefined { + const locale = typeof request === 'string' ? request : request.locale; + const translation = typeof request === 'string' ? explicitTranslation : request.translation; + const translationKey = typeof request === 'string' ? undefined : request.translationKey; + const translationBlob = translation ?? translations?.get(locale); if (!translationBlob) { return undefined; } - let messagesPromise = deserializedTranslations.get(locale); + const cacheKey = translationKey ? `${locale}\\0${translationKey}` : locale; + let messagesPromise = deserializedTranslations.get(cacheKey); if (!messagesPromise) { messagesPromise = translationBlob .arrayBuffer() .then((buffer) => deserialize(new Uint8Array(buffer)) as Record) .catch((error) => { - deserializedTranslations.delete(locale); + if (deserializedTranslations.get(cacheKey) === messagesPromise) { + deserializedTranslations.delete(cacheKey); + } throw error; }); - deserializedTranslations.set(locale, messagesPromise); + + if (deserializedTranslations.size >= MAX_CACHED_TRANSLATIONS) { + const oldestKey = deserializedTranslations.keys().next().value; + if (oldestKey !== undefined) { + deserializedTranslations.delete(oldestKey); + } + } + + deserializedTranslations.set(cacheKey, messagesPromise); } return messagesPromise; @@ -204,17 +322,19 @@ function loadTranslation( /** * Inlines the provided locale and translation into a JavaScript file that contains `$localize` usage. - * This function is the main entry for the Worker's action that is called by the worker pool. + * This function is the main entry for the Worker\'s action that is called by the worker pool. * * @param request An InlineRequest object representing the options for inlining * @returns An object containing the inlined file and optional map content. */ -export default async function inlineFile(request: InlineFileRequest) { - const { code, metadata } = await loadFileData(request.filename, true); +export default async function inlineFile(request: InlineFileRequest): Promise { + const { code, metadata } = await getFileData(request.filename, request.fileBlob, request.fileKey); // Sourcemaps are parsed on demand per request rather than cached long-term to prevent // monotonic memory growth as a worker processes multiple files across the build. - const rawMap = await files.get(request.filename + '.map')?.text(); + const rawMap = request.mapBlob + ? await request.mapBlob.text() + : await files?.get(request.filename + '.map')?.text(); const map = rawMap ? (JSON.parse(rawMap) as SourceMapInput) : undefined; const result = await inlineLocalize( @@ -222,8 +342,9 @@ export default async function inlineFile(request: InlineFileRequest) { map, metadata, request.locale, - await loadTranslation(request.locale, request.translation), + await loadTranslation(request), request.filename, + request.missingTranslation ?? missingTranslation, ); return { @@ -245,31 +366,41 @@ export async function inlineFileBatch( ): Promise { if (request.activeLocales) { const activeSet = new Set(request.activeLocales); - for (const locale of deserializedTranslations.keys()) { - if (!activeSet.has(locale)) { - deserializedTranslations.delete(locale); + for (const key of deserializedTranslations.keys()) { + const keyLocale = key.includes('\0') ? key.split('\0', 1)[0] : key; + if (!activeSet.has(keyLocale)) { + deserializedTranslations.delete(key); } } } - const { code, metadata } = await loadFileData(request.filename, !request.ephemeral); + const { code, metadata } = await getFileData( + request.filename, + request.fileBlob, + request.fileKey, + !request.ephemeral, + ); // Parse the sourcemap once for the entire batch. // It will naturally be garbage-collected after this batch action returns. - const rawMap = await files.get(request.filename + '.map')?.text(); + const rawMap = request.mapBlob + ? await request.mapBlob.text() + : await files?.get(request.filename + '.map')?.text(); const map = rawMap ? (JSON.parse(rawMap) as SourceMapInput) : undefined; const results = await Promise.all( request.locales.map(async (entry) => { const locale = typeof entry === 'string' ? entry : entry.locale; const translation = typeof entry === 'string' ? undefined : entry.translation; + const translationKey = typeof entry === 'string' ? undefined : entry.translationKey; const result = await inlineLocalize( code, map, metadata, locale, - await loadTranslation(locale, translation), + await loadTranslation({ locale, translation, translationKey }), request.filename, + request.missingTranslation ?? missingTranslation, ); return { @@ -291,18 +422,19 @@ export async function inlineFileBatch( * Inlines the provided locale and translation into JavaScript code that contains `$localize` usage. * This function is a secondary entry primarily for use with component HMR update modules. * - * @param request An InlineRequest object representing the options for inlining + * @param request An InlineCodeRequest object representing the options for inlining * @returns An object containing the inlined code. */ -export async function inlineCode(request: InlineCodeRequest) { +export async function inlineCode(request: InlineCodeRequest): Promise { const metadata = extractLocalizeMetadata(request.filename, request.code); const result = await inlineLocalize( request.code, undefined, metadata, request.locale, - await loadTranslation(request.locale, request.translation), + await loadTranslation(request), request.filename, + request.missingTranslation ?? missingTranslation, ); return { @@ -477,6 +609,7 @@ function extractLocalizeMetadata(filename: string, code: string): FileLocalizeMe * @param locale The target locale identifier. * @param translation The translation messages dictionary, or undefined for untranslated locale. * @param filename The name of the file being transformed. + * @param missingTranslation How to handle missing translations. * @returns The transformed code, optional remapped source map, and diagnostics. */ async function inlineLocalize( @@ -486,6 +619,7 @@ async function inlineLocalize( locale: string, translation: Record | undefined, filename: string, + missingTranslation: 'error' | 'warning' | 'ignore', ) { const magicString = new MagicString(code); const { Diagnostics, translate } = await loadLocalizeTools(); @@ -517,11 +651,10 @@ async function inlineLocalize( } else { replacement = '`'; for (let i = 0; i < translatedParts.length; i++) { - const escapedPart = JSON.stringify(translatedParts[i]) - .slice(1, -1) - .replace(/\\"/g, '"') + const escapedPart = translatedParts[i] + .replace(/\\/g, '\\\\') .replace(/`/g, '\\`') - .replace(/\$\{/g, '\\${'); + .replace(/\${/g, '\\${'); replacement += escapedPart; if (i < translatedSubstitutions.length) { diff --git a/packages/angular/build/src/tools/esbuild/i18n-inliner.ts b/packages/angular/build/src/tools/esbuild/i18n-inliner.ts index b8ad6cdedcab..c8f1048e6644 100644 --- a/packages/angular/build/src/tools/esbuild/i18n-inliner.ts +++ b/packages/angular/build/src/tools/esbuild/i18n-inliner.ts @@ -10,9 +10,10 @@ import assert from 'node:assert'; import { extname, join } from 'node:path'; import { serialize } from 'node:v8'; import { calculateHash, createContentHash, initializeHash } from '../../utils/hash'; -import { WorkerPool } from '../../utils/worker-pool'; +import { WorkerPool, getSharedBuildWorkerPool } from '../../utils/worker-pool'; import { type BuildOutputFile, BuildOutputFileType, createOutputFile } from './bundler-files'; import { type Cache, type PersistentCacheStore, createPersistentCacheStore } from './cache'; +import type { InlineCodeResult, InlineDiagnosticMessage } from './i18n-inliner-worker'; /** * A keyword used to indicate if a JavaScript file may require inlining of translations. @@ -116,6 +117,33 @@ interface CacheCheckItem { cachedResult: Promise; } +export interface InlineTemplateUpdateResult { + code: string; + errors: string[]; + warnings: string[]; +} + +/** + * Partitions diagnostic messages into error and warning strings. + */ +function partitionDiagnostics(messages: readonly InlineDiagnosticMessage[]): { + errors: string[]; + warnings: string[]; +} { + const errors: string[] = []; + const warnings: string[] = []; + + for (const message of messages) { + if (message.type === 'error') { + errors.push(message.message); + } else { + warnings.push(message.message); + } + } + + return { errors, warnings }; +} + /** * A class that performs i18n translation inlining of JavaScript code. * A worker pool is used to distribute the transformation actions and allow @@ -128,27 +156,27 @@ export class I18nInliner { #cacheStore: PersistentCacheStore | undefined; #cache: Cache | undefined; readonly #localizeFiles: ReadonlyMap; + readonly #filesBlobs: Map; readonly #unmodifiedFiles: Array; constructor( private readonly options: I18nInlinerOptions, - maxThreads?: number, + _maxThreads?: number, ) { this.#unmodifiedFiles = []; - const { outputFiles, shouldOptimize, missingTranslation, translations } = options; + const { outputFiles } = options; const files = new Map(); const pendingMaps = []; for (const file of outputFiles) { if (file.type === BuildOutputFileType.Root || file.type === BuildOutputFileType.ServerRoot) { // Skip also the server entry-point. - // Skip stats and similar files. + this.#unmodifiedFiles.push(file); continue; } const fileExtension = extname(file.path); if (fileExtension === '.js' || fileExtension === '.mjs') { - // Check if localizations are present const contentBuffer = Buffer.isBuffer(file.contents) ? file.contents : Buffer.from(file.contents.buffer, file.contents.byteOffset, file.contents.byteLength); @@ -179,24 +207,11 @@ export class I18nInliner { } this.#localizeFiles = files; + this.#filesBlobs = new Map( + Array.from(files, ([name, file]) => [name, new Blob([file.contents])]), + ); - this.#workerPool = new WorkerPool({ - filename: require.resolve('./i18n-inliner-worker'), - maxThreads, - // Extract options to ensure only the named options are serialized and sent to the worker - workerData: { - missingTranslation, - shouldOptimize, - translations, - // A Blob is an immutable data structure that allows sharing the data between workers - // without copying until the data is actually used within a Worker. This is useful here - // since each file may not actually be processed in each Worker and the Blob avoids - // unneeded repeat copying of potentially large JavaScript files. - files: new Map( - Array.from(files, ([name, file]) => [name, new Blob([file.contents])]), - ), - }, - }); + this.#workerPool = getSharedBuildWorkerPool(); } /** @@ -238,9 +253,15 @@ export class I18nInliner { // Pre-calculate cache key bases and serialized Blobs for each locale in this window const localeCacheBases = new Map(); const localeBlobs = new Map(); + const localeKeys = new Map(); for (const { locale, translation, translationIntegrity } of windowLocales) { localeBlobs.set(locale, serializeTranslation(translation)); + localeKeys.set( + locale, + translationIntegrity ?? + (translation ? calculateHash(JSON.stringify(translation)) : undefined), + ); if (this.#cacheStore) { localeCacheBases.set( @@ -304,7 +325,7 @@ export class I18nInliner { // Group uncached items by filename for this window const uncachedByFile = new Map< string, - Array<{ locale: string; cacheKey?: string; translation?: Blob }> + Array<{ locale: string; cacheKey?: string; translation?: Blob; translationKey?: string }> >(); for (const item of resolvedChecks) { @@ -322,6 +343,7 @@ export class I18nInliner { locale: item.locale, cacheKey: item.cacheKey, translation: localeBlobs.get(item.locale), + translationKey: localeKeys.get(item.locale), }); } } @@ -362,13 +384,11 @@ export class I18nInliner { outputFiles.push(createOutputFile(filename + '.map', fileResult.map, type)); } - for (const message of fileResult.messages) { - if (message.type === 'error') { - errors.push(message.message); - } else { - warnings.push(message.message); - } - } + const { errors: newErrors, warnings: newWarnings } = partitionDiagnostics( + fileResult.messages, + ); + errors.push(...newErrors); + warnings.push(...newWarnings); } } @@ -386,7 +406,10 @@ export class I18nInliner { } async #processUncachedBatches( - uncachedByFile: Map>, + uncachedByFile: Map< + string, + Array<{ locale: string; cacheKey?: string; translation?: Blob; translationKey?: string }> + >, localeCount: number, fileResultsByLocale: Map>, activeLocales?: string[], @@ -402,22 +425,32 @@ export class I18nInliner { const workerTasks: Promise[] = []; for (const [filename, entries] of uncachedByFile) { + const file = this.#localizeFiles.get(filename); + const fileBlob = this.#filesBlobs.get(filename); + const mapBlob = this.#filesBlobs.get(filename + '.map'); + const fileKey = file ? `${filename}\0${file.hash}` : undefined; const ephemeral = isLastWindow && entries.length <= localesPerBatch; + for (let i = 0; i < entries.length; i += localesPerBatch) { const batchEntries = entries.slice(i, i + localesPerBatch); const task = (async () => { - const batchResult = (await this.#workerPool.run( - { - filename, - locales: batchEntries.map((e) => ({ - locale: e.locale, - translation: e.translation, - })), - ephemeral, - activeLocales, - }, - { name: 'inlineFileBatch' }, - )) as { + const batchResult = (await this.#workerPool.run({ + tag: 'inline-i18n', + action: 'inlineFileBatch', + filename, + fileBlob, + fileKey, + mapBlob, + missingTranslation: this.options.missingTranslation, + shouldOptimize: this.options.shouldOptimize, + ephemeral, + activeLocales, + locales: batchEntries.map((e) => ({ + locale: e.locale, + translation: e.translation, + translationKey: e.translationKey, + })), + })) as { file: string; results: Array; }; @@ -476,7 +509,7 @@ export class I18nInliner { translation: Record | undefined, templateCode: string, templateId: string, - ): Promise<{ code: string; errors: string[]; warnings: string[] }> { + ): Promise { const hasLocalize = templateCode.includes(LOCALIZE_KEYWORD); if (!hasLocalize) { @@ -487,25 +520,19 @@ export class I18nInliner { }; } - const { output, messages } = await this.#workerPool.run( - { - code: templateCode, - filename: templateId, - locale, - translation: serializeTranslation(translation), - }, - { name: 'inlineCode' }, - ); + const { output, messages } = (await this.#workerPool.run({ + tag: 'inline-i18n', + action: 'inlineCode', + code: templateCode, + filename: templateId, + locale, + translation: serializeTranslation(translation), + translationKey: translation ? calculateHash(JSON.stringify(translation)) : undefined, + missingTranslation: this.options.missingTranslation, + shouldOptimize: this.options.shouldOptimize, + })) as InlineCodeResult; - const errors: string[] = []; - const warnings: string[] = []; - for (const message of messages) { - if (message.type === 'error') { - errors.push(message.message); - } else { - warnings.push(message.message); - } - } + const { errors, warnings } = partitionDiagnostics(messages); return { code: output, @@ -519,7 +546,11 @@ export class I18nInliner { * @returns A void promise that resolves when closing is complete. */ async close(): Promise { - await Promise.allSettled([this.#cacheStore?.close(), this.#workerPool.destroy()]); + if (this.#workerPool !== getSharedBuildWorkerPool()) { + await Promise.allSettled([this.#cacheStore?.close(), this.#workerPool.destroy()]); + } else { + await this.#cacheStore?.close(); + } } /** diff --git a/packages/angular/build/src/tools/esbuild/javascript-transformer-worker.ts b/packages/angular/build/src/tools/esbuild/javascript-transformer-worker.ts index f2ec7eccce6d..48a86b5590dd 100644 --- a/packages/angular/build/src/tools/esbuild/javascript-transformer-worker.ts +++ b/packages/angular/build/src/tools/esbuild/javascript-transformer-worker.ts @@ -22,17 +22,25 @@ import { import { transform as transformWithOxc } from '../oxc/oxc-transform.js'; import type { JavaScriptTransformerOptions } from './javascript-transformer'; -interface JavaScriptTransformRequest { +export interface JavaScriptTransformRequest { filename: string; data: string | Uint8Array; skipLinker?: boolean; sideEffects?: boolean; instrumentForCoverage?: boolean; + sourcemap?: boolean; + thirdPartySourcemaps?: boolean; + advancedOptimizations?: boolean; + jit?: boolean; } interface TransformOptions extends Omit { inputSourceMap?: EncodedSourceMap; isAlreadyStripped?: boolean; + sourcemap?: boolean; + thirdPartySourcemaps?: boolean; + advancedOptimizations?: boolean; + jit?: boolean; } const { @@ -92,10 +100,18 @@ async function instrumentCoverage( export default async function transformJavaScript( request: JavaScriptTransformRequest, ): Promise { - const { filename, data, ...options } = request; + const { + filename, + data, + sourcemap: reqSourcemap = sourcemap, + thirdPartySourcemaps: reqThirdPartySourcemaps = thirdPartySourcemaps, + advancedOptimizations: reqAdvancedOptimizations = advancedOptimizations, + jit: reqJit = jit, + ...options + } = request; const useInputSourcemap = - sourcemap && (!!thirdPartySourcemaps || !/[\\/]node_modules[\\/]/.test(filename)); + reqSourcemap && (!!reqThirdPartySourcemaps || !/[\\/]node_modules[\\/]/.test(filename)); let textData: string; let inputSourceMap: EncodedSourceMap | undefined; @@ -137,6 +153,10 @@ export default async function transformJavaScript( const transformedData = await transformJavaScriptImpl(filename, textData, { ...options, + sourcemap: reqSourcemap, + thirdPartySourcemaps: reqThirdPartySourcemaps, + advancedOptimizations: reqAdvancedOptimizations, + jit: reqJit, inputSourceMap, isAlreadyStripped, }); @@ -156,8 +176,13 @@ async function transformJavaScriptImpl( options: TransformOptions, ): Promise { const shouldLink = !options.skipLinker; + const optSourcemap = options.sourcemap ?? sourcemap; + const optThirdPartySourcemaps = options.thirdPartySourcemaps ?? thirdPartySourcemaps; + const optAdvancedOptimizations = options.advancedOptimizations ?? advancedOptimizations; + const optJit = options.jit ?? jit; + const useInputSourcemap = - sourcemap && (!!thirdPartySourcemaps || !/[\\/]node_modules[\\/]/.test(filename)); + optSourcemap && (!!optThirdPartySourcemaps || !/[\\/]node_modules[\\/]/.test(filename)); let code = data; const maps: (DecodedSourceMap | EncodedSourceMap)[] = []; @@ -191,7 +216,7 @@ async function transformJavaScriptImpl( relative: (_from: string, to: string) => to, } as never, logger: new ConsoleLogger(LogLevel.info), - linkerJitMode: jit, + linkerJitMode: optJit, // This is a workaround until https://github.com/angular/angular/issues/42769 is fixed. sourceMapping: false, }) as PluginItem, @@ -206,7 +231,7 @@ async function transformJavaScriptImpl( // Run Oxc linking and/or advanced optimizations in a single unified AST traversal pass const oxcLink = shouldLink && !useBabelLinker; - if (oxcLink || advancedOptimizations) { + if (oxcLink || optAdvancedOptimizations) { const sideEffectFree = options.sideEffects === false; const safeAngularPackage = sideEffectFree && /[\\/]node_modules[\\/]@angular[\\/]/.test(filename); @@ -214,8 +239,8 @@ async function transformJavaScriptImpl( const result = transformWithOxc(filename, code, { link: oxcLink, - jit, - advancedOptimizations, + jit: optJit, + advancedOptimizations: optAdvancedOptimizations, sourcemap: useInputSourcemap, sideEffects: options.sideEffects, topLevelSafeMode, diff --git a/packages/angular/build/src/tools/esbuild/javascript-transformer.ts b/packages/angular/build/src/tools/esbuild/javascript-transformer.ts index 3ba0dfff45b1..58f27b9fc895 100644 --- a/packages/angular/build/src/tools/esbuild/javascript-transformer.ts +++ b/packages/angular/build/src/tools/esbuild/javascript-transformer.ts @@ -8,9 +8,8 @@ import { readFile } from 'node:fs/promises'; import { createContentHash } from '../../utils/hash'; -import { IMPORT_EXEC_ARGV } from '../../utils/server-rendering/esm-in-memory-loader/utils'; import { removeSourceMappingURL } from '../../utils/source-map'; -import { WorkerPool, WorkerPoolOptions } from '../../utils/worker-pool'; +import { WorkerPool, getSharedBuildWorkerPool } from '../../utils/worker-pool'; import { Cache } from './cache'; const LINKER_DECLARATION_PREFIX = 'ɵɵngDeclare'; @@ -124,24 +123,7 @@ export class JavaScriptTransformer { } #ensureWorkerPool(): WorkerPool { - if (this.#workerPool) { - return this.#workerPool; - } - - const workerPoolOptions: WorkerPoolOptions = { - filename: require.resolve('./javascript-transformer-worker'), - maxThreads: this.maxThreads, - minThreads: this.maxThreads, - workerData: this.#commonOptions, - }; - - // Prevent passing SSR `--import` (loader-hooks) from parent to child worker. - const filteredExecArgv = process.execArgv.filter((v) => v !== IMPORT_EXEC_ARGV); - if (process.execArgv.length !== filteredExecArgv.length) { - workerPoolOptions.execArgv = filteredExecArgv; - } - - this.#workerPool = new WorkerPool(workerPoolOptions); + this.#workerPool ??= getSharedBuildWorkerPool(); return this.#workerPool; } @@ -248,11 +230,13 @@ export class JavaScriptTransformer { return this.#ensureWorkerPool().run( { + tag: 'transform-js', filename, data, skipLinker: !shouldLink, sideEffects, instrumentForCoverage, + ...this.#commonOptions, }, { transferList: isTransferable ? [data.buffer] : undefined, @@ -271,12 +255,14 @@ export class JavaScriptTransformer { task.reject(new Error('JavaScriptTransformer closed.')); } - if (this.#workerPool) { + if (this.#workerPool && this.#workerPool !== getSharedBuildWorkerPool()) { try { await this.#workerPool.destroy(); } finally { this.#workerPool = undefined; } + } else { + this.#workerPool = undefined; } } } diff --git a/packages/angular/build/src/tools/sass/sass-service.ts b/packages/angular/build/src/tools/sass/sass-service.ts index 2df6f85f7a52..b47c2e183552 100644 --- a/packages/angular/build/src/tools/sass/sass-service.ts +++ b/packages/angular/build/src/tools/sass/sass-service.ts @@ -21,7 +21,7 @@ import type { StringOptions, } from 'sass'; import { maxWorkers } from '../../utils/environment-options'; -import { WorkerPool } from '../../utils/worker-pool'; +import { WorkerPool, getSharedBuildWorkerPool } from '../../utils/worker-pool'; // Polyfill Symbol.dispose if not present // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -92,10 +92,7 @@ export class SassWorkerImplementation { ) {} #ensureWorkerPool(): WorkerPool { - this.#workerPool ??= new WorkerPool({ - filename: require.resolve('./worker'), - maxThreads: this.maxThreads, - }); + this.#workerPool ??= getSharedBuildWorkerPool(); return this.#workerPool; } @@ -138,6 +135,7 @@ export class SassWorkerImplementation { const response = (await this.#ensureWorkerPool().run( { + tag: 'render-sass', source, importerChannel, hasLogger: !!logger, @@ -192,12 +190,14 @@ export class SassWorkerImplementation { * @returns A void promise that resolves when closing is complete. */ async close(): Promise { - if (this.#workerPool) { + if (this.#workerPool && this.#workerPool !== getSharedBuildWorkerPool()) { try { await this.#workerPool.destroy(); } finally { this.#workerPool = undefined; } + } else { + this.#workerPool = undefined; } } diff --git a/packages/angular/build/src/tools/sass/worker.ts b/packages/angular/build/src/tools/sass/worker.ts index e4167a3d1c69..0cb15e8d4291 100644 --- a/packages/angular/build/src/tools/sass/worker.ts +++ b/packages/angular/build/src/tools/sass/worker.ts @@ -30,7 +30,7 @@ import type { SerializableDeprecation, SerializableWarningMessage } from './sass /** * A request to render a Sass stylesheet using the supplied options. */ -interface RenderRequestMessage { +export interface RenderRequestMessage { /** * The contents to compile. */ diff --git a/packages/angular/build/src/utils/shared-worker-router.ts b/packages/angular/build/src/utils/shared-worker-router.ts new file mode 100644 index 000000000000..06beb5b97deb --- /dev/null +++ b/packages/angular/build/src/utils/shared-worker-router.ts @@ -0,0 +1,72 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +import inlineFile, { + type InlineCodeRequest, + type InlineFileBatchRequest, + type InlineFileRequest, + inlineCode, + inlineFileBatch, +} from '../tools/esbuild/i18n-inliner-worker'; +import transformJavaScript, { + type JavaScriptTransformRequest, +} from '../tools/esbuild/javascript-transformer-worker'; +import renderSassStylesheet, { type RenderRequestMessage } from '../tools/sass/worker'; + +interface TransformJsTask extends JavaScriptTransformRequest { + tag: 'transform-js'; +} + +interface RenderSassTask extends RenderRequestMessage { + tag: 'render-sass'; +} + +interface InlineI18nFileTask extends InlineFileRequest { + tag: 'inline-i18n'; + action?: 'inlineFile'; +} + +interface InlineI18nFileBatchTask extends InlineFileBatchRequest { + tag: 'inline-i18n'; + action: 'inlineFileBatch'; +} + +interface InlineI18nCodeTask extends InlineCodeRequest { + tag: 'inline-i18n'; + action: 'inlineCode'; +} + +type InlineI18nTask = InlineI18nFileTask | InlineI18nFileBatchTask | InlineI18nCodeTask; + +type SharedWorkerTask = TransformJsTask | RenderSassTask | InlineI18nTask; + +/** + * Main worker dispatch function. Dispatches incoming tasks based on task tag. + * + * @param task The task payload dispatched to the shared build worker pool. + * @returns The resolved result of the corresponding task handler. + */ +export default function workerRouter(task: SharedWorkerTask): Promise { + switch (task.tag) { + case 'transform-js': + return transformJavaScript(task); + case 'render-sass': + return renderSassStylesheet(task); + case 'inline-i18n': + switch (task.action) { + case 'inlineCode': + return inlineCode(task); + case 'inlineFileBatch': + return inlineFileBatch(task); + default: + return inlineFile(task); + } + default: + throw new Error(`Unknown worker task tag: ${(task as { tag?: unknown })?.tag}`); + } +} diff --git a/packages/angular/build/src/utils/worker-pool.ts b/packages/angular/build/src/utils/worker-pool.ts index 907de66ba02f..9133478ba9e6 100644 --- a/packages/angular/build/src/utils/worker-pool.ts +++ b/packages/angular/build/src/utils/worker-pool.ts @@ -8,6 +8,8 @@ import { getCompileCacheDir } from 'node:module'; import { Piscina } from 'piscina'; +import { maxWorkers } from './environment-options'; +import { IMPORT_EXEC_ARGV } from './server-rendering/esm-in-memory-loader/utils'; export type WorkerPoolOptions = ConstructorParameters[0]; @@ -44,3 +46,37 @@ export class WorkerPool extends Piscina { super(piscinaOptions); } } + +/** + * The singleton shared build worker pool instance. + */ +let sharedBuildWorkerPool: WorkerPool | undefined; + +/** + * Returns the singleton shared build worker pool instance bounded by `maxWorkers`. + * The pool routes tasks using `shared-worker-router`. + */ +export function getSharedBuildWorkerPool(): WorkerPool { + if (!sharedBuildWorkerPool) { + const filteredExecArgv = process.execArgv.filter((v) => v !== IMPORT_EXEC_ARGV); + sharedBuildWorkerPool = new WorkerPool({ + filename: require.resolve('./shared-worker-router'), + maxThreads: maxWorkers, + minThreads: 1, + execArgv: filteredExecArgv.length !== process.execArgv.length ? filteredExecArgv : undefined, + }); + } + + return sharedBuildWorkerPool; +} + +/** + * Destroys and resets the singleton shared build worker pool. + */ +export async function shutdownSharedBuildWorkerPool(): Promise { + if (sharedBuildWorkerPool) { + const pool = sharedBuildWorkerPool; + sharedBuildWorkerPool = undefined; + await pool.destroy(); + } +} diff --git a/packages/angular/build/src/utils/worker-pool_spec.ts b/packages/angular/build/src/utils/worker-pool_spec.ts new file mode 100644 index 000000000000..c2b0fffb2f7a --- /dev/null +++ b/packages/angular/build/src/utils/worker-pool_spec.ts @@ -0,0 +1,244 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +import { serialize } from 'node:v8'; +import { initializeHash } from './hash'; +import { WorkerPool, getSharedBuildWorkerPool, shutdownSharedBuildWorkerPool } from './worker-pool'; + +describe('Singleton Shared Build Worker Pool', () => { + beforeAll(async () => { + await initializeHash(); + }); + + afterEach(async () => { + await shutdownSharedBuildWorkerPool(); + }); + + it('should return a WorkerPool instance from getSharedBuildWorkerPool', () => { + const pool = getSharedBuildWorkerPool(); + expect(pool).toBeDefined(); + expect(pool instanceof WorkerPool).toBeTrue(); + }); + + it('should return the identical singleton instance on multiple getSharedBuildWorkerPool calls', () => { + const pool1 = getSharedBuildWorkerPool(); + const pool2 = getSharedBuildWorkerPool(); + + expect(pool1).toBe(pool2); + }); + + it('should reset the singleton instance after shutdownSharedBuildWorkerPool is called', async () => { + const pool1 = getSharedBuildWorkerPool(); + await shutdownSharedBuildWorkerPool(); + + const pool2 = getSharedBuildWorkerPool(); + expect(pool2).not.toBe(pool1); + expect(pool2 instanceof WorkerPool).toBeTrue(); + }); + + it('should dispatch transform-js tasks to the shared worker router', async () => { + const pool = getSharedBuildWorkerPool(); + const code = 'export const value: number = 42;\n'; + + const result = (await pool.run({ + tag: 'transform-js', + filename: 'test.ts', + data: code, + skipLinker: true, + sourcemap: false, + })) as Uint8Array; + + expect(result).toBeDefined(); + const text = Buffer.from(result).toString('utf-8'); + expect(text).toContain('42'); + }); + + it('should dispatch render-sass tasks to the shared worker router', async () => { + const pool = getSharedBuildWorkerPool(); + const scss = '$color: red; .test { color: $color; }'; + + const response = (await pool.run({ + tag: 'render-sass', + source: scss, + hasLogger: false, + rebase: false, + options: { + url: '/test.scss', + }, + })) as { result?: { css: string } }; + + expect(response.result).toBeDefined(); + expect(response.result?.css).toContain('color: red'); + }); + + it('should dispatch inline-i18n inlineCode tasks to the shared worker router', async () => { + const pool = getSharedBuildWorkerPool(); + const code = 'export const greeting = $localize`:@@greeting:Hello`;\n'; + const translation = { + greeting: { + messageParts: ['Bonjour'], + placeholderNames: [], + text: 'Bonjour', + }, + }; + + const result = (await pool.run({ + tag: 'inline-i18n', + action: 'inlineCode', + code, + filename: 'main.js', + locale: 'fr', + translation: new Blob([serialize(translation)]), + missingTranslation: 'ignore', + })) as { output: string; messages: unknown[] }; + + expect(result.output).toContain('"Bonjour"'); + expect(result.output).not.toContain('$localize'); + }); + + it('should dispatch inline-i18n inlineFile tasks with fileBlobs to the shared worker router', async () => { + const pool = getSharedBuildWorkerPool(); + const code = 'export const greeting = $localize`:@@greeting:Hello`;\n'; + const fileBlob = new Blob([code]); + const translationEs = { + greeting: { + messageParts: ['Hola'], + placeholderNames: [], + text: 'Hola', + }, + }; + const translationFr = { + greeting: { + messageParts: ['Bonjour'], + placeholderNames: [], + text: 'Bonjour', + }, + }; + + const resultEs = (await pool.run({ + tag: 'inline-i18n', + action: 'inlineFile', + filename: 'main.js', + fileKey: 'main.js\0hash123', + locale: 'es', + translation: new Blob([serialize(translationEs)]), + translationKey: 'trans_es_123', + missingTranslation: 'ignore', + fileBlob, + })) as { file: string; code: string }; + + const resultFr = (await pool.run({ + tag: 'inline-i18n', + action: 'inlineFile', + filename: 'main.js', + fileKey: 'main.js\0hash123', + locale: 'fr', + translation: new Blob([serialize(translationFr)]), + translationKey: 'trans_fr_123', + missingTranslation: 'ignore', + fileBlob, + })) as { file: string; code: string }; + + expect(resultEs.code).toContain('"Hola"'); + expect(resultEs.code).not.toContain('$localize'); + expect(resultFr.code).toContain('"Bonjour"'); + expect(resultFr.code).not.toContain('$localize'); + }); + + it('should dispatch inline-i18n inlineFileBatch tasks to the shared worker router', async () => { + const pool = getSharedBuildWorkerPool(); + const code = 'export const greeting = $localize`:@@greeting:Hello`;\n'; + const fileBlob = new Blob([code]); + const translationEs = { + greeting: { + messageParts: ['Hola'], + placeholderNames: [], + text: 'Hola', + }, + }; + const translationFr = { + greeting: { + messageParts: ['Bonjour'], + placeholderNames: [], + text: 'Bonjour', + }, + }; + + const batchResult = (await pool.run({ + tag: 'inline-i18n', + action: 'inlineFileBatch', + filename: 'main.js', + fileKey: 'main.js\0hash123', + fileBlob, + missingTranslation: 'ignore', + locales: [ + { + locale: 'es', + translation: new Blob([serialize(translationEs)]), + translationKey: 'trans_es_123', + }, + { + locale: 'fr', + translation: new Blob([serialize(translationFr)]), + translationKey: 'trans_fr_123', + }, + ], + })) as { file: string; results: { locale: string; code: string }[] }; + + expect(batchResult.file).toBe('main.js'); + expect(batchResult.results.length).toBe(2); + expect(batchResult.results[0].locale).toBe('es'); + expect(batchResult.results[0].code).toContain('"Hola"'); + expect(batchResult.results[1].locale).toBe('fr'); + expect(batchResult.results[1].code).toContain('"Bonjour"'); + }); + + it('should execute mixed tasks concurrently across the shared pool without interference', async () => { + const pool = getSharedBuildWorkerPool(); + + const jsPromise = pool.run({ + tag: 'transform-js', + filename: 'concurrent.ts', + data: 'export const a: number = 1;\n', + skipLinker: true, + sourcemap: false, + }); + + const sassPromise = pool.run({ + tag: 'render-sass', + source: '$bg: blue; body { background: $bg; }', + hasLogger: false, + rebase: false, + options: { + url: '/concurrent.scss', + }, + }); + + const i18nPromise = pool.run({ + tag: 'inline-i18n', + action: 'inlineCode', + code: 'export const msg = $localize`:@@m:Hi`;\n', + filename: 'concurrent.js', + locale: 'de', + translation: new Blob([ + serialize({ m: { messageParts: ['Hallo'], placeholderNames: [], text: 'Hallo' } }), + ]), + missingTranslation: 'ignore', + }); + + const [jsResult, sassResult, i18nResult] = await Promise.all([ + jsPromise as Promise, + sassPromise as Promise<{ result?: { css: string } }>, + i18nPromise as Promise<{ output: string }>, + ]); + + expect(Buffer.from(jsResult).toString('utf-8')).toContain('1'); + expect(sassResult.result?.css).toContain('background: blue'); + expect(i18nResult.output).toContain('"Hallo"'); + }); +});