diff --git a/handwritten/storage/README.md b/handwritten/storage/README.md index c4f94576c1b3..52379fd0dda6 100644 --- a/handwritten/storage/README.md +++ b/handwritten/storage/README.md @@ -236,6 +236,12 @@ Samples are located in the [`storage/`](https://github.com/GoogleCloudPlatform/n The [Google Cloud Storage Node.js Client API Reference][client-docs] documentation also contains samples. +## Troubleshooting: Timeouts and 401 Invalid Credentials + +* In constrained environments like Kubernetes, severe CPU starvation can block the Node.js event loop for extended periods. +* This can cause in-flight authentication tokens to expire (resulting in intermittent `401 Invalid Credentials` errors) or cause TLS handshake timeouts (`fetch failed`). +* **Recommended solution:** Ensure your pods have adequate CPU limits allocated and ensure exponential backoff is configured for retryable requests. + ## Supported Node.js Versions Our client libraries follow the [Node.js release schedule](https://github.com/nodejs/release#release-schedule). diff --git a/handwritten/storage/src/resumable-upload.ts b/handwritten/storage/src/resumable-upload.ts index de3661951bfc..72d6e872f602 100644 --- a/handwritten/storage/src/resumable-upload.ts +++ b/handwritten/storage/src/resumable-upload.ts @@ -31,6 +31,7 @@ import * as crypto from 'crypto'; import {GCCL_GCS_CMD_KEY, decorateHeaders} from './nodejs-common/util.js'; import {FileExceptionMessages, FileMetadata, RequestError} from './file.js'; import {HashStreamValidator} from './hash-stream-validator.js'; +import {augmentError} from './util.js'; const NOT_FOUND_STATUS_CODE = 404; const RESUMABLE_INCOMPLETE_STATUS_CODE = 308; @@ -1135,7 +1136,7 @@ export class Upload extends Writable { // the response body. private async responseHandler(resp: GaxiosResponse) { if (resp.data.error) { - this.destroy(resp.data.error); + this.destroy(augmentError(resp.data.error)); return; } @@ -1370,13 +1371,17 @@ export class Upload extends Writable { } } - const res = await this.authClient.request<{error?: object}>( - combinedReqOpts, - ); - if (res.data && res.data.error) { - throw res.data.error; + try { + const res = await this.authClient.request<{error?: object}>( + combinedReqOpts, + ); + if (res.data && res.data.error) { + throw res.data.error; + } + return res; + } catch (e) { + throw augmentError(e); } - return res; } private async makeRequestStream(reqOpts: GaxiosOptions) { @@ -1413,7 +1418,9 @@ export class Upload extends Writable { } } - const res = await this.authClient.request(combinedReqOpts); + const res = await this.authClient.request(combinedReqOpts).catch(e => { + throw augmentError(e); + }); const successfulRequest = await this.onResponse(res); this.removeListener('error', errorCallback); diff --git a/handwritten/storage/src/storage-transport.ts b/handwritten/storage/src/storage-transport.ts index 33aa496ea6f2..442912e4b8e0 100644 --- a/handwritten/storage/src/storage-transport.ts +++ b/handwritten/storage/src/storage-transport.ts @@ -25,6 +25,7 @@ import {AuthClient, GoogleAuth, GoogleAuthOptions} from 'google-auth-library'; // @ts-ignore import {getPackageJSON} from './package-json-helper.cjs'; import {GCCL_GCS_CMD_KEY, decorateHeaders} from './nodejs-common/util.js'; +import {augmentError, setErrorProperty} from './util.js'; import { IdempotencyStrategy, RETRYABLE_ERR_FN_DEFAULT, @@ -273,32 +274,38 @@ export class StorageTransport { hasEtagInBody ); - // Helper to enrich GaxiosError objects with legacy ApiError properties + // Helper to enrich GaxiosError objects with legacy ApiError properties and troubleshooting hints // eslint-disable-next-line @typescript-eslint/no-explicit-any const decorateError = (err: any) => { if (err && typeof err === 'object') { - err.code = err.response?.status || err.status || err.code; + const resolvedCode = err.response?.status || err.status || err.code; + if (resolvedCode !== undefined) { + err = setErrorProperty(err, 'code', resolvedCode); + } if (err.response?.data?.error) { const apiError = err.response.data.error; if (typeof apiError === 'string') { - err.message = apiError; + err = setErrorProperty(err, 'message', apiError, false); } else if (apiError && typeof apiError === 'object') { if (typeof apiError.message === 'string') { - err.message = apiError.message; + err = setErrorProperty(err, 'message', apiError.message, false); } else if ( apiError.message && typeof apiError.message === 'object' ) { - err.message = + const extractedMessage = apiError.message.error?.message || apiError.message.message || JSON.stringify(apiError.message); + err = setErrorProperty(err, 'message', extractedMessage, false); + } + if (apiError.errors) { + err = setErrorProperty(err, 'errors', apiError.errors); } - if (apiError.errors) err.errors = apiError.errors; } } } - return err; + return augmentError(err); }; const isRetryDisabled = diff --git a/handwritten/storage/src/transfer-manager.ts b/handwritten/storage/src/transfer-manager.ts index ac0c080bac41..0a890222ae2e 100644 --- a/handwritten/storage/src/transfer-manager.ts +++ b/handwritten/storage/src/transfer-manager.ts @@ -34,6 +34,7 @@ import {GaxiosError, GaxiosResponse} from 'gaxios'; import {createHash} from 'crypto'; import {GCCL_GCS_CMD_KEY} from './nodejs-common/util.js'; import { + augmentError, getRuntimeTrackingString, getUserAgentString, getPLimit, @@ -410,9 +411,9 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { this.bucket.storage.retryOptions.autoRetry && this.bucket.storage.retryOptions.retryableErrorFn!(err as GaxiosError) ) { - throw err; + throw augmentError(err); } else { - bail(err); + bail(augmentError(err)); } } } diff --git a/handwritten/storage/src/util.ts b/handwritten/storage/src/util.ts index 007816c3fe77..e801f0ae477c 100644 --- a/handwritten/storage/src/util.ts +++ b/handwritten/storage/src/util.ts @@ -381,3 +381,144 @@ export function getPLimit(): Promise { } return pLimitPromise; } + +const FETCH_TIMEOUT_HINT = + ' (Hint: This may be caused by severe CPU starvation or network throttling in constrained environments like Kubernetes.)'; +const INVALID_CREDENTIALS_HINT = + ' (Hint: If this occurs intermittently under heavy load, check your container for CPU starvation which can cause tokens to expire in flight.)'; + +/** + * Safely assigns a property on an error object in place when writable/configurable, + * or reconstructs a descriptor-preserving clone (retaining prototype, stack, and cause) + * if the error object is frozen or non-configurable. + * + * @internal + */ +export function setErrorProperty( + err: T, + key: string, + value: unknown, + defaultEnumerable = true, +): T { + const record = err as Record; + try { + record[key] = value; + if (Object.is(record[key], value)) { + return err; + } + } catch { + // Property may be read-only or object may be frozen; fall through. + } + + try { + const desc = Object.getOwnPropertyDescriptor(err, key); + Object.defineProperty(err, key, { + value, + writable: true, + enumerable: desc?.enumerable ?? defaultEnumerable, + configurable: true, + }); + return err; + } catch { + // Object may be frozen or property may be non-configurable; reconstruct below. + } + + const originalStack = (err as {stack?: string}).stack; + const rawDescriptors = Object.getOwnPropertyDescriptors(err); + const descriptors: PropertyDescriptorMap = {}; + for (const propKey of Reflect.ownKeys(rawDescriptors)) { + const desc = (rawDescriptors as Record)[ + propKey + ]; + (descriptors as Record)[propKey] = + 'value' in desc + ? {...desc, writable: true, configurable: true} + : {...desc, configurable: true}; + } + descriptors[key] = { + value, + writable: true, + enumerable: rawDescriptors[key]?.enumerable ?? defaultEnumerable, + configurable: true, + }; + if (originalStack !== undefined) { + descriptors.stack = { + value: originalStack, + writable: true, + enumerable: false, + configurable: true, + }; + } + if (!descriptors.cause && 'cause' in err) { + descriptors.cause = { + value: (err as {cause?: unknown}).cause, + writable: true, + enumerable: false, + configurable: true, + }; + } + return Object.create(Object.getPrototypeOf(err), descriptors) as T; +} + +function appendErrorHint(err: T, hint: string): T { + const currentMessage = typeof err.message === 'string' ? err.message : ''; + if (currentMessage.includes(hint)) { + return err; + } + return setErrorProperty(err, 'message', `${currentMessage}${hint}`, false); +} + +/** + * Appends troubleshooting hints to native fetch/TLS timeout errors and 401 + * Invalid Credentials errors without rewrapping or losing prototype, stack, or cause. + * + * @internal + */ +export function augmentError(err: T): T { + if (!err || typeof err !== 'object') { + return err; + } + + const errObj = err as unknown as Error & { + code?: unknown; + status?: unknown; + cause?: unknown; + }; + + // 1. Check if the error is a native fetch/undici timeout (either directly a TypeError + // or a GaxiosError wrapping the native fetch TypeError as its cause). + const wrappedFetchErr = + errObj instanceof Error && + errObj.cause instanceof TypeError && + errObj.cause.message === 'fetch failed' + ? (errObj.cause as TypeError & {cause?: unknown}) + : undefined; + + const fetchErr: (TypeError & {cause?: unknown}) | undefined = + errObj instanceof TypeError && + errObj.message === 'fetch failed' && + errObj.cause + ? errObj + : wrappedFetchErr?.cause + ? wrappedFetchErr + : undefined; + + if (fetchErr) { + const cause = fetchErr.cause as {code?: unknown; message?: unknown}; + if ( + cause && + typeof cause === 'object' && + (cause.code === 'UND_ERR_CONNECT_TIMEOUT' || + (typeof cause.message === 'string' && cause.message.includes('TLS'))) + ) { + return appendErrorHint(errObj, FETCH_TIMEOUT_HINT) as unknown as T; + } + } + + // 2. Check if the error is a 401 Invalid Credentials. + if (errObj.code === 401 || errObj.status === 401) { + return appendErrorHint(errObj, INVALID_CREDENTIALS_HINT) as unknown as T; + } + + return err; +} diff --git a/handwritten/storage/test/storage-transport.ts b/handwritten/storage/test/storage-transport.ts index a91fe9307f70..31237f2db56d 100644 --- a/handwritten/storage/test/storage-transport.ts +++ b/handwritten/storage/test/storage-transport.ts @@ -575,4 +575,295 @@ describe('Storage Transport', () => { ); }); }); + + describe('error augmentation', () => { + const cpuStarvationHint = + ' (Hint: This may be caused by severe CPU starvation or network throttling in constrained environments like Kubernetes.)'; + const tokenExpirationHint = + ' (Hint: If this occurs intermittently under heavy load, check your container for CPU starvation which can cause tokens to expire in flight.)'; + + const createFetchError = (cause: unknown): TypeError & {cause?: unknown} => + new ( + TypeError as unknown as new ( + message: string, + options?: {cause?: unknown}, + ) => TypeError & {cause?: unknown} + )('fetch failed', {cause}); + + it('should append CPU starvation hint when fetch fails with UND_ERR_CONNECT_TIMEOUT', async () => { + const requestStub = authClientStub.request as sinon.SinonStub; + const cause = Object.assign(new Error('Connect Timeout Error'), { + code: 'UND_ERR_CONNECT_TIMEOUT', + }); + const fetchError = createFetchError(cause); + const originalStack = fetchError.stack; + requestStub.rejects(fetchError); + + await assert.rejects( + transport.makeRequest({url: '/test'}), + (err: Error & {cause?: unknown}) => { + assert.strictEqual(err, fetchError); + assert.ok(err instanceof TypeError); + assert.strictEqual(err.message, `fetch failed${cpuStarvationHint}`); + assert.strictEqual(err.cause, cause); + assert.strictEqual(err.stack, originalStack); + return true; + }, + ); + }); + + it('should append CPU starvation hint when fetch fails with a TLS cause message', async () => { + const requestStub = authClientStub.request as sinon.SinonStub; + const cause = new Error( + 'Client network socket disconnected before secure TLS connection was established', + ); + const fetchError = createFetchError(cause); + requestStub.rejects(fetchError); + + await assert.rejects( + transport.makeRequest({url: '/test'}), + (err: Error & {cause?: unknown}) => { + assert.ok(err instanceof TypeError); + assert.ok(err.message.includes(cpuStarvationHint)); + assert.strictEqual(err.cause, cause); + return true; + }, + ); + }); + + it('should append CPU starvation hint when GaxiosError wraps a fetch failed TypeError', async () => { + const requestStub = authClientStub.request as sinon.SinonStub; + const undiciCause = Object.assign(new Error('Connect Timeout Error'), { + code: 'UND_ERR_CONNECT_TIMEOUT', + }); + const fetchError = createFetchError(undiciCause); + const gaxiosError = new GaxiosError( + fetchError.message, + { + headers: new Headers(), + url: new URL('https://storage.googleapis.com/test'), + }, + undefined, + fetchError, + ); + requestStub.rejects(gaxiosError); + + await assert.rejects( + transport.makeRequest({url: '/test'}), + (err: Error & {cause?: unknown}) => { + assert.strictEqual(err.cause, fetchError); + assert.ok(err instanceof GaxiosError); + assert.strictEqual(err.message, `fetch failed${cpuStarvationHint}`); + return true; + }, + ); + }); + + it('should not append CPU starvation hint when fetch fails with an unrelated cause', async () => { + const requestStub = authClientStub.request as sinon.SinonStub; + const cause = Object.assign(new Error('connect ECONNREFUSED'), { + code: 'ECONNREFUSED', + }); + const fetchError = createFetchError(cause); + requestStub.rejects(fetchError); + + await assert.rejects( + transport.makeRequest({url: '/test'}), + (err: Error) => { + assert.strictEqual(err.message, 'fetch failed'); + return true; + }, + ); + }); + + it('should append in-flight token expiration hint when request returns a 401 status', async () => { + const requestStub = authClientStub.request as sinon.SinonStub; + const cause = new Error('Underlying auth failure'); + const gaxiosError = new GaxiosError( + 'Invalid Credentials', + { + headers: new Headers(), + url: new URL('https://storage.googleapis.com/test'), + }, + { + bodyUsed: true, + config: { + headers: new Headers(), + url: new URL('https://storage.googleapis.com/test'), + }, + data: { + error: { + code: 401, + message: 'Invalid Credentials', + }, + }, + headers: new Headers(), + status: 401, + statusText: 'Unauthorized', + } as unknown as GaxiosResponse, + cause, + ); + const originalStack = gaxiosError.stack; + requestStub.rejects(gaxiosError); + + await assert.rejects( + transport.makeRequest({url: '/test'}), + (err: Error & {code?: number; status?: number; cause?: unknown}) => { + assert.strictEqual(err, gaxiosError); + assert.ok(err instanceof GaxiosError); + assert.strictEqual( + err.message, + `Invalid Credentials${tokenExpirationHint}`, + ); + assert.strictEqual(err.status, 401); + assert.strictEqual(err.code, 401); + assert.strictEqual(err.cause, cause); + assert.strictEqual(err.stack, originalStack); + return true; + }, + ); + }); + + it('should preserve prototype, stack, and cause when error is frozen', async () => { + const requestStub = authClientStub.request as sinon.SinonStub; + const cause = Object.assign(new Error('Connect Timeout Error'), { + code: 'UND_ERR_CONNECT_TIMEOUT', + }); + const fetchError = Object.freeze(createFetchError(cause)); + const originalStack = fetchError.stack; + requestStub.rejects(fetchError); + + await assert.rejects( + transport.makeRequest({url: '/test'}), + (err: Error & {cause?: unknown}) => { + assert.ok(err instanceof TypeError); + assert.strictEqual(err.message, `fetch failed${cpuStarvationHint}`); + assert.strictEqual(err.cause, cause); + assert.strictEqual(err.stack, originalStack); + return true; + }, + ); + }); + + it('should safely decorate and augment a frozen GaxiosError with response error metadata', async () => { + const requestStub = authClientStub.request as sinon.SinonStub; + const cause = new Error('Underlying auth failure'); + const apiErrors = [{reason: 'authError', message: 'Invalid Credentials'}]; + const frozenGaxiosError = Object.freeze( + new GaxiosError( + 'Request failed with status code 401', + { + headers: new Headers(), + url: new URL('https://storage.googleapis.com/test'), + }, + { + bodyUsed: true, + config: { + headers: new Headers(), + url: new URL('https://storage.googleapis.com/test'), + }, + data: { + error: { + code: 401, + message: 'Invalid Credentials', + errors: apiErrors, + }, + }, + headers: new Headers(), + status: 401, + statusText: 'Unauthorized', + } as unknown as GaxiosResponse, + cause, + ), + ); + const originalStack = frozenGaxiosError.stack; + requestStub.rejects(frozenGaxiosError); + + await assert.rejects( + transport.makeRequest({url: '/test'}), + ( + err: Error & { + code?: number; + status?: number; + errors?: unknown[]; + cause?: unknown; + }, + ) => { + assert.strictEqual(err.cause, cause); + assert.strictEqual(err.code, 401); + assert.strictEqual(err.status, 401); + assert.deepStrictEqual(err.errors, apiErrors); + assert.strictEqual(err.stack, originalStack); + assert.ok(err instanceof GaxiosError); + assert.strictEqual( + err.message, + `Invalid Credentials${tokenExpirationHint}`, + ); + return true; + }, + ); + }); + + it('should append CPU starvation hint when wrapper modifies outer error message', async () => { + const requestStub = authClientStub.request as sinon.SinonStub; + const undiciCause = Object.assign(new Error('Connect Timeout Error'), { + code: 'UND_ERR_CONNECT_TIMEOUT', + }); + const fetchError = createFetchError(undiciCause); + const gaxiosError = new GaxiosError( + 'The operation was aborted due to timeout', + { + headers: new Headers(), + url: new URL('https://storage.googleapis.com/test'), + }, + undefined, + fetchError, + ); + requestStub.rejects(gaxiosError); + + await assert.rejects( + transport.makeRequest({url: '/test'}), + (err: Error & {cause?: unknown}) => { + assert.strictEqual(err.cause, fetchError); + assert.ok(err instanceof GaxiosError); + assert.strictEqual( + err.message, + `The operation was aborted due to timeout${cpuStarvationHint}`, + ); + return true; + }, + ); + }); + + it('should mark accessor descriptors as configurable when cloning a frozen error', async () => { + const requestStub = authClientStub.request as sinon.SinonStub; + const baseError = new Error(); + Object.defineProperty(baseError, 'message', { + get: () => 'Original getter message', + enumerable: false, + configurable: true, + }); + Object.defineProperty(baseError, 'status', { + value: 401, + writable: true, + enumerable: true, + configurable: true, + }); + const frozenAccessorError = Object.freeze(baseError); + requestStub.rejects(frozenAccessorError); + + await assert.rejects( + transport.makeRequest({url: '/test'}), + (err: Error & {code?: number; status?: number}) => { + assert.strictEqual(err.code, 401); + assert.strictEqual(err.status, 401); + assert.strictEqual( + err.message, + `Original getter message${tokenExpirationHint}`, + ); + return true; + }, + ); + }); + }); });