Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions handwritten/storage/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,12 @@ Samples are located in the [`storage/`](https://github.com/GoogleCloudPlatform/n
The [Google Cloud Storage Node.js Client API Reference][client-docs] documentation
also contains samples.

## Troubleshooting: Timeouts and 401 Invalid Credentials

* In constrained environments like Kubernetes, severe CPU starvation can block the Node.js event loop for extended periods.
* This can cause in-flight authentication tokens to expire (resulting in intermittent `401 Invalid Credentials` errors) or cause TLS handshake timeouts (`fetch failed`).
* **Recommended solution:** Ensure your pods have adequate CPU limits allocated and ensure exponential backoff is configured for retryable requests.

## Supported Node.js Versions

Our client libraries follow the [Node.js release schedule](https://github.com/nodejs/release#release-schedule).
Expand Down
23 changes: 15 additions & 8 deletions handwritten/storage/src/resumable-upload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import * as crypto from 'crypto';
import {GCCL_GCS_CMD_KEY, decorateHeaders} from './nodejs-common/util.js';
import {FileExceptionMessages, FileMetadata, RequestError} from './file.js';
import {HashStreamValidator} from './hash-stream-validator.js';
import {augmentError} from './util.js';

const NOT_FOUND_STATUS_CODE = 404;
const RESUMABLE_INCOMPLETE_STATUS_CODE = 308;
Expand Down Expand Up @@ -1135,7 +1136,7 @@ export class Upload extends Writable {
// the response body.
private async responseHandler(resp: GaxiosResponse) {
if (resp.data.error) {
this.destroy(resp.data.error);
this.destroy(augmentError(resp.data.error));
return;
}

Expand Down Expand Up @@ -1370,13 +1371,17 @@ export class Upload extends Writable {
}
}

const res = await this.authClient.request<{error?: object}>(
combinedReqOpts,
);
if (res.data && res.data.error) {
throw res.data.error;
try {
const res = await this.authClient.request<{error?: object}>(
combinedReqOpts,
);
if (res.data && res.data.error) {
throw res.data.error;
}
return res;
} catch (e) {
throw augmentError(e);
}
return res;
}

private async makeRequestStream(reqOpts: GaxiosOptions) {
Expand Down Expand Up @@ -1413,7 +1418,9 @@ export class Upload extends Writable {
}
}

const res = await this.authClient.request(combinedReqOpts);
const res = await this.authClient.request(combinedReqOpts).catch(e => {
throw augmentError(e);
});
const successfulRequest = await this.onResponse(res);
this.removeListener('error', errorCallback);

Expand Down
21 changes: 14 additions & 7 deletions handwritten/storage/src/storage-transport.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import {AuthClient, GoogleAuth, GoogleAuthOptions} from 'google-auth-library';
// @ts-ignore
import {getPackageJSON} from './package-json-helper.cjs';
import {GCCL_GCS_CMD_KEY, decorateHeaders} from './nodejs-common/util.js';
import {augmentError, setErrorProperty} from './util.js';
import {
IdempotencyStrategy,
RETRYABLE_ERR_FN_DEFAULT,
Expand Down Expand Up @@ -273,32 +274,38 @@ export class StorageTransport {
hasEtagInBody
);

// Helper to enrich GaxiosError objects with legacy ApiError properties
// Helper to enrich GaxiosError objects with legacy ApiError properties and troubleshooting hints
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const decorateError = (err: any) => {
if (err && typeof err === 'object') {
err.code = err.response?.status || err.status || err.code;
const resolvedCode = err.response?.status || err.status || err.code;
if (resolvedCode !== undefined) {
err = setErrorProperty(err, 'code', resolvedCode);
}
Comment thread
thiyaguk09 marked this conversation as resolved.
Resolved
if (err.response?.data?.error) {
const apiError = err.response.data.error;
if (typeof apiError === 'string') {
err.message = apiError;
err = setErrorProperty(err, 'message', apiError, false);
} else if (apiError && typeof apiError === 'object') {
if (typeof apiError.message === 'string') {
err.message = apiError.message;
err = setErrorProperty(err, 'message', apiError.message, false);
} else if (
apiError.message &&
typeof apiError.message === 'object'
) {
err.message =
const extractedMessage =
apiError.message.error?.message ||
apiError.message.message ||
JSON.stringify(apiError.message);
err = setErrorProperty(err, 'message', extractedMessage, false);
}
if (apiError.errors) {
err = setErrorProperty(err, 'errors', apiError.errors);
}
if (apiError.errors) err.errors = apiError.errors;
}
}
}
return err;
return augmentError(err);
};

const isRetryDisabled =
Expand Down
5 changes: 3 additions & 2 deletions handwritten/storage/src/transfer-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import {GaxiosError, GaxiosResponse} from 'gaxios';
import {createHash} from 'crypto';
import {GCCL_GCS_CMD_KEY} from './nodejs-common/util.js';
import {
augmentError,
getRuntimeTrackingString,
getUserAgentString,
getPLimit,
Expand Down Expand Up @@ -410,9 +411,9 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper {
this.bucket.storage.retryOptions.autoRetry &&
this.bucket.storage.retryOptions.retryableErrorFn!(err as GaxiosError)
) {
throw err;
throw augmentError(err);
} else {
bail(err);
bail(augmentError(err));
}
}
}
Expand Down
141 changes: 141 additions & 0 deletions handwritten/storage/src/util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -381,3 +381,144 @@ export function getPLimit(): Promise<PLimit> {
}
return pLimitPromise;
}

const FETCH_TIMEOUT_HINT =
' (Hint: This may be caused by severe CPU starvation or network throttling in constrained environments like Kubernetes.)';
const INVALID_CREDENTIALS_HINT =
' (Hint: If this occurs intermittently under heavy load, check your container for CPU starvation which can cause tokens to expire in flight.)';

/**
* Safely assigns a property on an error object in place when writable/configurable,
* or reconstructs a descriptor-preserving clone (retaining prototype, stack, and cause)
* if the error object is frozen or non-configurable.
*
* @internal
*/
export function setErrorProperty<T extends object>(
err: T,
key: string,
value: unknown,
defaultEnumerable = true,
): T {
const record = err as Record<string, unknown>;
try {
record[key] = value;
if (Object.is(record[key], value)) {
return err;
}
} catch {
// Property may be read-only or object may be frozen; fall through.
}

try {
const desc = Object.getOwnPropertyDescriptor(err, key);
Object.defineProperty(err, key, {
value,
writable: true,
enumerable: desc?.enumerable ?? defaultEnumerable,
configurable: true,
});
return err;
} catch {
// Object may be frozen or property may be non-configurable; reconstruct below.
}

const originalStack = (err as {stack?: string}).stack;
const rawDescriptors = Object.getOwnPropertyDescriptors(err);
const descriptors: PropertyDescriptorMap = {};
for (const propKey of Reflect.ownKeys(rawDescriptors)) {
const desc = (rawDescriptors as Record<PropertyKey, PropertyDescriptor>)[
propKey
];
(descriptors as Record<PropertyKey, PropertyDescriptor>)[propKey] =
'value' in desc
? {...desc, writable: true, configurable: true}
: {...desc, configurable: true};
}
descriptors[key] = {
value,
writable: true,
enumerable: rawDescriptors[key]?.enumerable ?? defaultEnumerable,
configurable: true,
};
if (originalStack !== undefined) {
descriptors.stack = {
value: originalStack,
writable: true,
enumerable: false,
configurable: true,
};
}
if (!descriptors.cause && 'cause' in err) {
descriptors.cause = {
value: (err as {cause?: unknown}).cause,
writable: true,
enumerable: false,
configurable: true,
};
}
return Object.create(Object.getPrototypeOf(err), descriptors) as T;
}

function appendErrorHint<T extends Error>(err: T, hint: string): T {
const currentMessage = typeof err.message === 'string' ? err.message : '';
if (currentMessage.includes(hint)) {
return err;
}
return setErrorProperty(err, 'message', `${currentMessage}${hint}`, false);
}

/**
* Appends troubleshooting hints to native fetch/TLS timeout errors and 401
* Invalid Credentials errors without rewrapping or losing prototype, stack, or cause.
*
* @internal
*/
export function augmentError<T>(err: T): T {
if (!err || typeof err !== 'object') {
return err;
}

const errObj = err as unknown as Error & {
code?: unknown;
status?: unknown;
cause?: unknown;
};

// 1. Check if the error is a native fetch/undici timeout (either directly a TypeError
// or a GaxiosError wrapping the native fetch TypeError as its cause).
const wrappedFetchErr =
errObj instanceof Error &&
errObj.cause instanceof TypeError &&
errObj.cause.message === 'fetch failed'
? (errObj.cause as TypeError & {cause?: unknown})
: undefined;
Comment thread
thiyaguk09 marked this conversation as resolved.
Resolved

const fetchErr: (TypeError & {cause?: unknown}) | undefined =
errObj instanceof TypeError &&
errObj.message === 'fetch failed' &&
errObj.cause
? errObj
: wrappedFetchErr?.cause
? wrappedFetchErr
: undefined;

if (fetchErr) {
const cause = fetchErr.cause as {code?: unknown; message?: unknown};
if (
cause &&
typeof cause === 'object' &&
(cause.code === 'UND_ERR_CONNECT_TIMEOUT' ||
(typeof cause.message === 'string' && cause.message.includes('TLS')))
) {
return appendErrorHint(errObj, FETCH_TIMEOUT_HINT) as unknown as T;
}
}

// 2. Check if the error is a 401 Invalid Credentials.
if (errObj.code === 401 || errObj.status === 401) {
return appendErrorHint(errObj, INVALID_CREDENTIALS_HINT) as unknown as T;
}

return err;
}
Loading
Loading