fix(knowledge): stop retrying an embedding key with no credit left

apps/sim/lib/embeddings/client.test.ts

@@ -822,6 +822,87 @@ describe('knowledge embedding transport fallback', () => {
     expect(result.embeddings).toEqual([[4, 4]])
   })
 
+  /**
+   * OpenAI returns 429 for an exhausted balance as well as for a rate limit, but
+   * only one of them reopens. Retrying a spent account cannot succeed, and since
+   * the sweep re-queues failed documents every sync it turns into permanent load
+   * — this was observed burning every attempt on thousands of documents for
+   * weeks against an account with no credit.
+   */
+  it('does not retry a 429 that reports an exhausted balance', async () => {
+    vi.useFakeTimers()
+    setEnv({ OPENAI_API_KEY: 'openai-test' })
+    const fetchMock = vi.fn().mockImplementation(
+      async () =>
+        ({
+          ok: false,
+          status: 429,
+          statusText: 'Too Many Requests',
+          headers: new Headers(),
+          json: async () => ({}),
+          text: async () =>
+            JSON.stringify({
+              error: {
+                message: 'You have no credits remaining.',
+                type: 'insufficient_quota',
+                code: 'credit_balance_exhausted',
+              },
+            }),
+        }) as Response
+    )
+    vi.stubGlobal('fetch', fetchMock)
+
+    const pending = embed(['hello'], { ...options, apiKey: 'openai-test' }).catch((e) => e)
+    await vi.runAllTimersAsync()
+    const error = await pending
+
+    expect(error).toBeInstanceOf(EmbeddingAPIError)
+    expect(fetchMock).toHaveBeenCalledTimes(1)
+  })
+
+  /**
+   * A rate limit with the same status must keep its retries — the two are only
+   * distinguishable by the body.
+   */
+  it('still retries a 429 that reports a rate limit', async () => {
+    vi.useFakeTimers()
+    setEnv({ OPENAI_API_KEY: 'openai-test' })
+    let call = 0
+    const fetchMock = vi.fn().mockImplementation(async () => {
+      call++
+      if (call === 1) {
+        return {
+          ok: false,
+          status: 429,
+          statusText: 'Too Many Requests',
+          headers: new Headers(),
+          json: async () => ({}),
+          text: async () =>
+            JSON.stringify({ error: { message: 'slow down', type: 'rate_limit_exceeded' } }),
+        } as Response
+      }
+      return jsonResponse(openAIBody([[5, 5]], 2))
+    })
+    vi.stubGlobal('fetch', fetchMock)
+
+    const pending = embed(['hello'], { ...options, apiKey: 'openai-test' })
+    await vi.runAllTimersAsync()
+    const result = await pending
+
+    expect(fetchMock).toHaveBeenCalledTimes(2)
+    expect(result.embeddings).toEqual([[5, 5]])
+  })
+
+  /**
+   * An exhausted key rules out the key just used, not the next one in the chain,
+   * so failover must still consider it.
+   */
+  it('keeps an exhausted-balance error eligible for failover', () => {
+    const error = new EmbeddingAPIError('Embedding API failed: 429', 429)
+    error.quotaExhausted = true
+    expect(isTransientEmbeddingError(error)).toBe(true)
+  })
+
   it('classifies only transient embedding failures for failover', () => {
     expect(isTransientEmbeddingError(new EmbeddingAPIError('unavailable', 503))).toBe(true)
     expect(isTransientEmbeddingError(new EmbeddingAPIError('rate limited', 429))).toBe(true)

apps/sim/lib/embeddings/client.ts

@@ -79,6 +79,12 @@ const EMBEDDING_RETRY_BUDGET_MS = EMBEDDING_MAX_RETRIES * EMBEDDING_MAX_RETRY_DE
 export class EmbeddingAPIError extends Error {
   public status: number
 
+  /**
+   * The provider rejected this for an exhausted balance rather than a rate that
+   * will recover. Both arrive as 429.
+   */
+  public quotaExhausted?: boolean
+
   /**
    * Wait the provider asked for, read from the rejected response. Consumed by
    * {@link retryWithExponentialBackoff}, which prefers it over its own backoff.
@@ -92,6 +98,31 @@ export class EmbeddingAPIError extends Error {
   }
 }
 
+/**
+ * True when a rejection body reports an exhausted balance rather than a rate
+ * limit.
+ *
+ * OpenAI returns 429 for both, but only one of them reopens. `insufficient_quota`
+ * stands until somebody adds credit, so retrying it cannot succeed no matter how
+ * long the loop waits — and because a failed document is re-queued by the sweep
+ * on every sync, an account that has run out turns into a permanent load: the
+ * budget is spent per document, per attempt, forever.
+ */
+function isQuotaExhaustionBody(errorText: string): boolean {
+  try {
+    const body = JSON.parse(errorText) as { error?: { type?: string; code?: string } }
+    const type = body.error?.type
+    const code = body.error?.code
+    return (
+      type === 'insufficient_quota' ||
+      code === 'insufficient_quota' ||
+      code === 'credit_balance_exhausted'
+    )
+  } catch {
+    return false
+  }
+}
+
 /**
  * True when the provider's stated wait outlasts the entire retry budget.
  *
@@ -116,6 +147,21 @@ function statedWaitOutlastsBudget(error: unknown): boolean {
   )
 }
 
+/**
+ * Whether another attempt against the same provider could plausibly succeed.
+ *
+ * Deliberately narrower than {@link isTransientEmbeddingError}, which also
+ * decides whether the fallback chain should try a *different* provider. Those
+ * two questions differ: an exhausted balance rules out the key we just used, but
+ * says nothing about the next one in the chain, so a quota rejection stops the
+ * retries here while remaining eligible for failover.
+ */
+function isWorthRetrying(error: unknown): boolean {
+  if (!isTransientEmbeddingError(error)) return false
+  if (error instanceof EmbeddingAPIError && error.quotaExhausted) return false
+  return !statedWaitOutlastsBudget(error)
+}
+
 export function isTransientEmbeddingError(error: unknown): boolean {
   if (error instanceof EmbeddingAPIError) {
     return error.status === 429 || error.status >= 500
@@ -251,6 +297,7 @@ async function callEmbeddingAPI(
           `Embedding API failed: ${response.status} ${response.statusText} - ${errorText}`,
           response.status
         )
+        error.quotaExhausted = isQuotaExhaustionBody(errorText)
 
         /**
          * Carry the provider's own answer to "when may I retry" onto the error,
@@ -298,8 +345,7 @@ async function callEmbeddingAPI(
       maxRetries: EMBEDDING_MAX_RETRIES,
       initialDelayMs: 1000,
       maxDelayMs: EMBEDDING_MAX_RETRY_DELAY_MS,
-      retryCondition: (error) =>
-        isTransientEmbeddingError(error) && !statedWaitOutlastsBudget(error),
+      retryCondition: isWorthRetrying,
     }
   )
 }