Skip to content

Commit 260de05

Browse files
waleedlatif1waleedlatif1
authored
improvement(media): record a hosted-key auth failure when fal refuses Sim's account (#9593)
* improvement(media): record a hosted-key auth failure when fal refuses Sim's account Mothership's generate_audio and generate_video call fal on Sim's hosted keys outside the tool executor, so a refused account (revoked key, or a lock for an exhausted balance) only reached logs. The submit path now records a Sim/HostedKey Failed point with reason auth on a 401, 402 or 403, the same series the hosted-key dashboard already plots for executor tools, so an alarm on that reason covers fal too. * improvement(media): label fal auth failures with the refused key and use the shared env-flags mock getFalApiKey now returns the env var its key came from, so a refused key in the numbered pool is named on the metric. The metrics module reads isHosted on each call instead of copying it at import, and the test sets it through the central env-flags mock under fake timers.
1 parent 86ad456 commit 260de05

7 files changed

Lines changed: 186 additions & 32 deletions

File tree

‎apps/sim/lib/media/falai-audio.ts‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ function buildInput(
141141

142142
export async function generateFalAudio(params: GenerateFalAudioParams): Promise<GeneratedAudio> {
143143
const type: AudioType = params.type || 'speech'
144-
const apiKey = getFalApiKey()
144+
const { apiKey, envVarName } = getFalApiKey()
145145

146146
// Voice cloning: a reference sample routes to a zero-shot clone model (F5-TTS),
147147
// which conditions on the sample (ref_audio_url) and speaks the prompt in that voice.
@@ -152,7 +152,13 @@ export async function generateFalAudio(params: GenerateFalAudioParams): Promise<
152152
ref_audio_url: params.voiceSampleDataUri,
153153
model_type: 'F5-TTS',
154154
}
155-
const { requestId, data } = await runFalQueue(model, input, apiKey, params.signal)
155+
const { requestId, data } = await runFalQueue(
156+
model,
157+
input,
158+
apiKey,
159+
{ tool: 'generate_audio', envVarName },
160+
params.signal
161+
)
156162
const cost = await recordFalCost(apiKey, model, requestId, params.onCost)
157163
const url = extractFalMediaUrl(data, ['audio', 'audio_url', 'audio_file', 'output'])
158164
if (!url) throw new Error('No audio URL in Fal.ai clone response')
@@ -172,7 +178,13 @@ export async function generateFalAudio(params: GenerateFalAudioParams): Promise<
172178
const input = buildInput(type, params, model)
173179

174180
// For fal audio models the model ID is the queue endpoint.
175-
const { requestId, data } = await runFalQueue(model, input, apiKey, params.signal)
181+
const { requestId, data } = await runFalQueue(
182+
model,
183+
input,
184+
apiKey,
185+
{ tool: 'generate_audio', envVarName },
186+
params.signal
187+
)
176188
const cost = await recordFalCost(apiKey, model, requestId, params.onCost)
177189
const url = extractFalMediaUrl(data, ['audio', 'audio_url', 'audio_file', 'output'])
178190
if (!url) throw new Error('No audio URL in Fal.ai response')

‎apps/sim/lib/media/falai-hosted-key-metrics.test.ts‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
import { jsonResponse } from '@sim/testing/helpers/http'
2+
import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks/env-flags.mock'
3+
import { inputValidationMock } from '@sim/testing/mocks/input-validation.mock'
4+
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
5+
6+
const { sentMetricData } = vi.hoisted(() => ({
7+
sentMetricData: [] as Array<{
8+
MetricName?: string
9+
Dimensions?: Array<{ Name?: string; Value?: string }>
10+
}>,
11+
}))
12+
13+
vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock)
14+
vi.mock('@aws-sdk/client-cloudwatch', () => ({
15+
StandardUnit: { Count: 'Count', None: 'None', Milliseconds: 'Milliseconds' },
16+
PutMetricDataCommand: class {
17+
constructor(readonly input: { MetricData: typeof sentMetricData }) {}
18+
},
19+
CloudWatchClient: class {
20+
async send(command: { input: { MetricData: typeof sentMetricData } }) {
21+
sentMetricData.push(...command.input.MetricData)
22+
}
23+
},
24+
}))
25+
26+
import { runFalQueue } from '@/lib/media/falai'
27+
import { flushHostedKeyMetrics } from '@/lib/monitoring/metrics'
28+
29+
function stubFalSubmit(status: number, body: unknown) {
30+
vi.stubGlobal(
31+
'fetch',
32+
vi.fn(async () => jsonResponse(body, status))
33+
)
34+
}
35+
36+
async function failedHostedKeyDimensions(): Promise<Array<Record<string, string | undefined>>> {
37+
await flushHostedKeyMetrics()
38+
return sentMetricData
39+
.filter((datum) => datum.MetricName === 'Failed')
40+
.map((datum) => Object.fromEntries((datum.Dimensions ?? []).map((d) => [d.Name, d.Value])))
41+
}
42+
43+
describe('a fal submit the hosted account refuses', () => {
44+
beforeEach(() => {
45+
vi.useFakeTimers()
46+
setEnvFlags({ isHosted: true })
47+
sentMetricData.length = 0
48+
})
49+
50+
afterEach(() => {
51+
vi.useRealTimers()
52+
resetEnvFlagsMock()
53+
})
54+
55+
it.each([
56+
[403, { detail: 'User is locked. Reason: Exhausted balance.' }],
57+
[401, { detail: 'Invalid API key' }],
58+
])('records a hosted-key auth failure against the refused key for a %i', async (status, body) => {
59+
stubFalSubmit(status, body)
60+
61+
await expect(
62+
runFalQueue('fal-ai/model', {}, 'key', {
63+
tool: 'generate_audio',
64+
envVarName: 'FALAI_API_KEY_2',
65+
})
66+
).rejects.toThrow(`Fal.ai API error: ${status}`)
67+
68+
expect(await failedHostedKeyDimensions()).toEqual([
69+
expect.objectContaining({
70+
Provider: 'falai',
71+
Tool: 'generate_audio',
72+
Key: 'FALAI_API_KEY_2',
73+
Reason: 'auth',
74+
}),
75+
])
76+
})
77+
78+
it('records nothing when fal rejects the request input', async () => {
79+
stubFalSubmit(422, { detail: [{ msg: 'field required' }] })
80+
81+
await expect(
82+
runFalQueue('fal-ai/model', {}, 'key', {
83+
tool: 'generate_video',
84+
envVarName: 'FALAI_API_KEY_1',
85+
})
86+
).rejects.toThrow('Fal.ai API error: 422')
87+
88+
expect(await failedHostedKeyDimensions()).toEqual([])
89+
})
90+
})

‎apps/sim/lib/media/falai-video.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ export async function generateFalVideo(params: GenerateFalVideoParams): Promise<
148148
)
149149
}
150150

151-
const apiKey = getFalApiKey()
151+
const { apiKey, envVarName } = getFalApiKey()
152152

153153
let endpoint = config.endpoint
154154
const input: Record<string, unknown> = { prompt: params.prompt }
@@ -177,7 +177,13 @@ export async function generateFalVideo(params: GenerateFalVideoParams): Promise<
177177
input.prompt_optimizer = params.promptOptimizer
178178
}
179179

180-
const { requestId, data } = await runFalQueue(endpoint, input, apiKey, params.signal)
180+
const { requestId, data } = await runFalQueue(
181+
endpoint,
182+
input,
183+
apiKey,
184+
{ tool: 'generate_video', envVarName },
185+
params.signal
186+
)
181187
const cost = await recordFalCost(apiKey, endpoint, requestId, params.onCost)
182188
const url = extractFalMediaUrl(data, ['video', 'output'])
183189
if (!url) throw new Error('No video URL in Fal.ai response')

‎apps/sim/lib/media/falai.test.ts‎

Lines changed: 38 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import { downloadFalMedia, type FalCostRecorder, runFalQueue } from '@/lib/media
1313
import { generateFalAudio } from '@/lib/media/falai-audio'
1414
import { generateFalVideo } from '@/lib/media/falai-video'
1515

16+
const FAL_KEY_USE = { tool: 'generate_video', envVarName: 'FALAI_API_KEY' } as const
1617
const CANCEL_URL = 'https://queue.fal.run/fal-ai/model/requests/req-1/cancel'
1718
const STATUS_URL = 'https://queue.fal.run/fal-ai/model/requests/req-1/status'
1819

@@ -67,7 +68,13 @@ describe('runFalQueue on a request that never finishes', () => {
6768
it('stops polling and cancels the request once its caller gives up', async () => {
6869
const fal = stubStuckFalQueue()
6970
const caller = new AbortController()
70-
const generation = runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', caller.signal)
71+
const generation = runFalQueue(
72+
'fal-ai/model',
73+
{ prompt: 'p' },
74+
'key',
75+
FAL_KEY_USE,
76+
caller.signal
77+
)
7178
const outcome = generation.then(
7279
() => 'resolved',
7380
() => 'rejected'
@@ -87,7 +94,13 @@ describe('runFalQueue on a request that never finishes', () => {
8794
it('ends a poll that never answers when its caller gives up', async () => {
8895
const fal = stubStuckFalQueue({ statusHangs: true })
8996
const caller = new AbortController()
90-
const generation = runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', caller.signal)
97+
const generation = runFalQueue(
98+
'fal-ai/model',
99+
{ prompt: 'p' },
100+
'key',
101+
FAL_KEY_USE,
102+
caller.signal
103+
)
91104
const outcome = generation.then(
92105
() => 'resolved',
93106
() => 'rejected'
@@ -104,7 +117,7 @@ describe('runFalQueue on a request that never finishes', () => {
104117

105118
it('fails and cancels after its own ceiling even when nothing aborts it', async () => {
106119
const fal = stubStuckFalQueue()
107-
const generation = runFalQueue('fal-ai/model', { prompt: 'p' }, 'key')
120+
const generation = runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', FAL_KEY_USE)
108121
const outcome = generation.then(
109122
() => 'resolved',
110123
(error: unknown) => String(error)
@@ -119,9 +132,13 @@ describe('runFalQueue on a request that never finishes', () => {
119132
it('warns when fal refuses to cancel a request', async () => {
120133
const fal = stubStuckFalQueue({ cancelStatus: 500 })
121134
const caller = new AbortController()
122-
const generation = runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', caller.signal).catch(
123-
() => undefined
124-
)
135+
const generation = runFalQueue(
136+
'fal-ai/model',
137+
{ prompt: 'p' },
138+
'key',
139+
FAL_KEY_USE,
140+
caller.signal
141+
).catch(() => undefined)
125142
await vi.advanceTimersByTimeAsync(3_500)
126143
caller.abort()
127144
await generation
@@ -137,7 +154,7 @@ describe('runFalQueue on a request that never finishes', () => {
137154
const fal = stubStuckFalQueue()
138155

139156
await expect(
140-
runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', AbortSignal.abort())
157+
runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', FAL_KEY_USE, AbortSignal.abort())
141158
).rejects.toThrow()
142159
expect(fal.calls).toEqual([])
143160
})
@@ -160,9 +177,13 @@ describe('runFalQueue on a request that never finishes', () => {
160177
})
161178
)
162179
const caller = new AbortController()
163-
const generation = runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', caller.signal).catch(
164-
() => 'rejected'
165-
)
180+
const generation = runFalQueue(
181+
'fal-ai/model',
182+
{ prompt: 'p' },
183+
'key',
184+
FAL_KEY_USE,
185+
caller.signal
186+
).catch(() => 'rejected')
166187

167188
await vi.advanceTimersByTimeAsync(0)
168189
caller.abort(new Error('watchdog'))
@@ -175,9 +196,13 @@ describe('runFalQueue on a request that never finishes', () => {
175196
it('asks fal never to start the request past that ceiling', async () => {
176197
const fal = stubStuckFalQueue()
177198
const caller = new AbortController()
178-
const generation = runFalQueue('fal-ai/model', { prompt: 'p' }, 'key', caller.signal).catch(
179-
() => undefined
180-
)
199+
const generation = runFalQueue(
200+
'fal-ai/model',
201+
{ prompt: 'p' },
202+
'key',
203+
FAL_KEY_USE,
204+
caller.signal
205+
).catch(() => undefined)
181206
await vi.advanceTimersByTimeAsync(0)
182207
caller.abort()
183208
await generation

‎apps/sim/lib/media/falai.ts‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
readResponseTextWithLimit,
1414
readResponseToBufferWithLimit,
1515
} from '@/lib/core/utils/stream-limits'
16+
import { hostedKeyMetrics } from '@/lib/monitoring/metrics'
1617
import { type FalAICostMetadata, getFalAICostMetadata } from '@/lib/tools/falai-pricing'
1718

1819
const logger = createLogger('FalMediaClient')
@@ -33,20 +34,33 @@ const FAL_CANCEL_TIMEOUT_MS = 5000
3334
*/
3435
const FAL_SUBMIT_TIMEOUT_MS = 60_000
3536

37+
/** Who spent a hosted Fal.ai key: the Mothership tool and the env var the key came from. */
38+
interface FalKeyUse {
39+
tool: 'generate_audio' | 'generate_video'
40+
envVarName: string
41+
}
42+
43+
/**
44+
* Statuses fal answers when the hosted account itself refuses the call (a revoked key, or an
45+
* account locked for an exhausted balance), so every generation fails until someone acts.
46+
*/
47+
const FAL_ACCOUNT_REFUSAL_STATUSES = new Set([401, 402, 403])
48+
3649
/**
3750
* Resolves a hosted Fal.ai API key from the numbered env pool
3851
* (FALAI_API_KEY_COUNT + FALAI_API_KEY_1..N), round-robined by minute,
3952
* mirroring getRotatingApiKey. Falls back to a single FALAI_API_KEY for dev.
4053
*/
41-
export function getFalApiKey(): string {
54+
export function getFalApiKey(): { apiKey: string; envVarName: string } {
4255
const count = Number.parseInt(process.env.FALAI_API_KEY_COUNT || '0', 10)
43-
const keys: string[] = []
56+
const keys: Array<{ apiKey: string; envVarName: string }> = []
4457
for (let i = 1; i <= count; i++) {
45-
const key = process.env[`FALAI_API_KEY_${i}`]
46-
if (key) keys.push(key)
58+
const envVarName = `FALAI_API_KEY_${i}`
59+
const apiKey = process.env[envVarName]
60+
if (apiKey) keys.push({ apiKey, envVarName })
4761
}
4862
if (keys.length === 0 && process.env.FALAI_API_KEY) {
49-
keys.push(process.env.FALAI_API_KEY)
63+
keys.push({ apiKey: process.env.FALAI_API_KEY, envVarName: 'FALAI_API_KEY' })
5064
}
5165
if (keys.length === 0) {
5266
throw new Error(
@@ -101,6 +115,7 @@ export async function runFalQueue(
101115
endpoint: string,
102116
input: Record<string, unknown>,
103117
apiKey: string,
118+
keyUse: FalKeyUse,
104119
signal?: AbortSignal
105120
): Promise<FalQueueResult> {
106121
const ceiling = new AbortController()
@@ -110,7 +125,7 @@ export async function runFalQueue(
110125
)
111126
const queueSignal = signal ? AbortSignal.any([signal, ceiling.signal]) : ceiling.signal
112127
try {
113-
return await submitAndPollFalQueue(endpoint, input, apiKey, queueSignal)
128+
return await submitAndPollFalQueue(endpoint, input, apiKey, keyUse, queueSignal)
114129
} finally {
115130
clearTimeout(ceilingTimer)
116131
}
@@ -120,6 +135,7 @@ async function submitAndPollFalQueue(
120135
endpoint: string,
121136
input: Record<string, unknown>,
122137
apiKey: string,
138+
keyUse: FalKeyUse,
123139
queueSignal: AbortSignal
124140
): Promise<FalQueueResult> {
125141
// Only a submit already under way outlives its caller; a stopped call never starts one.
@@ -135,6 +151,14 @@ async function submitAndPollFalQueue(
135151
signal: AbortSignal.timeout(FAL_SUBMIT_TIMEOUT_MS),
136152
})
137153
if (!createResponse.ok) {
154+
if (FAL_ACCOUNT_REFUSAL_STATUSES.has(createResponse.status)) {
155+
hostedKeyMetrics.recordFailed({
156+
provider: 'falai',
157+
tool: keyUse.tool,
158+
key: keyUse.envVarName,
159+
reason: 'auth',
160+
})
161+
}
138162
const err = await readResponseTextWithLimit(createResponse, {
139163
maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES,
140164
label: 'Fal.ai create error response',

‎apps/sim/lib/monitoring/metrics.ts‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -46,11 +46,6 @@ type QueueReason = 'actor_requests' | 'dimension' | 'queue_position'
4646
/** `metering` marks a provider call that succeeded but could not be priced. */
4747
type FailureReason = 'rate_limited' | 'auth' | 'other' | 'metering'
4848

49-
// Hosted keys exist only on Sim Cloud, whose app and workers can always reach
50-
// CloudWatch (task role or workload identity). Elsewhere recorders are no-ops,
51-
// so they stay always-safe to call.
52-
const ENABLED = isHosted
53-
5449
const ENVIRONMENT =
5550
process.env.OTEL_DEPLOYMENT_ENVIRONMENT ||
5651
process.env.DEPLOYMENT_ENVIRONMENT ||
@@ -98,13 +93,16 @@ function buildDimensions(labels: Record<string, string | undefined>) {
9893
return dimensions
9994
}
10095

96+
// Hosted keys exist only on Sim Cloud, whose app and workers can always reach
97+
// CloudWatch (task role or workload identity). Elsewhere recorders are no-ops,
98+
// so they stay always-safe to call.
10199
function enqueue(
102100
MetricName: string,
103101
Value: number,
104102
Unit: StandardUnit,
105103
labels: Record<string, string | undefined>
106104
): void {
107-
if (!ENABLED) return
105+
if (!isHosted) return
108106
buffer.push({
109107
MetricName,
110108
Value,
@@ -129,7 +127,7 @@ export async function flushHostedKeyMetrics(): Promise<void> {
129127
logger.warn('Dropped hosted-key metric points (buffer cap reached)', { dropped })
130128
dropped = 0
131129
}
132-
if (!ENABLED || buffer.length === 0) return
130+
if (!isHosted || buffer.length === 0) return
133131
const pending = buffer
134132
buffer = []
135133
for (let i = 0; i < pending.length; i += MAX_BATCH) {

‎scripts/check-unused-exports.baseline.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4702,7 +4702,6 @@
47024702
"apps/sim/lib/messaging/email/mailer.ts#sendBatchEmails",
47034703
"apps/sim/lib/messaging/email/types.ts#EmailAttachment",
47044704
"apps/sim/lib/messaging/email/utils.ts#extractEmailFromAddress",
4705-
"apps/sim/lib/monitoring/metrics.ts#flushHostedKeyMetrics",
47064705
"apps/sim/lib/mothership/agent-cli/types.ts#AgentCliClient",
47074706
"apps/sim/lib/mothership/application/execute-knowledge-use-case.ts#requireCopilotKnowledgeWorkspaceId",
47084707
"apps/sim/lib/mothership/application/execute-knowledge-use-case.ts#resolveCopilotKnowledgePrincipal",

0 commit comments

Comments
 (0)