From 9bab33e390a93bcb9eaf940496668b3ad4d9bf05 Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Sat, 24 Oct 2026 16:01:09 -0400 Subject: [PATCH 01/17] fix(db): start on databases that ran downstream migrations A downstream build released `$7_cache_entries_lookup_indexes` and `$8_upload_capability_hash` on top of the shared `$0`-`$6`. The migrator refuses such a database: the executed migrations are missing from this release. Before migrating, drop those two ledger rows and keep their schema: the `uploads.capabilityHash` column on every driver and, on SQLite, the composite lookup index, which are ensured after migrating so fresh and upgraded databases end up identical. The ledger then lists only this release's migrations, so later releases start on it unchanged. The NOCASE prefix index is dropped: with `case_sensitive_like = on` the binary composite index serves prefix matches. On SQLite, migrate while holding the write lock. The `node-cluster` primary and its worker both migrate on start and Kysely's SQLite migration lock is a no-op, so the loser failed with `duplicate column name: reservationHash`; under the lock it waits and finds nothing left to apply, and an interrupted migration leaves no partial schema. Signed-off-by: Rui Chen --- lib/db.ts | 19 +- lib/migrations.ts | 120 ++++++++++++- tests/migrations.test.ts | 367 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 487 insertions(+), 19 deletions(-) create mode 100644 tests/migrations.test.ts diff --git a/lib/db.ts b/lib/db.ts index 8558458..5413c96 100644 --- a/lib/db.ts +++ b/lib/db.ts @@ -4,14 +4,13 @@ import { createSingletonPromise } from '@antfu/utils' import SQLite from 'better-sqlite3' import { createHooks } from 'hookable' import { Kysely, MysqlDialect, PostgresDialect, SqliteDialect } from 'kysely' -import { Migrator } from 'kysely/migration' import { createPool } from 'mysql2' import pg from 'pg' import { match } from 'ts-pattern' import z from 'zod' import { env } from './env' import { logger } from './logger' -import { migrations } from './migrations' +import { migrateToLatest } from './migrations' // we only really use bigint for timestamps, so we can safely parse them as numbers pg.types.setTypeParser(20, Number) @@ -203,17 +202,11 @@ export const getDatabase = createSingletonPromise(async () => { const hooks = createHooks<{ afterMigrate: () => Promise }>() - const migrator = new Migrator({ - db, - provider: { - async getMigrations() { - return migrations(env.DB_DRIVER, hooks) - }, - }, - }) - const { error, results } = await migrator.migrateToLatest() - if (error) { - logger.error('Database migration failed', error) + let results + try { + results = await migrateToLatest(db, env.DB_DRIVER, hooks) + } catch (err) { + logger.error('Database migration failed', err) // eslint-disable-next-line unicorn/no-process-exit process.exit(1) } diff --git a/lib/migrations.ts b/lib/migrations.ts index ad7c9d7..907a2c9 100644 --- a/lib/migrations.ts +++ b/lib/migrations.ts @@ -2,16 +2,17 @@ import type { Hookable } from 'hookable' import type { Kysely } from 'kysely' import type { Migration } from 'kysely/migration' import type { Env } from './schemas' +import { setTimeout as sleep } from 'node:timers/promises' import { sql } from 'kysely' +import { Migrator } from 'kysely/migration' import { reservationHash } from './helpers' import { Storage } from './storage' -export function migrations( - driver: Env['DB_DRIVER'], - hooks: Hookable<{ - afterMigrate: () => Promise - }>, -) { +type MigrationHooks = Hookable<{ + afterMigrate: () => Promise +}> + +export function migrations(driver: Env['DB_DRIVER'], hooks: MigrationHooks) { // MySQL requires `DROP INDEX ... ON table`, other drivers reject it async function dropIndex(db: Kysely, index: string, table: string) { const query = db.schema.dropIndex(index) @@ -287,3 +288,110 @@ export function migrations( }, } satisfies Record } + +/** + * Migrations a downstream build released on top of the shared `$0`-`$6`. Their + * ledger rows are dropped, their schema is kept by `ensureDownstreamSchema`: an + * executed migration a build doesn't know makes its migrator refuse to start, + * so a ledger listing only this build's migrations keeps the database usable by + * any later release. + */ +const DOWNSTREAM_MIGRATIONS = ['$7_cache_entries_lookup_indexes', '$8_upload_capability_hash'] + +async function forgetDownstreamMigrations(db: Kysely) { + const tables = await db.introspection.getTables({ withInternalKyselyTables: true }) + if (tables.every((table) => table.name !== 'kysely_migration')) return + await db.deleteFrom('kysely_migration').where('name', 'in', DOWNSTREAM_MIGRATIONS).execute() +} + +async function hasColumn(db: Kysely, table: string, column: string) { + const tables = await db.introspection.getTables() + return tables.some((t) => t.name === table && t.columns.some((c) => c.name === column)) +} + +/** Idempotent: runs on every start, after the migrations. */ +async function ensureDownstreamSchema(db: Kysely, driver: Env['DB_DRIVER']) { + // SHA-256 of the capability carried by an Upload's URL; nullable, Uploads + // created without one have none. + if (!(await hasColumn(db, 'uploads', 'capabilityHash'))) { + try { + await db.schema.alterTable('uploads').addColumn('capabilityHash', 'varchar(64)').execute() + } catch (err) { + // another replica added it first + if (!(await hasColumn(db, 'uploads', 'capabilityHash'))) throw err + } + } + + // Every `matchCacheEntry` probe filters on repoId, scope and version and then + // on the key; with `case_sensitive_like = on` this binary index also serves + // the prefix matches. SQLite only: MySQL's four utf8mb4 varchar columns + // exceed InnoDB's 3072-byte index key limit, and Postgres limits a B-tree + // entry to about a third of a page, which four unbounded text columns of an + // existing row could exceed. + if (driver !== 'sqlite') return + await db.schema + .createIndex('idx_cache_entries_lookup') + .ifNotExists() + .on('cache_entries') + .columns(['repoId', 'scope', 'version', 'key']) + .execute() + // served prefix matches while `like` ignored case + await db.schema.dropIndex('idx_cache_entries_lookup_prefix').ifExists().execute() +} + +/** Upper bound for waiting on another process that is migrating the SQLite file. */ +const SQLITE_STARTUP_LOCK_TIMEOUT_MS = 120_000 + +/** + * Runs `run` while holding SQLite's write lock so that only one process + * migrates: the `node-cluster` primary and its workers all migrate on start, + * and Kysely's SQLite migration lock is a no-op. A process that waits here + * sees the finished schema and has nothing left to apply. SQLite DDL is + * transactional, so a failed or interrupted migration leaves no partial schema. + */ +async function withSqliteWriteLock(db: Kysely, run: () => Promise) { + const deadline = Date.now() + SQLITE_STARTUP_LOCK_TIMEOUT_MS + for (;;) { + try { + await sql`begin immediate`.execute(db) + break + } catch (err) { + // better-sqlite3 reports extended codes: SQLITE_BUSY, SQLITE_BUSY_SNAPSHOT, ... + const code = (err as { code?: unknown } | null)?.code + if (typeof code !== 'string' || !code.startsWith('SQLITE_BUSY') || Date.now() >= deadline) + throw err + await sleep(100) + } + } + try { + const result = await run() + await sql`commit`.execute(db) + return result + } catch (err) { + await sql`rollback`.execute(db) + throw err + } +} + +export async function migrateToLatest( + db: Kysely, + driver: Env['DB_DRIVER'], + hooks: MigrationHooks, +) { + const migrator = new Migrator({ + db, + provider: { + async getMigrations() { + return migrations(driver, hooks) + }, + }, + }) + const migrate = async () => { + await forgetDownstreamMigrations(db) + const { error, results } = await migrator.migrateToLatest() + if (error) throw error + await ensureDownstreamSchema(db, driver) + return results + } + return driver === 'sqlite' ? withSqliteWriteLock(db, migrate) : migrate() +} diff --git a/tests/migrations.test.ts b/tests/migrations.test.ts new file mode 100644 index 0000000..f5510a5 --- /dev/null +++ b/tests/migrations.test.ts @@ -0,0 +1,367 @@ +import type { Migration } from 'kysely/migration' +import path from 'node:path' +import { setTimeout as sleep } from 'node:timers/promises' + +import SQLite from 'better-sqlite3' +import { createHooks } from 'hookable' +import { Kysely, MysqlDialect, PostgresDialect, sql, SqliteDialect } from 'kysely' +import { Migrator } from 'kysely/migration' +import { createPool } from 'mysql2' +import pg from 'pg' +import { match } from 'ts-pattern' +import { afterAll, describe, expect, test } from 'vitest' +import { env } from '~/lib/env' +import { migrateToLatest, migrations } from '~/lib/migrations' +import { TEST_TEMP_DIR } from './setup' + +// never fired: `$3_repoId` would clear the shared test storage +const hooks = createHooks<{ afterMigrate: () => Promise }>() +const upstream = migrations(env.DB_DRIVER, hooks) +// code unit order, as Kysely sorts migration names +const byName = (a: string, b: string) => (a < b ? -1 : Number(a > b)) +const UPSTREAM_LEDGER = Object.keys(upstream).toSorted(byName) + +/** + * The migrations of a downstream build that released two migrations of its own + * on top of `$0`-`$6`, as it released them. + */ +const downstream: Record = { + ...Object.fromEntries(Object.entries(upstream).filter(([name]) => name < '$7')), + $7_cache_entries_lookup_indexes: { + async up(db) { + if (env.DB_DRIVER !== 'sqlite') return + await db.schema + .createIndex('idx_cache_entries_lookup') + .on('cache_entries') + .columns(['repoId', 'scope', 'version', 'key']) + .execute() + await sql`create index ${sql.ref('idx_cache_entries_lookup_prefix')} on ${sql.table('cache_entries')} (${sql.ref('repoId')}, ${sql.ref('scope')}, ${sql.ref('version')}, ${sql.ref('key')} collate nocase)`.execute( + db, + ) + }, + }, + $8_upload_capability_hash: { + async up(db) { + await db.schema.alterTable('uploads').addColumn('capabilityHash', 'varchar(64)').execute() + }, + }, +} +const DOWNSTREAM_LEDGER = Object.keys(downstream).toSorted(byName) + +/** Reverts a database migrated by this build to the downstream release (SQLite). */ +const ROLLBACK_TO_DOWNSTREAM_SQL = ` +begin immediate; +delete from uploads +where capabilityHash is null + and createdAt >= (select cast(strftime('%s', timestamp) as integer) * 1000 + from kysely_migration where name = '$7_uploads_reservationHash'); +drop index "idx_uploads_reservationHash"; +alter table "uploads" drop column "reservationHash"; +create index if not exists "idx_cache_entries_lookup" on "cache_entries" ("repoId", "scope", "version", "key"); +create index if not exists "idx_cache_entries_lookup_prefix" on "cache_entries" ("repoId", "scope", "version", "key" collate nocase); +delete from kysely_migration where name in ('$7_uploads_reservationHash', '$8_exact_key_scope_collation'); +insert into kysely_migration (name, timestamp) values + ('$7_cache_entries_lookup_indexes', strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + ('$8_upload_capability_hash', strftime('%Y-%m-%dT%H:%M:%fZ', 'now')); +commit; +` + +const databases: Kysely[] = [] + +afterAll(async () => { + for (const db of databases) await db.destroy() +}) + +function sqliteFile(name: string) { + return path.join(TEST_TEMP_DIR, `migrations_${name}.sqlite`) +} + +async function createDatabase(name: string) { + const dbName = `migrations_${name}_${Date.now()}` + const db = await match(env) + .with({ DB_DRIVER: 'sqlite' }, async () => { + const database = new SQLite(sqliteFile(dbName)) + return new Kysely({ dialect: new SqliteDialect({ database }) }) + }) + .with({ DB_DRIVER: 'postgres' }, async (env) => { + const connection = { + host: env.DB_POSTGRES_HOST, + port: env.DB_POSTGRES_PORT, + user: env.DB_POSTGRES_USER, + password: env.DB_POSTGRES_PASSWORD, + } + // dropped with the test container + const admin = new pg.Client({ ...connection, database: env.DB_POSTGRES_DATABASE }) + await admin.connect() + await admin.query(`create database "${dbName}"`) + await admin.end() + return new Kysely({ + dialect: new PostgresDialect({ pool: new pg.Pool({ ...connection, database: dbName }) }), + }) + }) + .with({ DB_DRIVER: 'mysql' }, async (env) => { + const connection = { + host: env.DB_MYSQL_HOST, + port: env.DB_MYSQL_PORT, + user: env.DB_MYSQL_USER, + password: env.DB_MYSQL_PASSWORD, + } + const admin = createPool(connection).promise() + await admin.query(`create database \`${dbName}\``) + await admin.end() + return new Kysely({ + dialect: new MysqlDialect({ pool: createPool({ ...connection, database: dbName }) }), + }) + }) + .exhaustive() + databases.push(db) + return db +} + +/** Kysely's default migrator, as the downstream release and upstream releases run it. */ +async function migrateStrictly(db: Kysely, list: Record) { + return new Migrator({ + db, + provider: { + async getMigrations() { + return list + }, + }, + }).migrateToLatest() +} + +async function ledger(db: Kysely) { + const rows = await db.selectFrom('kysely_migration').select('name').execute() + return rows.map((row) => row.name as string).toSorted(byName) +} + +/** Columns and indexes of the application tables, independent of column order. */ +async function schema(db: Kysely) { + return match(env.DB_DRIVER) + .with('sqlite', async () => { + const tables = await sql<{ name: string }>` + select name from sqlite_master + where type = 'table' and name not like 'sqlite_%' and name not like 'kysely_%' + order by name`.execute(db) + const columns = [] + for (const { name } of tables.rows) { + const info = await sql>` + select name, type, "notnull", dflt_value, pk from pragma_table_info(${name}) + order by name`.execute(db) + columns.push({ table: name, columns: info.rows }) + } + const indexes = await sql` + select name, tbl_name, sql from sqlite_master + where type = 'index' and name not like 'sqlite_%' + order by name`.execute(db) + return { columns, indexes: indexes.rows } + }) + .with('postgres', async () => { + const columns = await sql` + select table_name, column_name, data_type, character_maximum_length, is_nullable, column_default + from information_schema.columns + where table_schema = 'public' and table_name not like 'kysely_%' + order by table_name, column_name`.execute(db) + const indexes = await sql` + select tablename, indexname, indexdef from pg_indexes + where schemaname = 'public' and tablename not like 'kysely_%' + order by indexname`.execute(db) + return { columns: columns.rows, indexes: indexes.rows } + }) + .with('mysql', async () => { + const columns = await sql` + select table_name, column_name, column_type, is_nullable, column_default, collation_name + from information_schema.columns + where table_schema = database() and table_name not like 'kysely_%' + order by table_name, column_name`.execute(db) + const indexes = await sql` + select table_name, index_name, seq_in_index, column_name, non_unique + from information_schema.statistics + where table_schema = database() and table_name not like 'kysely_%' + order by table_name, index_name, seq_in_index`.execute(db) + return { columns: columns.rows, indexes: indexes.rows } + }) + .exhaustive() +} + +const reservation = { key: 'key', version: 'version', scope: 'refs/heads/main', repoId: '1' } +const idleUpload = { + lastPartUploadedAt: null, + finishedPartUploadCount: 0, + startedPartUploadCount: 0, +} + +/** A database migrated and used by the downstream release. */ +async function createDownstreamDatabase(name: string) { + const db = await createDatabase(name) + const { error } = await migrateStrictly(db, downstream) + expect(error).toBeUndefined() + + await db + .insertInto('storage_locations') + .values({ id: 'location', folderName: 'location', partCount: 1, sizeBytes: 3 }) + .execute() + await db + .insertInto('cache_entries') + .values({ id: 'entry', ...reservation, updatedAt: 1, locationId: 'location' }) + .execute() + // two Uploads holding one reservation (a lost check-then-insert race) and an unrelated one + await db + .insertInto('uploads') + .values([ + { + ...idleUpload, + ...reservation, + id: 1, + folderName: '1', + createdAt: 1, + capabilityHash: 'a'.repeat(64), + }, + { + ...idleUpload, + ...reservation, + id: 2, + folderName: '2', + createdAt: 2, + capabilityHash: 'b'.repeat(64), + }, + { + ...idleUpload, + ...reservation, + key: 'other', + id: 3, + folderName: '3', + createdAt: 3, + capabilityHash: null, + }, + ]) + .execute() + // keeps the upgrade's ledger timestamps after these, as separate deploys would + await sleep(10) + return db +} + +describe('migrations', () => { + test('a fresh database migrates to an upstream-only ledger', async () => { + const db = await createDatabase('fresh') + + const results = await migrateToLatest(db, env.DB_DRIVER, hooks) + + expect(results?.map((r) => [r.migrationName, r.status])).toEqual( + UPSTREAM_LEDGER.map((name) => [name, 'Success']), + ) + expect(await ledger(db)).toEqual(UPSTREAM_LEDGER) + const { error, results: strict } = await migrateStrictly(db, upstream) + expect(error).toBeUndefined() + expect(strict).toEqual([]) + }) + + test('a downstream database runs the upstream migrations and matches a fresh one', async () => { + const db = await createDownstreamDatabase('downstream') + expect(await ledger(db)).toEqual(DOWNSTREAM_LEDGER) + + const results = await migrateToLatest(db, env.DB_DRIVER, hooks) + + expect(results?.map((r) => [r.migrationName, r.status])).toEqual([ + ['$7_uploads_reservationHash', 'Success'], + ['$8_exact_key_scope_collation', 'Success'], + ]) + expect(await ledger(db)).toEqual(UPSTREAM_LEDGER) + // the newest Upload keeps the reservation, with its capability hash intact + const uploads = await db + .selectFrom('uploads') + .select(['id', 'capabilityHash', 'reservationHash']) + .orderBy('id') + .execute() + expect(uploads.map((u) => [u.id, u.capabilityHash])).toEqual([ + [2, 'b'.repeat(64)], + [3, null], + ]) + expect(uploads.every((u) => u.reservationHash?.length === 64)).toBe(true) + expect(await db.selectFrom('cache_entries').selectAll().execute()).toEqual([ + { id: 'entry', ...reservation, updatedAt: 1, locationId: 'location' }, + ]) + + const fresh = await createDatabase('reference') + await migrateToLatest(fresh, env.DB_DRIVER, hooks) + expect(await schema(db)).toEqual(await schema(fresh)) + + // an upstream release starts on it, and this build starts on it again + const { error, results: strict } = await migrateStrictly(db, upstream) + expect(error).toBeUndefined() + expect(strict).toEqual([]) + expect(await migrateToLatest(db, env.DB_DRIVER, hooks)).toEqual([]) + expect(await schema(db)).toEqual(await schema(fresh)) + }) + + test.runIf(env.DB_DRIVER === 'sqlite')( + 'the rollback returns a migrated database to the downstream release', + async () => { + const db = await createDownstreamDatabase('rollback') + const downstreamSchema = await schema(db) + await migrateToLatest(db, env.DB_DRIVER, hooks) + await db + .insertInto('uploads') + .values({ + ...idleUpload, + ...reservation, + key: 'new', + id: 4, + folderName: '4', + createdAt: Date.now(), + }) + .execute() + + for (const statement of ROLLBACK_TO_DOWNSTREAM_SQL.split(';')) + if (statement.trim()) await sql.raw(statement).execute(db) + + expect(await ledger(db)).toEqual(DOWNSTREAM_LEDGER) + expect(await schema(db)).toEqual(downstreamSchema) + // the Upload this build created has no capability, the downstream release could not use it + const uploads = await db.selectFrom('uploads').select('id').orderBy('id').execute() + expect(uploads.map((u) => u.id)).toEqual([2, 3]) + const { error, results } = await migrateStrictly(db, downstream) + expect(error).toBeUndefined() + expect(results).toEqual([]) + + await sleep(10) + await migrateToLatest(db, env.DB_DRIVER, hooks) + expect(await ledger(db)).toEqual(UPSTREAM_LEDGER) + }, + ) + + test.runIf(env.DB_DRIVER === 'sqlite')('processes starting together migrate once', async () => { + const file = sqliteFile(`concurrent_${Date.now()}`) + const seed = await createDownstreamDatabase('concurrent') + await sql`vacuum into ${file}`.execute(seed) + // no busy wait inside better-sqlite3: it would block this single-threaded test + const [a, b] = [0, 1].map(() => { + const db = new Kysely({ + dialect: new SqliteDialect({ database: new SQLite(file, { timeout: 0 }) }), + }) + databases.push(db) + return db + }) + + const results = await Promise.all([ + migrateToLatest(a, env.DB_DRIVER, hooks), + migrateToLatest(b, env.DB_DRIVER, hooks), + ]) + + expect(results.map((r) => r?.length ?? -1).toSorted((x, y) => x - y)).toEqual([0, 2]) + expect(await ledger(a)).toEqual(UPSTREAM_LEDGER) + }) + + test('a database with executed migrations unknown to this build is refused', async () => { + const db = await createDatabase('unknown') + await migrateToLatest(db, env.DB_DRIVER, hooks) + await db + .insertInto('kysely_migration') + .values({ name: '$9_from_the_future', timestamp: new Date().toISOString() }) + .execute() + + await expect(migrateToLatest(db, env.DB_DRIVER, hooks)).rejects.toThrow( + 'previously executed migration $9_from_the_future is missing', + ) + }) +}) From 5b7ab0af5eabf567dcf458dedca7f533bea7401e Mon Sep 17 00:00:00 2001 From: Louis Haftmann <30736553+LouisHaftmann@users.noreply.github.com> Date: Sat, 24 Oct 2026 16:56:42 +0200 Subject: [PATCH 02/17] fix(download): answer Range on an empty cache entry the same way on every storage driver (#291) Co-authored-by: Peter Svensson Signed-off-by: Rui Chen --- routes/download/[locationId].ts | 8 ++++++++ tests/proxy-backpressure.test.ts | 20 +++++++++++++++++++ tests/range-download.test.ts | 34 ++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/routes/download/[locationId].ts b/routes/download/[locationId].ts index ddc4909..c31513c 100644 --- a/routes/download/[locationId].ts +++ b/routes/download/[locationId].ts @@ -44,6 +44,14 @@ export default defineEventHandler(async (event) => { download = await storage.download(locationId, range) } catch (err) { if (!(err instanceof RangeNotSatisfiableError)) throw err + // An empty entry has no satisfiable range. The filesystem and GCS adapters raise + // for any Range on one and Azure for any start past 0, while RustFS (S3) answers + // `bytes=0-` with an empty 200. Ignore Range on empty entries (RFC 9110 ยง14.2) so + // the response does not depend on the storage driver. + if (err.size === 0) { + setHeader(event, 'content-length', 0) + return send(event) + } setResponseStatus(event, 416, 'Range Not Satisfiable') if (err.size !== undefined) setHeader(event, 'content-range', `bytes */${err.size}`) return send(event) diff --git a/tests/proxy-backpressure.test.ts b/tests/proxy-backpressure.test.ts index 2fc5af7..9a7ff4f 100644 --- a/tests/proxy-backpressure.test.ts +++ b/tests/proxy-backpressure.test.ts @@ -147,4 +147,24 @@ describe('proxy routes apply backpressure', () => { .toBeLessThan(MAX_BUFFERED_BYTES) expect.soft(queued, 'bytes queued in the response').toBeLessThan(MAX_BUFFERED_BYTES) }, 60_000) + + test('download destroys the storage stream when the client hangs up', async () => { + // Never ends on its own, so only the route tearing it down can stop the read + // (and release the reader lease that `Storage.download` ties to `close`). + function* endless() { + while (true) yield Buffer.alloc(CHUNK_BYTES) + } + const source = Readable.from(endless(), { objectMode: false, highWaterMark: CHUNK_BYTES }) + const storage = await getStorage() + vi.spyOn(storage, 'download').mockResolvedValue({ stream: source }) + + const { port } = server.address() as AddressInfo + const req = http.get(`http://localhost:${port}/download/${randomUUID()}`) + req.on('error', () => undefined) + const [res] = (await once(req, 'response')) as [IncomingMessage] + expect(res.statusCode).toBe(200) + res.once('data', () => req.destroy()) + + await vi.waitFor(() => expect(source.destroyed).toBe(true), { timeout: 5000, interval: 50 }) + }, 20_000) }) diff --git a/tests/range-download.test.ts b/tests/range-download.test.ts index d971a10..6dbe819 100644 --- a/tests/range-download.test.ts +++ b/tests/range-download.test.ts @@ -125,6 +125,40 @@ describe('proxy download route on unmerged entries', () => { }) }) +test.each(['bytes=0-', 'bytes=5-'])( + 'an empty merged entry answers %s with an empty 200', + async (range) => { + const storage = await getStorage() + const key = `range-empty-${crypto.randomUUID()}` + const upload = await storage.createUpload({ + key, + version: VERSION, + scope: SCOPE, + repoId: REPO_ID, + }) + await storage.uploadPart(upload.id, 0, Readable.from(Buffer.alloc(0))) + await storage.completeUpload({ key, version: VERSION, scope: SCOPE, repoId: REPO_ID }) + const matched = await storage.matchCacheEntry({ + keys: [key], + version: VERSION, + scopes: [SCOPE], + repoId: REPO_ID, + }) + const id = matched!.match.locationId + const first = await storage.download(id) + for await (const _ of first!.stream) { + /* drain */ + } + await storage.waitForOngoingMerges() + + const { res, body } = await get(id, range) + expect(res.status).toBe(200) + expect(res.headers.get('content-range')).toBeNull() + expect(res.headers.get('content-length')).toBe('0') + expect(body).toHaveLength(0) + }, +) + test('a re-save does not change the bytes behind a handed-out URL', async () => { const { id, key } = await createEntry() const resaved = await createEntry(key) From 951880a1a2a9a0b134fa267354df9626c6c015f0 Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Sat, 24 Oct 2026 15:35:21 -0400 Subject: [PATCH 03/17] fix(download): stop the merge pump waiting on a destroyed stream The first download of an unmerged entry tees its parts into the client response and the background merge upload. A destroyed stream returns false from write() and never emits 'drain' (nor 'error' when destroyed without one), so the pump parks forever when: - the merge upload fails: `download()` destroys the merger stream, and the client's download stalls mid-body until the client times out; - the client hangs up while the pump waits on the merger: the next write to the destroyed response stream parks the pump, the merge never completes, and its lease keeps renewing. Either way the current part's backend read stays open (one S3 socket per occurrence), so enough of them fill the S3 socket pool. Wait for 'drain' or 'close', skip destroyed sinks, keep feeding the one that is still open, and stop reading once both are gone so `streamParts` destroys the current part stream. The regression tests fail on the previous pump with the filesystem and S3 drivers: the merge never settles after the client leaves, the client never finishes after the merge upload fails, and the part stream is never destroyed when both are gone. Signed-off-by: Rui Chen --- lib/storage.ts | 38 +++++- tests/download-merge-sink-gone.test.ts | 158 +++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 6 deletions(-) create mode 100644 tests/download-merge-sink-gone.test.ts diff --git a/lib/storage.ts b/lib/storage.ts index d40ce61..afe6826 100644 --- a/lib/storage.ts +++ b/lib/storage.ts @@ -1,11 +1,11 @@ /* eslint-disable ts/method-signature-style */ import type { ContainerClient } from '@azure/storage-blob' import type { Kysely } from 'kysely' +import type { Writable } from 'node:stream' import type { Database, StorageLocation } from './db' import type { Reservation } from './helpers' import type { Env } from './schemas' import { randomUUID } from 'node:crypto' -import { once } from 'node:events' import { createReadStream, createWriteStream } from 'node:fs' import fs from 'node:fs/promises' import { Agent } from 'node:https' @@ -66,6 +66,23 @@ function escapeLikePattern(value: string) { .replaceAll('_', String.raw`\_`) } +/** + * Resolves once `stream` wants more data or is closed. A destroyed stream emits + * no 'drain', and no 'error' when destroyed without one, so waiting on 'drain' + * alone hangs forever once its reader is gone. + */ +function drainOrClose(stream: Writable) { + const { promise, resolve } = Promise.withResolvers() + const done = () => { + stream.off('drain', done) + stream.off('close', done) + resolve() + } + stream.on('drain', done) + stream.on('close', done) + return promise +} + export class ObjectNotFoundError extends Error { constructor(objectName: string) { super(`Object not found in storage: ${objectName}`) @@ -198,16 +215,25 @@ export class Storage { ) { if (location.partsDeletedAt) throw new Error('No parts to feed') + // Either sink can be destroyed mid-download: the route destroys the response + // when the client hangs up, and a failed merge upload destroys the merger. + // Waiting on a destroyed sink's 'drain' would park the pump forever, pinning + // the current part's backend read and renewing the merge lease. Keep feeding + // whichever sink is still open, and stop reading once both are gone. + const sinks = [responseStream, mergerStream] for await (const chunk of this.streamParts(location)) { - // attach both drain listeners simultaneously to prevent a race condition where the second stream being faster hangs forever + // attach every drain wait before awaiting any, so a faster sink's 'drain' is never missed const drains = [] - if (!responseStream.write(chunk)) drains.push(once(responseStream, 'drain')) - if (!mergerStream.write(chunk)) drains.push(once(mergerStream, 'drain')) + for (const sink of sinks) { + if (sink.destroyed) continue + if (!sink.write(chunk) && !sink.destroyed) drains.push(drainOrClose(sink)) + } await Promise.all(drains) + // Leaving the loop makes `streamParts` destroy the current part stream. + if (sinks.every((sink) => sink.destroyed)) return } - responseStream.end() - mergerStream.end() + for (const sink of sinks) if (!sink.destroyed) sink.end() await globalThis.gc?.() } diff --git a/tests/download-merge-sink-gone.test.ts b/tests/download-merge-sink-gone.test.ts new file mode 100644 index 0000000..151fbd7 --- /dev/null +++ b/tests/download-merge-sink-gone.test.ts @@ -0,0 +1,158 @@ +import { Buffer } from 'node:buffer' +import { randomUUID } from 'node:crypto' +import { once } from 'node:events' +import { Readable } from 'node:stream' +import { setTimeout as sleep } from 'node:timers/promises' +import { afterEach, describe, expect, test, vi } from 'vitest' +import { getDatabase } from '~/lib/db' +import { getStorage } from '~/lib/storage' + +const PART_COUNT = 3 +const PART_BYTES = 1024 * 1024 + +async function createUnmergedEntry() { + const storage = await getStorage() + const db = await getDatabase() + const key = `merge-sink-gone-${randomUUID()}` + const entry = { key, version: 'v1', scope: 'refs/heads/main', repoId: '123' } + const upload = await storage.createUpload(entry) + if (!upload) throw new Error('createUpload returned nothing') + for (let i = 0; i < PART_COUNT; i++) + await storage.uploadPart(upload.id, i, Readable.from([Buffer.alloc(PART_BYTES, i)])) + await storage.completeUpload(entry) + return db + .selectFrom('cache_entries') + .where('key', '=', key) + .select('locationId') + .executeTakeFirstOrThrow() +} + +/** Records every part stream the download opens, so the test can see each one released. */ +async function trackPartStreams() { + const storage = await getStorage() + const original = storage.adapter.createDownloadStream.bind(storage.adapter) + const parts: Readable[] = [] + vi.spyOn(storage.adapter, 'createDownloadStream').mockImplementation(async (name, range) => { + const download = await original(name, range) + if (name.includes('/parts/')) parts.push(download.stream) + return download + }) + return parts +} + +/** Fails the background merge upload without reading it, as a broken S3 upload would. */ +async function failMergeUpload() { + const storage = await getStorage() + vi.spyOn(storage.adapter, 'uploadStream').mockRejectedValue(new Error('merge upload failed')) +} + +/** Reads one chunk, then hangs up the way the route does when the client leaves. */ +async function leaveAfterFirstChunk(stream: Readable) { + await once(stream, 'readable') + expect(stream.read()).not.toBeNull() + stream.destroy() +} + +async function settles(promise: Promise, timeoutMs = 5000) { + return Promise.race([promise.then(() => 'settled'), sleep(timeoutMs, 'still pending')]) +} + +async function locationState(locationId: string) { + const db = await getDatabase() + const { mergedAt } = await db + .selectFrom('storage_locations') + .where('id', '=', locationId) + .select('mergedAt') + .executeTakeFirstOrThrow() + const mergeLeases = await db + .selectFrom('merge_leases') + .where('storageLocationId', '=', locationId) + .select('token') + .execute() + return { mergedAt, mergeLeases } +} + +afterEach(() => { + vi.restoreAllMocks() +}) + +/** + * The first download of an unmerged entry tees its parts into the client + * response and the background merge upload. When one of them is destroyed + * (client hung up, merge upload failed), a pump waiting for 'drain' parks + * forever: a destroyed stream emits neither 'drain' nor 'error'. That holds the + * part's backend read (an S3 socket), starves the merge, and keeps renewing + * the merge lease for the life of the process. + */ +describe('first download of an unmerged entry', () => { + test('client leaving mid-stream still completes the merge and releases the parts', async () => { + const storage = await getStorage() + const { locationId } = await createUnmergedEntry() + const parts = await trackPartStreams() + + const download = await storage.download(locationId) + if (!download) throw new Error('download returned nothing') + const { stream } = download + await leaveAfterFirstChunk(stream) + + expect(await settles(storage.waitForOngoingMerges()), 'merge settled').toBe('settled') + const { mergedAt, mergeLeases } = await locationState(locationId) + expect(mergedAt, 'merge completed').not.toBeNull() + expect(mergeLeases, 'merge lease released').toHaveLength(0) + expect(parts).toHaveLength(PART_COUNT) + expect( + parts.filter((part) => !part.destroyed), + 'part streams left open', + ).toHaveLength(0) + }, 20_000) + + test('a failed merge upload still serves the whole entry to the client', async () => { + const storage = await getStorage() + const { locationId } = await createUnmergedEntry() + await failMergeUpload() + + const download = await storage.download(locationId) + if (!download) throw new Error('download returned nothing') + const { stream } = download + let received = 0 + const consumed = (async () => { + for await (const chunk of stream) received += (chunk as Buffer).length + })() + + expect(await settles(consumed), 'client download finished').toBe('settled') + expect(received).toBe(PART_COUNT * PART_BYTES) + expect(await settles(storage.waitForOngoingMerges()), 'merge settled').toBe('settled') + const { mergedAt, mergeLeases } = await locationState(locationId) + expect(mergedAt, 'failed merge not marked complete').toBeNull() + expect(mergeLeases, 'merge lease released').toHaveLength(0) + }, 20_000) + + test('client and merge both gone stops reading and destroys the part stream', async () => { + const storage = await getStorage() + const { locationId } = await createUnmergedEntry() + await failMergeUpload() + // An unending part: only being destroyed lets it go, as with a GetObject body. + const part = Readable.from( + (async function* () { + for (;;) { + await sleep(1) + yield Buffer.alloc(64 * 1024) + } + })(), + ) + vi.spyOn(storage.adapter, 'createDownloadStream').mockResolvedValue({ stream: part }) + + const download = await storage.download(locationId) + if (!download) throw new Error('download returned nothing') + const { stream } = download + await leaveAfterFirstChunk(stream) + + await vi.waitFor(() => expect(part.destroyed, 'part stream destroyed').toBe(true), { + timeout: 5000, + interval: 25, + }) + expect(await settles(storage.waitForOngoingMerges()), 'merge settled').toBe('settled') + const { mergeLeases } = await locationState(locationId) + expect(mergeLeases, 'merge lease released').toHaveLength(0) + }, 20_000) +}) From fd4ddda5c96ba862557ce89633e5cba329aeb3de Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Sat, 24 Oct 2026 15:44:43 -0400 Subject: [PATCH 04/17] perf(storage): stop forcing a full GC after every part `streamParts()` awaited `globalThis.gc()` after every part, and the image starts Node with `--expose-gc`, so every part of every unmerged restore and streaming merge ran a synchronous full mark-compact on the event loop. With ~1 MiB parts (BuildKit) that caps all concurrent first restores combined at roughly one part per GC pause, and the pause grows with the heap, so more concurrent restores make every one of them slower. Memory no longer depends on it: the upload route lets storage pace the body (#275) and the download route pipes with backpressure (#282), so the bytes in flight are bounded by stream high-water marks and become ordinary garbage that V8 collects incrementally. Remove both calls, the `--expose-gc` flag and the startup warning about it. Signed-off-by: Rui Chen --- Dockerfile | 2 +- lib/storage.ts | 4 ---- plugins/setup.ts | 5 ----- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 72b6c0a..7a9aac9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,4 +36,4 @@ RUN if echo "$BASE_IMAGE" | grep -q "node-caged"; then \ echo "export NODE_CAGED=false" > /app/.runtime-env; \ fi -CMD ["sh", "-c", ". /app/.runtime-env && exec node --expose-gc /app/server/index.mjs"] +CMD ["sh", "-c", ". /app/.runtime-env && exec node /app/server/index.mjs"] diff --git a/lib/storage.ts b/lib/storage.ts index afe6826..45d60d2 100644 --- a/lib/storage.ts +++ b/lib/storage.ts @@ -234,8 +234,6 @@ export class Storage { } for (const sink of sinks) if (!sink.destroyed) sink.end() - - await globalThis.gc?.() } private async *streamParts(location: StorageLocation) { @@ -247,8 +245,6 @@ export class Storage { ) for await (const chunk of partStream) yield chunk - - await globalThis.gc?.() } } diff --git a/plugins/setup.ts b/plugins/setup.ts index 783d73e..b9975b3 100644 --- a/plugins/setup.ts +++ b/plugins/setup.ts @@ -12,11 +12,6 @@ export default defineNitroPlugin(async (nitro) => { if (cluster.isPrimary) { logger.info(`๐Ÿš€ Starting GitHub Actions Cache Server (${version})`) - if (!globalThis.gc) - logger.warn( - 'Garbage collection is not exposed. Start the process with `node --expose-gc` for improved memory usage under high load.', - ) - for (const signal of ['SIGTERM', 'SIGINT'] satisfies NodeJS.Signals[]) { process.on(signal, () => { const workers = Object.values(cluster.workers ?? {}) From ec55882762f5264ceb5fdf18341ba3878de13bef Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Sat, 24 Oct 2026 16:15:58 -0400 Subject: [PATCH 05/17] fix(s3): make the socket pool size configurable and apply it to http endpoints The S3 client's agent pool was hard-coded at 50 sockets, and the agent was passed only as `httpsAgent`. NodeHttpHandler picks the agent by endpoint scheme, so an http:// endpoint (MinIO, RustFS, an in-cluster gateway) silently fell back to the SDK's default agent and ignored the configured pool. Local tests and benchmarks against RustFS therefore never exercised the pool an https deployment uses. Add STORAGE_S3_MAX_SOCKETS (default 50, the previous value) and build both the http and the https agent from the same options. tests/s3-socket-pool.test.ts holds one GetObject body unread with a pool of one socket and checks that a second GetObject waits for it; against an http:// endpoint it did not wait before this change. Signed-off-by: Rui Chen --- README.md | 8 +++++++ lib/schemas.ts | 1 + lib/storage.ts | 16 +++++++++----- tests/s3-socket-pool.test.ts | 41 ++++++++++++++++++++++++++++++++++++ tests/setup.ts | 1 + 5 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 tests/s3-socket-pool.test.ts diff --git a/README.md b/README.md index 9030701..b276ff3 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,14 @@ volumes: cache-data: ``` +## Resource limits + +All optional. + +| Variable | Default | Effect | +| ------------------------ | ------- | ----------------------------------------------------------------------------- | +| `STORAGE_S3_MAX_SOCKETS` | `50` | Sockets per S3 connection pool, for `http://` and `https://` endpoints alike. | + ## Documentation ๐Ÿ‘‰ ๐Ÿ‘ˆ diff --git a/lib/schemas.ts b/lib/schemas.ts index bbae5f5..1fa283f 100644 --- a/lib/schemas.ts +++ b/lib/schemas.ts @@ -5,6 +5,7 @@ export const envStorageDriverSchema = type.or( 'STORAGE_DRIVER': type.unit('s3'), 'STORAGE_S3_BUCKET': 'string', 'STORAGE_S3_SOCKET_TIMEOUT_MS': 'number.integer >= 0 = 10000', + 'STORAGE_S3_MAX_SOCKETS': 'number.integer >= 1 = 50', 'AWS_REGION': "string = 'us-east-1'", 'AWS_ENDPOINT_URL?': 'string.url', 'AWS_ACCESS_KEY_ID?': 'string', diff --git a/lib/storage.ts b/lib/storage.ts index 45d60d2..a11e212 100644 --- a/lib/storage.ts +++ b/lib/storage.ts @@ -8,7 +8,8 @@ import type { Env } from './schemas' import { randomUUID } from 'node:crypto' import { createReadStream, createWriteStream } from 'node:fs' import fs from 'node:fs/promises' -import { Agent } from 'node:https' +import { Agent as HttpAgent } from 'node:http' +import { Agent as HttpsAgent } from 'node:https' import path from 'node:path' import { PassThrough, Readable } from 'node:stream' import { pipeline } from 'node:stream/promises' @@ -902,16 +903,21 @@ function accumulateFolder( class S3Adapter implements StorageAdapter { static async fromEnv(env: Extract) { const bucket = env.STORAGE_S3_BUCKET - const agent = new Agent({ + // Both protocols get the configured pool: NodeHttpHandler picks the agent by + // endpoint scheme, and an http:// endpoint (MinIO, RustFS, an in-cluster + // gateway) would otherwise fall back to the SDK's default agent and ignore + // STORAGE_S3_MAX_SOCKETS. + const agentOptions = { keepAlive: true, - maxSockets: 50, + maxSockets: env.STORAGE_S3_MAX_SOCKETS, keepAliveMsecs: 1000, - }) + } const s3 = new S3Client({ forcePathStyle: true, region: env.AWS_REGION, requestHandler: new NodeHttpHandler({ - httpsAgent: agent, + httpAgent: new HttpAgent(agentOptions), + httpsAgent: new HttpsAgent(agentOptions), socketTimeout: env.STORAGE_S3_SOCKET_TIMEOUT_MS, }), }) diff --git a/tests/s3-socket-pool.test.ts b/tests/s3-socket-pool.test.ts new file mode 100644 index 0000000..864f937 --- /dev/null +++ b/tests/s3-socket-pool.test.ts @@ -0,0 +1,41 @@ +import type { Env } from '~/lib/schemas' +import { randomUUID } from 'node:crypto' +import { Readable } from 'node:stream' +import { setTimeout as sleep } from 'node:timers/promises' + +import { afterEach, describe, expect, test } from 'vitest' +import { env } from '~/lib/env' +import { Storage } from '~/lib/storage' + +const s3Env = env as Extract + +// The test endpoint is http://, so this also covers the agent NodeHttpHandler +// picks for plain-http endpoints (MinIO, RustFS, in-cluster gateways). +describe.skipIf(process.env.VITEST_STORAGE_DRIVER !== 's3')('s3 socket pool', () => { + const originalMaxSockets = s3Env.STORAGE_S3_MAX_SOCKETS + afterEach(() => { + s3Env.STORAGE_S3_MAX_SOCKETS = originalMaxSockets + }) + + test('the configured pool size bounds the sockets used for an http endpoint', async () => { + s3Env.STORAGE_S3_MAX_SOCKETS = 1 + const storage = await Storage.fromEnv() + const folder = `socket-pool-${randomUUID()}` + await storage.adapter.uploadStream(`${folder}/a`, Readable.from([Buffer.from('a')])) + await storage.adapter.uploadStream(`${folder}/b`, Readable.from([Buffer.from('b')])) + + try { + // An unread body keeps its socket: the only one in the pool. + const held = await storage.adapter.createDownloadStream(`${folder}/a`) + const second = storage.adapter.createDownloadStream(`${folder}/b`) + const queued = await Promise.race([second.then(() => false), sleep(1000, true)]) + expect(queued, 'second GET queued for the only socket').toBe(true) + + held.stream.destroy() + const { stream } = await second + expect(Buffer.concat(await stream.toArray()).toString()).toBe('b') + } finally { + await storage.adapter.deleteFolder(folder) + } + }) +}) diff --git a/tests/setup.ts b/tests/setup.ts index 5ca471e..75f8c9e 100644 --- a/tests/setup.ts +++ b/tests/setup.ts @@ -85,6 +85,7 @@ const TESTING_ENV_BY_STORAGE_DRIVER = { AWS_REGION: 'us-east-1', STORAGE_S3_BUCKET: 'vitest', STORAGE_S3_SOCKET_TIMEOUT_MS: 10_000, + STORAGE_S3_MAX_SOCKETS: 50, AWS_ACCESS_KEY_ID: 'rustfsadmin', AWS_SECRET_ACCESS_KEY: 'rustfsadmin', AWS_ENDPOINT_URL: 'http://localhost:9000', From fe9af9a54a84d1613e5c07029ada740108de69c6 Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Sat, 24 Oct 2026 16:27:21 -0400 Subject: [PATCH 06/17] fix(s3): separate socket pools for reads, writes and control requests All S3 traffic shared one client and one agent pool. A streaming Merge (the first restore of an unmerged entry, or a streaming Eager Merge) holds a part GetObject open, paused by backpressure, while lib-storage waits in that same pool for a socket to upload the merged object. Once concurrent Merges reach maxSockets, every socket is a paused GET waiting for a socket that never frees: the pool deadlocks, Merge Leases renew forever, no Merge or restore completes, and every later S3 call queues behind them. That includes the HeadObject each cache lookup issues, which also waits behind every bulk transfer in the agent's FIFO queue even without a deadlock. Use one S3Client per kind of work: - reads: GetObject (and presigning) (STORAGE_S3_MAX_SOCKETS) - writes: PutObject, multipart uploads, UploadPartCopy compose (STORAGE_S3_MAX_SOCKETS) - control: HeadObject, HeadBucket, ListObjectsV2, DeleteObjects (STORAGE_S3_CONTROL_MAX_SOCKETS, default 16) Write-socket holders never wait on reads, and control requests hold a socket only briefly, so no wait cycle spans pools. The control client also sets connectionTimeout (STORAGE_S3_CONTROL_QUEUE_TIMEOUT_MS, default 5000), which bounds the wait for a socket plus the connect, so a starved lookup fails instead of hanging. tests/merge-socket-pool.test.ts runs as many concurrent first restores as pool sockets (4) against RustFS; with the shared pool it never finishes. tests/s3-socket-pool.test.ts checks that a HeadObject is answered while every data socket is held by an unread GetObject body. Signed-off-by: Rui Chen --- README.md | 8 +-- lib/schemas.ts | 2 + lib/storage.ts | 90 ++++++++++++++++++++------------ tests/merge-socket-pool.test.ts | 92 +++++++++++++++++++++++++++++++++ tests/s3-socket-pool.test.ts | 18 +++++++ tests/setup.ts | 2 + 6 files changed, 176 insertions(+), 36 deletions(-) create mode 100644 tests/merge-socket-pool.test.ts diff --git a/README.md b/README.md index b276ff3..6799d39 100644 --- a/README.md +++ b/README.md @@ -32,9 +32,11 @@ volumes: All optional. -| Variable | Default | Effect | -| ------------------------ | ------- | ----------------------------------------------------------------------------- | -| `STORAGE_S3_MAX_SOCKETS` | `50` | Sockets per S3 connection pool, for `http://` and `https://` endpoints alike. | +| Variable | Default | Effect | +| ------------------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| `STORAGE_S3_MAX_SOCKETS` | `50` | Sockets per S3 data pool, for `http://` and `https://` endpoints alike. Reads (GetObject) and writes (uploads, Merges) each get a pool this size. | +| `STORAGE_S3_CONTROL_MAX_SOCKETS` | `16` | Sockets for HEAD, LIST and DELETE requests (cache lookups, cleanup), kept apart from bulk transfers. | +| `STORAGE_S3_CONTROL_QUEUE_TIMEOUT_MS` | `5000` | How long a control request waits for a socket and connection before failing; `0` waits forever. | ## Documentation diff --git a/lib/schemas.ts b/lib/schemas.ts index 1fa283f..e64c979 100644 --- a/lib/schemas.ts +++ b/lib/schemas.ts @@ -6,6 +6,8 @@ export const envStorageDriverSchema = type.or( 'STORAGE_S3_BUCKET': 'string', 'STORAGE_S3_SOCKET_TIMEOUT_MS': 'number.integer >= 0 = 10000', 'STORAGE_S3_MAX_SOCKETS': 'number.integer >= 1 = 50', + 'STORAGE_S3_CONTROL_MAX_SOCKETS': 'number.integer >= 1 = 16', + 'STORAGE_S3_CONTROL_QUEUE_TIMEOUT_MS': 'number.integer >= 0 = 5000', 'AWS_REGION': "string = 'us-east-1'", 'AWS_ENDPOINT_URL?': 'string.url', 'AWS_ACCESS_KEY_ID?': 'string', diff --git a/lib/storage.ts b/lib/storage.ts index a11e212..7519ad3 100644 --- a/lib/storage.ts +++ b/lib/storage.ts @@ -900,30 +900,52 @@ function accumulateFolder( folders.set(folderName, existing) } +/** reads: GetObject; writes: uploads, multipart and part copies; control: HEAD, LIST, DELETE. */ +interface S3Clients { + reads: S3Client + writes: S3Client + control: S3Client +} + class S3Adapter implements StorageAdapter { static async fromEnv(env: Extract) { const bucket = env.STORAGE_S3_BUCKET - // Both protocols get the configured pool: NodeHttpHandler picks the agent by - // endpoint scheme, and an http:// endpoint (MinIO, RustFS, an in-cluster - // gateway) would otherwise fall back to the SDK's default agent and ignore - // STORAGE_S3_MAX_SOCKETS. - const agentOptions = { - keepAlive: true, - maxSockets: env.STORAGE_S3_MAX_SOCKETS, - keepAliveMsecs: 1000, + // One socket pool per kind of work. A streaming Merge holds a part GetObject + // open (paused by backpressure) while it waits for a socket to upload the + // merged object; in one shared pool, as many concurrent Merges as sockets + // deadlock it, and the HeadObject of every cache lookup queues behind all + // bulk transfers. Write holders never wait on reads and control requests + // hold a socket only briefly, so no wait cycle can span pools. + const createClient = (maxSockets: number, connectionTimeout?: number) => { + // Both protocols get the configured pool: NodeHttpHandler picks the agent by + // endpoint scheme, and an http:// endpoint (MinIO, RustFS, an in-cluster + // gateway) would otherwise fall back to the SDK's default agent and ignore + // the configured size. + const agentOptions = { keepAlive: true, maxSockets, keepAliveMsecs: 1000 } + return new S3Client({ + forcePathStyle: true, + region: env.AWS_REGION, + requestHandler: new NodeHttpHandler({ + httpAgent: new HttpAgent(agentOptions), + httpsAgent: new HttpsAgent(agentOptions), + socketTimeout: env.STORAGE_S3_SOCKET_TIMEOUT_MS, + connectionTimeout, + }), + }) + } + const clients: S3Clients = { + reads: createClient(env.STORAGE_S3_MAX_SOCKETS), + writes: createClient(env.STORAGE_S3_MAX_SOCKETS), + // Bounds the wait for a socket plus the connect, so a starved lookup fails + // instead of hanging; cleared once a kept-alive socket is assigned. + control: createClient( + env.STORAGE_S3_CONTROL_MAX_SOCKETS, + env.STORAGE_S3_CONTROL_QUEUE_TIMEOUT_MS, + ), } - const s3 = new S3Client({ - forcePathStyle: true, - region: env.AWS_REGION, - requestHandler: new NodeHttpHandler({ - httpAgent: new HttpAgent(agentOptions), - httpsAgent: new HttpsAgent(agentOptions), - socketTimeout: env.STORAGE_S3_SOCKET_TIMEOUT_MS, - }), - }) try { - await s3.send( + await clients.control.send( new HeadBucketCommand({ Bucket: bucket, }), @@ -935,10 +957,10 @@ class S3Adapter implements StorageAdapter { throw err } - return new S3Adapter({ s3, bucket }) + return new S3Adapter({ clients, bucket }) } - private s3 + private clients private bucket private keyPrefix = 'gh-actions-cache' @@ -950,12 +972,14 @@ class S3Adapter implements StorageAdapter { run: async (folderName: string, partCount: number) => { const Bucket = this.bucket const Key = `${this.keyPrefix}/${folderName}/merged` - const { UploadId } = await this.s3.send(new CreateMultipartUploadCommand({ Bucket, Key })) + const { UploadId } = await this.clients.writes.send( + new CreateMultipartUploadCommand({ Bucket, Key }), + ) if (!UploadId) throw new Error('S3 did not return an UploadId') try { const Parts = [] for (let PartNumber = 1; PartNumber <= partCount; PartNumber++) { - const copy = await this.s3.send( + const copy = await this.clients.writes.send( new UploadPartCopyCommand({ Bucket, Key, @@ -966,11 +990,11 @@ class S3Adapter implements StorageAdapter { ) Parts.push({ PartNumber, ETag: copy.CopyPartResult?.ETag }) } - await this.s3.send( + await this.clients.writes.send( new CompleteMultipartUploadCommand({ Bucket, Key, UploadId, MultipartUpload: { Parts } }), ) } catch (err) { - await this.s3 + await this.clients.writes .send(new AbortMultipartUploadCommand({ Bucket, Key, UploadId })) .catch(() => {}) throw err @@ -978,8 +1002,8 @@ class S3Adapter implements StorageAdapter { }, } - constructor({ bucket, s3 }: { s3: S3Client; bucket: string }) { - this.s3 = s3 + constructor({ bucket, clients }: { clients: S3Clients; bucket: string }) { + this.clients = clients this.bucket = bucket } @@ -987,7 +1011,7 @@ class S3Adapter implements StorageAdapter { let continuationToken: string | undefined do { - const response = await this.s3.send( + const response = await this.clients.control.send( new ListObjectsV2Command({ Bucket: this.bucket, Prefix: prefix, @@ -1010,7 +1034,7 @@ class S3Adapter implements StorageAdapter { private async deleteByPrefix(prefix: string) { const deleted = { objects: 0, bytes: 0 } while (true) { - const listResponse = await this.s3.send( + const listResponse = await this.clients.control.send( new ListObjectsV2Command({ Bucket: this.bucket, Prefix: prefix }), ) if (!listResponse.Contents || listResponse.Contents.length === 0) break @@ -1026,7 +1050,7 @@ class S3Adapter implements StorageAdapter { listResponse.Contents.filter((obj): obj is { Key: string } => !!obj.Key), 1000, ).map((chunkedObjects) => - this.s3.send( + this.clients.control.send( new DeleteObjectsCommand({ Bucket: this.bucket, Delete: { @@ -1052,7 +1076,7 @@ class S3Adapter implements StorageAdapter { async createDownloadStream(objectName: string, range?: RangeRequest): Promise { try { - const response = await this.s3.send( + const response = await this.clients.reads.send( new GetObjectCommand({ Bucket: this.bucket, Key: `${this.keyPrefix}/${objectName}`, @@ -1079,7 +1103,7 @@ class S3Adapter implements StorageAdapter { async objectExists(objectName: string) { try { - await this.s3.send( + await this.clients.control.send( new HeadObjectCommand({ Bucket: this.bucket, Key: `${this.keyPrefix}/${objectName}`, @@ -1107,7 +1131,7 @@ class S3Adapter implements StorageAdapter { async uploadStream(objectName: string, iterator: AsyncIterable) { await new S3Upload({ - client: this.s3, + client: this.clients.writes, params: { Bucket: this.bucket, Key: `${this.keyPrefix}/${objectName}`, @@ -1163,7 +1187,7 @@ class S3Adapter implements StorageAdapter { async createDownloadUrl(objectName: string, expiresAt: number) { return getSignedUrl( - this.s3, + this.clients.reads, new GetObjectCommand({ Bucket: this.bucket, Key: `${this.keyPrefix}/${objectName}`, diff --git a/tests/merge-socket-pool.test.ts b/tests/merge-socket-pool.test.ts new file mode 100644 index 0000000..2fc94cc --- /dev/null +++ b/tests/merge-socket-pool.test.ts @@ -0,0 +1,92 @@ +import type { Env } from '~/lib/schemas' +import { createHash, randomUUID } from 'node:crypto' +import { Readable } from 'node:stream' + +import { afterEach, describe, expect, test } from 'vitest' +import { getDatabase } from '~/lib/db' +import { env } from '~/lib/env' +import { Storage } from '~/lib/storage' + +const MiB = 1024 * 1024 +const POOL = 4 +const scope = { version: 'v1', scope: 'refs/heads/main', repoId: '123' } +const s3Env = env as Extract + +async function uploadEntry(storage: Storage, parts: Buffer[]) { + const key = `merge-pool-${randomUUID()}` + const upload = await storage.createUpload({ key, ...scope }) + for (const [index, part] of parts.entries()) + await storage.uploadPart(upload.id, index, Readable.from(part)) + await storage.completeUpload({ key, ...scope }) + const db = await getDatabase() + return db + .selectFrom('cache_entries') + .innerJoin('storage_locations', 'storage_locations.id', 'cache_entries.locationId') + .where('cache_entries.key', '=', key) + .select(['storage_locations.id as locationId', 'folderName']) + .executeTakeFirstOrThrow() +} + +async function sha256(stream: Readable) { + const hash = createHash('sha256') + for await (const chunk of stream) hash.update(chunk) + return hash.digest('hex') +} + +// A first restore merges its parts while it streams them: it holds a part +// GetObject open and needs another S3 socket to upload the merged object. With +// one shared pool, as many concurrent first restores as sockets each hold a GET +// and wait for an upload socket that never frees, so every restore and every +// later S3 call (lookup HEADs included) hangs. +describe.skipIf(process.env.VITEST_STORAGE_DRIVER !== 's3')('merges and the S3 socket pool', () => { + const originalMaxSockets = s3Env.STORAGE_S3_MAX_SOCKETS + const originalEagerMerge = env.EAGER_MERGE + afterEach(() => { + s3Env.STORAGE_S3_MAX_SOCKETS = originalMaxSockets + env.EAGER_MERGE = originalEagerMerge + }) + + // A deadlock never settles, so it fails as a test timeout. + test( + 'as many concurrent first restores as pool sockets complete and merge', + { timeout: 60_000 }, + async () => { + env.EAGER_MERGE = false + s3Env.STORAGE_S3_MAX_SOCKETS = POOL + const storage = await Storage.fromEnv() + + // Parts above lib-storage's 5 MiB part size, so each merge is a multipart upload. + const parts = [Buffer.alloc(6 * MiB, 'a'), Buffer.alloc(6 * MiB, 'b')] + const expected = createHash('sha256').update(Buffer.concat(parts)).digest('hex') + const entries = [] + for (let i = 0; i < POOL; i++) entries.push(await uploadEntry(storage, parts)) + + try { + const digests = await Promise.all( + entries.map(async ({ locationId }) => { + const download = await storage.download(locationId) + if (!download) throw new Error(`location ${locationId} not found`) + return sha256(download.stream) + }), + ) + expect(digests).toEqual(entries.map(() => expected)) + + await storage.waitForOngoingMerges() + const db = await getDatabase() + const merged = await db + .selectFrom('storage_locations') + .where( + 'id', + 'in', + entries.map((e) => e.locationId), + ) + .where('mergedAt', 'is not', null) + .select('id') + .execute() + expect(merged).toHaveLength(POOL) + } finally { + for (const { folderName } of entries) await storage.adapter.deleteFolder(folderName) + } + }, + ) +}) diff --git a/tests/s3-socket-pool.test.ts b/tests/s3-socket-pool.test.ts index 864f937..d876f57 100644 --- a/tests/s3-socket-pool.test.ts +++ b/tests/s3-socket-pool.test.ts @@ -38,4 +38,22 @@ describe.skipIf(process.env.VITEST_STORAGE_DRIVER !== 's3')('s3 socket pool', () await storage.adapter.deleteFolder(folder) } }) + + test('a lookup HEAD does not queue behind held reads', async () => { + s3Env.STORAGE_S3_MAX_SOCKETS = 1 + const storage = await Storage.fromEnv() + const folder = `socket-pool-${randomUUID()}` + await storage.adapter.uploadStream(`${folder}/a`, Readable.from([Buffer.from('a')])) + + // A paused restore (or a Merge waiting to upload) keeps its GET socket. + const held = await storage.adapter.createDownloadStream(`${folder}/a`) + try { + const lookup = storage.adapter.objectExists(`${folder}/a`) + const answered = await Promise.race([lookup, sleep(1000, 'queued')]) + expect(answered, 'HeadObject answered while every data socket is held').toBe(true) + } finally { + held.stream.destroy() + await storage.adapter.deleteFolder(folder) + } + }) }) diff --git a/tests/setup.ts b/tests/setup.ts index 75f8c9e..8dedff8 100644 --- a/tests/setup.ts +++ b/tests/setup.ts @@ -86,6 +86,8 @@ const TESTING_ENV_BY_STORAGE_DRIVER = { STORAGE_S3_BUCKET: 'vitest', STORAGE_S3_SOCKET_TIMEOUT_MS: 10_000, STORAGE_S3_MAX_SOCKETS: 50, + STORAGE_S3_CONTROL_MAX_SOCKETS: 16, + STORAGE_S3_CONTROL_QUEUE_TIMEOUT_MS: 5000, AWS_ACCESS_KEY_ID: 'rustfsadmin', AWS_SECRET_ACCESS_KEY: 'rustfsadmin', AWS_ENDPOINT_URL: 'http://localhost:9000', From d65f0969846b5dd9fe3b2e948bcd5867af0ea0b5 Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Sat, 24 Oct 2026 16:33:35 -0400 Subject: [PATCH 07/17] fix(storage): cap concurrent Merges per process Every first restore of an unmerged entry and, with EAGER_MERGE, every finalize started a Merge with no global limit. Each Merge holds storage connections, a Merge Lease and stream buffers for its whole duration, and a lazy Merge is paced by the restoring client's read speed. A burst of first restores or of finalizes therefore grew Merges, sockets and memory without bound. Start a Merge only while fewer than MAX_CONCURRENT_MERGES (default 16) run in this process. Beyond that, startMerge returns false: a restore streams the parts directly, exactly as when another worker holds the Merge Lease, an Eager Merge is left to the first download, and a later download merges the entry. The slot is reserved before the first await, so concurrent callers cannot overshoot it. tests/merge-concurrency.test.ts covers a first restore and an Eager Merge beyond the limit. Signed-off-by: Rui Chen --- README.md | 1 + lib/schemas.ts | 1 + lib/storage.ts | 40 ++++++++---- tests/merge-concurrency.test.ts | 111 ++++++++++++++++++++++++++++++++ tests/setup.ts | 1 + 5 files changed, 143 insertions(+), 11 deletions(-) create mode 100644 tests/merge-concurrency.test.ts diff --git a/README.md b/README.md index 6799d39..8c17dd4 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ All optional. | `STORAGE_S3_MAX_SOCKETS` | `50` | Sockets per S3 data pool, for `http://` and `https://` endpoints alike. Reads (GetObject) and writes (uploads, Merges) each get a pool this size. | | `STORAGE_S3_CONTROL_MAX_SOCKETS` | `16` | Sockets for HEAD, LIST and DELETE requests (cache lookups, cleanup), kept apart from bulk transfers. | | `STORAGE_S3_CONTROL_QUEUE_TIMEOUT_MS` | `5000` | How long a control request waits for a socket and connection before failing; `0` waits forever. | +| `MAX_CONCURRENT_MERGES` | `16` | Merges one process runs at once. Beyond it, restores stream the parts and a later download merges. | ## Documentation diff --git a/lib/schemas.ts b/lib/schemas.ts index e64c979..c0e2f0d 100644 --- a/lib/schemas.ts +++ b/lib/schemas.ts @@ -87,6 +87,7 @@ export const envBaseSchema = type({ 'DEBUG?': 'unknown', 'ENABLE_DIRECT_DOWNLOADS': 'boolean = false', 'EAGER_MERGE': 'boolean = false', + 'MAX_CONCURRENT_MERGES': 'number.integer >= 1 = 16', 'BENCHMARK': 'boolean = false', 'SKIP_TOKEN_VALIDATION': 'boolean = false', 'MANAGEMENT_API_KEY?': 'string', diff --git a/lib/storage.ts b/lib/storage.ts index 7519ad3..d2c7a36 100644 --- a/lib/storage.ts +++ b/lib/storage.ts @@ -148,6 +148,7 @@ export class Storage { private db private mergeStreamPromises = new Set>() + private activeMerges = 0 adapter private constructor({ db, adapter }: { adapter: StorageAdapter; db: Kysely }) { @@ -304,19 +305,35 @@ export class Storage { /** * Runs a Merge under a Merge Lease. `write` produces `${folderName}/merged`; * a lease-fenced transaction then marks the Merge complete. Returns false - * when another worker holds the lease. Writing straight to the final object - * is safe (ADR-0004). The returned promise never rejects: failures are logged - * and the merge state rolled back. + * when another worker holds the lease or this process already runs + * MAX_CONCURRENT_MERGES Merges; a skipped Merge is retried by a later + * download. Writing straight to the final object is safe (ADR-0004). The + * returned promise never rejects: failures are logged and the merge state + * rolled back. */ private async startMerge(location: StorageLocation, write: () => Promise) { - const mergeToken = await acquireMergeLease(this.db, location.id) - if (!mergeToken) return false - - await this.db - .updateTable('storage_locations') - .set({ mergeStartedAt: Date.now() }) - .where('id', '=', location.id) - .execute() + // Each Merge holds storage connections and buffers for its whole duration; + // unbounded, a burst of first restores or finalizes exhausts them. + if (this.activeMerges >= env.MAX_CONCURRENT_MERGES) return false + // Reserve the slot before the first await so concurrent callers cannot overshoot it. + this.activeMerges++ + let mergeToken: string | undefined + try { + mergeToken = await acquireMergeLease(this.db, location.id) + if (mergeToken) + await this.db + .updateTable('storage_locations') + .set({ mergeStartedAt: Date.now() }) + .where('id', '=', location.id) + .execute() + } catch (err) { + this.activeMerges-- + throw err + } + if (!mergeToken) { + this.activeMerges-- + return false + } const renewalTimer = setInterval(() => { void renewMergeLease(this.db, location.id, mergeToken) @@ -364,6 +381,7 @@ export class Storage { }) .finally(async () => { clearInterval(renewalTimer) + this.activeMerges-- await releaseMergeLease(this.db, location.id, mergeToken) }) this.mergeStreamPromises.add(mergePromise) diff --git a/tests/merge-concurrency.test.ts b/tests/merge-concurrency.test.ts new file mode 100644 index 0000000..85e3e71 --- /dev/null +++ b/tests/merge-concurrency.test.ts @@ -0,0 +1,111 @@ +import { createHash, randomUUID } from 'node:crypto' +import { Readable } from 'node:stream' + +import { afterEach, describe, expect, test } from 'vitest' +import { getDatabase } from '~/lib/db' +import { env } from '~/lib/env' +import { Storage } from '~/lib/storage' + +const scope = { version: 'v1', scope: 'refs/heads/main', repoId: '123' } +const parts = [Buffer.alloc(256 * 1024, 'a'), Buffer.alloc(256 * 1024, 'b')] +const expected = createHash('sha256').update(Buffer.concat(parts)).digest('hex') + +async function uploadEntry(storage: Storage) { + const key = `merge-cap-${randomUUID()}` + const upload = await storage.createUpload({ key, ...scope }) + for (const [index, part] of parts.entries()) + await storage.uploadPart(upload.id, index, Readable.from(part)) + await storage.completeUpload({ key, ...scope }) + const db = await getDatabase() + return db + .selectFrom('cache_entries') + .innerJoin('storage_locations', 'storage_locations.id', 'cache_entries.locationId') + .where('cache_entries.key', '=', key) + .select(['storage_locations.id as locationId', 'folderName']) + .executeTakeFirstOrThrow() +} + +async function download(storage: Storage, locationId: string) { + const result = await storage.download(locationId) + if (!result) throw new Error(`location ${locationId} not found`) + return result.stream +} + +async function sha256(stream: Readable) { + const hash = createHash('sha256') + for await (const chunk of stream) hash.update(chunk) + return hash.digest('hex') +} + +async function mergeState(locationId: string) { + const db = await getDatabase() + return db + .selectFrom('storage_locations') + .where('id', '=', locationId) + .select(['mergedAt', 'mergeStartedAt']) + .executeTakeFirstOrThrow() +} + +describe('merge concurrency limit', () => { + const originalMaxMerges = env.MAX_CONCURRENT_MERGES + const originalEagerMerge = env.EAGER_MERGE + afterEach(() => { + env.MAX_CONCURRENT_MERGES = originalMaxMerges + env.EAGER_MERGE = originalEagerMerge + }) + + test('a first restore beyond the limit is served from parts without merging', async () => { + env.EAGER_MERGE = false + env.MAX_CONCURRENT_MERGES = 1 + const storage = await Storage.fromEnv() + const first = await uploadEntry(storage) + const second = await uploadEntry(storage) + + try { + // The first restore's Merge is paced by its unread stream, so it still + // holds the only slot when the second restore starts. + const firstStream = await download(storage, first.locationId) + const secondStream = await download(storage, second.locationId) + expect(await sha256(secondStream)).toBe(expected) + expect(await sha256(firstStream)).toBe(expected) + await storage.waitForOngoingMerges() + + expect(await mergeState(first.locationId)).not.toMatchObject({ mergedAt: null }) + expect(await mergeState(second.locationId)).toMatchObject({ mergedAt: null }) + + // With the slot free again, the next restore merges the skipped entry. + expect(await sha256(await download(storage, second.locationId))).toBe(expected) + await storage.waitForOngoingMerges() + expect(await mergeState(second.locationId)).not.toMatchObject({ mergedAt: null }) + } finally { + await storage.adapter.deleteFolder(first.folderName) + await storage.adapter.deleteFolder(second.folderName) + } + }) + + test('an eager merge beyond the limit leaves the entry to a later download', async () => { + env.EAGER_MERGE = false + env.MAX_CONCURRENT_MERGES = 1 + const storage = await Storage.fromEnv() + const busy = await uploadEntry(storage) + // An unread first restore keeps its Merge, and the only slot, busy. + const busyStream = await download(storage, busy.locationId) + + env.EAGER_MERGE = true + const eager = await uploadEntry(storage) + try { + // Upload completion awaits the Merge Lease, so a started Merge would have + // stamped mergeStartedAt by now. + expect(await mergeState(eager.locationId)).toEqual({ mergedAt: null, mergeStartedAt: null }) + + expect(await sha256(busyStream)).toBe(expected) + await storage.waitForOngoingMerges() + expect(await sha256(await download(storage, eager.locationId))).toBe(expected) + await storage.waitForOngoingMerges() + expect(await mergeState(eager.locationId)).not.toMatchObject({ mergedAt: null }) + } finally { + await storage.adapter.deleteFolder(busy.folderName) + await storage.adapter.deleteFolder(eager.folderName) + } + }) +}) diff --git a/tests/setup.ts b/tests/setup.ts index 8dedff8..01bf3ee 100644 --- a/tests/setup.ts +++ b/tests/setup.ts @@ -44,6 +44,7 @@ const TESTING_ENV_BASE = { | 'ORPHANED_STORAGE_GRACE_PERIOD_HOURS' | 'ENABLE_DIRECT_DOWNLOADS' | 'EAGER_MERGE' + | 'MAX_CONCURRENT_MERGES' | 'BENCHMARK' | 'SKIP_TOKEN_VALIDATION' | 'ACTIONS_TOKEN_ISSUER' From 2dfddcfbca13d743e5268e43036e9409caf188e8 Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Sat, 24 Oct 2026 16:43:36 -0400 Subject: [PATCH 08/17] fix(download): close downloads whose client stopped reading Node's HTTP server has no idle timeout for a response in progress and the download route set none. A client that stays connected but stops reading parks the pipe forever: the backend read keeps its storage socket, the Storage Reader Lease keeps renewing, and on the first restore of an unmerged entry the Merge it paces never finishes and keeps its Merge Lease. Each such client permanently removes one socket from the storage pool. Destroy the download when no bytes have moved for DOWNLOAD_IDLE_TIMEOUT_MS (default 300000, 0 disables). The timer is refreshed on every chunk, so slow but progressing clients are unaffected. It covers whole and Range responses and the first restore of an unmerged entry, whose Merge keeps going after the client's side of the tee is closed. tests/proxy-backpressure.test.ts parks a client after the first chunk of a 200, a 206 and an unmerged first restore, and checks that the storage read is released, the response closed, and the Merge completed. Signed-off-by: Rui Chen --- README.md | 13 ++-- lib/schemas.ts | 1 + routes/download/[locationId].ts | 27 +++++++- tests/proxy-backpressure.test.ts | 110 +++++++++++++++++++++++++++++++ tests/setup.ts | 1 + 5 files changed, 145 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8c17dd4..960adf4 100644 --- a/README.md +++ b/README.md @@ -32,12 +32,13 @@ volumes: All optional. -| Variable | Default | Effect | -| ------------------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | -| `STORAGE_S3_MAX_SOCKETS` | `50` | Sockets per S3 data pool, for `http://` and `https://` endpoints alike. Reads (GetObject) and writes (uploads, Merges) each get a pool this size. | -| `STORAGE_S3_CONTROL_MAX_SOCKETS` | `16` | Sockets for HEAD, LIST and DELETE requests (cache lookups, cleanup), kept apart from bulk transfers. | -| `STORAGE_S3_CONTROL_QUEUE_TIMEOUT_MS` | `5000` | How long a control request waits for a socket and connection before failing; `0` waits forever. | -| `MAX_CONCURRENT_MERGES` | `16` | Merges one process runs at once. Beyond it, restores stream the parts and a later download merges. | +| Variable | Default | Effect | +| ------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| `STORAGE_S3_MAX_SOCKETS` | `50` | Sockets per S3 data pool, for `http://` and `https://` endpoints alike. Reads (GetObject) and writes (uploads, Merges) each get a pool this size. | +| `STORAGE_S3_CONTROL_MAX_SOCKETS` | `16` | Sockets for HEAD, LIST and DELETE requests (cache lookups, cleanup), kept apart from bulk transfers. | +| `STORAGE_S3_CONTROL_QUEUE_TIMEOUT_MS` | `5000` | How long a control request waits for a socket and connection before failing; `0` waits forever. | +| `MAX_CONCURRENT_MERGES` | `16` | Merges one process runs at once. Beyond it, restores stream the parts and a later download merges. | +| `DOWNLOAD_IDLE_TIMEOUT_MS` | `300000` | Closes a download whose client has not read a byte for this long; `0` disables. | ## Documentation diff --git a/lib/schemas.ts b/lib/schemas.ts index c0e2f0d..493e5bd 100644 --- a/lib/schemas.ts +++ b/lib/schemas.ts @@ -88,6 +88,7 @@ export const envBaseSchema = type({ 'ENABLE_DIRECT_DOWNLOADS': 'boolean = false', 'EAGER_MERGE': 'boolean = false', 'MAX_CONCURRENT_MERGES': 'number.integer >= 1 = 16', + 'DOWNLOAD_IDLE_TIMEOUT_MS': 'number.integer >= 0 = 300000', 'BENCHMARK': 'boolean = false', 'SKIP_TOKEN_VALIDATION': 'boolean = false', 'MANAGEMENT_API_KEY?': 'string', diff --git a/routes/download/[locationId].ts b/routes/download/[locationId].ts index c31513c..a9d4335 100644 --- a/routes/download/[locationId].ts +++ b/routes/download/[locationId].ts @@ -1,6 +1,7 @@ import type { RangeRequest } from '~/lib/storage' import { pipeline } from 'node:stream/promises' import { z } from 'zod' +import { env } from '~/lib/env' import { logger } from '~/lib/logger' import { getStorage, RangeNotSatisfiableError } from '~/lib/storage' @@ -77,11 +78,33 @@ export default defineEventHandler(async (event) => { // `sendStream` neither waits for `drain` nor notices client aborts; `pipeline` does both // and destroys the source, which releases its reader lease. `_handled` is an h3 v1 internal. event._handled = true + // The HTTP server has no idle timeout, so a client that stops reading but keeps + // the connection open would hold the backend read, its reader lease and, on a + // first restore, the Merge it paces forever. Bytes moving at any rate keep the + // download alive. + let idleTimer: NodeJS.Timeout | undefined + if (env.DOWNLOAD_IDLE_TIMEOUT_MS > 0) { + const body = download.stream + idleTimer = setTimeout(() => { + body.destroy( + Object.assign(new Error(`No bytes sent for ${env.DOWNLOAD_IDLE_TIMEOUT_MS} ms`), { + code: 'ERR_DOWNLOAD_IDLE_TIMEOUT', + }), + ) + }, env.DOWNLOAD_IDLE_TIMEOUT_MS) + idleTimer.unref() + body.on('data', () => idleTimer?.refresh()) + } try { await pipeline(download.stream, event.node.res) } catch (err) { + const { code } = err as NodeJS.ErrnoException + if (code === 'ERR_DOWNLOAD_IDLE_TIMEOUT') { + logger.warn(`Download /download/${locationId} closed: ${(err as Error).message}`) + return + } // Client went away mid-body. Expected on cancelled jobs and parallel runners. - if ((err as NodeJS.ErrnoException).code === 'ERR_STREAM_PREMATURE_CLOSE') { + if (code === 'ERR_STREAM_PREMATURE_CLOSE') { logger.debug(`Client aborted /download/${locationId}: ${(err as Error).message}`) return } @@ -91,5 +114,7 @@ export default defineEventHandler(async (event) => { return } throw err + } finally { + clearTimeout(idleTimer) } }) diff --git a/tests/proxy-backpressure.test.ts b/tests/proxy-backpressure.test.ts index 9a7ff4f..72cff49 100644 --- a/tests/proxy-backpressure.test.ts +++ b/tests/proxy-backpressure.test.ts @@ -9,6 +9,8 @@ import { setTimeout as sleep } from 'node:timers/promises' import * as h3 from 'h3' import { afterAll, afterEach, beforeAll, describe, expect, test, vi } from 'vitest' +import { getDatabase } from '~/lib/db' +import { env } from '~/lib/env' import { getStorage } from '~/lib/storage' const MiB = 1024 * 1024 @@ -167,4 +169,112 @@ describe('proxy routes apply backpressure', () => { await vi.waitFor(() => expect(source.destroyed).toBe(true), { timeout: 5000, interval: 50 }) }, 20_000) + + describe('download idle timeout', () => { + const originalIdleTimeout = env.DOWNLOAD_IDLE_TIMEOUT_MS + beforeAll(() => { + env.DOWNLOAD_IDLE_TIMEOUT_MS = 500 + }) + afterAll(() => { + env.DOWNLOAD_IDLE_TIMEOUT_MS = originalIdleTimeout + }) + + test.each([ + { response: 'a whole entry', status: 200, range: undefined }, + { response: 'a Range', status: 206, range: { start: 0, end: ENTRY_BYTES - 1 } }, + ])( + 'releases the storage read of a client that stopped reading $response', + async ({ status, range }) => { + // Real timers and sockets on purpose: the stall comes from TCP flow control, + // which fake timers cannot drive. + let pulled = 0 + function* entry() { + while (pulled < ENTRY_BYTES) { + pulled += CHUNK_BYTES + yield Buffer.alloc(CHUNK_BYTES) + } + } + const source = Readable.from(entry(), { objectMode: false, highWaterMark: CHUNK_BYTES }) + const storage = await getStorage() + vi.spyOn(storage, 'download').mockResolvedValue({ + stream: source, + size: ENTRY_BYTES, + range, + }) + + const { port } = server.address() as AddressInfo + const serverSide = once(server, 'request') as Promise<[IncomingMessage, ServerResponse]> + const req = http.get(`http://localhost:${port}/download/${randomUUID()}`, { + headers: range ? { range: `bytes=${range.start}-${range.end}` } : {}, + }) + req.on('error', () => undefined) + const [res] = (await once(req, 'response')) as [IncomingMessage] + expect(res.statusCode).toBe(status) + res.once('data', () => res.pause()) // stops reading, keeps the connection open + const [, serverRes] = await serverSide + const serverClosed = once(serverRes, 'close') + + // Without an idle timeout neither ever happens: the pipe just stays parked. + await expect(once(source, 'close')).rejects.toMatchObject({ + code: 'ERR_DOWNLOAD_IDLE_TIMEOUT', + }) + await serverClosed + expect(pulled).toBeLessThan(ENTRY_BYTES) + req.destroy() + }, + 20_000, + ) + + test('a stalled first restore is closed and its Merge still completes', async () => { + const storage = await getStorage() + const db = await getDatabase() + const reservation = { + key: `idle-unmerged-${randomUUID()}`, + version: 'v1', + scope: 'refs/heads/main', + repoId: '123', + } + const upload = await storage.createUpload(reservation) + // Far more than loopback socket buffers hold, so the stalled client really blocks the tee. + for (let i = 0; i < 4; i++) + await storage.uploadPart(upload.id, i, Readable.from([Buffer.alloc(8 * MiB, i)])) + await storage.completeUpload(reservation) + const { locationId } = await db + .selectFrom('cache_entries') + .where('key', '=', reservation.key) + .select('locationId') + .executeTakeFirstOrThrow() + + const { port } = server.address() as AddressInfo + const serverSide = once(server, 'request') as Promise<[IncomingMessage, ServerResponse]> + const req = http.get(`http://localhost:${port}/download/${locationId}`) + req.on('error', () => undefined) + const [res] = (await once(req, 'response')) as [IncomingMessage] + expect(res.statusCode).toBe(200) + res.once('data', () => res.pause()) + const [, serverRes] = await serverSide + await once(serverRes, 'close') + req.destroy() + + // The pump keeps feeding the merge once the client's side of the tee is gone. + await storage.waitForOngoingMerges() + const location = await db + .selectFrom('storage_locations') + .where('id', '=', locationId) + .select('mergedAt') + .executeTakeFirstOrThrow() + expect(location.mergedAt).not.toBeNull() + await vi.waitFor( + async () => { + const leases = await db + .selectFrom('storage_reader_leases') + .where('storageLocationId', '=', locationId) + .select('id') + .execute() + expect(leases, 'reader leases left behind').toHaveLength(0) + }, + { timeout: 5000, interval: 50 }, + ) + }, 30_000) + }) }) diff --git a/tests/setup.ts b/tests/setup.ts index 01bf3ee..8e8660a 100644 --- a/tests/setup.ts +++ b/tests/setup.ts @@ -45,6 +45,7 @@ const TESTING_ENV_BASE = { | 'ENABLE_DIRECT_DOWNLOADS' | 'EAGER_MERGE' | 'MAX_CONCURRENT_MERGES' + | 'DOWNLOAD_IDLE_TIMEOUT_MS' | 'BENCHMARK' | 'SKIP_TOKEN_VALIDATION' | 'ACTIONS_TOKEN_ISSUER' From 783e9ed39edb09ab9a82b1fec42a8f9bab38c456 Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Sat, 24 Oct 2026 16:47:08 -0400 Subject: [PATCH 09/17] fix(download): release the backend read when the client leaves before storage answers A client can give up on /download while its storage read is still waiting, for example a GetObject queued behind a saturated socket pool. The body then arrives for a response that is already closed. `pipeline` rejects a closed destination with ERR_STREAM_UNABLE_TO_PIPE before it takes ownership of the source, and the route rethrew without destroying it, so the body pinned its backend socket and the Storage Reader Lease for the life of the process. Enough of them fill the pool and every storage call queues. Check the response before piping and destroy the body if the client is gone, and destroy it on every error exit of the pipe as well. #282 and #291 already cover a client leaving mid-body. tests/download-client-gone.test.ts holds the storage read until the client has hung up, then checks that the backend socket and the reader lease are released; on Node 26.12.1 the socket stayed held before this change. Signed-off-by: Rui Chen --- routes/download/[locationId].ts | 19 +++- tests/download-client-gone.test.ts | 158 +++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 tests/download-client-gone.test.ts diff --git a/routes/download/[locationId].ts b/routes/download/[locationId].ts index a9d4335..bed49d7 100644 --- a/routes/download/[locationId].ts +++ b/routes/download/[locationId].ts @@ -78,6 +78,16 @@ export default defineEventHandler(async (event) => { // `sendStream` neither waits for `drain` nor notices client aborts; `pipeline` does both // and destroys the source, which releases its reader lease. `_handled` is an h3 v1 internal. event._handled = true + // The client can hang up while storage is still answering (a read queued behind + // a full socket pool). `pipeline` refuses an already-closed destination with + // ERR_STREAM_UNABLE_TO_PIPE without taking ownership of the source, so release + // the backend read here instead of leaving its socket and reader lease pinned. + const { res } = event.node + if (res.destroyed || res.closed) { + download.stream.destroy() + logger.debug(`Client aborted /download/${locationId} before storage answered`) + return + } // The HTTP server has no idle timeout, so a client that stops reading but keeps // the connection open would hold the backend read, its reader lease and, on a // first restore, the Merge it paces forever. Bytes moving at any rate keep the @@ -96,20 +106,23 @@ export default defineEventHandler(async (event) => { body.on('data', () => idleTimer?.refresh()) } try { - await pipeline(download.stream, event.node.res) + await pipeline(download.stream, res) } catch (err) { + // Every exit that did not finish the pipe must release the backend read: an + // unread body pins its storage connection and the reader lease it carries. + download.stream.destroy() const { code } = err as NodeJS.ErrnoException if (code === 'ERR_DOWNLOAD_IDLE_TIMEOUT') { logger.warn(`Download /download/${locationId} closed: ${(err as Error).message}`) return } // Client went away mid-body. Expected on cancelled jobs and parallel runners. - if (code === 'ERR_STREAM_PREMATURE_CLOSE') { + if (code === 'ERR_STREAM_PREMATURE_CLOSE' || code === 'ERR_STREAM_UNABLE_TO_PIPE') { logger.debug(`Client aborted /download/${locationId}: ${(err as Error).message}`) return } // Headers are out, so Nitro's error handler would crash with ERR_HTTP_HEADERS_SENT. - if (event.node.res.headersSent) { + if (res.headersSent) { logger.error(`Download stream failed for ${locationId}`, { error: err }) return } diff --git a/tests/download-client-gone.test.ts b/tests/download-client-gone.test.ts new file mode 100644 index 0000000..90d8aeb --- /dev/null +++ b/tests/download-client-gone.test.ts @@ -0,0 +1,158 @@ +import type { IncomingMessage, ServerResponse } from 'node:http' +import type { AddressInfo } from 'node:net' +import { Buffer } from 'node:buffer' +import { randomUUID } from 'node:crypto' +import { once } from 'node:events' +import http from 'node:http' +import { Readable } from 'node:stream' + +import * as h3 from 'h3' +import { afterAll, afterEach, beforeAll, describe, expect, test, vi } from 'vitest' +import { getDatabase } from '~/lib/db' +import { getStorage } from '~/lib/storage' + +/** + * A stand-in for the object store: answers every GET with an unending body, so + * a response only gives its socket back by being read to the end or destroyed. + */ +async function startOrigin() { + const server = http.createServer((_req, res) => { + res.writeHead(200, { 'content-type': 'application/octet-stream' }) + const chunk = Buffer.alloc(64 * 1024) + const pump = () => { + while (!res.destroyed && res.write(chunk)); + } + res.on('drain', pump) + pump() + }) + server.listen(0) + await once(server, 'listening') + const { port } = server.address() as AddressInfo + return { + url: `http://localhost:${port}`, + close: async () => { + server.closeAllConnections() + server.close() + await once(server, 'close') + }, + } +} + +// The real route on an in-process h3 app, as in proxy-backpressure.test.ts, so +// the storage it calls and the sockets it holds are observable here. +const router = h3.createRouter({ preemptive: true }) +const server = http.createServer(h3.toNodeListener(h3.createApp().use(router))) + +beforeAll(async () => { + for (const [name, value] of Object.entries(h3)) vi.stubGlobal(name, value) + // Not a static import: the route calls Nitro auto-imports at evaluation time, + // so it can only load after the stubs above. + const { default: download } = await import('~/routes/download/[locationId]') + router.get('/download/:locationId', download) + server.listen(0) + await once(server, 'listening') +}) + +afterEach(() => { + vi.restoreAllMocks() + server.closeAllConnections() +}) + +afterAll(async () => { + vi.unstubAllGlobals() + server.close() + await once(server, 'close') +}) + +async function createMergedEntry() { + const storage = await getStorage() + const db = await getDatabase() + const key = `client-gone-${randomUUID()}` + const entry = { key, version: 'v1', scope: 'refs/heads/main', repoId: '123' } + const upload = await storage.createUpload(entry) + await storage.uploadPart(upload.id, 0, Readable.from('payload')) + await storage.completeUpload(entry) + const { locationId } = await db + .selectFrom('cache_entries') + .where('key', '=', key) + .select('locationId') + .executeTakeFirstOrThrow() + // Merged, so download() goes straight to one read of the merged object. + await db + .updateTable('storage_locations') + .set({ mergedAt: Date.now() }) + .where('id', '=', locationId) + .execute() + return locationId +} + +/** + * A client can give up on /download while the storage read is still waiting + * for a socket in a saturated pool. The body then arrives for a response that + * is already closed, and `pipeline` rejects a closed destination + * (ERR_STREAM_UNABLE_TO_PIPE) before it takes ownership of the source. Unless + * the route destroys it, the body pins its backend socket and reader lease for + * the life of the process. + */ +describe('download for a client that left before storage answered', () => { + test('releases the backend socket and the reader lease', async () => { + const origin = await startOrigin() + const agent = new http.Agent({ keepAlive: true, maxSockets: 1 }) + const storage = await getStorage() + const db = await getDatabase() + const locationId = await createMergedEntry() + + const { promise: clientGone, resolve: markClientGone } = Promise.withResolvers() + const { promise: adapterSettled, resolve: markAdapterSettled } = Promise.withResolvers() + let body: IncomingMessage | undefined + vi.spyOn(storage.adapter, 'createDownloadStream').mockImplementation(async () => { + try { + await clientGone + const req = http.get(origin.url, { agent }) + const [res] = (await once(req, 'response')) as [IncomingMessage] + body = res + return { stream: res } + } finally { + markAdapterSettled() + } + }) + + try { + const { port } = server.address() as AddressInfo + const serverSide = once(server, 'request') as Promise<[IncomingMessage, ServerResponse]> + const req = http.get(`http://localhost:${port}/download/${locationId}`) + req.on('error', () => {}) + const [, serverRes] = await serverSide + req.destroy() + await once(serverRes, 'close') + markClientGone() + await adapterSettled + + expect(body).toBeDefined() + await vi.waitFor( + () => { + expect( + Object.values(agent.sockets).flat(), + 'sockets held for the gone client', + ).toHaveLength(0) + expect(body!.destroyed, 'backend body destroyed').toBe(true) + }, + { timeout: 5000, interval: 25 }, + ) + await vi.waitFor( + async () => { + const leases = await db + .selectFrom('storage_reader_leases') + .where('storageLocationId', '=', locationId) + .select('id') + .execute() + expect(leases, 'reader leases left behind').toHaveLength(0) + }, + { timeout: 5000, interval: 25 }, + ) + } finally { + agent.destroy() + await origin.close() + } + }, 20_000) +}) From f8d500c84b4801117f845d01bc649f9d8a817d54 Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Sat, 24 Oct 2026 16:54:09 -0400 Subject: [PATCH 10/17] feat(metrics): expose S3 pool, S3 command, merge, lease and download metrics A starved S3 socket pool (paused GetObject sockets, a deep agent queue, HeadObject lookups timing out, Merge Leases never finishing) was invisible on /metrics. Add low-cardinality series that separate S3 latency, socket-pool queueing, Merge contention and client behaviour: - s3_http_sockets{pool,state}, s3_http_queued_requests{pool}, s3_http_max_sockets{pool}: agent pool state at scrape time, per pool (reads, writes, control). - s3_socket_wait_seconds{pool}: time a request waited for an agent socket. - s3_requests_total{command,outcome}, s3_request_duration_seconds{command}, s3_request_retries_total{command}: per-command outcome and latency from an SDK initialize-step middleware (includes retries and socket wait). - cache_merges_total{kind,result}, cache_merge_duration_seconds{kind,result}, cache_merges_in_progress, cache_merge_oldest_age_seconds, cache_merges_skipped_total (MAX_CONCURRENT_MERGES reached). - cache_leases{type,state}: Merge and Storage Reader Lease rows, active vs expired. - cache_downloads_total{result}: completed, client_gone, idle_timeout, not_found, error. - cache_lookup_duration_seconds{result}: download-URL lookup latency. Nothing is recorded per chunk; pool and lease gauges are computed at scrape. The registry stays per process (ADR-0007). Signed-off-by: Rui Chen --- lib/metrics.ts | 252 +++++++++++++++++- lib/storage.ts | 52 +++- routes/download/[locationId].ts | 16 +- .../GetCacheEntryDownloadURL.post.ts | 22 +- tests/merge-concurrency.test.ts | 6 + tests/metrics.test.ts | 52 ++++ 6 files changed, 380 insertions(+), 20 deletions(-) diff --git a/lib/metrics.ts b/lib/metrics.ts index 012d35f..f1190da 100644 --- a/lib/metrics.ts +++ b/lib/metrics.ts @@ -1,5 +1,7 @@ +import type { Agent, ClientRequest } from 'node:http' +import { performance } from 'node:perf_hooks' import { sql } from 'kysely' -import { collectDefaultMetrics, Counter, Gauge, Registry } from 'prom-client' +import { collectDefaultMetrics, Counter, Gauge, Histogram, Registry } from 'prom-client' import { getDatabase } from './db' // Per-process registry. Correct for the shipped single-worker-per-container @@ -42,3 +44,251 @@ export const cacheStorageBytes = new Gauge({ cacheStorageBytes.set(Number(bytes)) }, }) + +const SECONDS_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60] + +export const cacheLookupDurationSeconds = new Histogram({ + name: 'cache_lookup_duration_seconds', + help: 'Download-URL lookup handling time by result, including the storage existence check.', + labelNames: ['result'], + buckets: SECONDS_BUCKETS, + registers: [metricsRegistry], +}) + +export const cacheDownloadsTotal = new Counter({ + name: 'cache_downloads_total', + help: 'Proxied /download requests by outcome. client_gone: the client hung up before the body finished; idle_timeout: no bytes moved for DOWNLOAD_IDLE_TIMEOUT_MS.', + labelNames: ['result'], + registers: [metricsRegistry], +}) +for (const result of ['completed', 'client_gone', 'idle_timeout', 'not_found', 'error']) + cacheDownloadsTotal.inc({ result }, 0) + +export type MergeKind = 'stream' | 'compose' + +export const cacheMergesTotal = new Counter({ + name: 'cache_merges_total', + help: 'Finished Merges by kind (stream: parts piped through this process; compose: server-side copy) and result.', + labelNames: ['kind', 'result'], + registers: [metricsRegistry], +}) + +export const cacheMergesSkippedTotal = new Counter({ + name: 'cache_merges_skipped_total', + help: 'Merges not started because this process already runs MAX_CONCURRENT_MERGES; a later download retries them.', + registers: [metricsRegistry], +}) + +export const cacheMergeDurationSeconds = new Histogram({ + name: 'cache_merge_duration_seconds', + help: 'Wall time of finished Merges by kind and result.', + labelNames: ['kind', 'result'], + buckets: [1, 5, 15, 30, 60, 120, 300, 600, 1800, 3600], + registers: [metricsRegistry], +}) + +const mergeStartTimes = new Set<{ startedAt: number }>() + +export const cacheMergesInProgress = new Gauge({ + name: 'cache_merges_in_progress', + help: 'Merges running now in this process.', + registers: [metricsRegistry], + collect() { + cacheMergesInProgress.set(mergeStartTimes.size) + }, +}) + +export const cacheMergeOldestAgeSeconds = new Gauge({ + name: 'cache_merge_oldest_age_seconds', + help: 'Age of the longest-running in-progress Merge; 0 when none is running.', + registers: [metricsRegistry], + collect() { + let oldest = Infinity + for (const { startedAt } of mergeStartTimes) oldest = Math.min(oldest, startedAt) + cacheMergeOldestAgeSeconds.set(oldest === Infinity ? 0 : (performance.now() - oldest) / 1000) + }, +}) + +/** Call when a Merge starts; call the returned function exactly once when it settles. */ +export function trackMerge(kind: MergeKind) { + const entry = { startedAt: performance.now() } + mergeStartTimes.add(entry) + return (result: 'success' | 'failure') => { + if (!mergeStartTimes.delete(entry)) return + cacheMergesTotal.inc({ kind, result }) + cacheMergeDurationSeconds.observe( + { kind, result }, + (performance.now() - entry.startedAt) / 1000, + ) + } +} + +// Lease rows, computed at scrape time. Expired rows mean a holder died or stopped +// renewing; a growing active count with no finished Merges means Merges are stuck. +export const cacheLeases = new Gauge({ + name: 'cache_leases', + help: 'Merge and Storage Reader Lease rows by state (active: not yet expired).', + labelNames: ['type', 'state'], + registers: [metricsRegistry], + async collect() { + const db = await getDatabase() + const now = Date.now() + const count = (table: 'merge_leases' | 'storage_reader_leases') => + db + .selectFrom(table) + .select([ + sql`count(*)`.as('total'), + sql`coalesce(sum(case when ${sql.ref('expiresAt')} < ${now} then 1 else 0 end), 0)`.as( + 'expired', + ), + ]) + .executeTakeFirstOrThrow() + const [merge, reader] = await Promise.all([ + count('merge_leases'), + count('storage_reader_leases'), + ]) + for (const [type, row] of [ + ['merge', merge], + ['reader', reader], + ] as const) { + const expired = Number(row.expired) + cacheLeases.set({ type, state: 'expired' }, expired) + cacheLeases.set({ type, state: 'active' }, Number(row.total) - expired) + } + }, +}) + +// --- S3 client --------------------------------------------------------------- + +/** reads: GetObject bodies; writes: uploads and Merges; control: HEAD/LIST/DELETE. */ +export type S3Pool = 'reads' | 'writes' | 'control' + +const s3Agents: { pool: S3Pool; agent: Agent }[] = [] + +export const s3HttpSockets = new Gauge({ + name: 's3_http_sockets', + help: 'S3 agent sockets by pool and state (active: assigned to a request, idle: kept alive). Counts the agent view; a socket the remote end already closed still reads active until it is read or destroyed.', + labelNames: ['pool', 'state'], + registers: [metricsRegistry], + collect() { + const counts = new Map() + for (const { pool, agent } of s3Agents) { + const count = counts.get(pool) ?? { active: 0, idle: 0 } + for (const sockets of Object.values(agent.sockets)) count.active += sockets?.length ?? 0 + for (const sockets of Object.values(agent.freeSockets)) count.idle += sockets?.length ?? 0 + counts.set(pool, count) + } + for (const [pool, { active, idle }] of counts) { + s3HttpSockets.set({ pool, state: 'active' }, active) + s3HttpSockets.set({ pool, state: 'idle' }, idle) + } + }, +}) + +export const s3HttpQueuedRequests = new Gauge({ + name: 's3_http_queued_requests', + help: 'Live S3 requests waiting for a free agent socket, by pool (maxSockets reached). Sustained > 0 means pool starvation.', + labelNames: ['pool'], + registers: [metricsRegistry], + collect() { + const counts = new Map() + for (const { pool, agent } of s3Agents) { + let queued = counts.get(pool) ?? 0 + // Aborted requests stay in the queue until a socket frees; Node then skips them. + for (const requests of Object.values(agent.requests)) + queued += requests?.filter((req) => !req.destroyed).length ?? 0 + counts.set(pool, queued) + } + for (const [pool, queued] of counts) s3HttpQueuedRequests.set({ pool }, queued) + }, +}) + +export const s3HttpMaxSockets = new Gauge({ + name: 's3_http_max_sockets', + help: 'Configured S3 agent socket limit by pool (STORAGE_S3_MAX_SOCKETS, STORAGE_S3_CONTROL_MAX_SOCKETS).', + labelNames: ['pool'], + registers: [metricsRegistry], +}) + +export const s3SocketWaitSeconds = new Histogram({ + name: 's3_socket_wait_seconds', + help: 'Time an S3 request waited for an agent socket, by pool. Requests aborted while queued are not observed; pair with s3_http_queued_requests.', + labelNames: ['pool'], + buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60], + registers: [metricsRegistry], +}) + +/** Exposes `agent`'s pool on /metrics and times how long each request waits for a socket. */ +export function instrumentS3Agent(pool: S3Pool, agent: Agent) { + s3Agents.push({ pool, agent }) + s3HttpMaxSockets.set({ pool }, agent.maxSockets) + // `addRequest` is where Node's Agent either hands out a socket or queues the + // request; it is undocumented but the hook every agent wrapper relies on. + const pooled = agent as Agent & { addRequest: (req: ClientRequest, options: unknown) => void } + const addRequest = pooled.addRequest.bind(agent) + pooled.addRequest = (req, options) => { + const queuedAt = performance.now() + req.once('socket', () => + s3SocketWaitSeconds.observe({ pool }, (performance.now() - queuedAt) / 1000), + ) + addRequest(req, options) + } +} + +export const s3RequestsTotal = new Counter({ + name: 's3_requests_total', + help: 'S3 commands by outcome (ok, not_found, aborted, timeout, network, error).', + labelNames: ['command', 'outcome'], + registers: [metricsRegistry], +}) + +export const s3RequestDurationSeconds = new Histogram({ + name: 's3_request_duration_seconds', + help: 'S3 command time until the response headers, including socket wait and SDK retries; excludes streaming a GetObject body.', + labelNames: ['command'], + buckets: SECONDS_BUCKETS, + registers: [metricsRegistry], +}) + +export const s3RequestRetriesTotal = new Counter({ + name: 's3_request_retries_total', + help: 'SDK retry attempts beyond the first, by command.', + labelNames: ['command'], + registers: [metricsRegistry], +}) + +const S3_NETWORK_ERROR_CODES: Record = { + ECONNRESET: true, + ECONNREFUSED: true, + EPIPE: true, + ENOTFOUND: true, + EAI_AGAIN: true, +} + +function s3Outcome(err: unknown) { + const { name, code, $metadata } = (err ?? {}) as { + name?: string + code?: string + $metadata?: { httpStatusCode?: number } + } + if (name === 'AbortError') return 'aborted' + // The SDK renames ECONNRESET/EPIPE to TimeoutError, so check the code first. + if (S3_NETWORK_ERROR_CODES[code ?? '']) return 'network' + if (name === 'TimeoutError' || name === 'RequestTimeout' || code === 'ETIMEDOUT') return 'timeout' + if (name === 'NotFound' || name === 'NoSuchKey' || $metadata?.httpStatusCode === 404) + return 'not_found' + return 'error' +} + +/** Records one S3 command: `err` is the thrown error, absent on success. */ +export function observeS3Request( + commandName: string, + startedAt: number, + attempts: number | undefined, + err?: unknown, +) { + const command = commandName.replace(/Command$/, '') + s3RequestDurationSeconds.observe({ command }, (performance.now() - startedAt) / 1000) + s3RequestsTotal.inc({ command, outcome: err === undefined ? 'ok' : s3Outcome(err) }) + if (attempts && attempts > 1) s3RequestRetriesTotal.inc({ command }, attempts - 1) +} diff --git a/lib/storage.ts b/lib/storage.ts index d2c7a36..9238826 100644 --- a/lib/storage.ts +++ b/lib/storage.ts @@ -4,6 +4,7 @@ import type { Kysely } from 'kysely' import type { Writable } from 'node:stream' import type { Database, StorageLocation } from './db' import type { Reservation } from './helpers' +import type { MergeKind, S3Pool } from './metrics' import type { Env } from './schemas' import { randomUUID } from 'node:crypto' import { createReadStream, createWriteStream } from 'node:fs' @@ -44,6 +45,7 @@ import { getDatabase, isUniqueViolation, retryOnLockConflict } from './db' import { env } from './env' import { generateNumberId, reservationHash } from './helpers' import { logger } from './logger' +import { cacheMergesSkippedTotal, instrumentS3Agent, observeS3Request, trackMerge } from './metrics' import { acquireMergeLease, createReaderLease, @@ -311,10 +313,13 @@ export class Storage { * returned promise never rejects: failures are logged and the merge state * rolled back. */ - private async startMerge(location: StorageLocation, write: () => Promise) { + private async startMerge(location: StorageLocation, kind: MergeKind, write: () => Promise) { // Each Merge holds storage connections and buffers for its whole duration; // unbounded, a burst of first restores or finalizes exhausts them. - if (this.activeMerges >= env.MAX_CONCURRENT_MERGES) return false + if (this.activeMerges >= env.MAX_CONCURRENT_MERGES) { + cacheMergesSkippedTotal.inc() + return false + } // Reserve the slot before the first await so concurrent callers cannot overshoot it. this.activeMerges++ let mergeToken: string | undefined @@ -340,6 +345,7 @@ export class Storage { }, LEASE_RENEWAL_MS) renewalTimer.unref() + const finishMerge = trackMerge(kind) const mergePromise = write() .then(async () => { // The merged object is already written, so losing a deadlock here must @@ -361,8 +367,10 @@ export class Storage { .execute() }), ) + finishMerge('success') }) .catch(async (err) => { + finishMerge('failure') logger.error(`Merge failed for storage location ${location.id}`, { error: err }) await this.db .updateTable('storage_locations') @@ -404,7 +412,7 @@ export class Storage { bytes <= compose.maxPartBytes && (index === partSizes.length - 1 || bytes >= compose.minPartBytes), ) - await this.startMerge(location, () => + await this.startMerge(location, composable ? 'compose' : 'stream', () => composable ? compose.run(location.folderName, location.partCount) : this.adapter.uploadStream( @@ -642,7 +650,7 @@ export class Storage { const responseStream = new PassThrough() const mergerStream = new PassThrough() - const merge = await this.startMerge(storageLocation, () => + const merge = await this.startMerge(storageLocation, 'stream', () => this.adapter .uploadStream(`${storageLocation.folderName}/merged`, mergerStream) .catch((err) => { @@ -934,29 +942,53 @@ class S3Adapter implements StorageAdapter { // deadlock it, and the HeadObject of every cache lookup queues behind all // bulk transfers. Write holders never wait on reads and control requests // hold a socket only briefly, so no wait cycle can span pools. - const createClient = (maxSockets: number, connectionTimeout?: number) => { + const createClient = (pool: S3Pool, maxSockets: number, connectionTimeout?: number) => { // Both protocols get the configured pool: NodeHttpHandler picks the agent by // endpoint scheme, and an http:// endpoint (MinIO, RustFS, an in-cluster // gateway) would otherwise fall back to the SDK's default agent and ignore // the configured size. const agentOptions = { keepAlive: true, maxSockets, keepAliveMsecs: 1000 } - return new S3Client({ + const httpAgent = new HttpAgent(agentOptions) + const httpsAgent = new HttpsAgent(agentOptions) + instrumentS3Agent(pool, httpAgent) + instrumentS3Agent(pool, httpsAgent) + const client = new S3Client({ forcePathStyle: true, region: env.AWS_REGION, requestHandler: new NodeHttpHandler({ - httpAgent: new HttpAgent(agentOptions), - httpsAgent: new HttpsAgent(agentOptions), + httpAgent, + httpsAgent, socketTimeout: env.STORAGE_S3_SOCKET_TIMEOUT_MS, connectionTimeout, }), }) + // `initialize` wraps the whole command (retries, socket wait, signing), so the + // duration is what callers wait for; a GetObject settles at response headers. + client.middlewareStack.add( + (next, context) => async (args) => { + const startedAt = performance.now() + try { + const result = await next(args) + const { $metadata } = result.output as { $metadata?: { attempts?: number } } + observeS3Request(context.commandName ?? 'unknown', startedAt, $metadata?.attempts) + return result + } catch (err) { + const { $metadata } = err as { $metadata?: { attempts?: number } } + observeS3Request(context.commandName ?? 'unknown', startedAt, $metadata?.attempts, err) + throw err + } + }, + { step: 'initialize', name: 'cacheServerMetrics' }, + ) + return client } const clients: S3Clients = { - reads: createClient(env.STORAGE_S3_MAX_SOCKETS), - writes: createClient(env.STORAGE_S3_MAX_SOCKETS), + reads: createClient('reads', env.STORAGE_S3_MAX_SOCKETS), + writes: createClient('writes', env.STORAGE_S3_MAX_SOCKETS), // Bounds the wait for a socket plus the connect, so a starved lookup fails // instead of hanging; cleared once a kept-alive socket is assigned. control: createClient( + 'control', env.STORAGE_S3_CONTROL_MAX_SOCKETS, env.STORAGE_S3_CONTROL_QUEUE_TIMEOUT_MS, ), diff --git a/routes/download/[locationId].ts b/routes/download/[locationId].ts index bed49d7..eec603a 100644 --- a/routes/download/[locationId].ts +++ b/routes/download/[locationId].ts @@ -3,6 +3,7 @@ import { pipeline } from 'node:stream/promises' import { z } from 'zod' import { env } from '~/lib/env' import { logger } from '~/lib/logger' +import { cacheDownloadsTotal } from '~/lib/metrics' import { getStorage, RangeNotSatisfiableError } from '~/lib/storage' const pathParamsSchema = z.object({ @@ -44,12 +45,16 @@ export default defineEventHandler(async (event) => { try { download = await storage.download(locationId, range) } catch (err) { - if (!(err instanceof RangeNotSatisfiableError)) throw err + if (!(err instanceof RangeNotSatisfiableError)) { + cacheDownloadsTotal.inc({ result: 'error' }) + throw err + } // An empty entry has no satisfiable range. The filesystem and GCS adapters raise // for any Range on one and Azure for any start past 0, while RustFS (S3) answers // `bytes=0-` with an empty 200. Ignore Range on empty entries (RFC 9110 ยง14.2) so // the response does not depend on the storage driver. if (err.size === 0) { + cacheDownloadsTotal.inc({ result: 'completed' }) setHeader(event, 'content-length', 0) return send(event) } @@ -57,11 +62,13 @@ export default defineEventHandler(async (event) => { if (err.size !== undefined) setHeader(event, 'content-range', `bytes */${err.size}`) return send(event) } - if (!download) + if (!download) { + cacheDownloadsTotal.inc({ result: 'not_found' }) throw createError({ statusCode: 404, message: 'Cache file not found', }) + } if (download.range && download.size !== undefined) { setResponseStatus(event, 206) @@ -85,6 +92,7 @@ export default defineEventHandler(async (event) => { const { res } = event.node if (res.destroyed || res.closed) { download.stream.destroy() + cacheDownloadsTotal.inc({ result: 'client_gone' }) logger.debug(`Client aborted /download/${locationId} before storage answered`) return } @@ -107,20 +115,24 @@ export default defineEventHandler(async (event) => { } try { await pipeline(download.stream, res) + cacheDownloadsTotal.inc({ result: 'completed' }) } catch (err) { // Every exit that did not finish the pipe must release the backend read: an // unread body pins its storage connection and the reader lease it carries. download.stream.destroy() const { code } = err as NodeJS.ErrnoException if (code === 'ERR_DOWNLOAD_IDLE_TIMEOUT') { + cacheDownloadsTotal.inc({ result: 'idle_timeout' }) logger.warn(`Download /download/${locationId} closed: ${(err as Error).message}`) return } // Client went away mid-body. Expected on cancelled jobs and parallel runners. if (code === 'ERR_STREAM_PREMATURE_CLOSE' || code === 'ERR_STREAM_UNABLE_TO_PIPE') { + cacheDownloadsTotal.inc({ result: 'client_gone' }) logger.debug(`Client aborted /download/${locationId}: ${(err as Error).message}`) return } + cacheDownloadsTotal.inc({ result: 'error' }) // Headers are out, so Nitro's error handler would crash with ERR_HTTP_HEADERS_SENT. if (res.headersSent) { logger.error(`Download stream failed for ${locationId}`, { error: err }) diff --git a/routes/twirp/github.actions.results.api.v1.CacheService/GetCacheEntryDownloadURL.post.ts b/routes/twirp/github.actions.results.api.v1.CacheService/GetCacheEntryDownloadURL.post.ts index feb91d0..1c45cbf 100644 --- a/routes/twirp/github.actions.results.api.v1.CacheService/GetCacheEntryDownloadURL.post.ts +++ b/routes/twirp/github.actions.results.api.v1.CacheService/GetCacheEntryDownloadURL.post.ts @@ -1,6 +1,6 @@ import { map, pipe, prop, sortBy } from 'remeda' import { z } from 'zod' -import { cacheRequestsTotal } from '~/lib/metrics' +import { cacheLookupDurationSeconds, cacheRequestsTotal } from '~/lib/metrics' import { getCacheScope } from '~/lib/scope' import { getStorage } from '~/lib/storage' import { readTwirpRequest, sendTwirpResponse, TwirpMessage } from '~/lib/twirp' @@ -21,12 +21,20 @@ export default defineEventHandler(async (event) => { ) const storage = await getStorage() - const match = await storage.getCacheEntryWithDownloadUrl({ - keys: [key, ...(restore_keys ?? [])], - version, - scopes: pipe(scopes, sortBy([prop('Permission'), 'desc']), map(prop('Scope'))), - repoId, - }) + const lookupTimer = cacheLookupDurationSeconds.startTimer() + let match + try { + match = await storage.getCacheEntryWithDownloadUrl({ + keys: [key, ...(restore_keys ?? [])], + version, + scopes: pipe(scopes, sortBy([prop('Permission'), 'desc']), map(prop('Scope'))), + repoId, + }) + } catch (err) { + lookupTimer({ result: 'error' }) + throw err + } + lookupTimer({ result: match ? 'hit' : 'miss' }) cacheRequestsTotal.inc({ result: match ? 'hit' : 'miss' }) diff --git a/tests/merge-concurrency.test.ts b/tests/merge-concurrency.test.ts index 85e3e71..66e6852 100644 --- a/tests/merge-concurrency.test.ts +++ b/tests/merge-concurrency.test.ts @@ -4,6 +4,7 @@ import { Readable } from 'node:stream' import { afterEach, describe, expect, test } from 'vitest' import { getDatabase } from '~/lib/db' import { env } from '~/lib/env' +import { cacheMergesSkippedTotal } from '~/lib/metrics' import { Storage } from '~/lib/storage' const scope = { version: 'v1', scope: 'refs/heads/main', repoId: '123' } @@ -64,8 +65,13 @@ describe('merge concurrency limit', () => { try { // The first restore's Merge is paced by its unread stream, so it still // holds the only slot when the second restore starts. + // `get()` hands out live value objects, so read the number right away. + const before = await cacheMergesSkippedTotal.get() + const skippedBefore = before.values[0]?.value ?? 0 const firstStream = await download(storage, first.locationId) const secondStream = await download(storage, second.locationId) + const after = await cacheMergesSkippedTotal.get() + expect((after.values[0]?.value ?? 0) - skippedBefore, 'merges skipped').toBe(1) expect(await sha256(secondStream)).toBe(expected) expect(await sha256(firstStream)).toBe(expected) await storage.waitForOngoingMerges() diff --git a/tests/metrics.test.ts b/tests/metrics.test.ts index b072108..9959203 100644 --- a/tests/metrics.test.ts +++ b/tests/metrics.test.ts @@ -71,5 +71,57 @@ describe('prometheus metrics', () => { // Scrape-time gauge: present and non-negative (collect() ran without throwing). expect(after).toContain('cache_storage_bytes') expect(readCounter(after, 'cache_storage_bytes')).toBeGreaterThanOrEqual(0) + + // Every series the dashboards read is exposed, with or without observations. + for (const name of [ + 'cache_lookup_duration_seconds', + 'cache_downloads_total', + 'cache_merges_total', + 'cache_merges_skipped_total', + 'cache_merge_duration_seconds', + 'cache_merges_in_progress', + 'cache_merge_oldest_age_seconds', + 'cache_leases', + 's3_http_sockets', + 's3_http_queued_requests', + 's3_http_max_sockets', + 's3_socket_wait_seconds', + 's3_requests_total', + 's3_request_duration_seconds', + 's3_request_retries_total', + ]) + expect(after, name).toMatch(new RegExp(`^# TYPE ${name} `, 'm')) + + for (const series of [ + 'cache_lookup_duration_seconds_count{result="hit"}', + 'cache_lookup_duration_seconds_count{result="miss"}', + 'cache_downloads_total{result="completed"}', + ]) + expect(readCounter(after, series), series).toBeGreaterThan(readCounter(before, series)) + expect(after).toContain('cache_downloads_total{result="idle_timeout"}') + expect(after).toContain('cache_leases{type="merge",state="active"}') + expect(after).toContain('cache_leases{type="reader",state="expired"}') + + // The first restore of an unmerged entry runs a streaming Merge in the background. + const merged = 'cache_merges_total{kind="stream",result="success"}' + await expect + .poll(async () => readCounter(await scrape(), merged), { timeout: 10_000 }) + .toBeGreaterThan(readCounter(before, merged)) + + if (process.env.VITEST_STORAGE_DRIVER === 's3') { + const headOk = 's3_requests_total{command="HeadObject",outcome="ok"}' + expect(readCounter(after, headOk)).toBeGreaterThan(readCounter(before, headOk)) + // Upload, restore and lookup each went through their own configured, + // instrumented pool (the test endpoint is http://, so this also proves the + // SDK's default agent is not used). + for (const pool of ['writes', 'reads', 'control']) { + const pooled = + readCounter(after, `s3_http_sockets{pool="${pool}",state="active"}`) + + readCounter(after, `s3_http_sockets{pool="${pool}",state="idle"}`) + expect(pooled, `${pool} sockets`).toBeGreaterThan(0) + } + expect(readCounter(after, 's3_http_max_sockets{pool="reads"}')).toBe(50) + expect(readCounter(after, 's3_http_max_sockets{pool="control"}')).toBe(16) + } }) }) From 12b6ac43dc163a961adb7b5a99ea7b0acbda12d5 Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Sat, 24 Oct 2026 16:54:47 -0400 Subject: [PATCH 11/17] ci: build patched images from patch-* tags as OCI archives Add a workflow that runs only for `patch-` tags and builds the multi-arch image (linux/amd64, linux/arm64) into an OCI archive uploaded as a workflow artifact (`image-`, kept 7 days). It logs in nowhere and pushes nothing; the archive is published by hand. ci-cd.yml (branch pushes, dev deploys) and release.yml (`v*` tags) are unchanged and do not match these tags. Signed-off-by: Rui Chen --- .github/workflows/patch-image.yml | 42 +++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/patch-image.yml diff --git a/.github/workflows/patch-image.yml b/.github/workflows/patch-image.yml new file mode 100644 index 0000000..4be22a0 --- /dev/null +++ b/.github/workflows/patch-image.yml @@ -0,0 +1,42 @@ +name: Patch image + +# Builds a multi-arch OCI image archive for each `patch-` tag +# (`patch-9.9.0-backpressure.1` -> artifact `image-9.9.0-backpressure.1`). +# The ghcr.io package predates this repo link, so GITHUB_TOKEN cannot push to it; +# publish the archive with `crane push --index ghcr.io//github-actions-cache-server:`. +# Kept separate from release.yml, which only releases from dev and moves `latest`. +on: + push: + tags: + - 'patch-*' + +permissions: + contents: read + +jobs: + image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Image tag + id: tag + run: echo "TAG=${GITHUB_REF_NAME#patch-}" >> "$GITHUB_OUTPUT" + + - uses: docker/setup-qemu-action@v4 + + - uses: docker/setup-buildx-action@v4 + + - uses: docker/build-push-action@v7 + with: + context: . + platforms: linux/amd64,linux/arm64 + outputs: type=oci,dest=${{ runner.temp }}/image.tar,tar=true + build-args: | + BUILD_HASH=${{ github.sha }} + + - uses: actions/upload-artifact@v5 + with: + name: image-${{ steps.tag.outputs.TAG }} + path: ${{ runner.temp }}/image.tar + retention-days: 7 From 43970c286d1d36a24165918c241a3f7e6f380fbc Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Sat, 24 Oct 2026 16:57:58 -0400 Subject: [PATCH 12/17] fix(db): release the Postgres startup probe client getDatabase checks a client out of the pool to fail fast on bad connection settings and never released it. That client stays checked out for the life of the process, leaving the pool one connection short, and Kysely's destroy() (pool.end()) would wait for it forever. Signed-off-by: Rui Chen --- lib/db.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/db.ts b/lib/db.ts index 5413c96..781ff64 100644 --- a/lib/db.ts +++ b/lib/db.ts @@ -140,7 +140,10 @@ export const getDatabase = createSingletonPromise(async () => { max: 10, }, ) - await pool.connect() + // Fail fast on bad connection settings, then hand the client back: a + // checked-out client keeps `pool.end()` (Kysely's `destroy()`) waiting forever. + const probe = await pool.connect() + probe.release() return new PostgresDialect({ pool, From 7f83ffb3657bfd6562f2436bd6c1339cd8ac5b67 Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Sat, 24 Oct 2026 15:58:25 -0400 Subject: [PATCH 13/17] fix(management-api): honor repoId when deleting cache entries `DELETE /management-api/cache-entries` accepted a `repoId` filter and documented it, but never applied it, so a delete filtered only by repository removed every cache entry of every repository. Apply it like the other filters. The Rust rewrite in #288 already fixes this; this is the same fix for the current TypeScript server. Signed-off-by: Rui Chen (cherry picked from commit 62e763eddeaa1ae22044060f169cb12a48595572) --- lib/api/cache-entries.ts | 1 + tests/management-api.test.ts | 41 ++++++++++++++++++++++++++++++++++++ tests/setup.ts | 1 + 3 files changed, 43 insertions(+) create mode 100644 tests/management-api.test.ts diff --git a/lib/api/cache-entries.ts b/lib/api/cache-entries.ts index b5b8828..1d2c3c9 100644 --- a/lib/api/cache-entries.ts +++ b/lib/api/cache-entries.ts @@ -164,6 +164,7 @@ export const cacheEntriesRouter = base if (input.key) query = query.where('key', '=', input.key) if (input.version) query = query.where('version', '=', input.version) if (input.scope) query = query.where('scope', '=', input.scope) + if (input.repoId) query = query.where('repoId', '=', input.repoId) await query.execute() context.event.waitUntil(runTask('cleanup:storage-locations')) diff --git a/tests/management-api.test.ts b/tests/management-api.test.ts new file mode 100644 index 0000000..5dd7a3c --- /dev/null +++ b/tests/management-api.test.ts @@ -0,0 +1,41 @@ +import { randomUUID } from 'node:crypto' +import { Readable } from 'node:stream' + +import { expect, test } from 'vitest' +import { getDatabase } from '~/lib/db' +import { getStorage } from '~/lib/storage' + +async function saveEntry(entry: { key: string; version: string; scope: string; repoId: string }) { + const storage = await getStorage() + const upload = await storage.createUpload(entry) + await storage.uploadPart(upload.id, 0, Readable.from(['payload'])) + await storage.completeUpload(entry) +} + +async function countEntries(key: string, repoId: string) { + const db = await getDatabase() + const { count } = await db + .selectFrom('cache_entries') + .where('key', '=', key) + .where('repoId', '=', repoId) + .select((eb) => eb.fn.countAll().as('count')) + .executeTakeFirstOrThrow() + return Number(count) +} + +test('deleting cache entries by repoId leaves other repositories alone', async () => { + const key = `management-delete-${randomUUID()}` + const entry = { key, version: 'v1', scope: 'refs/heads/main' } + await saveEntry({ ...entry, repoId: 'kept' }) + await saveEntry({ ...entry, repoId: 'deleted' }) + + const res = await fetch('http://localhost:3000/management-api/cache-entries', { + method: 'DELETE', + headers: { 'content-type': 'application/json', 'x-api-key': 'vitest-management-api-key' }, + body: JSON.stringify({ repoId: 'deleted' }), + }) + expect(res.ok).toBe(true) + + expect(await countEntries(key, 'deleted')).toBe(0) + expect(await countEntries(key, 'kept')).toBe(1) +}) diff --git a/tests/setup.ts b/tests/setup.ts index 8e8660a..009f771 100644 --- a/tests/setup.ts +++ b/tests/setup.ts @@ -37,6 +37,7 @@ const TESTING_ENV_BASE = { ACTIONS_RESULTS_URL: 'http://localhost:3000/', ACTIONS_CACHE_URL: 'http://localhost:3000/', SKIP_TOKEN_VALIDATION: 'true', + MANAGEMENT_API_KEY: 'vitest-management-api-key', } satisfies Omit< typeof envBaseSchema.infer, | 'CACHE_CLEANUP_OLDER_THAN_DAYS' From 85189b700573312b237ae4aabe0d885900a125bb Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Sat, 24 Oct 2026 16:21:24 -0400 Subject: [PATCH 14/17] feat(auth): require verified tokens for direct downloads A direct-download URL is a bearer credential for the stored object, and the only gate in front of it is the token's `repository_id` and `ac` claims. With SKIP_TOKEN_VALIDATION those claims are never verified, so the server now refuses to start with ENABLE_DIRECT_DOWNLOADS and SKIP_TOKEN_VALIDATION together, unless UNSAFE_ALLOW_DIRECT_DOWNLOADS_WITHOUT_TOKEN_VALIDATION is set for local testing. - DIRECT_DOWNLOAD_URL_TTL_SECONDS (default 300, 1-3600) sets the signed URL lifetime. The reader lease outlives the URL by a fixed margin, since storage authenticates a GET only when it starts. - Token verification requires `exp`; ACTIONS_TOKEN_AUDIENCE optionally pins `aud`, and a value that lists no audience stops startup. - Token rejections report the jose error code and claim name instead of attaching the jose error, whose decoded payload was logged with it. - `ac` scopes are schema-checked and treated as bit flags: lookups use only readable scopes, uploads and finalizes only writable ones. The tests run a second server against a local OIDC issuer (discovery and JWKS), covering key rotation, a JWKS outage, fork pull request scopes and the signed URL lifetime on S3, GCS and Azure Blob. Signed-off-by: Rui Chen --- README.md | 9 + ...-reader-leases-protect-active-downloads.md | 2 +- ...irect-downloads-require-verified-tokens.md | 7 + lib/env.ts | 13 +- lib/schemas.ts | 48 ++ lib/scope.ts | 57 +- lib/storage-leases.ts | 7 +- lib/storage.ts | 18 +- .../CreateCacheEntry.post.ts | 4 +- .../FinalizeCacheEntryUpload.ts | 4 +- .../GetCacheEntryDownloadURL.post.ts | 15 +- tests/cleanup-lifecycle.test.ts | 8 +- tests/direct-download-auth.test.ts | 573 ++++++++++++++++++ tests/setup.ts | 24 +- tests/write-authorization.test.ts | 114 ++++ 15 files changed, 865 insertions(+), 38 deletions(-) create mode 100644 docs/adr/0012-direct-downloads-require-verified-tokens.md create mode 100644 tests/direct-download-auth.test.ts create mode 100644 tests/write-authorization.test.ts diff --git a/README.md b/README.md index 960adf4..4c05fe9 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,15 @@ All optional. | `MAX_CONCURRENT_MERGES` | `16` | Merges one process runs at once. Beyond it, restores stream the parts and a later download merges. | | `DOWNLOAD_IDLE_TIMEOUT_MS` | `300000` | Closes a download whose client has not read a byte for this long; `0` disables. | +Direct downloads (`ENABLE_DIRECT_DOWNLOADS=true`) hand the runner a signed storage URL for merged entries instead of proxying the bytes. +They require token validation: the server refuses to start with `SKIP_TOKEN_VALIDATION=true`, because unverified callers could then obtain signed URLs for any repository's cache. + +| Variable | Default | Effect | +| -------------------------------------------------------- | ------- | ------------------------------------------------------------------------------------------------- | +| `DIRECT_DOWNLOAD_URL_TTL_SECONDS` | `300` | Lifetime of a signed download URL, `1` to `3600`. Raise it only for clients that fetch URLs late. | +| `ACTIONS_TOKEN_AUDIENCE` | unset | Comma-separated `aud` values a token must carry; unset skips the check, an empty list is refused. | +| `UNSAFE_ALLOW_DIRECT_DOWNLOADS_WITHOUT_TOKEN_VALIDATION` | `false` | Allows direct downloads with `SKIP_TOKEN_VALIDATION=true`. Local testing only. | + ## Documentation ๐Ÿ‘‰ ๐Ÿ‘ˆ diff --git a/docs/adr/0002-part-reader-leases-protect-active-downloads.md b/docs/adr/0002-part-reader-leases-protect-active-downloads.md index b4a9309..7640afe 100644 --- a/docs/adr/0002-part-reader-leases-protect-active-downloads.md +++ b/docs/adr/0002-part-reader-leases-protect-active-downloads.md @@ -1,3 +1,3 @@ # Storage Reader Leases protect active downloads -Concurrent downloads hold durable, expiring Storage Reader Leases, including direct downloads whose lease lasts no longer than the signed URL. A renewable, fenced Merge Lease grants a single worker authority to complete the Merge; Part Reader Leases protect concurrent readers of Parts. Both expiring lease types renew every 30 seconds while active and expire after two minutes, except direct-download leases, which expire with their signed URLs. Parts are deleted only after the merge completes and no active Part Reader Leases remain; a Storage Location is deleted only after no Storage Reader Leases remain. A reader acquires its lease in the same locked transaction that reads the Storage Location's merge state, so its scope is chosen atomically: a download arriving before the merge takes a Part Reader Lease that blocks parts deletion, and one arriving after takes a Storage Reader Lease over the merged representation. There is therefore no window in which a reader can fail to acquire a Part Reader Lease against parts that are being deleted. +Concurrent downloads hold durable, expiring Storage Reader Leases, including direct downloads, whose lease outlives the signed URL by a fixed margin. A renewable, fenced Merge Lease grants a single worker authority to complete the Merge; Part Reader Leases protect concurrent readers of Parts. Both expiring lease types renew every 30 seconds while active and expire after two minutes, except direct-download leases, which expire a fixed margin after their signed URLs: storage authenticates a GET when it starts, so a transfer begun just before the URL expires keeps reading after it. Parts are deleted only after the merge completes and no active Part Reader Leases remain; a Storage Location is deleted only after no Storage Reader Leases remain. A reader acquires its lease in the same locked transaction that reads the Storage Location's merge state, so its scope is chosen atomically: a download arriving before the merge takes a Part Reader Lease that blocks parts deletion, and one arriving after takes a Storage Reader Lease over the merged representation. There is therefore no window in which a reader can fail to acquire a Part Reader Lease against parts that are being deleted. diff --git a/docs/adr/0012-direct-downloads-require-verified-tokens.md b/docs/adr/0012-direct-downloads-require-verified-tokens.md new file mode 100644 index 0000000..a57cb33 --- /dev/null +++ b/docs/adr/0012-direct-downloads-require-verified-tokens.md @@ -0,0 +1,7 @@ +# Direct downloads require verified tokens + +A direct-download URL is a bearer credential: whoever holds it can read the merged object from storage until it expires, without asking the server again. The only thing standing between a caller and that URL is `getCacheScope`, which authorizes a lookup from the token's `repository_id` and `ac` claims. With `SKIP_TOKEN_VALIDATION=true` those claims are decoded but not verified, so any client that can reach the server can name any repository and scope and walk away with a signed URL for its cache. The server therefore refuses to start with `ENABLE_DIRECT_DOWNLOADS=true` and `SKIP_TOKEN_VALIDATION=true` together. `UNSAFE_ALLOW_DIRECT_DOWNLOADS_WITHOUT_TOKEN_VALIDATION=true` overrides the check for local testing only; it is named so that it cannot be set by accident. This extends ADR-0006: skipping verification stays a dev/test escape hatch, and the one feature that turns an unverified claim into a storage credential is now unreachable with it. + +A signed URL lives `DIRECT_DOWNLOAD_URL_TTL_SECONDS` (default 300, at most 3600). Clients request the URL immediately before fetching it, and storage authenticates a GET when it starts rather than for its whole duration, so the lifetime only has to cover the start of the transfer and the client's retries, not a large restore. A short lifetime limits replay of a URL that leaks into job logs; it also stays inside the window in which the AWS SDK refreshes temporary credentials (5 minutes before they expire), so a URL is never signed by credentials that lapse before it does. Clients that cache a URL and fetch it later, such as BuildKit's `type=gha` cache importing a layer late in a build, need a longer lifetime; raise the setting for them. The Storage Reader Lease outlives the URL by a fixed margin (ADR-0002), so cleanup cannot delete an object while a transfer begun just before expiry is still reading it. + +Token verification requires an `exp` claim, because jose only checks expiry when the claim is present. The optional `ACTIONS_TOKEN_AUDIENCE` pins the expected `aud`, restricting the server to tokens issued for one installation; a value that lists no audience (blank, or only commas) stops startup rather than silently disabling the check. Token rejections return and log the jose error code and failing claim name, never the token or its decoded claims, which jose attaches to its errors. Cache scope permissions are bit flags (read = 1, write = 2): lookups use only scopes with the read bit, and uploads only scopes with the write bit. diff --git a/lib/env.ts b/lib/env.ts index 587caa4..aace2fd 100644 --- a/lib/env.ts +++ b/lib/env.ts @@ -1,6 +1,10 @@ import type { envDbDriverSchema, envStorageDriverSchema } from './schemas' import arkenv from 'arkenv' -import { envSchema } from './schemas' +import { + assertDirectDownloadsRequireTokenValidation, + envSchema, + parseActionsTokenAudience, +} from './schemas' export const env = arkenv(envSchema, { env: Object.assign( @@ -13,3 +17,10 @@ export const env = arkenv(envSchema, { process.env, ), }) + +assertDirectDownloadsRequireTokenValidation(env) + +// GitHub-issued runtime tokens carry a deployment-specific `aud`, so pinning it +// restricts the server to one installation's tokens. Parsed here so a bad value +// stops startup. +export const actionsTokenAudience = parseActionsTokenAudience(env.ACTIONS_TOKEN_AUDIENCE) diff --git a/lib/schemas.ts b/lib/schemas.ts index 493e5bd..d66cd92 100644 --- a/lib/schemas.ts +++ b/lib/schemas.ts @@ -79,6 +79,7 @@ export const envBaseSchema = type({ "string.url = 'https://results-receiver.actions.githubusercontent.com'", 'ACTIONS_TOKEN_ISSUER': "string.url = 'https://token.actions.githubusercontent.com'", 'ACTIONS_TOKEN_JWKS_URL?': 'string.url', + 'ACTIONS_TOKEN_AUDIENCE?': 'string', 'CACHE_CLEANUP_OLDER_THAN_DAYS': 'number = 90', 'CACHE_MAX_SIZE_BYTES?': 'number.integer > 0', 'CACHE_FILESYSTEM_MAX_USAGE_PERCENT': 'number > 0 & number <= 100 = 90', @@ -86,6 +87,8 @@ export const envBaseSchema = type({ 'DISABLE_CLEANUP_JOBS?': 'boolean', 'DEBUG?': 'unknown', 'ENABLE_DIRECT_DOWNLOADS': 'boolean = false', + 'DIRECT_DOWNLOAD_URL_TTL_SECONDS': '1 <= number.integer <= 3600 = 300', + 'UNSAFE_ALLOW_DIRECT_DOWNLOADS_WITHOUT_TOKEN_VALIDATION': 'boolean = false', 'EAGER_MERGE': 'boolean = false', 'MAX_CONCURRENT_MERGES': 'number.integer >= 1 = 16', 'DOWNLOAD_IDLE_TIMEOUT_MS': 'number.integer >= 0 = 300000', @@ -96,3 +99,48 @@ export const envBaseSchema = type({ export const envSchema = envBaseSchema.and(envStorageDriverSchema).and(envDbDriverSchema) export type Env = typeof envSchema.infer + +/** + * A direct-download URL is a bearer credential for the stored object. Handing + * one out to a caller whose token was never verified lets anyone who can reach + * the server read any repository's cache straight from storage, so the two + * settings are refused together unless explicitly overridden for local testing. + */ +export function assertDirectDownloadsRequireTokenValidation( + env: Pick< + Env, + | 'ENABLE_DIRECT_DOWNLOADS' + | 'SKIP_TOKEN_VALIDATION' + | 'UNSAFE_ALLOW_DIRECT_DOWNLOADS_WITHOUT_TOKEN_VALIDATION' + >, +) { + if ( + env.ENABLE_DIRECT_DOWNLOADS && + env.SKIP_TOKEN_VALIDATION && + !env.UNSAFE_ALLOW_DIRECT_DOWNLOADS_WITHOUT_TOKEN_VALIDATION + ) + throw new Error( + 'ENABLE_DIRECT_DOWNLOADS=true requires token validation, but SKIP_TOKEN_VALIDATION=true. ' + + 'Unverified callers would receive signed storage URLs for any cache entry. ' + + 'Unset SKIP_TOKEN_VALIDATION, or set UNSAFE_ALLOW_DIRECT_DOWNLOADS_WITHOUT_TOKEN_VALIDATION=true for local testing only.', + ) +} + +/** + * The `aud` values a token must carry. Unset means no audience check; set but + * listing no audience (blank, or only commas) is refused, so a typo cannot + * silently switch the check off. + */ +export function parseActionsTokenAudience(value: string | undefined) { + if (value === undefined) return + const audience = value + .split(',') + .map((entry) => entry.trim()) + .filter(Boolean) + if (audience.length === 0) + throw new Error( + 'ACTIONS_TOKEN_AUDIENCE is set but lists no audience. ' + + 'Set one or more comma-separated audiences, or unset it to skip the audience check.', + ) + return audience +} diff --git a/lib/scope.ts b/lib/scope.ts index bddbb61..3b611ed 100644 --- a/lib/scope.ts +++ b/lib/scope.ts @@ -1,7 +1,8 @@ import type { H3Event } from 'h3' import * as jose from 'jose' import { hasAtLeast } from 'remeda' -import { env } from './env' +import { z } from 'zod' +import { actionsTokenAudience, env } from './env' import { logger } from './logger' const issuer = env.ACTIONS_TOKEN_ISSUER.replace(/\/$/, '') @@ -64,20 +65,50 @@ async function verifyGitHubActionsToken(token: string) { return jose.decodeJwt(token) } - const { payload } = await jose.jwtVerify(token, await getJwks(), { issuer }) + const { payload } = await jose.jwtVerify(token, await getJwks(), { + issuer, + audience: actionsTokenAudience, + // jose checks `exp` and `nbf` only when present; a token without `exp` + // would otherwise never expire. + requiredClaims: ['exp'], + }) return payload } +/** + * The reason a token was rejected, safe to log and return: never the token or + * its claims. jose errors carry the decoded payload, so they must not become + * the `cause` of a logged error. + */ +function describeTokenError(err: unknown) { + if (err instanceof jose.errors.JWTClaimValidationFailed || err instanceof jose.errors.JWTExpired) + return `${err.code} (${err.claim})` + if (err instanceof jose.errors.JOSEError) return err.code + return err instanceof Error ? err.name : 'unknown error' +} + +export const CACHE_PERMISSION_READ = 1 +export const CACHE_PERMISSION_WRITE = 2 + +const cacheScopesSchema = z.array( + z.object({ + Scope: z.string().min(1), + Permission: z.number().int().nonnegative(), + }), +) + function parseJsonScopes(json: string) { + let value: unknown try { - return JSON.parse(json) - } catch (err) { - throw createError({ - statusCode: 401, - message: 'Invalid JSON in cache scopes', - cause: err, - }) + value = JSON.parse(json) + } catch { + // no `cause`: the parser's message quotes the claim + throw createError({ statusCode: 401, message: 'Invalid JSON in cache scopes' }) } + + const parsed = cacheScopesSchema.safeParse(value) + if (!parsed.success) throw createError({ statusCode: 401, message: 'Invalid cache scopes' }) + return parsed.data } export async function getCacheScope(event: H3Event) { @@ -86,18 +117,14 @@ export async function getCacheScope(event: H3Event) { throw createError({ statusCode: 401, message: 'Authorization header missing or malformed' }) const decoded = await verifyGitHubActionsToken(token).catch((err) => { - throw createError({ - statusCode: 401, - message: 'Invalid token', - cause: err, - }) + throw createError({ statusCode: 401, message: `Invalid token: ${describeTokenError(err)}` }) }) const scopesJson = decoded.ac if (!scopesJson || typeof scopesJson !== 'string') throw createError({ statusCode: 401, message: 'Token does not contain cache scopes' }) - const scopes = parseJsonScopes(scopesJson) as Array<{ Scope: string; Permission: number }> + const scopes = parseJsonScopes(scopesJson) if (!hasAtLeast(scopes, 1)) throw createError({ statusCode: 401, message: 'Token does not contain any cache scopes' }) diff --git a/lib/storage-leases.ts b/lib/storage-leases.ts index c06dc8e..5079f45 100644 --- a/lib/storage-leases.ts +++ b/lib/storage-leases.ts @@ -6,7 +6,12 @@ import { env } from './env' export const LEASE_DURATION_MS = 2 * 60 * 1000 export const LEASE_RENEWAL_MS = 30 * 1000 -export const DIRECT_DOWNLOAD_LEASE_DURATION_MS = 10 * 60 * 1000 +/** + * How long a direct-download reader lease outlives its signed URL. A download + * that starts just before the URL expires keeps streaming, so the lease must + * cover that transfer too. + */ +export const DIRECT_DOWNLOAD_LEASE_MARGIN_MS = 10 * 60 * 1000 export async function acquireMergeLease( db: Kysely, diff --git a/lib/storage.ts b/lib/storage.ts index 9238826..f6b781b 100644 --- a/lib/storage.ts +++ b/lib/storage.ts @@ -49,7 +49,7 @@ import { cacheMergesSkippedTotal, instrumentS3Agent, observeS3Request, trackMerg import { acquireMergeLease, createReaderLease, - DIRECT_DOWNLOAD_LEASE_DURATION_MS, + DIRECT_DOWNLOAD_LEASE_MARGIN_MS, LEASE_RENEWAL_MS, releaseMergeLease, releaseReaderLease, @@ -823,9 +823,12 @@ export class Storage { cacheEntry: cacheEntry.match, } - // Merged entry with direct downloads enabled: take a reader lease bounded - // by the signed URL lifetime, then sign. - const directDownloadExpiresAt = Date.now() + DIRECT_DOWNLOAD_LEASE_DURATION_MS + // Merged entry with direct downloads enabled: take a reader lease that + // outlives the signed URL, then sign. Storage authenticates a GET when it + // starts, so a download begun just before the URL expires keeps reading + // after it; the margin keeps cleanup from deleting the object under it. + const urlExpiresAt = Date.now() + env.DIRECT_DOWNLOAD_URL_TTL_SECONDS * 1000 + const leaseExpiresAt = urlExpiresAt + DIRECT_DOWNLOAD_LEASE_MARGIN_MS const leased = await this.db.transaction().execute(async (tx) => { let query = tx .selectFrom('storage_locations') @@ -834,7 +837,7 @@ export class Storage { if (env.DB_DRIVER !== 'sqlite') query = query.forUpdate() const current = await query.executeTakeFirst() if (!current?.mergedAt) return current - await createReaderLease(tx, current.id, 'storage', directDownloadExpiresAt) + await createReaderLease(tx, current.id, 'storage', leaseExpiresAt) await tx .updateTable('storage_locations') .set({ lastDownloadedAt: Date.now() }) @@ -846,10 +849,7 @@ export class Storage { // The merge could have been undone between validation and the lease read. const downloadUrl = leased.mergedAt - ? await this.adapter.createDownloadUrl( - `${leased.folderName}/merged`, - directDownloadExpiresAt, - ) + ? await this.adapter.createDownloadUrl(`${leased.folderName}/merged`, urlExpiresAt) : defaultUrl return { diff --git a/routes/twirp/github.actions.results.api.v1.CacheService/CreateCacheEntry.post.ts b/routes/twirp/github.actions.results.api.v1.CacheService/CreateCacheEntry.post.ts index 077b7f4..ff51f39 100644 --- a/routes/twirp/github.actions.results.api.v1.CacheService/CreateCacheEntry.post.ts +++ b/routes/twirp/github.actions.results.api.v1.CacheService/CreateCacheEntry.post.ts @@ -1,6 +1,6 @@ import { z } from 'zod' import { env } from '~/lib/env' -import { getCacheScope } from '~/lib/scope' +import { CACHE_PERMISSION_WRITE, getCacheScope } from '~/lib/scope' import { AlreadyExistsError, getStorage } from '~/lib/storage' import { readTwirpRequest, sendTwirpResponse, TwirpMessage } from '~/lib/twirp' @@ -19,7 +19,7 @@ export default defineEventHandler(async (event) => { ) const storage = await getStorage() - const writeScope = scopes.find((s) => s.Permission >= 2) + const writeScope = scopes.find((s) => (s.Permission & CACHE_PERMISSION_WRITE) !== 0) if (!writeScope) throw createError({ statusCode: 403, message: 'No scope with write permission found' }) diff --git a/routes/twirp/github.actions.results.api.v1.CacheService/FinalizeCacheEntryUpload.ts b/routes/twirp/github.actions.results.api.v1.CacheService/FinalizeCacheEntryUpload.ts index fedca6c..5dc2033 100644 --- a/routes/twirp/github.actions.results.api.v1.CacheService/FinalizeCacheEntryUpload.ts +++ b/routes/twirp/github.actions.results.api.v1.CacheService/FinalizeCacheEntryUpload.ts @@ -1,6 +1,6 @@ import { z } from 'zod' import { cacheUploadsTotal } from '~/lib/metrics' -import { getCacheScope } from '~/lib/scope' +import { CACHE_PERMISSION_WRITE, getCacheScope } from '~/lib/scope' import { getStorage } from '~/lib/storage' import { readTwirpRequest, sendTwirpResponse, TwirpMessage } from '~/lib/twirp' @@ -19,7 +19,7 @@ export default defineEventHandler(async (event) => { ) const storage = await getStorage() - const writeScope = scopes.find((s) => s.Permission >= 2) + const writeScope = scopes.find((s) => (s.Permission & CACHE_PERMISSION_WRITE) !== 0) if (!writeScope) throw createError({ statusCode: 403, message: 'No scope with write permission found' }) diff --git a/routes/twirp/github.actions.results.api.v1.CacheService/GetCacheEntryDownloadURL.post.ts b/routes/twirp/github.actions.results.api.v1.CacheService/GetCacheEntryDownloadURL.post.ts index 1c45cbf..c08675c 100644 --- a/routes/twirp/github.actions.results.api.v1.CacheService/GetCacheEntryDownloadURL.post.ts +++ b/routes/twirp/github.actions.results.api.v1.CacheService/GetCacheEntryDownloadURL.post.ts @@ -1,7 +1,7 @@ -import { map, pipe, prop, sortBy } from 'remeda' +import { filter, map, pipe, prop, sortBy } from 'remeda' import { z } from 'zod' import { cacheLookupDurationSeconds, cacheRequestsTotal } from '~/lib/metrics' -import { getCacheScope } from '~/lib/scope' +import { CACHE_PERMISSION_READ, getCacheScope } from '~/lib/scope' import { getStorage } from '~/lib/storage' import { readTwirpRequest, sendTwirpResponse, TwirpMessage } from '~/lib/twirp' @@ -13,6 +13,15 @@ const bodySchema = z.object({ export default defineEventHandler(async (event) => { const { scopes, repoId } = await getCacheScope(event) + // Scopes are bit flags; a write-only scope must not be read from. + const readScopes = pipe( + scopes, + filter((s) => (s.Permission & CACHE_PERMISSION_READ) !== 0), + sortBy([prop('Permission'), 'desc']), + map(prop('Scope')), + ) + if (readScopes.length === 0) + throw createError({ statusCode: 403, message: 'No scope with read permission found' }) const { key, restore_keys, version } = await readTwirpRequest( event, @@ -27,7 +36,7 @@ export default defineEventHandler(async (event) => { match = await storage.getCacheEntryWithDownloadUrl({ keys: [key, ...(restore_keys ?? [])], version, - scopes: pipe(scopes, sortBy([prop('Permission'), 'desc']), map(prop('Scope'))), + scopes: readScopes, repoId, }) } catch (err) { diff --git a/tests/cleanup-lifecycle.test.ts b/tests/cleanup-lifecycle.test.ts index 49ddeca..ef9ea50 100644 --- a/tests/cleanup-lifecycle.test.ts +++ b/tests/cleanup-lifecycle.test.ts @@ -5,6 +5,7 @@ import { afterAll, beforeAll, describe, expect, test, vi } from 'vitest' import { getDatabase } from '~/lib/db' import { env } from '~/lib/env' import { Storage } from '~/lib/storage' +import { DIRECT_DOWNLOAD_LEASE_MARGIN_MS } from '~/lib/storage-leases' describe('cleanup lifecycle', () => { beforeAll(() => { @@ -327,7 +328,7 @@ describe('cleanup lifecycle', () => { }, 30_000) test.skipIf(process.env.VITEST_STORAGE_DRIVER !== 's3')( - 'a direct-download reader lease covers the signed URL lifetime', + 'a direct-download reader lease outlives the signed URL, which honors the TTL', async () => { const db = await getDatabase() const storage = await Storage.fromEnv() @@ -387,7 +388,10 @@ describe('cleanup lifecycle', () => { ) expect(signedLifetimeSeconds).toBeGreaterThan(0) - expect(lease.expiresAt - startedAt).toBeGreaterThanOrEqual(signedLifetimeSeconds * 1000) + expect(signedLifetimeSeconds).toBeLessThanOrEqual(env.DIRECT_DOWNLOAD_URL_TTL_SECONDS) + expect(lease.expiresAt - startedAt).toBeGreaterThanOrEqual( + signedLifetimeSeconds * 1000 + DIRECT_DOWNLOAD_LEASE_MARGIN_MS, + ) expect(accessed.lastDownloadedAt).toBeGreaterThanOrEqual(startedAt) } finally { env.ENABLE_DIRECT_DOWNLOADS = directDownloadsEnabled diff --git a/tests/direct-download-auth.test.ts b/tests/direct-download-auth.test.ts new file mode 100644 index 0000000..0b61775 --- /dev/null +++ b/tests/direct-download-auth.test.ts @@ -0,0 +1,573 @@ +import type { ResultPromise } from 'execa' +import type { KeyObject } from 'node:crypto' +import type { Server } from 'node:http' +import type { AddressInfo } from 'node:net' + +import { generateKeyPairSync, randomBytes, randomUUID } from 'node:crypto' +import { createServer } from 'node:http' +import { setTimeout as sleep } from 'node:timers/promises' + +import { type } from 'arkenv' +import { execa } from 'execa' +import { exportJWK, SignJWT } from 'jose' +import { afterAll, beforeAll, describe, expect, test, vi } from 'vitest' +import { + assertDirectDownloadsRequireTokenValidation, + envBaseSchema, + parseActionsTokenAudience, +} from '~/lib/schemas' + +// A second server with token validation on, verifying against a local issuer +// whose keys are generated here, so these tests never reach GitHub. +const PORT = 3011 +const BASE_URL = `http://localhost:${PORT}` +const TWIRP = `${BASE_URL}/twirp/github.actions.results.api.v1.CacheService` +const URL_TTL_SECONDS = 120 +const KID = 'test-key' +const AUDIENCE = 'test-audience' + +const trusted = generateKeyPairSync('rsa', { modulusLength: 2048 }) +const untrusted = generateKeyPairSync('rsa', { modulusLength: 2048 }) + +const repoA = `repo-a-${randomUUID()}` +const repoB = `repo-b-${randomUUID()}` +const scopeMarker = `refs/heads/marker-${randomUUID()}` +// Only ever inside a token, so finding it in the logs means decoded claims were logged. +const claimMarker = `claim-marker-${randomUUID()}` + +// holder object instead of bare `let`s (`unicorn/no-top-level-assignment-in-function`) +const issuer: { + url: string + server?: Server + keys: object[] + jwksFetchedAt: number[] +} = { url: '', keys: [], jwksFetchedAt: [] } + +async function publicJwk(key: KeyObject, kid: string) { + return { ...(await exportJWK(key)), kid, alg: 'RS256', use: 'sig' } +} + +async function startIssuer() { + issuer.keys.push(await publicJwk(trusted.publicKey, KID)) + const server = createServer((req, res) => { + res.setHeader('content-type', 'application/json') + if (req.url === '/.well-known/openid-configuration') + return res.end( + JSON.stringify({ issuer: issuer.url, jwks_uri: `${issuer.url}/.well-known/jwks` }), + ) + if (req.url === '/.well-known/jwks') { + issuer.jwksFetchedAt.push(Date.now()) + return res.end(JSON.stringify({ keys: issuer.keys })) + } + // a JWKS endpoint that is down + res.statusCode = req.url === '/broken-jwks' ? 503 : 404 + res.end('{}') + }) + const listening = Promise.withResolvers() + server.listen(0, '127.0.0.1', listening.resolve) + await listening.promise + issuer.server = server + issuer.url = `http://127.0.0.1:${(server.address() as AddressInfo).port}` +} + +function encodeSegment(value: object) { + return Buffer.from(JSON.stringify(value)).toString('base64url') +} + +// Same database and storage as the main test server, so whatever this server +// stores stays authorized for the other suites' orphaned-storage checks. +function serverEnv(overrides: Record) { + return { + ...process.env, + NITRO_PORT: String(PORT), + API_BASE_URL: BASE_URL, + DISABLE_CLEANUP_JOBS: 'true', + ...overrides, + } +} + +type ServerProcess = ResultPromise<{ node: true; ipc: true; all: true; reject: false }> + +function startServer(overrides: Record): ServerProcess { + return execa({ + node: true, + ipc: true, + all: true, + reject: false, + env: serverEnv(overrides), + })`.output/server/index.mjs` +} + +/** Resolves `ready` once the server signals readiness, or `exited` if it dies first. */ +async function readyOrExited(server: ServerProcess) { + return Promise.race([ + server + .getOneMessage({ filter: (message) => message === 'nitro:ready' }) + .then(() => 'ready' as const) + .catch(() => 'exited' as const), + server.then(() => 'exited' as const), + ]) +} + +type Claims = Record + +async function mint( + claims: Claims, + { + key = trusted.privateKey, + kid = KID, + iss = issuer.url, + aud = AUDIENCE, + exp = '10m', + nbf, + }: { + key?: KeyObject + kid?: string + iss?: string + aud?: string | null + exp?: string | number | null + nbf?: string | number + } = {}, +) { + const jwt = new SignJWT({ ...claims, run_id: claimMarker }) + .setProtectedHeader({ alg: 'RS256', kid }) + .setIssuedAt() + if (iss) jwt.setIssuer(iss) + if (aud !== null) jwt.setAudience(aud) + if (exp !== null) jwt.setExpirationTime(exp) + if (nbf !== undefined) jwt.setNotBefore(nbf) + return jwt.sign(key) +} + +function claimsFor(repoId: string, scopes = [{ Scope: scopeMarker, Permission: 3 }]): Claims { + return { ac: JSON.stringify(scopes), repository_id: repoId } +} + +async function twirp(method: string, token: string | undefined, body: object) { + const res = await fetch(`${TWIRP}/${method}`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }), + }, + body: JSON.stringify(body), + }) + const text = await res.text() + let json: Record = {} + try { + json = JSON.parse(text) + } catch {} + return { status: res.status, json } +} + +async function save(token: string, key: string, data: Buffer) { + const created = await twirp('CreateCacheEntry', token, { key, version: 'v1' }) + expect(created.status).toBe(200) + const uploadUrl = created.json.signed_upload_url as string + const part = await fetch(uploadUrl, { method: 'PUT', body: new Uint8Array(data) }) + expect(part.status).toBe(201) + // Keep the signed URL's query and append to it, as Azure Blob clients do. + const blocklistUrl = new URL(uploadUrl) + blocklistUrl.searchParams.set('comp', 'blocklist') + const blocklist = await fetch(blocklistUrl, { method: 'PUT' }) + expect(blocklist.status).toBe(201) + const finalized = await twirp('FinalizeCacheEntryUpload', token, { + key, + version: 'v1', + size_bytes: data.length, + }) + expect(finalized.status).toBe(200) +} + +function lookup(token: string | undefined, key: string) { + return twirp('GetCacheEntryDownloadURL', token, { key, version: 'v1' }) +} + +function parseEnvBase(input: object) { + return envBaseSchema({ API_BASE_URL: 'http://localhost', ...input }) +} + +describe('direct downloads require token validation (config)', () => { + const base = { + ENABLE_DIRECT_DOWNLOADS: true, + SKIP_TOKEN_VALIDATION: true, + UNSAFE_ALLOW_DIRECT_DOWNLOADS_WITHOUT_TOKEN_VALIDATION: false, + } + + test('refuses direct downloads with token validation skipped', () => { + expect(() => assertDirectDownloadsRequireTokenValidation(base)).toThrow( + 'requires token validation', + ) + }) + + test('allows the combination only with the explicit unsafe override', () => { + expect(() => + assertDirectDownloadsRequireTokenValidation({ + ...base, + UNSAFE_ALLOW_DIRECT_DOWNLOADS_WITHOUT_TOKEN_VALIDATION: true, + }), + ).not.toThrow() + expect(() => + assertDirectDownloadsRequireTokenValidation({ ...base, SKIP_TOKEN_VALIDATION: false }), + ).not.toThrow() + }) + + test('direct downloads are off by default; the URL TTL defaults to 300 s within 1โ€“3600', () => { + expect(parseEnvBase({})).toMatchObject({ + ENABLE_DIRECT_DOWNLOADS: false, + SKIP_TOKEN_VALIDATION: false, + UNSAFE_ALLOW_DIRECT_DOWNLOADS_WITHOUT_TOKEN_VALIDATION: false, + DIRECT_DOWNLOAD_URL_TTL_SECONDS: 300, + }) + for (const ttl of [1, 3600]) + expect(parseEnvBase({ DIRECT_DOWNLOAD_URL_TTL_SECONDS: ttl })).toMatchObject({ + DIRECT_DOWNLOAD_URL_TTL_SECONDS: ttl, + }) + for (const ttl of [0, 3601, 1.5]) + expect(parseEnvBase({ DIRECT_DOWNLOAD_URL_TTL_SECONDS: ttl })).toBeInstanceOf(type.errors) + }) + + test('the server refuses to start', { timeout: 30_000 }, async () => { + const server = startServer({ ENABLE_DIRECT_DOWNLOADS: 'true', SKIP_TOKEN_VALIDATION: 'true' }) + if ((await readyOrExited(server)) === 'ready') { + server.kill() + throw new Error('server started with direct downloads and skipped token validation') + } + const result = await server + expect(result.exitCode).not.toBe(0) + expect(result.all).toContain('ENABLE_DIRECT_DOWNLOADS=true requires token validation') + }) +}) + +describe('token audience (config)', () => { + test('unset skips the audience check; a list is trimmed', () => { + expect(parseActionsTokenAudience(undefined)).toBeUndefined() + expect(parseActionsTokenAudience(' a, b ,')).toEqual(['a', 'b']) + }) + + test.each(['', ' ', ',', ' , ,'])('refuses %j, which lists no audience', (value) => { + expect(() => parseActionsTokenAudience(value)).toThrow('lists no audience') + }) + + test('the server refuses to start with an empty audience list', { timeout: 30_000 }, async () => { + const server = startServer({ ACTIONS_TOKEN_AUDIENCE: ' , ' }) + if ((await readyOrExited(server)) === 'ready') { + server.kill() + throw new Error('server started with an empty ACTIONS_TOKEN_AUDIENCE') + } + const result = await server + expect(result.exitCode).not.toBe(0) + expect(result.all).toContain('ACTIONS_TOKEN_AUDIENCE is set but lists no audience') + }) +}) + +describe('token validation and direct downloads', () => { + let server: ServerProcess + let logs = '' + const issuedTokens: string[] = [] + const track = async (token: Promise) => { + const value = await token + issuedTokens.push(value) + return value + } + + beforeAll(async () => { + await startIssuer() + server = startServer({ + SKIP_TOKEN_VALIDATION: 'false', + ACTIONS_TOKEN_ISSUER: issuer.url, + ACTIONS_TOKEN_AUDIENCE: `other-audience, ${AUDIENCE}`, + ENABLE_DIRECT_DOWNLOADS: 'true', + EAGER_MERGE: 'true', + DIRECT_DOWNLOAD_URL_TTL_SECONDS: String(URL_TTL_SECONDS), + }) + server.all?.on('data', (chunk: Buffer) => { + logs += chunk.toString() + }) + if ((await readyOrExited(server)) !== 'ready') + throw new Error(`server failed to start:\n${logs}`) + }, 60_000) + + afterAll(async () => { + server?.kill() + await server + const closed = Promise.withResolvers() + issuer.server?.close(() => closed.resolve()) + await closed.promise + }) + + test('accepts a token signed by the issuer', async () => { + const token = await track(mint(claimsFor(repoA))) + const res = await lookup(token, randomUUID()) + expect(res.status).toBe(200) + expect(res.json.ok).toBe(false) + }) + + test.each([ + ['missing bearer', async () => undefined], + [ + 'signature from an untrusted key', + () => mint(claimsFor(repoA), { key: untrusted.privateKey }), + ], + ['expired', () => mint(claimsFor(repoA), { exp: Math.floor(Date.now() / 1000) - 60 })], + ['not yet valid', () => mint(claimsFor(repoA), { nbf: Math.floor(Date.now() / 1000) + 600 })], + ['wrong issuer', () => mint(claimsFor(repoA), { iss: 'https://issuer.invalid' })], + ['no expiry', () => mint(claimsFor(repoA), { exp: null })], + ['wrong audience', () => mint(claimsFor(repoA), { aud: 'someone-else' })], + ['no audience', () => mint(claimsFor(repoA), { aud: null })], + ['no cache scopes', () => mint({ repository_id: repoA })], + [ + 'no repository id', + () => mint({ ac: JSON.stringify([{ Scope: scopeMarker, Permission: 3 }]) }), + ], + ['malformed cache scopes', () => mint({ ac: '[{"Scope":1}]', repository_id: repoA })], + [ + 'unsigned token', + async () => { + const claims = { + ...claimsFor(repoA), + iss: issuer.url, + exp: Math.floor(Date.now() / 1000) + 600, + } + return `${encodeSegment({ alg: 'none' })}.${encodeSegment(claims)}.` + }, + ], + ])('rejects %s with 401 on lookup, create and finalize', async (_, makeToken) => { + const token = await makeToken() + if (token) issuedTokens.push(token) + const key = randomUUID() + const res = await lookup(token, key) + expect(res.status).toBe(401) + expect(res.json.signed_download_url).toBeUndefined() + + const create = await twirp('CreateCacheEntry', token, { key, version: 'v1' }) + expect(create.status).toBe(401) + expect(create.json.signed_upload_url).toBeUndefined() + const finalize = await twirp('FinalizeCacheEntryUpload', token, { key, version: 'v1' }) + expect(finalize.status).toBe(401) + }) + + test( + 'a forged token gets no URL for an existing entry, direct or proxied', + { timeout: 30_000 }, + async () => { + const key = `forged-${randomUUID()}` + const owner = await track(mint(claimsFor(repoA))) + await save(owner, key, randomBytes(1024)) + + const forged = await mint(claimsFor(repoA), { key: untrusted.privateKey }) + issuedTokens.push(forged) + const res = await lookup(forged, key) + expect(res.status).toBe(401) + expect(JSON.stringify(res.json)).not.toContain('/download/') + expect(JSON.stringify(res.json)).not.toContain('X-Amz-') + }, + ) + + test( + 'another repository cannot match the entry, even by prefix', + { timeout: 30_000 }, + async () => { + const key = `cross-repo-${randomUUID()}` + await save(await track(mint(claimsFor(repoA))), key, randomBytes(1024)) + + const other = await track(mint(claimsFor(repoB))) + const exact = await lookup(other, key) + expect(exact.json.ok).toBe(false) + const prefix = await twirp('GetCacheEntryDownloadURL', other, { + key: 'no-such-key', + restore_keys: [key.slice(0, 12)], + version: 'v1', + }) + expect(prefix.json.ok).toBe(false) + + const owner = await lookup(await track(mint(claimsFor(repoA))), key) + expect(owner.json.ok).toBe(true) + }, + ) + + test('scope permissions are bit flags', { timeout: 30_000 }, async () => { + const key = `scopes-${randomUUID()}` + await save(await track(mint(claimsFor(repoA))), key, randomBytes(1024)) + + const readOnly = await track(mint(claimsFor(repoA, [{ Scope: scopeMarker, Permission: 1 }]))) + const read = await lookup(readOnly, key) + expect(read.json.ok).toBe(true) + const write = await twirp('CreateCacheEntry', readOnly, { key: randomUUID(), version: 'v1' }) + expect(write.status).toBe(403) + + const writeOnly = await track(mint(claimsFor(repoA, [{ Scope: scopeMarker, Permission: 2 }]))) + const writeOnlyRead = await lookup(writeOnly, key) + expect(writeOnlyRead.status).toBe(403) + + const otherScope = await track( + mint(claimsFor(repoA, [{ Scope: `refs/heads/other-${randomUUID()}`, Permission: 3 }])), + ) + const otherScopeRead = await lookup(otherScope, key) + expect(otherScopeRead.json.ok).toBe(false) + }) + + test( + 'a fork pull request writes only to its own ref and reads the base scopes it lists', + { timeout: 30_000 }, + async () => { + const baseRef = `refs/heads/base-${randomUUID()}` + const prRef = `refs/pull/${randomUUID()}/merge` + const baseKey = `base-${randomUUID()}` + await save( + await track(mint(claimsFor(repoA, [{ Scope: baseRef, Permission: 3 }]))), + baseKey, + randomBytes(1024), + ) + + // the read-only base scope is listed first; writes must still land on the PR ref + const pr = await track( + mint( + claimsFor(repoA, [ + { Scope: baseRef, Permission: 1 }, + { Scope: prRef, Permission: 3 }, + ]), + ), + ) + const baseFromPr = await lookup(pr, baseKey) + expect(baseFromPr.json.ok).toBe(true) + + const prKey = `pr-${randomUUID()}` + await save(pr, prKey, randomBytes(1024)) + const prFromPr = await lookup(pr, prKey) + expect(prFromPr.json.ok).toBe(true) + const baseOnly = await track(mint(claimsFor(repoA, [{ Scope: baseRef, Permission: 3 }]))) + const prFromBase = await lookup(baseOnly, prKey) + expect(prFromBase.json.ok).toBe(false) + + // a read-only token cannot finalize an upload either + const readOnly = await track(mint(claimsFor(repoA, [{ Scope: prRef, Permission: 1 }]))) + const finalize = await twirp('FinalizeCacheEntryUpload', readOnly, { + key: prKey, + version: 'v1', + size_bytes: 1024, + }) + expect(finalize.status).toBe(403) + }, + ) + + test('accepts a key rotated in by the issuer (new kid)', { timeout: 60_000 }, async () => { + const rotated = generateKeyPairSync('rsa', { modulusLength: 2048 }) + const rotatedKid = 'rotated-key' + const token = await track(mint(claimsFor(repoA), { key: rotated.privateKey, kid: rotatedKid })) + issuer.keys.push(await publicJwk(rotated.publicKey, rotatedKid)) + + // A real wait: the server process refetches the JWKS for an unknown kid at + // most once per 30 s (jose's cooldown), and fake timers can't reach into it. + const fetches = issuer.jwksFetchedAt.length + expect(fetches).toBeGreaterThan(0) + await sleep(Math.max(0, issuer.jwksFetchedAt.at(-1)! + 31_000 - Date.now())) + + const res = await lookup(token, randomUUID()) + expect(res.status).toBe(200) + expect(issuer.jwksFetchedAt.length).toBeGreaterThan(fetches) + }) + + test('fails closed with 401 when the JWKS cannot be fetched', { timeout: 30_000 }, async () => { + const port = PORT + 1 + const broken = startServer({ + NITRO_PORT: String(port), + API_BASE_URL: `http://localhost:${port}`, + SKIP_TOKEN_VALIDATION: 'false', + ACTIONS_TOKEN_ISSUER: issuer.url, + ACTIONS_TOKEN_JWKS_URL: `${issuer.url}/broken-jwks`, + }) + let brokenLogs = '' + broken.all?.on('data', (chunk: Buffer) => { + brokenLogs += chunk.toString() + }) + try { + if ((await readyOrExited(broken)) !== 'ready') + throw new Error(`server failed to start:\n${brokenLogs}`) + const token = await track(mint(claimsFor(repoA))) + const res = await fetch( + `http://localhost:${port}/twirp/github.actions.results.api.v1.CacheService/GetCacheEntryDownloadURL`, + { + method: 'POST', + headers: { 'content-type': 'application/json', 'authorization': `Bearer ${token}` }, + body: JSON.stringify({ key: randomUUID(), version: 'v1' }), + }, + ) + expect(res.status).toBe(401) + expect(brokenLogs).not.toContain(token) + expect(brokenLogs).not.toContain(claimMarker) + } finally { + broken.kill() + await broken + } + }) + + test( + 'an authorized lookup of a merged entry gets a URL that expires within the TTL', + { timeout: 60_000 }, + async () => { + const key = `direct-${randomUUID()}` + const data = randomBytes(64 * 1024) + const token = await track(mint(claimsFor(repoA))) + await save(token, key, data) + + const signedUrl = async () => { + const res = await lookup(token, key) + expect(res.json.ok).toBe(true) + return new URL(res.json.signed_download_url as string) + } + + if (process.env.STORAGE_DRIVER === 'filesystem') { + // No signed URLs on the filesystem driver: always the proxied route. + const url = await signedUrl() + expect(url.origin).toBe(BASE_URL) + expect(url.pathname).toMatch(/^\/download\//) + return + } + + // Eager Merge runs after finalize; until it lands the lookup is proxied. + const url = await vi.waitFor( + async () => { + const candidate = await signedUrl() + expect(candidate.origin).not.toBe(BASE_URL) + return candidate + }, + { timeout: 30_000, interval: 200 }, + ) + // S3 signs a relative lifetime; GCS (V2 signing) an absolute epoch second, + // Azure (SAS) an absolute ISO time + const now = Date.now() + const gcsExpiresAt = url.searchParams.get('Expires') + const azureExpiresAt = url.searchParams.get('se') + const expires = gcsExpiresAt + ? Number(gcsExpiresAt) - Math.floor(now / 1000) + : azureExpiresAt + ? Math.floor((Date.parse(azureExpiresAt) - now) / 1000) + : Number(url.searchParams.get('X-Amz-Expires')) + expect(expires).toBeGreaterThan(0) + expect(expires).toBeLessThanOrEqual(URL_TTL_SECONDS) + + const download = await fetch(url) + const body = Buffer.from(await download.arrayBuffer()) + expect(body.equals(data)).toBe(true) + }, + ) + + test('logs contain neither tokens, claims nor signed URLs', () => { + expect(issuedTokens.length).toBeGreaterThan(0) + for (const token of issuedTokens) { + expect(logs).not.toContain(token) + // nor its signature on its own + const signature = token.slice(token.lastIndexOf('.') + 1) + if (signature) expect(logs).not.toContain(signature) + } + // Repository ids, scopes and keys do appear in DEBUG query logs; the claims + // jose decodes from a rejected token must not. + expect(logs).not.toContain(claimMarker) + expect(logs).not.toContain('X-Amz-Signature') + expect(logs).not.toContain('GoogleAccessId=') + expect(logs).not.toMatch(/[?&]sig=/) + // rejections are still diagnosable + expect(logs).toContain('ERR_JWT_EXPIRED') + }) +}) diff --git a/tests/setup.ts b/tests/setup.ts index 009f771..d73f058 100644 --- a/tests/setup.ts +++ b/tests/setup.ts @@ -7,6 +7,7 @@ import type { Server } from 'node:http' import type { StartedTestContainer } from 'testcontainers' import type { Env, envBaseSchema, envDbDriverSchema, envStorageDriverSchema } from '~/lib/schemas' +import { generateKeyPairSync } from 'node:crypto' import fs from 'node:fs/promises' import path from 'node:path' @@ -44,6 +45,8 @@ const TESTING_ENV_BASE = { | 'CACHE_FILESYSTEM_MAX_USAGE_PERCENT' | 'ORPHANED_STORAGE_GRACE_PERIOD_HOURS' | 'ENABLE_DIRECT_DOWNLOADS' + | 'DIRECT_DOWNLOAD_URL_TTL_SECONDS' + | 'UNSAFE_ALLOW_DIRECT_DOWNLOADS_WITHOUT_TOKEN_VALIDATION' | 'EAGER_MERGE' | 'MAX_CONCURRENT_MERGES' | 'DOWNLOAD_IDLE_TIMEOUT_MS' @@ -99,7 +102,9 @@ const TESTING_ENV_BY_STORAGE_DRIVER = { STORAGE_DRIVER: 'gcs', STORAGE_GCS_BUCKET: 'vitest', STORAGE_GCS_ENDPOINT: 'http://localhost:9000', - STORAGE_GCS_SERVICE_ACCOUNT_KEY: 'tests/gcs-service-account-key.json', + // generated in `setup`: fake-gcs-server needs no credentials, but signing a + // direct-download URL does + STORAGE_GCS_SERVICE_ACCOUNT_KEY: `${TEST_TEMP_DIR}/gcs-service-account-key.json`, }, azblob: { STORAGE_DRIVER: 'azblob', @@ -138,6 +143,20 @@ export async function setup() { recursive: true, }) await fs.mkdir(TEST_TEMP_DIR, { recursive: true }) + if (env.VITEST_STORAGE_DRIVER === 'gcs') { + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }) + await fs.writeFile( + TESTING_ENV_BY_STORAGE_DRIVER.gcs.STORAGE_GCS_SERVICE_ACCOUNT_KEY, + JSON.stringify({ + type: 'service_account', + project_id: 'vitest', + private_key_id: 'vitest', + private_key: privateKey.export({ type: 'pkcs8', format: 'pem' }), + client_email: 'vitest@vitest.iam.gserviceaccount.com', + token_uri: `${TESTING_ENV_BY_STORAGE_DRIVER.gcs.STORAGE_GCS_ENDPOINT}/token`, + }), + ) + } // eslint-disable-next-line no-console console.log('Starting test containers for', env.VITEST_DB_DRIVER, env.VITEST_STORAGE_DRIVER) @@ -216,7 +235,8 @@ export async function setup() { .withEntrypoint(['sh']) .withCommand([ `-c`, - `mkdir -p /data/${env.STORAGE_GCS_BUCKET} && /bin/fake-gcs-server -scheme http -port 9000 -data /data`, + // `-public-host` routes path-style object GETs, the shape of a signed URL + `mkdir -p /data/${env.STORAGE_GCS_BUCKET} && /bin/fake-gcs-server -scheme http -port 9000 -public-host ${new URL(env.STORAGE_GCS_ENDPOINT).host} -data /data`, ]) .withExposedPorts({ container: 9000, diff --git a/tests/write-authorization.test.ts b/tests/write-authorization.test.ts new file mode 100644 index 0000000..f3b9a54 --- /dev/null +++ b/tests/write-authorization.test.ts @@ -0,0 +1,114 @@ +import { Buffer } from 'node:buffer' +import crypto from 'node:crypto' + +import { SignJWT } from 'jose' +import { describe, expect, test } from 'vitest' + +const BASE = 'http://localhost:3000' +const TWIRP_BASE = `${BASE}/twirp/github.actions.results.api.v1.CacheService` +const MANAGEMENT_API_KEY = 'vitest-management-api-key' + +function signToken({ + repoId = '123', + scopes = [{ Scope: 'refs/heads/main', Permission: 3 }], +}: { repoId?: string; scopes?: { Scope: string; Permission: number }[] } = {}) { + return new SignJWT({ ac: JSON.stringify(scopes), repository_id: repoId }) + .setProtectedHeader({ alg: 'HS256' }) + .sign(crypto.createSecretKey('mock-secret-key', 'ascii')) +} + +async function twirp(method: string, token: string, body: object) { + const response = await fetch(`${TWIRP_BASE}/${method}`, { + method: 'POST', + headers: { 'authorization': `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify(body), + }) + return { status: response.status, body: (await response.json()) as T } +} + +async function createUpload(token: string, key: string) { + const { status, body } = await twirp<{ ok: boolean; signed_upload_url: string }>( + 'CreateCacheEntry', + token, + { key, version: 'v1' }, + ) + expect(status).toBe(200) + expect(body.ok).toBe(true) + return body.signed_upload_url +} + +async function save(token: string, key: string, data: Uint8Array) { + const url = await createUpload(token, key) + const put = await fetch(url, { method: 'PUT', body: data as BodyInit }) + expect(put.status).toBe(201) + const { status } = await twirp('FinalizeCacheEntryUpload', token, { key, version: 'v1' }) + expect(status).toBe(200) +} + +async function restore(token: string, key: string) { + const { body } = await twirp<{ ok: boolean; signed_download_url?: string }>( + 'GetCacheEntryDownloadURL', + token, + { key, version: 'v1' }, + ) + if (!body.ok || !body.signed_download_url) return + const response = await fetch(body.signed_download_url) + expect(response.status).toBe(200) + return Buffer.from(await response.arrayBuffer()) +} + +function management(path: string, init: RequestInit = {}, apiKey = MANAGEMENT_API_KEY) { + return fetch(`${BASE}/management-api${path}`, { + ...init, + headers: { 'x-api-key': apiKey, ...init.headers }, + }) +} + +describe('write path authorization', () => { + test('another repository cannot replace or read an entry with the same key and scope', async () => { + const tokenA = await signToken({ repoId: 'write-auth-owner' }) + const tokenB = await signToken({ repoId: 'write-auth-other' }) + const key = `write-auth-${crypto.randomUUID()}` + const original = crypto.randomBytes(64) + + await save(tokenA, key, original) + expect(await restore(tokenB, key)).toBeUndefined() + + await save(tokenB, key, crypto.randomBytes(64)) + const restored = await restore(tokenA, key) + expect(restored?.equals(original)).toBe(true) + }) + + test('a pull request ref write scope cannot replace a base branch entry', async () => { + const base = await signToken({ repoId: 'write-auth-fork' }) + const pullRequest = await signToken({ + repoId: 'write-auth-fork', + scopes: [ + { Scope: 'refs/pull/1/merge', Permission: 3 }, + { Scope: 'refs/heads/main', Permission: 1 }, + ], + }) + const key = `write-auth-${crypto.randomUUID()}` + const original = crypto.randomBytes(64) + await save(base, key, original) + + // the pull request restores the base entry but saves into its own ref + const restored = await restore(pullRequest, key) + expect(restored?.equals(original)).toBe(true) + await save(pullRequest, key, crypto.randomBytes(64)) + const baseRestored = await restore(base, key) + expect(baseRestored?.equals(original)).toBe(true) + }) +}) + +describe('management API authorization', () => { + test('rejects a missing or wrong API key', async () => { + const wrongKey = `${MANAGEMENT_API_KEY}x` + const missing = await management('/cache-entries', {}, '') + expect(missing.status).toBe(401) + const wrong = await management('/cache-entries', {}, wrongKey) + expect(wrong.status).toBe(401) + const wrongDelete = await management('/cache-entries', { method: 'DELETE' }, wrongKey) + expect(wrongDelete.status).toBe(401) + }) +}) From 4943b575350547fe467a9028d52c147734592d6b Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Sat, 24 Oct 2026 16:24:15 -0400 Subject: [PATCH 15/17] fix(logging): keep query strings out of request logs The error hook, the debug request/response hooks and the Results Passthrough logged `event.path` with its query string. Upload URLs are used by Azure Blob-style clients that put SAS-style parameters in the query, so those logs could record credentials. Log the path without its query string. Storage errors in the upload route are logged by the route and rethrown as a handled error, because Nitro logs unhandled errors with the full request URL. The request-logging hooks move from `plugins/setup.ts` into their own plugin so the test can drive them against the real upload route. Signed-off-by: Rui Chen --- lib/helpers.ts | 9 ++ plugins/request-logging.ts | 27 +++++ plugins/setup.ts | 23 ---- routes/[...path].ts | 3 +- .../devstoreaccount1/upload/[uploadId].put.ts | 17 ++- tests/request-logging.test.ts | 114 ++++++++++++++++++ 6 files changed, 165 insertions(+), 28 deletions(-) create mode 100644 plugins/request-logging.ts create mode 100644 tests/request-logging.test.ts diff --git a/lib/helpers.ts b/lib/helpers.ts index 8906e43..add3b06 100644 --- a/lib/helpers.ts +++ b/lib/helpers.ts @@ -1,3 +1,4 @@ +import type { H3Event } from 'h3' import { createHash } from 'node:crypto' import { customAlphabet } from 'nanoid' @@ -7,6 +8,14 @@ export function generateNumberId() { return Number.parseInt(generate()) } +/** + * The request path without its query string, for logging. A query can carry + * credentials (signed URLs, SAS-style parameters), so it must never reach the logs. + */ +export function loggedPath(event: H3Event) { + return event.path.split('?', 1)[0] +} + /** The key, version, scope and repo that at most one Upload may hold at a time. */ export interface Reservation { key: string diff --git a/plugins/request-logging.ts b/plugins/request-logging.ts new file mode 100644 index 0000000..605389c --- /dev/null +++ b/plugins/request-logging.ts @@ -0,0 +1,27 @@ +import { H3Error } from 'h3' +import { env } from '~/lib/env' +import { loggedPath } from '~/lib/helpers' +import { logger } from '~/lib/logger' + +export default defineNitroPlugin((nitro) => { + nitro.hooks.hook('error', (error, { event }) => { + if (!event) { + logger.error(error) + return + } + + logger.error( + `Response: ${event.method} ${loggedPath(event)} > ${error instanceof H3Error ? error.statusCode : '[no status code]'}\n`, + error, + ) + }) + + if (env.DEBUG) { + nitro.hooks.hook('request', (event) => { + logger.debug(`Request: ${event.method} ${loggedPath(event)}`) + }) + nitro.hooks.hook('afterResponse', (event) => { + logger.debug(`Response: ${event.method} ${loggedPath(event)} > ${getResponseStatus(event)}`) + }) + } +}) diff --git a/plugins/setup.ts b/plugins/setup.ts index b9975b3..c93ba8a 100644 --- a/plugins/setup.ts +++ b/plugins/setup.ts @@ -1,9 +1,7 @@ /* eslint-disable unicorn/no-process-exit */ import cluster from 'node:cluster' -import { H3Error } from 'h3' import { getDatabase } from '~/lib/db' -import { env } from '~/lib/env' import { logger } from '~/lib/logger' import { getStorage } from '~/lib/storage' @@ -35,26 +33,5 @@ export default defineNitroPlugin(async (nitro) => { await storage.waitForOngoingMerges() }) - nitro.hooks.hook('error', (error, { event }) => { - if (!event) { - logger.error(error) - return - } - - logger.error( - `Response: ${event.method} ${event.path} > ${error instanceof H3Error ? error.statusCode : '[no status code]'}\n`, - error, - ) - }) - - if (env.DEBUG) { - nitro.hooks.hook('request', (event) => { - logger.debug(`Request: ${event.method} ${event.path}`) - }) - nitro.hooks.hook('afterResponse', (event) => { - logger.debug(`Response: ${event.method} ${event.path} > ${getResponseStatus(event)}`) - }) - } - if (process.send && cluster.isPrimary) process.send('nitro:ready') }) diff --git a/routes/[...path].ts b/routes/[...path].ts index 1806232..65c2a8c 100644 --- a/routes/[...path].ts +++ b/routes/[...path].ts @@ -1,7 +1,8 @@ import { env } from '~/lib/env' +import { loggedPath } from '~/lib/helpers' import { logger } from '~/lib/logger' export default defineEventHandler(async (event) => { - logger.debug('proxying unknown path', event.path, 'to', env.DEFAULT_ACTIONS_RESULTS_URL) + logger.debug('proxying unknown path', loggedPath(event), 'to', env.DEFAULT_ACTIONS_RESULTS_URL) return proxyRequest(event, `${env.DEFAULT_ACTIONS_RESULTS_URL}${event.path}`) }) diff --git a/routes/devstoreaccount1/upload/[uploadId].put.ts b/routes/devstoreaccount1/upload/[uploadId].put.ts index 9eb7fdd..ba2c812 100644 --- a/routes/devstoreaccount1/upload/[uploadId].put.ts +++ b/routes/devstoreaccount1/upload/[uploadId].put.ts @@ -3,6 +3,7 @@ import { randomUUID } from 'node:crypto' import { z } from 'zod' +import { logger } from '~/lib/logger' import { getStorage } from '~/lib/storage' const pathParamsSchema = z.object({ @@ -35,10 +36,18 @@ export default defineEventHandler(async (event) => { const { uploadId } = parsedPathParams.data - // Not h3's `getRequestWebStream`: it enqueues every `data` event and never pauses the - // request, so a body arriving faster than storage consumes it is buffered in full. - const storage = await getStorage() - const stored = await storage.uploadPart(uploadId, chunkIndex, event.node.req) + let stored: boolean + try { + // Not h3's `getRequestWebStream`: it enqueues every `data` event and never pauses the + // request, so a body arriving faster than storage consumes it is buffered in full. + const storage = await getStorage() + stored = await storage.uploadPart(uploadId, chunkIndex, event.node.req) + } catch (err) { + // Logged here and rethrown as handled: Nitro logs unhandled errors with the + // full request URL, query string included. + logger.error(`Failed to store upload ${uploadId}`, err) + throw createError({ statusCode: 500, statusMessage: 'Failed to store upload' }) + } if (!stored) throw createError({ statusCode: 404, diff --git a/tests/request-logging.test.ts b/tests/request-logging.test.ts new file mode 100644 index 0000000..243eea4 --- /dev/null +++ b/tests/request-logging.test.ts @@ -0,0 +1,114 @@ +import type { ConsolaReporter } from 'consola' +import type { NitroApp, NitroRuntimeHooks } from 'nitropack' +import type { AddressInfo } from 'node:net' +import { Buffer } from 'node:buffer' +import { randomBytes, randomUUID } from 'node:crypto' +import { once } from 'node:events' +import http from 'node:http' +import { inspect } from 'node:util' + +import { LogLevels } from 'consola' +import * as h3 from 'h3' +import { createHooks } from 'hookable' +import { afterAll, beforeAll, expect, test, vi } from 'vitest' +import { env } from '~/lib/env' +import { logger } from '~/lib/logger' +import { getStorage } from '~/lib/storage' + +// The real upload route and request-logging plugin on an in-process h3 app, +// wired to the plugin's hooks the way Nitro's app is, so the server log is observable. +const hooks = createHooks() +const router = h3.createRouter({ preemptive: true }) +const server = http.createServer( + h3.toNodeListener( + h3 + .createApp({ + onError: (error, event) => hooks.callHook('error', error, { event, tags: ['request'] }), + onRequest: (event) => hooks.callHook('request', event), + onAfterResponse: (event, response) => hooks.callHook('afterResponse', event, response), + }) + .use(router), + ), +) + +const logs: string[] = [] +const reporter: ConsolaReporter = { + log: (entry) => { + logs.push(entry.args.map((arg) => inspect(arg)).join(' ')) + }, +} +const originalLevel = logger.level +const originalDebug = env.DEBUG +const proxyRequest = vi.fn(() => null) + +beforeAll(async () => { + for (const [name, value] of Object.entries(h3)) vi.stubGlobal(name, value) + vi.stubGlobal('defineNitroPlugin', (plugin: unknown) => plugin) + // Every request-logging hook, including the debug ones, must keep the query out. + env.DEBUG = true + logger.level = LogLevels.debug + logger.addReporter(reporter) + + // Not static imports: the plugin and route call Nitro auto-imports + // (`defineNitroPlugin`, `defineEventHandler`, โ€ฆ) at evaluation time. + const { default: requestLogging } = await import('~/plugins/request-logging') + // `hookable` here is the app's copy, not Nitro's; the runtime API is the same. + requestLogging({ hooks } as unknown as NitroApp) + const { default: upload } = await import('~/routes/devstoreaccount1/upload/[uploadId].put') + const { default: passthrough } = await import('~/routes/[...path]') + // The upstream Results service is not under test; only what is logged and forwarded is. + vi.stubGlobal('proxyRequest', proxyRequest) + router.put('/upload/:uploadId', upload).use('/**', passthrough) + server.listen(0) + await once(server, 'listening') +}) + +afterAll(async () => { + logger.removeReporter(reporter) + logger.level = originalLevel + env.DEBUG = originalDebug + vi.restoreAllMocks() + vi.unstubAllGlobals() + server.close() + await once(server, 'close') +}) + +test('query strings stay out of request, error and passthrough logs', async () => { + const storage = await getStorage() + const created = await storage.createUpload({ + key: `request-logging-${randomUUID()}`, + version: 'v1', + scope: 'refs/heads/main', + repoId: '123', + }) + + const { port } = server.address() as AddressInfo + // Azure Blob clients send SAS-style credentials in the query. + const secret = randomBytes(32).toString('hex') + const url = `http://localhost:${port}/upload/${created.id}?sig=${secret}` + + const stored = await fetch(url, { method: 'PUT', body: 'part' }) + expect(stored.status).toBe(201) + + // Storage failing mid-upload (e.g. the backend is unreachable) is logged as an error. + vi.spyOn(storage.adapter, 'uploadStream').mockRejectedValue(new Error('storage unavailable')) + const blockId = Buffer.from(`${randomUUID()}000000000001`).toString('base64') + const failed = await fetch(`${url}&comp=block&blockid=${encodeURIComponent(blockId)}`, { + method: 'PUT', + body: 'part', + }) + expect(failed.status).toBe(500) + await vi.waitFor(() => expect(logs.join('\n')).toContain('> 500')) + + // An unknown path falls through to the Results Passthrough. + const unknown = await fetch(`http://localhost:${port}/uploads/${created.id}?sig=${secret}`) + expect(unknown.status).toBe(204) + await vi.waitFor(() => expect(logs.join('\n')).toContain(`proxying unknown path`)) + expect(proxyRequest).toHaveBeenCalledTimes(1) + + const logged = logs.join('\n') + expect(logged).toContain(`/upload/${created.id}`) + expect(logged).toContain('storage unavailable') + expect(logged).not.toContain('sig=') + expect(logged).not.toContain(secret) +}) From 5ca1331fb9bd6e1330a866b107f5edd886909fcc Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Sat, 24 Oct 2026 16:27:55 -0400 Subject: [PATCH 16/17] fix(upload): require an unguessable capability for upload URLs Upload URLs are used by Azure Blob-style clients that send no Actions token, and until now they carried only the numeric upload id. Issue a random 256-bit capability with every upload, return it in the `signed_upload_url` query (`?sig=`), and require it on single-shot, block and blocklist requests. Missing or wrong capabilities, unknown uploads and uploads created without a capability all get the same 403. An authorized part whose upload is deleted while it streams still gets 404. Only the SHA-256 of the capability is stored, in `uploads.capabilityHash` (the column the migration bridge ensures), and compared in constant time. URL shape, parameter and hash are the ones the previous release issued, so uploads begun before an upgrade can be completed after it. Upload URLs requested with a method other than PUT now get 405 instead of falling through to the Results Passthrough, which forwarded the URL, capability included, to the upstream Results service. Clients already keep the signed URL's query and append their own parameters (as with an Azure SAS), so no client change is needed. See ADR 0013. Signed-off-by: Rui Chen --- .../0013-upload-urls-carry-a-capability.md | 11 + lib/db.ts | 2 + lib/helpers.ts | 28 +- lib/storage.ts | 27 +- routes/[...path].ts | 10 +- .../devstoreaccount1/upload/[uploadId].put.ts | 45 +-- .../CreateCacheEntry.post.ts | 5 +- tests/migrations.test.ts | 15 +- tests/proxy-backpressure.test.ts | 11 +- tests/request-logging.test.ts | 30 +- tests/upload-authorization.test.ts | 263 ++++++++++++++++++ tests/upload-part.test.ts | 80 ++++-- tests/write-authorization.test.ts | 14 + 13 files changed, 487 insertions(+), 54 deletions(-) create mode 100644 docs/adr/0013-upload-urls-carry-a-capability.md create mode 100644 tests/upload-authorization.test.ts diff --git a/docs/adr/0013-upload-urls-carry-a-capability.md b/docs/adr/0013-upload-urls-carry-a-capability.md new file mode 100644 index 0000000..83dc4fa --- /dev/null +++ b/docs/adr/0013-upload-urls-carry-a-capability.md @@ -0,0 +1,11 @@ +--- +status: proposed +--- + +# Upload URLs carry a capability + +Twirp calls are authorized by the Actions runtime token (`getCacheScope`), but the Upload URL returned in `signed_upload_url` is fetched by Azure Blob-style clients that send no token. Until now that URL held only the numeric Upload id, so it was the sole thing standing between any client that can reach the server and the Upload's Parts. We now issue a random 256-bit capability with each Upload, put it in the URL's query (`?sig=โ€ฆ`), and require it on every block, single-shot and blocklist request. A missing or wrong capability and an unknown Upload all get the same `403`, so the response reveals nothing about which Uploads exist. An authorized Part whose Upload is deleted while it streams still gets `404`. + +The capability lives in the query because every client we support treats the signed URL as an Azure SAS URL: `@actions/cache` and `go-actions-cache` keep the query and append `comp`/`blockid`, and OpenDAL (sccache) requires a query and re-sends it as the SAS. Only its SHA-256 is stored (`uploads.capabilityHash`), compared in constant time, so neither the database nor its debug query log can reproduce a working URL. Request logs record paths without their query string, and upload errors are logged by the route rather than left to Nitro, whose unhandled-error log includes the request URL. Upload URLs requested with a method other than `PUT` get `405` instead of falling through to the Results Passthrough, which would forward the capability upstream. + +We rejected an HMAC over the Upload id: it needs a secret shared by every worker and kept across restarts, which this server has no place for, while a stored random value needs nothing new. Uploads created without a capability (by a release that did not issue one) have none to check, so their URLs do not accept Parts; they are short-lived and `cleanup:uploads` removes them once idle. Upload URLs keep the same shape (`/devstoreaccount1/upload/{id}?sig={capability}`) and the same hash across releases, so an Upload begun before an upgrade can still be completed after it. diff --git a/lib/db.ts b/lib/db.ts index 781ff64..3c311e6 100644 --- a/lib/db.ts +++ b/lib/db.ts @@ -68,6 +68,8 @@ export const uploadSchema = z.object({ finishedPartUploadCount: z.number(), folderName: z.string(), reservationHash: z.string().nullable(), + // SHA-256 of the Upload capability; `null` for Uploads created without one + capabilityHash: z.string().nullable(), }) type Upload = z.infer diff --git a/lib/helpers.ts b/lib/helpers.ts index add3b06..ef286db 100644 --- a/lib/helpers.ts +++ b/lib/helpers.ts @@ -1,5 +1,6 @@ import type { H3Event } from 'h3' -import { createHash } from 'node:crypto' +import { Buffer } from 'node:buffer' +import { createHash, randomBytes, timingSafeEqual } from 'node:crypto' import { customAlphabet } from 'nanoid' const generate = customAlphabet('0123456789', 10) @@ -29,3 +30,28 @@ export function reservationHash({ key, version, scope, repoId }: Reservation) { .update(JSON.stringify([key, version, scope, repoId])) .digest('hex') } + +/** + * A new Upload capability: 256 random bits, sent in the Upload URL's query + * (`?sig=`). Upload URLs are fetched by Azure Blob-style clients that send no + * Actions token, so the capability is their only authorization. + */ +export function generateUploadCapability() { + return randomBytes(32).toString('hex') +} + +/** What is stored of a capability, so neither the database nor its query log can reproduce a working Upload URL. */ +export function hashUploadCapability(capability: string) { + return createHash('sha256').update(capability).digest('hex') +} + +/** Whether `capability` is the one whose hash was stored, compared in constant time. */ +export function uploadCapabilityMatches( + capabilityHash: string | null | undefined, + capability: string | undefined, +) { + if (!capabilityHash || !capability) return false + const expected = Buffer.from(capabilityHash, 'hex') + const actual = createHash('sha256').update(capability).digest() + return expected.length === actual.length && timingSafeEqual(expected, actual) +} diff --git a/lib/storage.ts b/lib/storage.ts index f6b781b..48ace1d 100644 --- a/lib/storage.ts +++ b/lib/storage.ts @@ -43,7 +43,13 @@ import { chunk } from 'remeda' import { match } from 'ts-pattern' import { getDatabase, isUniqueViolation, retryOnLockConflict } from './db' import { env } from './env' -import { generateNumberId, reservationHash } from './helpers' +import { + generateNumberId, + generateUploadCapability, + hashUploadCapability, + reservationHash, + uploadCapabilityMatches, +} from './helpers' import { logger } from './logger' import { cacheMergesSkippedTotal, instrumentS3Agent, observeS3Request, trackMerge } from './metrics' import { @@ -426,6 +432,21 @@ export class Storage { return Promise.all(this.mergeStreamPromises) } + /** + * Whether `capability` is the one issued with the Upload. Upload URLs are not + * covered by the Actions runtime token, so this is their only authorization. + * Unknown Uploads, Uploads created without a capability, and missing or wrong + * capabilities are indistinguishable. + */ + async authorizeUpload(uploadId: number, capability: string | undefined) { + const upload = await this.db + .selectFrom('uploads') + .where('id', '=', uploadId) + .select('capabilityHash') + .executeTakeFirst() + return uploadCapabilityMatches(upload?.capabilityHash, capability) + } + /** * `false` when the upload does not exist, or was deleted while the part was streaming * (the written part is then Orphaned Storage, ADR-0001). @@ -685,6 +706,7 @@ export class Storage { async createUpload(reservation: Reservation) { const uploadId = generateNumberId() + const capability = generateUploadCapability() try { await this.db .insertInto('uploads') @@ -694,6 +716,7 @@ export class Storage { folderName: uploadId.toString(), createdAt: Date.now(), reservationHash: reservationHash(reservation), + capabilityHash: hashUploadCapability(capability), lastPartUploadedAt: null, finishedPartUploadCount: 0, startedPartUploadCount: 0, @@ -704,7 +727,7 @@ export class Storage { throw err } - return { id: uploadId } + return { id: uploadId, capability } } async matchCacheEntry({ diff --git a/routes/[...path].ts b/routes/[...path].ts index 65c2a8c..24f5b78 100644 --- a/routes/[...path].ts +++ b/routes/[...path].ts @@ -2,7 +2,15 @@ import { env } from '~/lib/env' import { loggedPath } from '~/lib/helpers' import { logger } from '~/lib/logger' +// This server's own Upload URLs. Only `PUT` is routed; any other method lands +// here, and forwarding it would hand the URL's capability to the upstream. +const UPLOAD_PATH = /^\/(?:devstoreaccount1\/)?upload\// + export default defineEventHandler(async (event) => { - logger.debug('proxying unknown path', loggedPath(event), 'to', env.DEFAULT_ACTIONS_RESULTS_URL) + const path = loggedPath(event) + if (UPLOAD_PATH.test(path)) + throw createError({ statusCode: 405, statusMessage: 'Method Not Allowed' }) + + logger.debug('proxying unknown path', path, 'to', env.DEFAULT_ACTIONS_RESULTS_URL) return proxyRequest(event, `${env.DEFAULT_ACTIONS_RESULTS_URL}${event.path}`) }) diff --git a/routes/devstoreaccount1/upload/[uploadId].put.ts b/routes/devstoreaccount1/upload/[uploadId].put.ts index ba2c812..284f40e 100644 --- a/routes/devstoreaccount1/upload/[uploadId].put.ts +++ b/routes/devstoreaccount1/upload/[uploadId].put.ts @@ -18,36 +18,43 @@ export default defineEventHandler(async (event) => { statusMessage: `Invalid path parameters: ${parsedPathParams.error.message}`, }) - if (getQuery(event).comp === 'blocklist') { - setResponseStatus(event, 201) - // prevent random EOF error with in tonistiigi/go-actions-cache caused by missing request id - setHeader(event, 'x-ms-request-id', randomUUID()) - return - } - - const blockId = getQuery(event)?.blockid as string - // if no block id, upload smaller than chunk size - const chunkIndex = blockId ? getChunkIndexFromBlockId(blockId) : 0 - if (chunkIndex === undefined) - throw createError({ - statusCode: 400, - statusMessage: `Invalid block id: ${blockId}`, - }) - const { uploadId } = parsedPathParams.data + const query = getQuery(event) + // The capability issued in `signed_upload_url`. Clients keep the URL's query + // and append their own parameters, as they do with an Azure SAS. + const capability = typeof query.sig === 'string' ? query.sig : undefined + + // A blocklist commits the blocks already stored; there is no part to write. + let chunkIndex: number | undefined + if (query.comp !== 'blocklist') { + const blockId = query.blockid as string + // if no block id, upload smaller than chunk size + chunkIndex = blockId ? getChunkIndexFromBlockId(blockId) : 0 + if (chunkIndex === undefined) + throw createError({ + statusCode: 400, + statusMessage: `Invalid block id: ${blockId}`, + }) + } - let stored: boolean + let authorized: boolean + let stored = true try { + const storage = await getStorage() + authorized = await storage.authorizeUpload(uploadId, capability) // Not h3's `getRequestWebStream`: it enqueues every `data` event and never pauses the // request, so a body arriving faster than storage consumes it is buffered in full. - const storage = await getStorage() - stored = await storage.uploadPart(uploadId, chunkIndex, event.node.req) + if (authorized && chunkIndex !== undefined) + stored = await storage.uploadPart(uploadId, chunkIndex, event.node.req) } catch (err) { // Logged here and rethrown as handled: Nitro logs unhandled errors with the // full request URL, query string included. logger.error(`Failed to store upload ${uploadId}`, err) throw createError({ statusCode: 500, statusMessage: 'Failed to store upload' }) } + // Unknown Uploads and missing or wrong capabilities get the same answer. + if (!authorized) throw createError({ statusCode: 403, statusMessage: 'Forbidden' }) + // The Upload was deleted while the part was streaming (e.g. finalized or cleaned up). if (!stored) throw createError({ statusCode: 404, diff --git a/routes/twirp/github.actions.results.api.v1.CacheService/CreateCacheEntry.post.ts b/routes/twirp/github.actions.results.api.v1.CacheService/CreateCacheEntry.post.ts index ff51f39..80bd5d5 100644 --- a/routes/twirp/github.actions.results.api.v1.CacheService/CreateCacheEntry.post.ts +++ b/routes/twirp/github.actions.results.api.v1.CacheService/CreateCacheEntry.post.ts @@ -39,7 +39,10 @@ export default defineEventHandler(async (event) => { return sendTwirpResponse( event, - { ok: true, signed_upload_url: `${env.API_BASE_URL}/devstoreaccount1/upload/${upload.id}` }, + { + ok: true, + signed_upload_url: `${env.API_BASE_URL}/devstoreaccount1/upload/${upload.id}?sig=${upload.capability}`, + }, TwirpMessage.CreateCacheEntryResponse, ) }) diff --git a/tests/migrations.test.ts b/tests/migrations.test.ts index f5510a5..dcdcd8f 100644 --- a/tests/migrations.test.ts +++ b/tests/migrations.test.ts @@ -1,4 +1,5 @@ import type { Migration } from 'kysely/migration' +import { createHash, randomBytes } from 'node:crypto' import path from 'node:path' import { setTimeout as sleep } from 'node:timers/promises' @@ -11,6 +12,7 @@ import pg from 'pg' import { match } from 'ts-pattern' import { afterAll, describe, expect, test } from 'vitest' import { env } from '~/lib/env' +import { uploadCapabilityMatches } from '~/lib/helpers' import { migrateToLatest, migrations } from '~/lib/migrations' import { TEST_TEMP_DIR } from './setup' @@ -190,6 +192,10 @@ const idleUpload = { finishedPartUploadCount: 0, startedPartUploadCount: 0, } +// The capability the downstream release issued for the surviving Upload, and +// the hash it stored for it (SHA-256, hex), as it computed them. +const downstreamCapability = randomBytes(32).toString('hex') +const downstreamCapabilityHash = createHash('sha256').update(downstreamCapability).digest('hex') /** A database migrated and used by the downstream release. */ async function createDownstreamDatabase(name: string) { @@ -223,7 +229,7 @@ async function createDownstreamDatabase(name: string) { id: 2, folderName: '2', createdAt: 2, - capabilityHash: 'b'.repeat(64), + capabilityHash: downstreamCapabilityHash, }, { ...idleUpload, @@ -274,9 +280,14 @@ describe('migrations', () => { .orderBy('id') .execute() expect(uploads.map((u) => [u.id, u.capabilityHash])).toEqual([ - [2, 'b'.repeat(64)], + [2, downstreamCapabilityHash], [3, null], ]) + // so the Upload URL the downstream release issued still authorizes its parts + expect(uploadCapabilityMatches(uploads[0]!.capabilityHash, downstreamCapability)).toBe(true) + expect( + uploadCapabilityMatches(uploads[0]!.capabilityHash, randomBytes(32).toString('hex')), + ).toBe(false) expect(uploads.every((u) => u.reservationHash?.length === 64)).toBe(true) expect(await db.selectFrom('cache_entries').selectAll().execute()).toEqual([ { id: 'entry', ...reservation, updatedAt: 1, locationId: 'location' }, diff --git a/tests/proxy-backpressure.test.ts b/tests/proxy-backpressure.test.ts index 72cff49..564062c 100644 --- a/tests/proxy-backpressure.test.ts +++ b/tests/proxy-backpressure.test.ts @@ -82,10 +82,13 @@ describe('proxy routes apply backpressure', () => { const { port } = server.address() as AddressInfo const serverSide = once(server, 'request') as Promise<[IncomingMessage, ServerResponse]> - const req = http.request(`http://localhost:${port}/upload/${created.id}`, { - method: 'PUT', - headers: { 'content-length': ENTRY_BYTES }, - }) + const req = http.request( + `http://localhost:${port}/upload/${created.id}?sig=${created.capability}`, + { + method: 'PUT', + headers: { 'content-length': ENTRY_BYTES }, + }, + ) const response = once(req, 'response') as Promise<[IncomingMessage]> const chunk = Buffer.alloc(CHUNK_BYTES) let sent = 0 diff --git a/tests/request-logging.test.ts b/tests/request-logging.test.ts index 243eea4..ae21ef9 100644 --- a/tests/request-logging.test.ts +++ b/tests/request-logging.test.ts @@ -73,7 +73,7 @@ afterAll(async () => { await once(server, 'close') }) -test('query strings stay out of request, error and passthrough logs', async () => { +test('upload capabilities stay out of request, error and passthrough logs', async () => { const storage = await getStorage() const created = await storage.createUpload({ key: `request-logging-${randomUUID()}`, @@ -83,12 +83,16 @@ test('query strings stay out of request, error and passthrough logs', async () = }) const { port } = server.address() as AddressInfo - // Azure Blob clients send SAS-style credentials in the query. - const secret = randomBytes(32).toString('hex') - const url = `http://localhost:${port}/upload/${created.id}?sig=${secret}` + const url = `http://localhost:${port}/upload/${created.id}?sig=${created.capability}` + const forged = randomBytes(32).toString('hex') const stored = await fetch(url, { method: 'PUT', body: 'part' }) expect(stored.status).toBe(201) + const rejected = await fetch(`http://localhost:${port}/upload/${created.id}?sig=${forged}`, { + method: 'PUT', + body: 'part', + }) + expect(rejected.status).toBe(403) // Storage failing mid-upload (e.g. the backend is unreachable) is logged as an error. vi.spyOn(storage.adapter, 'uploadStream').mockRejectedValue(new Error('storage unavailable')) @@ -100,15 +104,25 @@ test('query strings stay out of request, error and passthrough logs', async () = expect(failed.status).toBe(500) await vi.waitFor(() => expect(logs.join('\n')).toContain('> 500')) - // An unknown path falls through to the Results Passthrough. - const unknown = await fetch(`http://localhost:${port}/uploads/${created.id}?sig=${secret}`) - expect(unknown.status).toBe(204) + // A mistyped Upload URL falls through to the Results Passthrough. + const mistyped = await fetch( + `http://localhost:${port}/uploads/${created.id}?sig=${created.capability}`, + ) + expect(mistyped.status).toBe(204) await vi.waitFor(() => expect(logs.join('\n')).toContain(`proxying unknown path`)) expect(proxyRequest).toHaveBeenCalledTimes(1) + // An Upload URL with an unsupported method is refused, not forwarded upstream. + for (const method of ['POST', 'HEAD', 'GET']) { + const response = await fetch(url, { method }) + expect(response.status).toBe(405) + } + expect(proxyRequest).toHaveBeenCalledTimes(1) + const logged = logs.join('\n') expect(logged).toContain(`/upload/${created.id}`) expect(logged).toContain('storage unavailable') expect(logged).not.toContain('sig=') - expect(logged).not.toContain(secret) + expect(logged).not.toContain(created.capability) + expect(logged).not.toContain(forged) }) diff --git a/tests/upload-authorization.test.ts b/tests/upload-authorization.test.ts new file mode 100644 index 0000000..dc9fb48 --- /dev/null +++ b/tests/upload-authorization.test.ts @@ -0,0 +1,263 @@ +import { Buffer } from 'node:buffer' +import crypto from 'node:crypto' + +import { SignJWT } from 'jose' +import { beforeAll, describe, expect, test } from 'vitest' +import { getDatabase } from '~/lib/db' +import { reservationHash } from '~/lib/helpers' + +const BASE = 'http://localhost:3000' +const TWIRP_BASE = `${BASE}/twirp/github.actions.results.api.v1.CacheService` + +function signToken({ repoId = '123', scope = 'refs/heads/main' } = {}) { + return new SignJWT({ + ac: JSON.stringify([{ Scope: scope, Permission: 3 }]), + repository_id: repoId, + }) + .setProtectedHeader({ alg: 'HS256' }) + .sign(crypto.createSecretKey('mock-secret-key', 'ascii')) +} + +async function twirp(method: string, token: string, body: object) { + const response = await fetch(`${TWIRP_BASE}/${method}`, { + method: 'POST', + headers: { 'authorization': `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify(body), + }) + const json = (await response.json()) as T + return { status: response.status, body: json } +} + +async function createUpload(token: string, key: string) { + const { status, body } = await twirp<{ ok: boolean; signed_upload_url: string }>( + 'CreateCacheEntry', + token, + { key, version: 'v1' }, + ) + expect(status).toBe(200) + expect(body.ok).toBe(true) + return body.signed_upload_url +} + +/** Block id in the 48-byte format (uuid + zero-padded index) used by `@actions/cache`. */ +function blockId(index: number) { + const padded = + index < 0 ? `-${String(-index).padStart(11, '0')}` : String(index).padStart(12, '0') + return Buffer.from(`${crypto.randomUUID()}${padded}`).toString('base64') +} + +/** Block id in the 64-byte format used by docker buildx (index at byte 16). */ +function buildxBlockId(index: number) { + const id = crypto.randomBytes(64) + id.writeUInt32BE(index, 16) + return id.toString('base64') +} + +/** + * An Upload as a release that already issued capabilities created it before an + * upgrade: the row it inserted (with the reservation hash the upgrade adds) and + * the Upload URL it returned. It stored the SHA-256 (hex) of a random 256-bit + * capability and returned `/devstoreaccount1/upload/{id}?sig={capability}`. + */ +async function createPreviousReleaseUpload( + key: string, + { withCapability = true }: { withCapability?: boolean } = {}, +) { + const reservation = { key, version: 'v1', scope: 'refs/heads/main', repoId: '123' } + const id = crypto.randomInt(1, 2 ** 31) + const capability = crypto.randomBytes(32).toString('hex') + const db = await getDatabase() + await db + .insertInto('uploads') + .values({ + ...reservation, + id, + folderName: id.toString(), + createdAt: Date.now(), + lastPartUploadedAt: null, + finishedPartUploadCount: 0, + startedPartUploadCount: 0, + reservationHash: reservationHash(reservation), + capabilityHash: withCapability + ? crypto.createHash('sha256').update(capability).digest('hex') + : null, + }) + .execute() + return `${BASE}/devstoreaccount1/upload/${id}?sig=${capability}` +} + +/** Appends Azure Blob-style query parameters the way the client SDKs do. */ +function withParams(url: string, params: Record) { + const parsed = new URL(url) + for (const [name, value] of Object.entries(params)) parsed.searchParams.set(name, value) + return parsed.href +} + +function withoutQuery(url: string) { + const parsed = new URL(url) + parsed.search = '' + return parsed.href +} + +/** PUTs `body` and returns the response status and error message, if any. */ +async function put(url: string, body?: Uint8Array) { + const response = await fetch(url, { method: 'PUT', body: body as BodyInit | undefined }) + const text = await response.text() + const message = response.ok ? undefined : (JSON.parse(text) as { statusMessage: string }) + return { status: response.status, statusMessage: message?.statusMessage } +} + +async function finalize(token: string, key: string) { + const { status } = await twirp<{ ok: boolean }>('FinalizeCacheEntryUpload', token, { + key, + version: 'v1', + }) + return status +} + +async function downloadEntry(token: string, key: string) { + const { body } = await twirp<{ ok: boolean; signed_download_url: string }>( + 'GetCacheEntryDownloadURL', + token, + { key, version: 'v1' }, + ) + expect(body.ok).toBe(true) + const response = await fetch(body.signed_download_url) + expect(response.status).toBe(200) + return Buffer.from(await response.arrayBuffer()) +} + +describe('upload URL authorization', () => { + let token: string + beforeAll(async () => { + token = await signToken() + }) + + test('an upload URL without its capability cannot add data to an in-flight upload', async () => { + const key = `upload-auth-${crypto.randomUUID()}` + const url = await createUpload(token, key) + const expected = crypto.randomBytes(1024) + + expect(await put(url, expected)).toMatchObject({ status: 201 }) + for (const index of [-1, 1]) { + const foreign = await put( + withParams(withoutQuery(url), { comp: 'block', blockid: blockId(index) }), + crypto.randomBytes(512), + ) + expect(foreign.status).toBe(403) + } + + expect(await finalize(token, key)).toBe(200) + const downloaded = await downloadEntry(token, key) + expect(downloaded.equals(expected)).toBe(true) + }) + + test('a wrong capability and an unknown upload get the same rejection', async () => { + const key = `upload-auth-${crypto.randomUUID()}` + const url = await createUpload(token, key) + const forged = crypto.randomBytes(32).toString('hex') + + const rejected = [ + await put(withParams(url, { sig: forged }), crypto.randomBytes(16)), + await put( + withParams(url.replace(/\/upload\/\d+/, '/upload/1'), { sig: forged }), + crypto.randomBytes(16), + ), + await put(withParams(withoutQuery(url), { comp: 'blocklist' })), + ] + for (const response of rejected) + expect(response).toEqual({ status: 403, statusMessage: 'Forbidden' }) + + // The rejected requests left no parts behind: the owner's upload still finalizes. + const expected = crypto.randomBytes(64) + expect(await put(url, expected)).toMatchObject({ status: 201 }) + expect(await finalize(token, key)).toBe(200) + const downloaded = await downloadEntry(token, key) + expect(downloaded.equals(expected)).toBe(true) + }) + + test('chunked block and blocklist uploads work with the capability', async () => { + const key = `upload-auth-${crypto.randomUUID()}` + const url = await createUpload(token, key) + const chunks = [crypto.randomBytes(2048), crypto.randomBytes(1024)] + + for (const [index, chunk] of chunks.entries()) { + const response = await put(withParams(url, { comp: 'block', blockid: blockId(index) }), chunk) + expect(response.status).toBe(201) + } + expect(await put(withParams(url, { comp: 'blocklist' }))).toMatchObject({ status: 201 }) + + expect(await finalize(token, key)).toBe(200) + const downloaded = await downloadEntry(token, key) + expect(downloaded.equals(Buffer.concat(chunks))).toBe(true) + }) + + test('only the creating repository and scope can finalize an upload', async () => { + const key = `upload-auth-${crypto.randomUUID()}` + const url = await createUpload(token, key) + const expected = crypto.randomBytes(256) + expect(await put(url, expected)).toMatchObject({ status: 201 }) + + const otherRepo = await signToken({ repoId: '456' }) + const otherScope = await signToken({ scope: 'refs/heads/other' }) + expect(await finalize(otherRepo, key)).toBe(404) + expect(await finalize(otherScope, key)).toBe(404) + + expect(await finalize(token, key)).toBe(200) + const downloaded = await downloadEntry(token, key) + expect(downloaded.equals(expected)).toBe(true) + }) + + test('chunked uploads with docker buildx block ids work with the capability', async () => { + const key = `upload-auth-${crypto.randomUUID()}` + const url = await createUpload(token, key) + const chunks = [crypto.randomBytes(1024), crypto.randomBytes(512), crypto.randomBytes(256)] + + for (const [index, chunk] of chunks.entries()) { + const block = withParams(url, { comp: 'block', blockid: buildxBlockId(index) }) + expect(await put(block, chunk)).toMatchObject({ status: 201 }) + expect(await put(withoutQuery(block), chunk)).toMatchObject({ status: 403 }) + } + expect(await put(withParams(url, { comp: 'blocklist' }))).toMatchObject({ status: 201 }) + + expect(await finalize(token, key)).toBe(200) + const downloaded = await downloadEntry(token, key) + expect(downloaded.equals(Buffer.concat(chunks))).toBe(true) + }) + + test('an upload URL issued before an upgrade still accepts parts', async () => { + const key = `upload-auth-${crypto.randomUUID()}` + const url = await createPreviousReleaseUpload(key) + const chunks = [crypto.randomBytes(2048), crypto.randomBytes(1024)] + + for (const [index, chunk] of chunks.entries()) { + const response = await put(withParams(url, { comp: 'block', blockid: blockId(index) }), chunk) + expect(response.status).toBe(201) + } + expect(await put(withParams(url, { comp: 'blocklist' }))).toMatchObject({ status: 201 }) + const forged = withParams(url, { + sig: crypto.randomBytes(32).toString('hex'), + comp: 'block', + blockid: blockId(2), + }) + expect(await put(forged, crypto.randomBytes(16))).toEqual({ + status: 403, + statusMessage: 'Forbidden', + }) + + expect(await finalize(token, key)).toBe(200) + const downloaded = await downloadEntry(token, key) + expect(downloaded.equals(Buffer.concat(chunks))).toBe(true) + }) + + test('an upload created without a capability accepts no parts', async () => { + const key = `upload-auth-${crypto.randomUUID()}` + const url = await createPreviousReleaseUpload(key, { withCapability: false }) + + for (const target of [url, withoutQuery(url), withParams(url, { comp: 'blocklist' })]) + expect(await put(target, crypto.randomBytes(16))).toEqual({ + status: 403, + statusMessage: 'Forbidden', + }) + }) +}) diff --git a/tests/upload-part.test.ts b/tests/upload-part.test.ts index 29bdefe..491f407 100644 --- a/tests/upload-part.test.ts +++ b/tests/upload-part.test.ts @@ -1,21 +1,57 @@ +import type { AddressInfo } from 'node:net' import { randomUUID } from 'node:crypto' +import { once } from 'node:events' +import http from 'node:http' import { Readable } from 'node:stream' -import { afterEach, expect, test, vi } from 'vitest' +import * as h3 from 'h3' +import { afterAll, afterEach, beforeAll, expect, test, vi } from 'vitest' import { getDatabase } from '~/lib/db' import { getStorage } from '~/lib/storage' // `generateNumberId` never yields a negative id const UNKNOWN_UPLOAD_ID = -1 +// The real upload route on an in-process h3 app, so storage can be made to fail +// under it. +const router = h3.createRouter({ preemptive: true }) +const server = http.createServer(h3.toNodeListener(h3.createApp().use(router))) + +beforeAll(async () => { + for (const [name, value] of Object.entries(h3)) vi.stubGlobal(name, value) + // Not a static import: the route calls Nitro auto-imports (`defineEventHandler`, + // โ€ฆ) at evaluation time, so it can only load after the stubs above. + const { default: upload } = await import('~/routes/devstoreaccount1/upload/[uploadId].put') + router.put('/upload/:uploadId', upload) + server.listen(0) + await once(server, 'listening') +}) + afterEach(() => vi.restoreAllMocks()) -test('a block PUT for an unknown upload answers 404', async () => { - const res = await fetch(`http://localhost:3000/devstoreaccount1/upload/${UNKNOWN_UPLOAD_ID}`, { - method: 'PUT', - body: 'payload', - }) - expect(res.status).toBe(404) +afterAll(async () => { + vi.unstubAllGlobals() + server.close() + await once(server, 'close') +}) + +function createUpload() { + return getStorage().then((storage) => + storage.createUpload({ + key: `upload-part-${randomUUID()}`, + version: 'v1', + scope: 'refs/heads/main', + repoId: '123', + }), + ) +} + +test('a block PUT for an unknown upload is refused like a wrong capability', async () => { + const res = await fetch( + `http://localhost:3000/devstoreaccount1/upload/${UNKNOWN_UPLOAD_ID}?sig=${'0'.repeat(64)}`, + { method: 'PUT', body: 'payload' }, + ) + expect(res.status).toBe(403) }) test('a part for an unknown upload is not written', async () => { @@ -26,19 +62,31 @@ test('a part for an unknown upload is not written', async () => { expect(uploadStream).not.toHaveBeenCalled() }) -test('a part whose upload is deleted mid-stream reports failure', async () => { +/** `cleanup:uploads` removing the row while a part streams */ +async function deleteUploadMidStream(uploadId: number) { const [storage, db] = await Promise.all([getStorage(), getDatabase()]) - const upload = await storage.createUpload({ - key: `upload-part-${randomUUID()}`, - version: 'v1', - scope: 'refs/heads/main', - repoId: '123', - }) - // `cleanup:uploads` removing the row mid-part vi.spyOn(storage.adapter, 'uploadStream').mockImplementation(async (_name, stream) => { - await db.deleteFrom('uploads').where('id', '=', upload.id).execute() + await db.deleteFrom('uploads').where('id', '=', uploadId).execute() for await (const _chunk of stream); }) +} + +test('a part whose upload is deleted mid-stream reports failure', async () => { + const storage = await getStorage() + const upload = await createUpload() + await deleteUploadMidStream(upload.id) expect(await storage.uploadPart(upload.id, 0, Readable.from('payload'))).toBe(false) }) + +test('an authorized block PUT whose upload is deleted mid-stream answers 404', async () => { + const upload = await createUpload() + await deleteUploadMidStream(upload.id) + + const { port } = server.address() as AddressInfo + const res = await fetch(`http://localhost:${port}/upload/${upload.id}?sig=${upload.capability}`, { + method: 'PUT', + body: 'payload', + }) + expect(res.status).toBe(404) +}) diff --git a/tests/write-authorization.test.ts b/tests/write-authorization.test.ts index f3b9a54..e5a1c87 100644 --- a/tests/write-authorization.test.ts +++ b/tests/write-authorization.test.ts @@ -65,6 +65,20 @@ function management(path: string, init: RequestInit = {}, apiKey = MANAGEMENT_AP } describe('write path authorization', () => { + test('an upload capability only authorizes its own upload', async () => { + const tokenA = await signToken({ repoId: 'write-auth-a' }) + const tokenB = await signToken({ repoId: 'write-auth-b' }) + const key = `write-auth-${crypto.randomUUID()}` + const urlA = new URL(await createUpload(tokenA, key)) + const urlB = new URL(await createUpload(tokenB, key)) + + // A's capability on B's upload id + const crossed = new URL(urlB) + crossed.searchParams.set('sig', urlA.searchParams.get('sig')!) + const response = await fetch(crossed, { method: 'PUT', body: crypto.randomBytes(16) }) + expect(response.status).toBe(403) + }) + test('another repository cannot replace or read an entry with the same key and scope', async () => { const tokenA = await signToken({ repoId: 'write-auth-owner' }) const tokenB = await signToken({ repoId: 'write-auth-other' }) From 110269cedf2363ecd049b88bb2cb3d4d498655e9 Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Sat, 24 Oct 2026 18:31:32 -0400 Subject: [PATCH 17/17] perf(storage): load the Azure and GCS SDKs only when selected `lib/storage.ts` imported `@azure/storage-blob`, `@azure/identity` and `@google-cloud/storage` at the top level, so every server loaded all three SDKs whatever `STORAGE_DRIVER` said. Since the Azure Blob driver (#280) that adds about 40 MB of live heap to an idle S3 or filesystem server, and every GC cycle has to walk it. Import each SDK inside its adapter's `fromEnv`; only types stay as static imports. Signed-off-by: Rui Chen (cherry picked from commit 00a7d6cb3f2a8e3db1b9e963c8a947b5d80d86f4) --- lib/storage.ts | 53 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/lib/storage.ts b/lib/storage.ts index 48ace1d..7fbdb78 100644 --- a/lib/storage.ts +++ b/lib/storage.ts @@ -1,5 +1,6 @@ /* eslint-disable ts/method-signature-style */ -import type { ContainerClient } from '@azure/storage-blob' +import type { BlobServiceClient, ContainerClient } from '@azure/storage-blob' +import type { Storage as GcsClient } from '@google-cloud/storage' import type { Kysely } from 'kysely' import type { Writable } from 'node:stream' import type { Database, StorageLocation } from './db' @@ -29,14 +30,6 @@ import { } from '@aws-sdk/client-s3' import { Upload as S3Upload } from '@aws-sdk/lib-storage' import { getSignedUrl } from '@aws-sdk/s3-request-presigner' -import { DefaultAzureCredential } from '@azure/identity' -import { - BlobSASPermissions, - BlobServiceClient, - generateBlobSASQueryParameters, - StorageSharedKeyCredential, -} from '@azure/storage-blob' -import { Storage as GcsClient } from '@google-cloud/storage' import { NodeHttpHandler } from '@smithy/node-http-handler' import { sql } from 'kysely' import { chunk } from 'remeda' @@ -1435,6 +1428,8 @@ class GcsAdapter implements StorageAdapter { static async fromEnv(env: Extract) { const bucketName = env.STORAGE_GCS_BUCKET + // Loaded only when selected: the SDKs of unused drivers stay out of the heap. + const { Storage: GcsClient } = await import('@google-cloud/storage') const gcs = new GcsClient({ keyFilename: env.STORAGE_GCS_SERVICE_ACCOUNT_KEY, apiEndpoint: env.STORAGE_GCS_ENDPOINT, @@ -1587,20 +1582,27 @@ class GcsAdapter implements StorageAdapter { class AzBlobAdapter implements StorageAdapter { static async fromEnv(env: Extract) { - const service = env.STORAGE_AZBLOB_CONNECTION_STRING - ? BlobServiceClient.fromConnectionString(env.STORAGE_AZBLOB_CONNECTION_STRING) - : new BlobServiceClient( - env.STORAGE_AZBLOB_ENDPOINT ?? - `https://${env.STORAGE_AZBLOB_ACCOUNT}.blob.core.windows.net`, - new DefaultAzureCredential(), - ) + // Loaded only when selected: the SDKs of unused drivers stay out of the heap. + const sdk = await import('@azure/storage-blob') + let service + if (env.STORAGE_AZBLOB_CONNECTION_STRING) { + service = sdk.BlobServiceClient.fromConnectionString(env.STORAGE_AZBLOB_CONNECTION_STRING) + } else { + const { DefaultAzureCredential } = await import('@azure/identity') + service = new sdk.BlobServiceClient( + env.STORAGE_AZBLOB_ENDPOINT ?? + `https://${env.STORAGE_AZBLOB_ACCOUNT}.blob.core.windows.net`, + new DefaultAzureCredential(), + ) + } const container = service.getContainerClient(env.STORAGE_AZBLOB_CONTAINER) if (!(await container.exists())) throw new Error(`Container ${env.STORAGE_AZBLOB_CONTAINER} does not exist`) - return new AzBlobAdapter({ service, container }) + return new AzBlobAdapter({ sdk, service, container }) } + private sdk private service private container private keyPrefix = 'gh-actions-cache' @@ -1627,7 +1629,16 @@ class AzBlobAdapter implements StorageAdapter { }, } - constructor({ service, container }: { service: BlobServiceClient; container: ContainerClient }) { + constructor({ + sdk, + service, + container, + }: { + sdk: typeof import('@azure/storage-blob') + service: BlobServiceClient + container: ContainerClient + }) { + this.sdk = sdk this.service = service this.container = container } @@ -1641,17 +1652,17 @@ class AzBlobAdapter implements StorageAdapter { // Backdated so clock skew doesn't make the SAS "not yet valid". const startsOn = new Date(Date.now() - 5 * 60 * 1000) const expiresOn = new Date(expiresAt) - const permissions = BlobSASPermissions.parse('r') + const permissions = this.sdk.BlobSASPermissions.parse('r') // A shared key signs directly; Entra ID credentials need a user delegation key. const delegationKey = - this.service.credential instanceof StorageSharedKeyCredential + this.service.credential instanceof this.sdk.StorageSharedKeyCredential ? undefined : await this.service.getUserDelegationKey(startsOn, expiresOn) return (objectName: string) => { const blob = this.blob(objectName) if (!delegationKey) return blob.generateSasUrl({ permissions, startsOn, expiresOn }) - const sas = generateBlobSASQueryParameters( + const sas = this.sdk.generateBlobSASQueryParameters( { containerName: blob.containerName, blobName: blob.name,