Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
9bab33e
fix(db): start on databases that ran downstream migrations
chenrui333 Oct 24, 2026
5b7ab0a
fix(download): answer Range on an empty cache entry the same way on e…
LouisHaftmann Oct 24, 2026
951880a
fix(download): stop the merge pump waiting on a destroyed stream
chenrui333 Oct 24, 2026
fd4ddda
perf(storage): stop forcing a full GC after every part
chenrui333 Oct 24, 2026
ec55882
fix(s3): make the socket pool size configurable and apply it to http …
chenrui333 Oct 24, 2026
fe9af9a
fix(s3): separate socket pools for reads, writes and control requests
chenrui333 Oct 24, 2026
d65f096
fix(storage): cap concurrent Merges per process
chenrui333 Oct 24, 2026
2dfddcf
fix(download): close downloads whose client stopped reading
chenrui333 Oct 24, 2026
783e9ed
fix(download): release the backend read when the client leaves before…
chenrui333 Oct 24, 2026
f8d500c
feat(metrics): expose S3 pool, S3 command, merge, lease and download …
chenrui333 Oct 24, 2026
12b6ac4
ci: build patched images from patch-* tags as OCI archives
chenrui333 Oct 24, 2026
43970c2
fix(db): release the Postgres startup probe client
chenrui333 Oct 24, 2026
7f83ffb
fix(management-api): honor repoId when deleting cache entries
chenrui333 Oct 24, 2026
85189b7
feat(auth): require verified tokens for direct downloads
chenrui333 Oct 24, 2026
4943b57
fix(logging): keep query strings out of request logs
chenrui333 Oct 24, 2026
5ca1331
fix(upload): require an unguessable capability for upload URLs
chenrui333 Oct 24, 2026
110269c
perf(storage): load the Azure and GCS SDKs only when selected
chenrui333 Oct 24, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions .github/workflows/patch-image.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
name: Patch image

# Builds a multi-arch OCI image archive for each `patch-<version>` tag
# (`patch-9.9.0-backpressure.1` -> artifact `image-9.9.0-backpressure.1`).
# The ghcr.io package predates this repo link, so GITHUB_TOKEN cannot push to it;
# publish the archive with `crane push --index <dir> ghcr.io/<owner>/github-actions-cache-server:<tag>`.
# Kept separate from release.yml, which only releases from dev and moves `latest`.
on:
push:
tags:
- 'patch-*'

permissions:
contents: read

jobs:
image:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6

- name: Image tag
id: tag
run: echo "TAG=${GITHUB_REF_NAME#patch-}" >> "$GITHUB_OUTPUT"

- uses: docker/setup-qemu-action@v4

- uses: docker/setup-buildx-action@v4

- uses: docker/build-push-action@v7
with:
context: .
platforms: linux/amd64,linux/arm64
outputs: type=oci,dest=${{ runner.temp }}/image.tar,tar=true
build-args: |
BUILD_HASH=${{ github.sha }}

- uses: actions/upload-artifact@v5
with:
name: image-${{ steps.tag.outputs.TAG }}
path: ${{ runner.temp }}/image.tar
retention-days: 7
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,4 @@ RUN if echo "$BASE_IMAGE" | grep -q "node-caged"; then \
echo "export NODE_CAGED=false" > /app/.runtime-env; \
fi

CMD ["sh", "-c", ". /app/.runtime-env && exec node --expose-gc /app/server/index.mjs"]
CMD ["sh", "-c", ". /app/.runtime-env && exec node /app/server/index.mjs"]
21 changes: 21 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,27 @@ volumes:
cache-data:
```

## Resource limits

All optional.

| Variable | Default | Effect |
| ------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------- |
| `STORAGE_S3_MAX_SOCKETS` | `50` | Sockets per S3 data pool, for `http://` and `https://` endpoints alike. Reads (GetObject) and writes (uploads, Merges) each get a pool this size. |
| `STORAGE_S3_CONTROL_MAX_SOCKETS` | `16` | Sockets for HEAD, LIST and DELETE requests (cache lookups, cleanup), kept apart from bulk transfers. |
| `STORAGE_S3_CONTROL_QUEUE_TIMEOUT_MS` | `5000` | How long a control request waits for a socket and connection before failing; `0` waits forever. |
| `MAX_CONCURRENT_MERGES` | `16` | Merges one process runs at once. Beyond it, restores stream the parts and a later download merges. |
| `DOWNLOAD_IDLE_TIMEOUT_MS` | `300000` | Closes a download whose client has not read a byte for this long; `0` disables. |

Direct downloads (`ENABLE_DIRECT_DOWNLOADS=true`) hand the runner a signed storage URL for merged entries instead of proxying the bytes.
They require token validation: the server refuses to start with `SKIP_TOKEN_VALIDATION=true`, because unverified callers could then obtain signed URLs for any repository's cache.

| Variable | Default | Effect |
| -------------------------------------------------------- | ------- | ------------------------------------------------------------------------------------------------- |
| `DIRECT_DOWNLOAD_URL_TTL_SECONDS` | `300` | Lifetime of a signed download URL, `1` to `3600`. Raise it only for clients that fetch URLs late. |
| `ACTIONS_TOKEN_AUDIENCE` | unset | Comma-separated `aud` values a token must carry; unset skips the check, an empty list is refused. |
| `UNSAFE_ALLOW_DIRECT_DOWNLOADS_WITHOUT_TOKEN_VALIDATION` | `false` | Allows direct downloads with `SKIP_TOKEN_VALIDATION=true`. Local testing only. |

## Documentation

👉 <https://gha-cache-server.falcondev.io/getting-started> 👈
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
# Storage Reader Leases protect active downloads

Concurrent downloads hold durable, expiring Storage Reader Leases, including direct downloads whose lease lasts no longer than the signed URL. A renewable, fenced Merge Lease grants a single worker authority to complete the Merge; Part Reader Leases protect concurrent readers of Parts. Both expiring lease types renew every 30 seconds while active and expire after two minutes, except direct-download leases, which expire with their signed URLs. Parts are deleted only after the merge completes and no active Part Reader Leases remain; a Storage Location is deleted only after no Storage Reader Leases remain. A reader acquires its lease in the same locked transaction that reads the Storage Location's merge state, so its scope is chosen atomically: a download arriving before the merge takes a Part Reader Lease that blocks parts deletion, and one arriving after takes a Storage Reader Lease over the merged representation. There is therefore no window in which a reader can fail to acquire a Part Reader Lease against parts that are being deleted.
Concurrent downloads hold durable, expiring Storage Reader Leases, including direct downloads, whose lease outlives the signed URL by a fixed margin. A renewable, fenced Merge Lease grants a single worker authority to complete the Merge; Part Reader Leases protect concurrent readers of Parts. Both expiring lease types renew every 30 seconds while active and expire after two minutes, except direct-download leases, which expire a fixed margin after their signed URLs: storage authenticates a GET when it starts, so a transfer begun just before the URL expires keeps reading after it. Parts are deleted only after the merge completes and no active Part Reader Leases remain; a Storage Location is deleted only after no Storage Reader Leases remain. A reader acquires its lease in the same locked transaction that reads the Storage Location's merge state, so its scope is chosen atomically: a download arriving before the merge takes a Part Reader Lease that blocks parts deletion, and one arriving after takes a Storage Reader Lease over the merged representation. There is therefore no window in which a reader can fail to acquire a Part Reader Lease against parts that are being deleted.
7 changes: 7 additions & 0 deletions docs/adr/0012-direct-downloads-require-verified-tokens.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Direct downloads require verified tokens

A direct-download URL is a bearer credential: whoever holds it can read the merged object from storage until it expires, without asking the server again. The only thing standing between a caller and that URL is `getCacheScope`, which authorizes a lookup from the token's `repository_id` and `ac` claims. With `SKIP_TOKEN_VALIDATION=true` those claims are decoded but not verified, so any client that can reach the server can name any repository and scope and walk away with a signed URL for its cache. The server therefore refuses to start with `ENABLE_DIRECT_DOWNLOADS=true` and `SKIP_TOKEN_VALIDATION=true` together. `UNSAFE_ALLOW_DIRECT_DOWNLOADS_WITHOUT_TOKEN_VALIDATION=true` overrides the check for local testing only; it is named so that it cannot be set by accident. This extends ADR-0006: skipping verification stays a dev/test escape hatch, and the one feature that turns an unverified claim into a storage credential is now unreachable with it.

A signed URL lives `DIRECT_DOWNLOAD_URL_TTL_SECONDS` (default 300, at most 3600). Clients request the URL immediately before fetching it, and storage authenticates a GET when it starts rather than for its whole duration, so the lifetime only has to cover the start of the transfer and the client's retries, not a large restore. A short lifetime limits replay of a URL that leaks into job logs; it also stays inside the window in which the AWS SDK refreshes temporary credentials (5 minutes before they expire), so a URL is never signed by credentials that lapse before it does. Clients that cache a URL and fetch it later, such as BuildKit's `type=gha` cache importing a layer late in a build, need a longer lifetime; raise the setting for them. The Storage Reader Lease outlives the URL by a fixed margin (ADR-0002), so cleanup cannot delete an object while a transfer begun just before expiry is still reading it.

Token verification requires an `exp` claim, because jose only checks expiry when the claim is present. The optional `ACTIONS_TOKEN_AUDIENCE` pins the expected `aud`, restricting the server to tokens issued for one installation; a value that lists no audience (blank, or only commas) stops startup rather than silently disabling the check. Token rejections return and log the jose error code and failing claim name, never the token or its decoded claims, which jose attaches to its errors. Cache scope permissions are bit flags (read = 1, write = 2): lookups use only scopes with the read bit, and uploads only scopes with the write bit.
11 changes: 11 additions & 0 deletions docs/adr/0013-upload-urls-carry-a-capability.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
---
status: proposed
---

# Upload URLs carry a capability

Twirp calls are authorized by the Actions runtime token (`getCacheScope`), but the Upload URL returned in `signed_upload_url` is fetched by Azure Blob-style clients that send no token. Until now that URL held only the numeric Upload id, so it was the sole thing standing between any client that can reach the server and the Upload's Parts. We now issue a random 256-bit capability with each Upload, put it in the URL's query (`?sig=…`), and require it on every block, single-shot and blocklist request. A missing or wrong capability and an unknown Upload all get the same `403`, so the response reveals nothing about which Uploads exist. An authorized Part whose Upload is deleted while it streams still gets `404`.

The capability lives in the query because every client we support treats the signed URL as an Azure SAS URL: `@actions/cache` and `go-actions-cache` keep the query and append `comp`/`blockid`, and OpenDAL (sccache) requires a query and re-sends it as the SAS. Only its SHA-256 is stored (`uploads.capabilityHash`), compared in constant time, so neither the database nor its debug query log can reproduce a working URL. Request logs record paths without their query string, and upload errors are logged by the route rather than left to Nitro, whose unhandled-error log includes the request URL. Upload URLs requested with a method other than `PUT` get `405` instead of falling through to the Results Passthrough, which would forward the capability upstream.

We rejected an HMAC over the Upload id: it needs a secret shared by every worker and kept across restarts, which this server has no place for, while a stored random value needs nothing new. Uploads created without a capability (by a release that did not issue one) have none to check, so their URLs do not accept Parts; they are short-lived and `cleanup:uploads` removes them once idle. Upload URLs keep the same shape (`/devstoreaccount1/upload/{id}?sig={capability}`) and the same hash across releases, so an Upload begun before an upgrade can still be completed after it.
1 change: 1 addition & 0 deletions lib/api/cache-entries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@ export const cacheEntriesRouter = base
if (input.key) query = query.where('key', '=', input.key)
if (input.version) query = query.where('version', '=', input.version)
if (input.scope) query = query.where('scope', '=', input.scope)
if (input.repoId) query = query.where('repoId', '=', input.repoId)

await query.execute()
context.event.waitUntil(runTask('cleanup:storage-locations'))
Expand Down
26 changes: 12 additions & 14 deletions lib/db.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,13 @@ import { createSingletonPromise } from '@antfu/utils'
import SQLite from 'better-sqlite3'
import { createHooks } from 'hookable'
import { Kysely, MysqlDialect, PostgresDialect, SqliteDialect } from 'kysely'
import { Migrator } from 'kysely/migration'
import { createPool } from 'mysql2'
import pg from 'pg'
import { match } from 'ts-pattern'
import z from 'zod'
import { env } from './env'
import { logger } from './logger'
import { migrations } from './migrations'
import { migrateToLatest } from './migrations'

// we only really use bigint for timestamps, so we can safely parse them as numbers
pg.types.setTypeParser(20, Number)
Expand Down Expand Up @@ -69,6 +68,8 @@ export const uploadSchema = z.object({
finishedPartUploadCount: z.number(),
folderName: z.string(),
reservationHash: z.string().nullable(),
// SHA-256 of the Upload capability; `null` for Uploads created without one
capabilityHash: z.string().nullable(),
})
type Upload = z.infer<typeof uploadSchema>

Expand Down Expand Up @@ -141,7 +142,10 @@ export const getDatabase = createSingletonPromise(async () => {
max: 10,
},
)
await pool.connect()
// Fail fast on bad connection settings, then hand the client back: a
// checked-out client keeps `pool.end()` (Kysely's `destroy()`) waiting forever.
const probe = await pool.connect()
probe.release()

return new PostgresDialect({
pool,
Expand Down Expand Up @@ -203,17 +207,11 @@ export const getDatabase = createSingletonPromise(async () => {
const hooks = createHooks<{
afterMigrate: () => Promise<void>
}>()
const migrator = new Migrator({
db,
provider: {
async getMigrations() {
return migrations(env.DB_DRIVER, hooks)
},
},
})
const { error, results } = await migrator.migrateToLatest()
if (error) {
logger.error('Database migration failed', error)
let results
try {
results = await migrateToLatest(db, env.DB_DRIVER, hooks)
} catch (err) {
logger.error('Database migration failed', err)
// eslint-disable-next-line unicorn/no-process-exit
process.exit(1)
}
Expand Down
13 changes: 12 additions & 1 deletion lib/env.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
import type { envDbDriverSchema, envStorageDriverSchema } from './schemas'
import arkenv from 'arkenv'
import { envSchema } from './schemas'
import {
assertDirectDownloadsRequireTokenValidation,
envSchema,
parseActionsTokenAudience,
} from './schemas'

export const env = arkenv(envSchema, {
env: Object.assign(
Expand All @@ -13,3 +17,10 @@ export const env = arkenv(envSchema, {
process.env,
),
})

assertDirectDownloadsRequireTokenValidation(env)

// GitHub-issued runtime tokens carry a deployment-specific `aud`, so pinning it
// restricts the server to one installation's tokens. Parsed here so a bad value
// stops startup.
export const actionsTokenAudience = parseActionsTokenAudience(env.ACTIONS_TOKEN_AUDIENCE)
37 changes: 36 additions & 1 deletion lib/helpers.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import { createHash } from 'node:crypto'
import type { H3Event } from 'h3'
import { Buffer } from 'node:buffer'
import { createHash, randomBytes, timingSafeEqual } from 'node:crypto'
import { customAlphabet } from 'nanoid'

const generate = customAlphabet('0123456789', 10)
Expand All @@ -7,6 +9,14 @@ export function generateNumberId() {
return Number.parseInt(generate())
}

/**
* The request path without its query string, for logging. A query can carry
* credentials (signed URLs, SAS-style parameters), so it must never reach the logs.
*/
export function loggedPath(event: H3Event) {
return event.path.split('?', 1)[0]
}

/** The key, version, scope and repo that at most one Upload may hold at a time. */
export interface Reservation {
key: string
Expand All @@ -20,3 +30,28 @@ export function reservationHash({ key, version, scope, repoId }: Reservation) {
.update(JSON.stringify([key, version, scope, repoId]))
.digest('hex')
}

/**
* A new Upload capability: 256 random bits, sent in the Upload URL's query
* (`?sig=`). Upload URLs are fetched by Azure Blob-style clients that send no
* Actions token, so the capability is their only authorization.
*/
export function generateUploadCapability() {
return randomBytes(32).toString('hex')
}

/** What is stored of a capability, so neither the database nor its query log can reproduce a working Upload URL. */
export function hashUploadCapability(capability: string) {
return createHash('sha256').update(capability).digest('hex')
}

/** Whether `capability` is the one whose hash was stored, compared in constant time. */
export function uploadCapabilityMatches(
capabilityHash: string | null | undefined,
capability: string | undefined,
) {
if (!capabilityHash || !capability) return false
const expected = Buffer.from(capabilityHash, 'hex')
const actual = createHash('sha256').update(capability).digest()
return expected.length === actual.length && timingSafeEqual(expected, actual)
}
Loading
Loading