From 26ece88762aeabb68e2f9dd20c3bec64e89042f7 Mon Sep 17 00:00:00 2001 From: Rami Date: Tue, 20 Oct 2026 14:31:56 +0100 Subject: [PATCH 1/3] feat(secrets): the store's three failures, a refusing or silent custodian failing the call alone A value the store cannot read now fails the call that asked for it, in words that name no value and no key, and the row stays as it was for the next call to read. - secret_key_unknown stays: a sealed row whose kid is neither custodian's. - CustodianRefused, custodian_refused, { kid, detail }: a custodian whose wrap or unwrap fails, "The custodian of the key refused to unwrap a data key: ". localCustodian refuses a wrapping it cannot open in words of its own, "its wrapped key does not open under this key". - KeeperUnavailable, keeper_unavailable, { detail }: a keeper, or the custodian it asks, that does not answer within the bound the store sets, 30 seconds through Effect's timeoutOrElse, around each hold and reveal of a keep, a read and a replacement. - The Custodian and Keeper ports take the error channel these come from: a custodian and a keeper's hold fail with a KeepingFailure, the last two, and a keeper's reveal with a ReadingFailure, all three. A keep or a replacement that fails keeps nothing. - A rotation bounds each row's unwrapping and wrapping again as a read is, and fails with the custodian's KeepingFailure. The server's start stops on it, as an InvalidSettingsError that names the previous key's setting before the custodian's words. - recordingCustodian is told to refuse(), in the words custodianRefusal, to stall(), whose promise settles once it is first asked, and to answer() again. - storeBehaviour's failures are a suite of their own: secret_key_unknown, custodian_refused and keeper_unavailable through read, at 30 seconds of TestClock and not before, read again once the custodian answers, and custodian_refused through keep and replace, the row as it was. The README says the three failures, the bound and what the recording custodian is told. Co-Authored-By: Claude Fable 5.1 --- packages/secrets/README.md | 34 ++++-- .../src/custodians/custodian-refused.ts | 15 +++ packages/secrets/src/custodians/custodian.ts | 6 +- .../src/custodians/local-custodian.test.ts | 14 ++- .../secrets/src/custodians/local-custodian.ts | 12 +- packages/secrets/src/index.ts | 4 +- .../secrets/src/keepers/keeper-unavailable.ts | 11 ++ packages/secrets/src/keepers/keeper.ts | 10 +- packages/secrets/src/keepers/sealed-keeper.ts | 4 +- packages/secrets/src/store/answer-bound.ts | 14 +++ packages/secrets/src/store/rotation.ts | 21 +++- .../secrets/src/store/secret-store.test.ts | 23 +++- packages/secrets/src/store/secret-store.ts | 50 ++++++--- .../secrets/src/testing/failure-behaviour.ts | 103 ++++++++++++++++++ .../src/testing/recording-custodian.ts | 65 +++++++++-- .../secrets/src/testing/store-behaviour.ts | 39 +------ .../database/ledger-stores-rotation.test.ts | 49 +++++++++ packages/server/src/database/ledger-stores.ts | 33 +++++- 18 files changed, 408 insertions(+), 99 deletions(-) create mode 100644 packages/secrets/src/custodians/custodian-refused.ts create mode 100644 packages/secrets/src/keepers/keeper-unavailable.ts create mode 100644 packages/secrets/src/store/answer-bound.ts create mode 100644 packages/secrets/src/testing/failure-behaviour.ts create mode 100644 packages/server/src/database/ledger-stores-rotation.test.ts diff --git a/packages/secrets/README.md b/packages/secrets/README.md index ec29374e8..3108d5c87 100644 --- a/packages/secrets/README.md +++ b/packages/secrets/README.md @@ -17,8 +17,8 @@ The secret store of the runtime: what it must hold to act for someone, such as a `@on.auto/secrets` (`src/index.ts`) exports: -- the ports, each an interface of functions: `SecretRows`, with `Holding`, `SecretRow`, `HeldRow`, `Holder` and the answers `Replacement` and `Claim`; `Custodian`, `{ kid, wrap(dataKey), unwrap(wrapped) }`, and `Custodians`, `{ current, previous? }`; and `Keeper`, `{ name, hold(id, value), reveal(holding), release(holding) }`; -- `secretStoreOn(rows, keeper, custodians)`, the `SecretStore` over them, and its failures, `SecretRefused` and `SecretKeyUnknown`; +- the ports, each an interface of functions: `SecretRows`, with `Holding`, `SecretRow`, `HeldRow`, `Holder` and the answers `Replacement` and `Claim`; `Custodian`, `{ kid, wrap(dataKey), unwrap(wrapped) }`, each failing with a `KeepingFailure`, and `Custodians`, `{ current, previous? }`; and `Keeper`, `{ name, hold(id, value), reveal(holding), release(holding) }`, whose `hold` fails with a `KeepingFailure` and whose `reveal` with a `ReadingFailure`; +- `secretStoreOn(rows, keeper, custodians)`, the `SecretStore` over them, its refusal of a value, `SecretRefused`, and its three failures, `SecretKeyUnknown`, `CustodianRefused` and `KeeperUnavailable`, with `KeepingFailure`, the last two, and `ReadingFailure`, all three; - `localCustodian(key)`, the custodian of a key in hand, and `secretKeyBytes`, the length of such a key, 32; - `sealedKeeper(custodians)`, the keeper that seals a value into its row. @@ -48,23 +48,33 @@ A value the `ledger` keeper holds is sealed by `jose`, the library that verifies ## The custodian -`localCustodian(key)` takes a key of 32 bytes as a `Redacted` byte array, and a key of another length stops it, naming the length and never the bytes: `A key that seals secrets is 32 bytes, and this one is 16`. Its `kid` is the first 16 hex characters of the key's SHA-256, `sha256Of` of `@on.auto/operations`, so no one names a key and one key always has one kid. `wrap(dataKey)` answers one JWE in its compact form, `{ alg: 'A256KW', enc: 'A256GCM' }`, of the data key under the key, `jose`'s envelope of AES key wrap around a content key of its own; `unwrap(wrapped)` answers the data key, taking those two algorithms alone and no compression, so a wrapping under another key, or of another algorithm under this one, fails it. +`localCustodian(key)` takes a key of 32 bytes as a `Redacted` byte array, and a key of another length stops it, naming the length and never the bytes: `A key that seals secrets is 32 bytes, and this one is 16`. Its `kid` is the first 16 hex characters of the key's SHA-256, `sha256Of` of `@on.auto/operations`, so no one names a key and one key always has one kid. `wrap(dataKey)` answers one JWE in its compact form, `{ alg: 'A256KW', enc: 'A256GCM' }`, of the data key under the key, `jose`'s envelope of AES key wrap around a content key of its own; `unwrap(wrapped)` answers the data key, taking those two algorithms alone and no compression, so a wrapping under another key, or of another algorithm under this one, is refused as a `CustodianRefused` in its own words, `The custodian of the key refused to unwrap a data key: its wrapped key does not open under this key`. Under the kid it names, that means a wrapping changed outside the runtime. The store takes `custodians: { current, previous? }`: the current one wraps every data key, and a read asks the one its row's kid names. A previous custodian with the current one's kid is no previous at all. Each read unwraps its data key again and nothing keeps one unwrapped, since a row is read when a connection is first used in a process and when its tokens are renewed. ## The keeper -`sealedKeeper(custodians)` is the keeper named `ledger`. `hold(id, value)` makes a data key, seals the value under it, has the current custodian wrap it, and answers `{ keptBy: 'ledger', held, wrappedKey, kid }`; the id names nothing in what it holds. `reveal(holding)` asks the custodian the holding's kid names to unwrap the data key, and opens the value with `dir` and `A256GCM` alone and no compression; a kid that is neither the current custodian's nor the previous one's, or a holding that names none, is refused as a `SecretKeyUnknown`, `{ kid, detail }`, `The secret is sealed under the key , which is neither the current key nor the previous one`, which names the kid and never the value. `release(holding)` does nothing, since the row is the value. +`sealedKeeper(custodians)` is the keeper named `ledger`. `hold(id, value)` makes a data key, seals the value under it, has the current custodian wrap it, and answers `{ keptBy: 'ledger', held, wrappedKey, kid }`; the id names nothing in what it holds. `reveal(holding)` asks the custodian the holding's kid names to unwrap the data key, and opens the value with `dir` and `A256GCM` alone and no compression; a kid that is neither the current custodian's nor the previous one's, or a holding that names none, is refused as a `SecretKeyUnknown`. `hold` and `reveal` fail as their custodian does. `release(holding)` does nothing, since the row is the value. + +## The failures + +A value the store cannot read now fails the call that asked for it, and nothing else: the store keeps the row as it was, and reads it again at the next call. There are three such failures, each a tagged error that names no value and no key, its words in `detail`: + +- `SecretKeyUnknown`, `secret_key_unknown`, `{ kid, detail }`: the row is sealed under a kid that is neither the current custodian's nor the previous one's, `The secret is sealed under the key , which is neither the current key nor the previous one`. +- `CustodianRefused`, `custodian_refused`, `{ kid, detail }`: the custodian of the kid failed to wrap or unwrap a data key, and says why, `The custodian of the key refused to unwrap a data key: `. +- `KeeperUnavailable`, `keeper_unavailable`, `{ detail }`: the keeper, or the custodian it asks, did not answer within the bound the store sets, 30 seconds through Effect's `timeoutOrElse`, `The keeper of the secret, or the custodian of its key, did not answer within 30 seconds`. + +A read fails with any of the three, a `ReadingFailure`; a keep and a replacement with the last two, a `KeepingFailure`, and keep nothing then, so a value is kept or replaced only when its keeper and its custodian answered. A custodian or a keeper answers the last two itself where it can tell, and the store answers `KeeperUnavailable` for one that is still silent at the bound. ## The store `secretStoreOn(rows, keeper, custodians)` answers the `SecretStore`. -- `keep(value, endsWith?)` takes the value as a `Redacted` string, and the ending the runtime may show where it is given one, makes an id, a UUIDv7 as `randomUUIDv7` of `@on.auto/operations` makes the ledger's other ids, has the keeper hold the value under it and keeps the row, and answers the id. An empty value, or one of more than 8 KiB of UTF-8, is refused before anything is sealed, as a `SecretRefused` that is `empty` or `too_large` and says nothing of the value: `A secret to keep is empty`, `A secret to keep holds more than 8 KiB`. -- `read(id)` answers a `StoredSecret`: the value as a `Redacted`, which prints and serializes as ``, so a value cannot reach a log by accident, the row's `version`, and `claimedUntil`; it answers nothing for an id the store does not keep, never an error, and fails with the keeper's `SecretKeyUnknown`. -- `replace(id, version, value)` has the keeper hold the new value and replaces the row's holding under the version read, refusing a value as `keep` does; `claim`, `endingsOf` and `close` are the rows' own. +- `keep(value, endsWith?)` takes the value as a `Redacted` string, and the ending the runtime may show where it is given one, makes an id, a UUIDv7 as `randomUUIDv7` of `@on.auto/operations` makes the ledger's other ids, has the keeper hold the value under it and keeps the row, and answers the id. An empty value, or one of more than 8 KiB of UTF-8, is refused before anything is sealed, as a `SecretRefused` that is `empty` or `too_large` and says nothing of the value: `A secret to keep is empty`, `A secret to keep holds more than 8 KiB`. A keeper that does not hold the value fails it with a `KeepingFailure`, and no row is kept. +- `read(id)` answers a `StoredSecret`: the value as a `Redacted`, which prints and serializes as ``, so a value cannot reach a log by accident, the row's `version`, and `claimedUntil`; it answers nothing for an id the store does not keep, never an error, and fails with one of the three failures above. +- `replace(id, version, value)` has the keeper hold the new value and replaces the row's holding under the version read, refusing a value and failing as `keep` does, the row then as it was; `claim`, `endingsOf` and `close` are the rows' own. - `forget(id)` forgets the row and has the keeper release the holding it held. -- `rotated()` holds again under the current custodian what the previous one wrapped: it reads with `heldBy({ keptBy: 'ledger', kid })` the rows of the previous custodian's kid, 100 at a time, has the previous custodian unwrap each data key and the current one wrap it, writes the new `wrappedKey` and `kid` with `reheld`, and answers how many rows it held again. Rows of the current kid, of any other kid and of any other keeper are left as they are, and so is each row's value, its version, its claim and the time it was kept. Without a previous custodian, or with the current one given as the previous, it holds nothing again. Once a rotation has run, nothing is wrapped under the previous key any more, so it can be dropped. +- `rotated()` holds again under the current custodian what the previous one wrapped: it reads with `heldBy({ keptBy: 'ledger', kid })` the rows of the previous custodian's kid, 100 at a time, has the previous custodian unwrap each data key and the current one wrap it, writes the new `wrappedKey` and `kid` with `reheld`, and answers how many rows it held again. Rows of the current kid, of any other kid and of any other keeper are left as they are, and so is each row's value, its version, its claim and the time it was kept. Without a previous custodian, or with the current one given as the previous, it holds nothing again. Once a rotation has run, nothing is wrapped under the previous key any more, so it can be dropped. Each row's unwrapping and wrapping again is bounded as a read is, and a custodian that refuses or does not answer fails the rotation with its `KeepingFailure`, the rows held again before it under the current custodian and the rest as they were; the server stops its start on it, naming the setting of the previous key before the custodian's words. A failure of the rows, of the keeper or of the custodian other than these is a defect, as a failure of the ledger's database is. @@ -76,12 +86,12 @@ A secret is only as recoverable as its key. The store keeps no copy of a key and - `memoryRows({ now })`, rows kept in memory under the rule of the lease, whose claims and times are read from `now`, and which fail as a defect for an id they keep already, as the ledger's table does: the one copy of the lease beside the ledger's SQL. `kept()` answers each row by its columns, in the order of its ids. - `valuesInTheClear({ now, mostCharacters })`, values a test holds in the clear over `memoryRows`, `{ keptBy: 'memory', held }`, under ids it gives: it keeps, reads, replaces, claims, answers endings and forgets as the rows do, refuses a value to keep or a replacement past its bound, `A secret to keep holds more than characters`, and answers what it holds, `values()` and `endings()`. The tool-servers package's `memoryTokens` and the connections package's harness keep their values in it. -- `recordingCustodian(key?)`, a custodian of a key in hand, a random one unless it is given one, that counts its wraps and unwraps, `wraps()` and `unwraps()`, holds the data keys it wrapped and unwrapped, `dataKeys()`, so a test sees them filled with zeros, and fails both as a defect, `The custodian refuses to wrap or unwrap a data key`, once it is told to `refuse()`. +- `recordingCustodian(key?)`, a custodian of a key in hand, a random one unless it is given one, that counts its wraps and unwraps, `wraps()` and `unwraps()`, and holds the data keys it wrapped and unwrapped, `dataKeys()`, so a test sees them filled with zeros. Told to `refuse()`, it refuses both as a `CustodianRefused` in the words `custodianRefusal`, `the key is disabled`; told to `stall()`, it answers neither, and the promise `stall()` answers settles once it is first asked; told to `answer()`, it answers again. - `rowsBehaviour(entry)`, every claim on a `SecretRows`: a holding kept, with and without a wrapped key and a kid, read back and forgotten twice as once, no second row under an id kept, the endings, a replacement at the version read, after which the next version may be claimed at once, and none at another or of a row that is gone, a claim a read answers while it is open and none while one is open, of a moved version or of a row that is gone, the rows a keeper holds under a kid, in order and at most as many as asked, a row held again only while its version and its holding are the ones read, its version, claim, ending and time kept, and a sealed row replaced or held again with the holding of another keeper, its wrapped key and kid then empty. -- `storeBehaviour(entry)`, what the store adds over any rows, the claims of the rows themselves being `rowsBehaviour`'s: a value kept under a UUIDv7 with its ending and read back redacted, sealed as a `dir` JWE under a data key of its own that the row's `wrapped_key` holds under the row's `kid`, a wrap at each keep and an unwrap at each read with each data key then filled with zeros, a forget twice as once, through which the keeper releases, a value under another key refused with that key's kid, the empty and the oversized value refused before anything is wrapped, as a replacement, a custodian that refuses, a replacement at the next version under a new data key, and a rotation over more than one batch that wraps again the data keys of the previous key's rows, touches no value, keeps each row's version and claim and leaves a stray kid, and none without a previous key. +- `storeBehaviour(entry)`, what the store adds over any rows, the claims of the rows themselves being `rowsBehaviour`'s: a value kept under a UUIDv7 with its ending and read back redacted, sealed as a `dir` JWE under a data key of its own that the row's `wrapped_key` holds under the row's `kid`, a wrap at each keep and an unwrap at each read with each data key then filled with zeros, a forget twice as once, through which the keeper releases, the empty and the oversized value refused before anything is wrapped, as a replacement, a replacement at the next version under a new data key, and a rotation over more than one batch that wraps again the data keys of the previous key's rows, touches no value, keeps each row's version and claim and leaves a stray kid, and none without a previous key. Its failures are a suite of their own in `src/testing/failure-behaviour.ts`: a value under another key not read, `secret_key_unknown` with that key's kid; a value and a replacement a refusing custodian does not wrap, `custodian_refused`, the row as it was; and a value a refusing custodian does not unwrap, `custodian_refused`, and one a stalled custodian leaves unanswered, `keeper_unavailable` at 30 seconds of `TestClock` and not a millisecond before, read again once the custodian answers, its row as it was and no failure naming the value. -An entry, `SecretsEntry`, is `{ aDatabase, rowsOn, rowsIn }`: a fresh database, the rows on one, and the rows of one as its table holds them, by their columns in the order of their ids. `src/testing/memory-rows.test.ts` runs both suites over `memoryRows`, and the ledger runs them over SQLite and PostgreSQL. `src/store/secret-store.test.ts` adds what only a keeper or rows of a test's own show: the id the store hands its keeper and the holding it releases, a replacement made between a rotation's read of a row and its write, which the rotation leaves as it is, and the data key a rotation unwrapped, filled with zeros once wrapped again. +An entry, `SecretsEntry`, is `{ aDatabase, rowsOn, rowsIn }`: a fresh database, the rows on one, and the rows of one as its table holds them, by their columns in the order of their ids. `src/testing/memory-rows.test.ts` runs both suites over `memoryRows`, and the ledger runs them over SQLite and PostgreSQL. `src/store/secret-store.test.ts` adds what only a keeper or rows of a test's own show: the id the store hands its keeper and the holding it releases, a replacement made between a rotation's read of a row and its write, which the rotation leaves as it is, the data key a rotation unwrapped, filled with zeros once wrapped again, and a rotation whose previous custodian refuses, which fails with that refusal and holds nothing again. ## Source -`src/rows` holds the port of the rows, `src/custodians` the port of the custodian and the custodian of a key in hand, `src/keepers` the port of the keeper, the ledger's keeper and its data keys, `src/store` the store, its rotation and its refusal, and `src/testing` the rows in memory, the values in the clear, the recording custodian, the suites and the entry of the tests. +`src/rows` holds the port of the rows, `src/custodians` the port of the custodian, its refusal and the custodian of a key in hand, `src/keepers` the port of the keeper, the ledger's keeper, its data keys, the unknown kid and the keeper that does not answer, `src/store` the store, the bound of an answer, its rotation and its refusal, and `src/testing` the rows in memory, the values in the clear, the recording custodian, the suites and the entry of the tests. diff --git a/packages/secrets/src/custodians/custodian-refused.ts b/packages/secrets/src/custodians/custodian-refused.ts new file mode 100644 index 000000000..67847cfc6 --- /dev/null +++ b/packages/secrets/src/custodians/custodian-refused.ts @@ -0,0 +1,15 @@ +import { Data } from 'effect'; + +export type CustodianAct = 'wrap' | 'unwrap'; + +export class CustodianRefused extends Data.TaggedError('custodian_refused')<{ + readonly kid: string; + readonly detail: string; +}> {} + +export function custodianRefused(kid: string, act: CustodianAct, words: string): CustodianRefused { + return new CustodianRefused({ + kid, + detail: `The custodian of the key ${kid} refused to ${act} a data key: ${words}`, + }); +} diff --git a/packages/secrets/src/custodians/custodian.ts b/packages/secrets/src/custodians/custodian.ts index f04aead1e..d43c3915d 100644 --- a/packages/secrets/src/custodians/custodian.ts +++ b/packages/secrets/src/custodians/custodian.ts @@ -1,9 +1,11 @@ import type { Effect, Redacted } from 'effect'; +import type { KeepingFailure } from '../keepers/keeper.ts'; + export interface Custodian { readonly kid: string; - readonly wrap: (dataKey: Redacted.Redacted) => Effect.Effect; - readonly unwrap: (wrapped: string) => Effect.Effect>; + readonly wrap: (dataKey: Redacted.Redacted) => Effect.Effect; + readonly unwrap: (wrapped: string) => Effect.Effect, KeepingFailure>; } export interface Custodians { diff --git a/packages/secrets/src/custodians/local-custodian.test.ts b/packages/secrets/src/custodians/local-custodian.test.ts index 95dcd5115..894316883 100644 --- a/packages/secrets/src/custodians/local-custodian.test.ts +++ b/packages/secrets/src/custodians/local-custodian.test.ts @@ -30,14 +30,22 @@ describe('the custodian of a key in hand', () => { expect(Array.from(Redacted.value(unwrapped))).toEqual(Array.from(Redacted.value(dataKey))); }); - it('unwraps nothing wrapped under another key, or by another algorithm under its own', async () => { + it('refuses to unwrap what was wrapped under another key, or by another algorithm under its own, in words of its own that name its kid', async () => { const custodian = localCustodian(aKeyOf(5)); const elsewhere = await Effect.runPromise(localCustodian(aKeyOf(6)).wrap(aKeyOf(9))); const direct = await new CompactEncrypt(new Uint8Array(32).fill(9)) .setProtectedHeader({ alg: 'dir', enc: 'A256GCM' }) .encrypt(new Uint8Array(32).fill(5)); - await expect(Effect.runPromise(custodian.unwrap(elsewhere))).rejects.toThrow('decryption operation failed'); - await expect(Effect.runPromise(custodian.unwrap(direct))).rejects.toThrow('not allowed'); + const refusals = await Effect.runPromise( + Effect.forEach([elsewhere, direct], (wrapped) => Effect.flip(custodian.unwrap(wrapped))), + ); + + const refusal = { + _tag: 'custodian_refused', + kid: custodian.kid, + detail: `The custodian of the key ${custodian.kid} refused to unwrap a data key: its wrapped key does not open under this key`, + }; + expect(refusals).toMatchObject([refusal, refusal]); }); }); diff --git a/packages/secrets/src/custodians/local-custodian.ts b/packages/secrets/src/custodians/local-custodian.ts index 4a40dfbde..8583f7024 100644 --- a/packages/secrets/src/custodians/local-custodian.ts +++ b/packages/secrets/src/custodians/local-custodian.ts @@ -2,12 +2,15 @@ import { sha256Of } from '@on.auto/operations'; import { Effect, Redacted } from 'effect'; import { CompactEncrypt, compactDecrypt, type DecryptOptions } from 'jose'; +import { custodianRefused } from './custodian-refused.ts'; import type { Custodian } from './custodian.ts'; export const secretKeyBytes = 32; const kidLength = 16; +const wrappingUnopened = 'its wrapped key does not open under this key'; + const keyWrapping = 'A256KW'; const contentEncryption = 'A256GCM'; @@ -37,9 +40,14 @@ export function localCustodian(key: Redacted.Redacted): Custodian { if (byteLength !== secretKeyBytes) { throw new Error(`A key that seals secrets is ${secretKeyBytes} bytes, and this one is ${byteLength}`); } + const kid = sha256Of(Redacted.value(key)).slice(0, kidLength); return { - kid: sha256Of(Redacted.value(key)).slice(0, kidLength), + kid, wrap: (dataKey) => Effect.promise(() => wrappedUnder(key, dataKey)), - unwrap: (wrapped) => Effect.promise(() => unwrappedWith(key, wrapped)), + unwrap: (wrapped) => + Effect.tryPromise({ + try: () => unwrappedWith(key, wrapped), + catch: () => custodianRefused(kid, 'unwrap', wrappingUnopened), + }), }; } diff --git a/packages/secrets/src/index.ts b/packages/secrets/src/index.ts index e7afc117a..c7836bd39 100644 --- a/packages/secrets/src/index.ts +++ b/packages/secrets/src/index.ts @@ -1,6 +1,8 @@ +export { CustodianRefused } from './custodians/custodian-refused.ts'; export type { Custodian, Custodians } from './custodians/custodian.ts'; export { localCustodian, secretKeyBytes } from './custodians/local-custodian.ts'; -export type { Keeper } from './keepers/keeper.ts'; +export { KeeperUnavailable } from './keepers/keeper-unavailable.ts'; +export type { Keeper, KeepingFailure, ReadingFailure } from './keepers/keeper.ts'; export { sealedKeeper } from './keepers/sealed-keeper.ts'; export { SecretKeyUnknown } from './keepers/secret-key-unknown.ts'; export type { Claim, HeldRow, Holder, Holding, Replacement, SecretRow, SecretRows } from './rows/secret-rows.ts'; diff --git a/packages/secrets/src/keepers/keeper-unavailable.ts b/packages/secrets/src/keepers/keeper-unavailable.ts new file mode 100644 index 000000000..f1d894528 --- /dev/null +++ b/packages/secrets/src/keepers/keeper-unavailable.ts @@ -0,0 +1,11 @@ +import { Data } from 'effect'; + +export class KeeperUnavailable extends Data.TaggedError('keeper_unavailable')<{ + readonly detail: string; +}> {} + +export function keeperUnavailable(seconds: number): KeeperUnavailable { + return new KeeperUnavailable({ + detail: `The keeper of the secret, or the custodian of its key, did not answer within ${seconds} seconds`, + }); +} diff --git a/packages/secrets/src/keepers/keeper.ts b/packages/secrets/src/keepers/keeper.ts index b0918e5e7..e6e689333 100644 --- a/packages/secrets/src/keepers/keeper.ts +++ b/packages/secrets/src/keepers/keeper.ts @@ -1,11 +1,17 @@ import type { Effect, Redacted } from 'effect'; +import type { CustodianRefused } from '../custodians/custodian-refused.ts'; import type { Holding } from '../rows/secret-rows.ts'; +import type { KeeperUnavailable } from './keeper-unavailable.ts'; import type { SecretKeyUnknown } from './secret-key-unknown.ts'; +export type KeepingFailure = CustodianRefused | KeeperUnavailable; + +export type ReadingFailure = SecretKeyUnknown | KeepingFailure; + export interface Keeper { readonly name: string; - readonly hold: (id: string, value: Redacted.Redacted) => Effect.Effect; - readonly reveal: (holding: Holding) => Effect.Effect; + readonly hold: (id: string, value: Redacted.Redacted) => Effect.Effect; + readonly reveal: (holding: Holding) => Effect.Effect; readonly release: (holding: Holding) => Effect.Effect; } diff --git a/packages/secrets/src/keepers/sealed-keeper.ts b/packages/secrets/src/keepers/sealed-keeper.ts index 3deb4456a..268cb3133 100644 --- a/packages/secrets/src/keepers/sealed-keeper.ts +++ b/packages/secrets/src/keepers/sealed-keeper.ts @@ -4,7 +4,7 @@ import { CompactEncrypt, compactDecrypt, type DecryptOptions } from 'jose'; import { custodianNamed, type Custodians } from '../custodians/custodian.ts'; import type { Holding } from '../rows/secret-rows.ts'; import { aDataKey, zeroedAfter } from './data-key.ts'; -import type { Keeper } from './keeper.ts'; +import type { Keeper, ReadingFailure } from './keeper.ts'; import { secretKeyUnknown } from './secret-key-unknown.ts'; export const ledgerKeeper = 'ledger'; @@ -48,7 +48,7 @@ export function sealedKeeper(custodians: Custodians): Keeper { ); }), reveal: ({ held, wrappedKey = '', kid = '' }) => - Effect.suspend(() => { + Effect.suspend((): Effect.Effect => { const custodian = custodianNamed(custodians, kid); return custodian === undefined ? Effect.fail(secretKeyUnknown(kid)) diff --git a/packages/secrets/src/store/answer-bound.ts b/packages/secrets/src/store/answer-bound.ts new file mode 100644 index 000000000..fdff4a1bb --- /dev/null +++ b/packages/secrets/src/store/answer-bound.ts @@ -0,0 +1,14 @@ +import { Duration, Effect } from 'effect'; + +import { keeperUnavailable, type KeeperUnavailable } from '../keepers/keeper-unavailable.ts'; + +const answerBoundSeconds = 30; + +const answerBound = Duration.seconds(answerBoundSeconds); + +export function answeredWithinItsBound(asked: Effect.Effect): Effect.Effect { + return Effect.timeoutOrElse(asked, { + duration: answerBound, + orElse: () => Effect.fail(keeperUnavailable(answerBoundSeconds)), + }); +} diff --git a/packages/secrets/src/store/rotation.ts b/packages/secrets/src/store/rotation.ts index 417530cc6..6b7f1b98d 100644 --- a/packages/secrets/src/store/rotation.ts +++ b/packages/secrets/src/store/rotation.ts @@ -2,8 +2,10 @@ import { Effect } from 'effect'; import { anotherPrevious, type Custodian, type Custodians } from '../custodians/custodian.ts'; import { zeroedAfter } from '../keepers/data-key.ts'; +import type { KeepingFailure } from '../keepers/keeper.ts'; import { ledgerKeeper } from '../keepers/sealed-keeper.ts'; import type { HeldRow, SecretRows } from '../rows/secret-rows.ts'; +import { answeredWithinItsBound } from './answer-bound.ts'; const rowsInABatch = 100; @@ -12,16 +14,25 @@ interface Rotation { readonly previous: Custodian; } -function rewrapped(rows: SecretRows, { current, previous }: Rotation, read: HeldRow): Effect.Effect { +function rewrapped( + rows: SecretRows, + { current, previous }: Rotation, + read: HeldRow, +): Effect.Effect { return Effect.gen(function* () { const { wrappedKey = '' } = read.holding; - const dataKey = yield* previous.unwrap(wrappedKey); - const rewrapping = yield* zeroedAfter(dataKey, current.wrap(dataKey)); + const rewrapping = yield* answeredWithinItsBound( + Effect.flatMap(previous.unwrap(wrappedKey), (dataKey) => zeroedAfter(dataKey, current.wrap(dataKey))), + ); return yield* rows.reheld(read, { ...read.holding, wrappedKey: rewrapping, kid: current.kid }); }); } -function rewrappedInBatches(rows: SecretRows, rotation: Rotation, before: number): Effect.Effect { +function rewrappedInBatches( + rows: SecretRows, + rotation: Rotation, + before: number, +): Effect.Effect { return Effect.gen(function* () { const held = yield* rows.heldBy({ keptBy: ledgerKeeper, kid: rotation.previous.kid }, rowsInABatch); const reheld = yield* Effect.forEach(held, (read) => rewrapped(rows, rotation, read)); @@ -30,7 +41,7 @@ function rewrappedInBatches(rows: SecretRows, rotation: Rotation, before: number }); } -export function rotatedOn(rows: SecretRows, custodians: Custodians): Effect.Effect { +export function rotatedOn(rows: SecretRows, custodians: Custodians): Effect.Effect { const previous = anotherPrevious(custodians); return previous === undefined ? Effect.succeed(0) diff --git a/packages/secrets/src/store/secret-store.test.ts b/packages/secrets/src/store/secret-store.test.ts index e9a308b6c..6da554e9d 100644 --- a/packages/secrets/src/store/secret-store.test.ts +++ b/packages/secrets/src/store/secret-store.test.ts @@ -6,7 +6,7 @@ import type { Keeper } from '../keepers/keeper.ts'; import { sealedKeeper } from '../keepers/sealed-keeper.ts'; import type { Holding, SecretRows } from '../rows/secret-rows.ts'; import { memoryRows } from '../testing/memory-rows.ts'; -import { recordingCustodian } from '../testing/recording-custodian.ts'; +import { custodianRefusal, recordingCustodian } from '../testing/recording-custodian.ts'; import { secretStoreOn, type SecretStore } from './secret-store.ts'; const now = Effect.map(DateTime.now, DateTime.toEpochMillis); @@ -117,3 +117,24 @@ describe('the rotation of the secret store', () => { expect(await stateOf(rotating, id)).toEqual({ value: 'pasted-key', version: 1 }); }); }); + +describe('a rotation whose previous custodian refuses', () => { + it('fails with the refusal of the custodian in its words, holding nothing again', async () => { + const rows = memoryRows({ now }); + const [previous, current] = [recordingCustodian(), recordingCustodian()]; + const before = secretStoreOn(rows, sealedKeeper({ current: previous }), { current: previous }); + await Effect.runPromise(before.keep(Redacted.make('pasted-key'))); + const rotating = secretStoreOn(rows, sealedKeeper({ current, previous }), { current, previous }); + + previous.refuse(); + + const refused = await Effect.runPromise(Effect.flip(rotating.rotated())); + + expect(refused).toMatchObject({ + _tag: 'custodian_refused', + kid: previous.kid, + detail: `The custodian of the key ${previous.kid} refused to unwrap a data key: ${custodianRefusal}`, + }); + expect(rows.kept().map(({ kid }) => kid)).toEqual([previous.kid]); + }); +}); diff --git a/packages/secrets/src/store/secret-store.ts b/packages/secrets/src/store/secret-store.ts index 28ab0dd79..edb8436bf 100644 --- a/packages/secrets/src/store/secret-store.ts +++ b/packages/secrets/src/store/secret-store.ts @@ -2,9 +2,9 @@ import { randomUUIDv7 } from '@on.auto/operations'; import { Effect, Redacted } from 'effect'; import type { Custodians } from '../custodians/custodian.ts'; -import type { Keeper } from '../keepers/keeper.ts'; -import type { SecretKeyUnknown } from '../keepers/secret-key-unknown.ts'; +import type { Keeper, KeepingFailure, ReadingFailure } from '../keepers/keeper.ts'; import type { Claim, Replacement, SecretRows } from '../rows/secret-rows.ts'; +import { answeredWithinItsBound } from './answer-bound.ts'; import { rotatedOn } from './rotation.ts'; import { SecretRefused } from './secret-refused.ts'; @@ -17,20 +17,25 @@ export interface StoredSecret { } export interface SecretStore { - readonly keep: (value: Redacted.Redacted, endsWith?: string) => Effect.Effect; - readonly read: (id: string) => Effect.Effect; + readonly keep: (value: Redacted.Redacted, endsWith?: string) => Effect.Effect; + readonly read: (id: string) => Effect.Effect; readonly replace: ( id: string, version: number, value: Redacted.Redacted, - ) => Effect.Effect; + ) => Effect.Effect; readonly claim: (id: string, version: number, lastingMs: number) => Effect.Effect; readonly endingsOf: (ids: readonly string[]) => Effect.Effect>; readonly forget: (id: string) => Effect.Effect; - readonly rotated: () => Effect.Effect; + readonly rotated: () => Effect.Effect; readonly close: () => Promise; } +interface Asked { + readonly id: string; + readonly version: number; +} + function refusalOf(value: string): SecretRefused | undefined { if (value === '') { return new SecretRefused({ because: 'empty', detail: 'A secret to keep is empty' }); @@ -43,8 +48,11 @@ function refusalOf(value: string): SecretRefused | undefined { : undefined; } -function refusedOr(value: Redacted.Redacted, kept: () => Effect.Effect): Effect.Effect { - return Effect.suspend(() => { +function refusedOr( + value: Redacted.Redacted, + kept: () => Effect.Effect, +): Effect.Effect { + return Effect.suspend((): Effect.Effect => { const refusal = refusalOf(Redacted.value(value)); return refusal === undefined ? kept() : Effect.fail(refusal); }); @@ -55,24 +63,20 @@ function keptBy( keeper: Keeper, value: Redacted.Redacted, endsWith: string | undefined, -): Effect.Effect { +): Effect.Effect { return Effect.gen(function* () { const id = randomUUIDv7(); - const holding = yield* keeper.hold(id, value); + const holding = yield* answeredWithinItsBound(keeper.hold(id, value)); yield* rows.keep(id, holding, endsWith); return id; }); } -function readBy( - rows: SecretRows, - keeper: Keeper, - id: string, -): Effect.Effect { +function readBy(rows: SecretRows, keeper: Keeper, id: string): Effect.Effect { return Effect.flatMap(rows.read(id), (row) => row === undefined ? Effect.undefined - : Effect.map(keeper.reveal(row.holding), (value) => ({ + : Effect.map(answeredWithinItsBound(keeper.reveal(row.holding)), (value) => ({ value, version: row.version, claimedUntil: row.claimedUntil, @@ -80,6 +84,17 @@ function readBy( ); } +function replacedBy( + rows: SecretRows, + keeper: Keeper, + { id, version }: Asked, + value: Redacted.Redacted, +): Effect.Effect { + return Effect.flatMap(answeredWithinItsBound(keeper.hold(id, value)), (holding) => + rows.replace(id, version, holding), + ); +} + function forgottenBy(rows: SecretRows, keeper: Keeper, id: string): Effect.Effect { return Effect.flatMap(rows.forget(id), (holding) => (holding === undefined ? Effect.void : keeper.release(holding))); } @@ -88,8 +103,7 @@ export function secretStoreOn(rows: SecretRows, keeper: Keeper, custodians: Cust return { keep: (value, endsWith) => refusedOr(value, () => keptBy(rows, keeper, value, endsWith)), read: (id) => readBy(rows, keeper, id), - replace: (id, version, value) => - refusedOr(value, () => Effect.flatMap(keeper.hold(id, value), (holding) => rows.replace(id, version, holding))), + replace: (id, version, value) => refusedOr(value, () => replacedBy(rows, keeper, { id, version }, value)), claim: rows.claim, endingsOf: rows.endingsOf, forget: (id) => forgottenBy(rows, keeper, id), diff --git a/packages/secrets/src/testing/failure-behaviour.ts b/packages/secrets/src/testing/failure-behaviour.ts new file mode 100644 index 000000000..f4ff32bf4 --- /dev/null +++ b/packages/secrets/src/testing/failure-behaviour.ts @@ -0,0 +1,103 @@ +import { Effect, Fiber, Redacted } from 'effect'; +import { TestClock } from 'effect/testing'; +import { describe, expect, it } from 'vitest'; + +import type { SecretStore } from '../store/secret-store.ts'; +import { aSecretsKey, custodianRefusal, recordingCustodian, type RecordingCustodian } from './recording-custodian.ts'; +import { kept, kidOf, readBack, rowsIn, storeOn, storeWith, type SecretsEntry } from './secrets-entry.ts'; + +function unknownKeyRefused(entry: SecretsEntry): void { + it('cannot read a value sealed under another key, and names that key and never the value', async () => { + const database = await entry.aDatabase(); + const sealing = aSecretsKey(); + const id = await kept(await storeOn(entry, database, { current: sealing }), 'pasted-key'); + const other = await storeOn(entry, database, { current: aSecretsKey() }); + + const refused = await Effect.runPromise(Effect.flip(other.read(id))); + + expect(refused).toMatchObject({ + _tag: 'secret_key_unknown', + kid: kidOf(sealing), + detail: `The secret is sealed under the key ${kidOf(sealing)}, which is neither the current key nor the previous one`, + }); + expect(JSON.stringify(refused)).not.toContain('pasted-key'); + }); +} + +function refusedTo(custodian: RecordingCustodian, act: string) { + return { + _tag: 'custodian_refused', + kid: custodian.kid, + detail: `The custodian of the key ${custodian.kid} refused to ${act} a data key: ${custodianRefusal}`, + }; +} + +function custodianRefusing(entry: SecretsEntry): void { + it('keeps nothing, as a value or as a replacement, whose data key the custodian refuses to wrap, failing in its words', async () => { + const database = await entry.aDatabase(); + const custodian = recordingCustodian(); + const store = await storeWith(entry, database, { current: custodian }); + const id = await kept(store, 'pasted-key'); + const before = await rowsIn(entry, database); + + custodian.refuse(); + const [keeping, replacing] = await Effect.runPromise( + Effect.all([ + Effect.flip(store.keep(Redacted.make('second-key'))), + Effect.flip(store.replace(id, 1, Redacted.make('renewed-key'))), + ]), + ); + + expect(keeping).toMatchObject(refusedTo(custodian, 'wrap')); + expect(replacing).toMatchObject(refusedTo(custodian, 'wrap')); + expect(await rowsIn(entry, database)).toEqual(before); + }); +} + +function readStalled(store: SecretStore, id: string, custodian: RecordingCustodian) { + return Effect.runPromise( + Effect.gen(function* () { + const asked = custodian.stall(); + const reading = yield* Effect.forkChild(Effect.flip(store.read(id))); + yield* Effect.promise(() => asked); + yield* TestClock.adjust('29999 millis'); + const before = reading.pollUnsafe(); + yield* TestClock.adjust('1 millis'); + return { before, failure: yield* Fiber.join(reading) }; + }).pipe(Effect.provide(TestClock.layer())), + ); +} + +function unreadableNow(entry: SecretsEntry): void { + it('cannot read now, in words that name no value, a value whose custodian refuses or does not answer within 30 seconds, and reads it once the custodian answers again', async () => { + const database = await entry.aDatabase(); + const custodian = recordingCustodian(); + const store = await storeWith(entry, database, { current: custodian }); + const id = await kept(store, 'pasted-key'); + const before = await rowsIn(entry, database); + + custodian.refuse(); + const refused = await Effect.runPromise(Effect.flip(store.read(id))); + const stalled = await readStalled(store, id, custodian); + custodian.answer(); + const answered = await readBack(store, id); + + expect(refused).toMatchObject(refusedTo(custodian, 'unwrap')); + expect(stalled.before).toBeUndefined(); + expect(stalled.failure).toMatchObject({ + _tag: 'keeper_unavailable', + detail: 'The keeper of the secret, or the custodian of its key, did not answer within 30 seconds', + }); + expect(answered).toBe('pasted-key'); + expect(JSON.stringify([refused, stalled])).not.toContain('pasted-key'); + expect(await rowsIn(entry, database)).toEqual(before); + }); +} + +export function storeFailureBehaviour(entry: SecretsEntry): void { + describe('the failures of the secret store', () => { + unknownKeyRefused(entry); + custodianRefusing(entry); + unreadableNow(entry); + }); +} diff --git a/packages/secrets/src/testing/recording-custodian.ts b/packages/secrets/src/testing/recording-custodian.ts index b2ad6d036..6f1508770 100644 --- a/packages/secrets/src/testing/recording-custodian.ts +++ b/packages/secrets/src/testing/recording-custodian.ts @@ -2,49 +2,96 @@ import { randomBytes } from 'node:crypto'; import { Effect, Redacted } from 'effect'; +import { custodianRefused, type CustodianAct } from '../custodians/custodian-refused.ts'; import type { Custodian } from '../custodians/custodian.ts'; import { localCustodian } from '../custodians/local-custodian.ts'; +import type { KeepingFailure } from '../keepers/keeper.ts'; export interface RecordingCustodian extends Custodian { readonly wraps: () => number; readonly unwraps: () => number; readonly dataKeys: () => readonly Readonly[]; readonly refuse: () => void; + readonly stall: () => Promise; + readonly answer: () => void; } -export const custodianRefusal = 'The custodian refuses to wrap or unwrap a data key'; +type Answering = + | { readonly by: 'answering' } + | { readonly by: 'refusing' } + | { readonly by: 'stalling'; readonly asked: () => void }; + +interface Answers extends Pick { + readonly answered: ( + act: CustodianAct, + answer: Effect.Effect, + ) => Effect.Effect; +} + +export const custodianRefusal = 'the key is disabled'; export function aSecretsKey(): Redacted.Redacted { return Redacted.make(new Uint8Array(randomBytes(32))); } +function answersOf(kid: string): Answers { + const now: { answering: Answering } = { answering: { by: 'answering' } }; + return { + answered: (act, answer) => { + const { answering } = now; + if (answering.by === 'refusing') { + return Effect.fail(custodianRefused(kid, act, custodianRefusal)); + } + if (answering.by === 'stalling') { + answering.asked(); + return Effect.never; + } + return answer; + }, + refuse: () => { + now.answering = { by: 'refusing' }; + }, + stall: () => + new Promise((resolve) => { + now.answering = { + by: 'stalling', + asked: () => { + resolve(); + }, + }; + }), + answer: () => { + now.answering = { by: 'answering' }; + }, + }; +} + export function recordingCustodian(key: Redacted.Redacted = aSecretsKey()): RecordingCustodian { const custodian = localCustodian(key); - const asked = { wraps: 0, unwraps: 0, refusing: false }; + const { answered, refuse, stall, answer } = answersOf(custodian.kid); + const asked = { wraps: 0, unwraps: 0 }; const dataKeys: Readonly[] = []; const noted = (dataKey: Redacted.Redacted) => Effect.sync(() => { dataKeys.push(Redacted.value(dataKey)); }); - const answered = (answer: Effect.Effect): Effect.Effect => - asked.refusing ? Effect.die(new Error(custodianRefusal)) : answer; return { kid: custodian.kid, wrap: (dataKey) => Effect.suspend(() => { asked.wraps += 1; - return Effect.andThen(noted(dataKey), answered(custodian.wrap(dataKey))); + return Effect.andThen(noted(dataKey), answered('wrap', custodian.wrap(dataKey))); }), unwrap: (wrapped) => Effect.suspend(() => { asked.unwraps += 1; - return Effect.tap(answered(custodian.unwrap(wrapped)), noted); + return Effect.tap(answered('unwrap', custodian.unwrap(wrapped)), noted); }), wraps: () => asked.wraps, unwraps: () => asked.unwraps, dataKeys: () => [...dataKeys], - refuse: () => { - asked.refusing = true; - }, + refuse, + stall, + answer, }; } diff --git a/packages/secrets/src/testing/store-behaviour.ts b/packages/secrets/src/testing/store-behaviour.ts index 4ad1dd78f..8626736e8 100644 --- a/packages/secrets/src/testing/store-behaviour.ts +++ b/packages/secrets/src/testing/store-behaviour.ts @@ -4,8 +4,9 @@ import { compactDecrypt, decodeProtectedHeader } from 'jose'; import { describe, expect, it } from 'vitest'; import { localCustodian } from '../custodians/local-custodian.ts'; +import { storeFailureBehaviour } from './failure-behaviour.ts'; import { storeLeaseBehaviour } from './lease-behaviour.ts'; -import { aSecretsKey, custodianRefusal, recordingCustodian } from './recording-custodian.ts'; +import { aSecretsKey, recordingCustodian } from './recording-custodian.ts'; import { at, firstMoment, @@ -138,24 +139,6 @@ function forgottenTwiceAsOnce(entry: SecretsEntry): void { }); } -function unknownKeyRefused(entry: SecretsEntry): void { - it('cannot read a value sealed under another key, and names that key and never the value', async () => { - const database = await entry.aDatabase(); - const sealing = aSecretsKey(); - const id = await kept(await storeOn(entry, database, { current: sealing }), 'pasted-key'); - const other = await storeOn(entry, database, { current: aSecretsKey() }); - - const refused = await Effect.runPromise(Effect.flip(other.read(id))); - - expect(refused).toMatchObject({ - _tag: 'secret_key_unknown', - kid: kidOf(sealing), - detail: `The secret is sealed under the key ${kidOf(sealing)}, which is neither the current key nor the previous one`, - }); - expect(JSON.stringify(refused)).not.toContain('pasted-key'); - }); -} - function valuesRefused(entry: SecretsEntry): void { it('refuses an empty value and one of more than 8 KiB of UTF-8 before anything is sealed, and keeps neither', async () => { const database = await entry.aDatabase(); @@ -181,21 +164,6 @@ function valuesRefused(entry: SecretsEntry): void { }); } -function custodianRefusing(entry: SecretsEntry): void { - it('keeps nothing whose data key the custodian refuses to wrap, and reads nothing whose data key it refuses to unwrap', async () => { - const database = await entry.aDatabase(); - const custodian = recordingCustodian(); - const store = await storeWith(entry, database, { current: custodian }); - const id = await kept(store, 'pasted-key'); - - custodian.refuse(); - - await expect(kept(store, 'second-key')).rejects.toThrow(custodianRefusal); - await expect(readBack(store, id)).rejects.toThrow(custodianRefusal); - expect((await rowsIn(entry, database)).map((row) => row.id)).toEqual([id]); - }); -} - function rewrappedInBatches(entry: SecretsEntry): void { it('reads with its previous key, and wraps again in batches under the current one the data keys the previous wrapped, touching no value', async () => { const database = await entry.aDatabase(); @@ -242,11 +210,10 @@ export function storeBehaviour(entry: SecretsEntry): void { sealedUnderADataKeyOfItsOwn(entry); unwrappedAtEachRead(entry); forgottenTwiceAsOnce(entry); - unknownKeyRefused(entry); valuesRefused(entry); - custodianRefusing(entry); rewrappedInBatches(entry); rewrappedNothing(entry); + storeFailureBehaviour(entry); storeLeaseBehaviour(entry); }); } diff --git a/packages/server/src/database/ledger-stores-rotation.test.ts b/packages/server/src/database/ledger-stores-rotation.test.ts new file mode 100644 index 000000000..f4b4f7db5 --- /dev/null +++ b/packages/server/src/database/ledger-stores-rotation.test.ts @@ -0,0 +1,49 @@ +import { sqliteSecretRows } from '@on.auto/ledger/sqlite3'; +import { randomUUIDv7, sha256Of } from '@on.auto/operations'; +import { Effect } from 'effect'; +import { describe, expect, it, onTestFinished } from 'vitest'; + +import { readSettings } from '../settings/settings.ts'; +import { temporaryLedger } from '../testing/records/temporary-ledger.ts'; +import { ledgerStoresOpenedBy } from './ledger-stores.ts'; + +const [previousKey, currentKey] = [Buffer.alloc(32, 1), Buffer.alloc(32, 2)]; + +function aLedgerFile(): string { + const { fileName, remove } = temporaryLedger(); + onTestFinished(remove); + return fileName; +} + +async function keptUnderThePreviousKey(fileName: string): Promise { + const rows = await Effect.runPromise(sqliteSecretRows({ fileName })); + const holding = { + keptBy: 'ledger', + held: 'a value', + wrappedKey: 'no wrapping', + kid: sha256Of(previousKey).slice(0, 16), + }; + await Effect.runPromise(rows.keep(randomUUIDv7(), holding)); + await rows.close(); +} + +describe('the stores of a ledger whose secrets the previous key cannot unwrap', () => { + it('stop the start, naming the setting of the previous key before the words of its custodian', async () => { + const fileName = aLedgerFile(); + await keptUnderThePreviousKey(fileName); + const settings = readSettings({ + LEDGER_FILE: fileName, + SECRETS_KEY: currentKey.toString('base64'), + SECRETS_KEY_PREVIOUS: previousKey.toString('base64'), + }); + + const opened = ledgerStoresOpenedBy(await Effect.runPromise(Effect.context()), settings, { + store: 'sqlite', + file: fileName, + }); + + await expect(opened).rejects.toThrow( + `The secrets settings are invalid. SECRETS_KEY_PREVIOUS: The secrets its key sealed could not be sealed again under the current key: The custodian of the key ${sha256Of(previousKey).slice(0, 16)} refused to unwrap a data key: its wrapped key does not open under this key`, + ); + }); +}); diff --git a/packages/server/src/database/ledger-stores.ts b/packages/server/src/database/ledger-stores.ts index c640f795b..79728c7f1 100644 --- a/packages/server/src/database/ledger-stores.ts +++ b/packages/server/src/database/ledger-stores.ts @@ -5,7 +5,7 @@ import { Effect, type Context } from 'effect'; import type { Served } from '../lifecycle/lifecycle.ts'; import { logContentStore, logLedger, logSecretStore } from '../logging/logging.ts'; -import type { SecretsKeySource } from '../settings/secrets-settings.ts'; +import { InvalidSettingsError } from '../settings/invalid-settings-error.ts'; import type { Settings } from '../settings/settings.ts'; import { custodiansOf, secretRowsOn, secretStoreOver } from './secret-store.ts'; import { retryStateKeysOf, secretsKeysOf, type SecretKeys } from './secrets-keys.ts'; @@ -16,16 +16,37 @@ export interface LedgerStores { readonly withSecretsClosed: (served: Served) => Served; } +type SecretsSettings = Settings['secrets']; + +type PreviousKeyGiven = NonNullable; + +function sealedAgainRefused({ setting }: PreviousKeyGiven, detail: string): InvalidSettingsError { + return new InvalidSettingsError({ + message: `The secrets settings are invalid. ${setting}: The secrets its key sealed could not be sealed again under the current key: ${detail}`, + }); +} + +function rotatedUnder( + secrets: SecretStore, + { previous }: SecretsSettings, +): Effect.Effect { + return previous === undefined + ? Effect.orDie(secrets.rotated()) + : Effect.mapError(secrets.rotated(), (failure: { readonly detail: string }) => + sealedAgainRefused(previous, failure.detail), + ); +} + function openedAndRotated( database: DatabaseSettings, keys: SecretKeys, - keyFrom: SecretsKeySource, -): Effect.Effect { + settings: SecretsSettings, +): Effect.Effect { return Effect.gen(function* () { const custodians = custodiansOf(keys); const secrets = secretStoreOver(yield* secretRowsOn(database), custodians); - const resealed = yield* secrets.rotated(); - yield* logSecretStore({ kid: custodians.current.kid, resealed, keyFrom }); + const resealed = yield* rotatedUnder(secrets, settings); + yield* logSecretStore({ kid: custodians.current.kid, resealed, keyFrom: settings.current }); return secrets; }); } @@ -38,7 +59,7 @@ export async function ledgerStoresOpenedBy( const run = Effect.runPromiseWith(context); await run(Effect.andThen(logLedger(ledger), logContentStore(content))); const keys = secretsKeysOf(secrets); - const store = await run(openedAndRotated(database, keys, secrets.current)); + const store = await run(openedAndRotated(database, keys, secrets)); return { secrets: store, retryStateKeys: retryStateKeysOf(keys), From 3033d7ab16210f06cc432c680aaab7125a515eb8 Mon Sep 17 00:00:00 2001 From: Rami Date: Tue, 20 Oct 2026 14:31:56 +0100 Subject: [PATCH 2/3] feat(connections): a credential the store cannot read now fails the call and breaks nothing A row the secret store cannot read now, for any of its three failures, fails the call that needed it and leaves the connection connected, so the next call reads it again; only a row that is gone, and a value that is not the tokens the connection expects, break the connection as before. - given-credentials answers such a read as ConnectionUnready { because: 'connection_unavailable' }, "The credential of the connection cannot be read now: ", records nothing, and logs that sentence as a warning, annotated with the org and the connection's id, once for each cause of each connection in the process: again when the store's words change, and again after a read that succeeded. - A value that is no tokens breaks the connection in words of its own, "The credential of the connection is not the tokens of a consent the runtime keeps, so the person who added it removes it and adds it again". - connection_unavailable joins the reasons a connection is not ready in @on.auto/tool-servers and the rejection's because in @on.auto/operations, with its words and its remedy for an agent. A link is not kept for the process on it, as on connection_broken, so the next use reads the credential again and calls once the store can read it. - The descriptions of search, dry_run and a run's because name it, connection_broken is said in one wording where search, the operations README and the reference pages name it, and docs/assets/openapi.json is written again by docs:openapi. The API README says why tool_not_offered still carries no Retry-After. - Tests: a tool call through a recording custodian told to refuse, refused twice in those words with one line logged and the connection connected, then answered once the custodian answers; the log of each cause; a custodian that stalls, unavailable at 30 seconds; a connection prepared again at each use in tool-servers; a restarted server whose row is sealed under another key, refusing an execute and a run as 503 tool_not_offered connection_unavailable without Retry-After, then calling once the row is sealed again under its key; a row gone breaking it as before, which the test of the providers a search lists now uses for its broken connection. - The connections README's step 3 says the new rule, and the reference and self-host pages name connection_unavailable beside the other reasons. Co-Authored-By: Claude Fable 5.1 --- capabilities/reasoning/README.md | 2 +- docs/assets/openapi.json | 29 ++- .../engineering/reference/reasoning-format.md | 2 +- docs/reference/http.md | 27 +-- docs/reference/mcp.md | 2 +- docs/reference/reasoning-format.md | 25 +-- docs/self-host/connections.md | 25 +-- packages/api/README.md | 2 +- packages/connections/README.md | 4 +- .../src/serving/broken-connections.test.ts | 36 ++-- .../src/serving/connection-source.ts | 5 +- .../src/serving/consent-serving.test.ts | 2 +- .../src/serving/given-credentials.ts | 43 ++++- .../src/serving/serving-services.ts | 6 + .../connections/src/serving/serving-words.ts | 7 +- .../serving/unreadable-credentials.test.ts | 179 ++++++++++++++++++ .../src/testing/sealed-connections.ts | 60 ++++++ packages/definitions/src/runs/run.ts | 2 +- packages/operations/README.md | 24 +-- .../src/outcome/rejection-because.ts | 1 + .../plain-language/connection-explanations.ts | 5 +- .../plain-language/tool-explanations.test.ts | 6 + .../connections-across-restarts.test.ts | 62 ++++-- .../connections/providers-searched.test.ts | 16 +- packages/tool-servers/README.md | 8 +- .../src/org-servers/connection-links.test.ts | 24 +++ .../src/org-servers/connection-links.ts | 12 +- .../src/org-servers/connection-source.ts | 7 +- .../tool-servers/src/search/search-schemas.ts | 2 +- .../src/testing/fake-connections.ts | 71 +++++-- .../src/tool-reads/dry-run-schema.ts | 2 +- 31 files changed, 547 insertions(+), 151 deletions(-) create mode 100644 packages/connections/src/serving/unreadable-credentials.test.ts create mode 100644 packages/connections/src/testing/sealed-connections.ts diff --git a/capabilities/reasoning/README.md b/capabilities/reasoning/README.md index 1eefddf33..9163fe307 100644 --- a/capabilities/reasoning/README.md +++ b/capabilities/reasoning/README.md @@ -44,7 +44,7 @@ A definition whose `output.shape` names an [output shape](../../packages/definit A reasoning function whose front matter names `tools` (each `server/tool`, or `server/*`) runs with the tools of the tool servers configured for its playbook, MCP servers and HTTP APIs alike. `makeReasoningFunctionAdapter` takes the server's `ToolAccess` from [`@on.auto/tool-servers`](../../packages/tool-servers) as `tools`, as a type only: it imports the helpers it needs to parse `tools` and bound a run from `@on.auto/tool-servers/policy`, which loads no transport code; `tools` is required, and a test with no tool server gives `noToolServers` of `@on.auto/tool-servers/testing`; with no server configured, a function that names tools is rejected as `unavailable`, kind `tool_not_offered`, in the words of `namedLinks`. The adapter then says it may change something outside (`mayChangeOutside`), so `run_definition` is destructive. The capability declares as the stream kind it owns `tool-calls`, with the presenter of `@on.auto/tool-servers`, `handCallPresenter`, since the server serves `execute`, which calls a tool as a run of a reasoning function would, beside it: the playbook shows each call made by hand in its events, under the types of a run's calls, `tool_call_started`, `tool_call_answered` and `tool_call_failed`, on a stream of its own whose kind tells it from a run's, as the declarations allow, since a stream kind is checked apart from the facts of a run. -A function that names tools calls tools (`callsTools`), so the start of its run records that, and a run of it that is still `started` is not run again under its id. A run first admits its request with the model's `admit`, so a run whose model is not offered reaches no server and starts no process; it then opens its tools with its id, org, playbook and journal before the model is called, and rejects as `unavailable` with kind `tool_not_offered` (because `tool_server_not_configured`, `tool_not_allowed` or `tool_not_listed`, or for a connection a person added `connection_pending`, `connection_broken` or `connection_not_allowed`) or `tool_server_failed` (because `unreachable`, `failing` or `rate_limited`). Otherwise the request carries `tools`: the offered tools under their model-facing names, `server__tool`, with their input schemas unchanged, whether the calls have ended, a signal that ends them, the bound of the whole run, the most input tokens its model calls may add up to, `mostInputTokens`, and the model's context window, `contextWindow`, where the catalog knows it. Both come from the adapter's `reading`, a `ModelReading` the server gives: `mostInputTokens` from `REASONING_MAX_INPUT_TOKENS`, 2,000,000 unless set (`defaultMostInputTokens`), and `contextWindowOf`, the catalog's `contextWindowOf`, which resolves the model's alias and reads the `context_window` its provider lists for it, through the listing's cache; a run without tools reads no window, and a test that gives no `reading` has the default bound and no window. In the adapter, the AI SDK's loop: +A function that names tools calls tools (`callsTools`), so the start of its run records that, and a run of it that is still `started` is not run again under its id. A run first admits its request with the model's `admit`, so a run whose model is not offered reaches no server and starts no process; it then opens its tools with its id, org, playbook and journal before the model is called, and rejects as `unavailable` with kind `tool_not_offered` (because `tool_server_not_configured`, `tool_not_allowed` or `tool_not_listed`, or for a connection a person added `connection_pending`, `connection_broken`, `connection_not_allowed` or `connection_unavailable`) or `tool_server_failed` (because `unreachable`, `failing` or `rate_limited`). Otherwise the request carries `tools`: the offered tools under their model-facing names, `server__tool`, with their input schemas unchanged, whether the calls have ended, a signal that ends them, the bound of the whole run, the most input tokens its model calls may add up to, `mostInputTokens`, and the model's context window, `contextWindow`, where the catalog knows it. Both come from the adapter's `reading`, a `ModelReading` the server gives: `mostInputTokens` from `REASONING_MAX_INPUT_TOKENS`, 2,000,000 unless set (`defaultMostInputTokens`), and `contextWindowOf`, the catalog's `contextWindowOf`, which resolves the model's alias and reads the `context_window` its provider lists for it, through the listing's cache; a run without tools reads no window, and a test that gives no `reading` has the default bound and no window. In the adapter, the AI SDK's loop: - gives the model the tools in the function's own output mode, text or JSON, beside the `json` tool the Anthropic and Bedrock providers add for a JSON output they do not take in the output mode itself (`@ai-sdk/anthropic` 4.0.70 and `@ai-sdk/amazon-bedrock` 5.0.103 name it `json`), a name with no `__`, so no tool of a tool server takes it (`src/tools/tool-loop.test.ts` pins both), and returns each answer to the model as a tool result, an error result for a tool error, giving each call the room its answer has (`src/tools/turn-meter.ts`): the step is measured where the AI SDK announces it, the bytes of its instructions, messages and tools as it starts, and the input and output tokens the provider counted and the tool calls it holds as it ends, before any tool runs; the step's own bytes a token, its bytes sent over its input tokens, turn the room into bytes, the smaller of the window and what `mostInputTokens` leaves after every step made, less the step's input and output tokens, shared equally by the step's calls, so an answer the model's context cannot hold is cut where it stays valid JSON, with a note, and read whole when no window is known or the provider counted no input tokens; nothing is taken from the window for the output, since the catalog's window is the most input tokens a model takes; - gives every model call its own deadline, the request's `timeout_ms`, which the time the tools take does not count against, armed when the call starts and ended however it settles, even when the AI SDK reports no end for a call that threw, and bounds the whole run by ten minutes, or one call's deadline when that is longer; diff --git a/docs/assets/openapi.json b/docs/assets/openapi.json index a8618a772..14fc427a5 100644 --- a/docs/assets/openapi.json +++ b/docs/assets/openapi.json @@ -1165,9 +1165,10 @@ "key_refused", "connection_pending", "connection_broken", - "connection_not_allowed" + "connection_not_allowed", + "connection_unavailable" ], - "description": "key_refused when the server did not accept the key this server gives it, which only whoever runs this server can put right; failing, rate_limited or unreachable when asking again later may work; connection_pending when nobody has approved the connection yet, connection_broken when its credential cannot be read, so it is removed and added again, and connection_not_allowed when a rule of whoever runs this server no longer allows it" + "description": "key_refused when the server did not accept the key this server gives it, which only whoever runs this server can put right; failing, rate_limited or unreachable when asking again later may work; connection_pending when nobody has approved the connection yet, connection_broken when its credential is gone from the secret store or is not the tokens of a consent, its consent was revoked or has ended, or its client is no longer registered, so it is removed and added again, connection_not_allowed when a rule of whoever runs this server no longer allows it, and connection_unavailable when its credential cannot be read just now, so asking again later may work" } }, "required": [ @@ -6807,6 +6808,7 @@ "connection_pending", "connection_broken", "connection_not_allowed", + "connection_unavailable", "failing", "rate_limited", "unreachable", @@ -6823,7 +6825,7 @@ "not_in_order", "baseline_check" ], - "description": "Why. With model_not_offered: provider_not_configured when its provider is not set up on this server while others are, model_not_allowed when it is outside the models the operator allows. With tool_not_offered: tool_server_not_configured, tool_not_allowed or tool_not_listed, or connection_pending, connection_broken or connection_not_allowed for a connection a person added. With tool_server_failed: failing, rate_limited, unreachable or key_refused. With tools_unfinished: server_failed, tool_error, model_unavailable, run_bound or no_answer" + "description": "Why. With model_not_offered: provider_not_configured when its provider is not set up on this server while others are, model_not_allowed when it is outside the models the operator allows. With tool_not_offered: tool_server_not_configured, tool_not_allowed or tool_not_listed, or connection_pending, connection_broken, connection_not_allowed or connection_unavailable for a connection a person added. With tool_server_failed: failing, rate_limited, unreachable or key_refused. With tools_unfinished: server_failed, tool_error, model_unavailable, run_bound or no_answer" } }, "required": [ @@ -6879,6 +6881,7 @@ "connection_pending", "connection_broken", "connection_not_allowed", + "connection_unavailable", "failing", "rate_limited", "unreachable", @@ -7211,6 +7214,7 @@ "connection_pending", "connection_broken", "connection_not_allowed", + "connection_unavailable", "failing", "rate_limited", "unreachable", @@ -7227,7 +7231,7 @@ "not_in_order", "baseline_check" ], - "description": "Why. With model_not_offered: provider_not_configured when its provider is not set up on this server while others are, model_not_allowed when it is outside the models the operator allows. With tool_not_offered: tool_server_not_configured, tool_not_allowed or tool_not_listed, or connection_pending, connection_broken or connection_not_allowed for a connection a person added. With tool_server_failed: failing, rate_limited, unreachable or key_refused. With tools_unfinished: server_failed, tool_error, model_unavailable, run_bound or no_answer" + "description": "Why. With model_not_offered: provider_not_configured when its provider is not set up on this server while others are, model_not_allowed when it is outside the models the operator allows. With tool_not_offered: tool_server_not_configured, tool_not_allowed or tool_not_listed, or connection_pending, connection_broken, connection_not_allowed or connection_unavailable for a connection a person added. With tool_server_failed: failing, rate_limited, unreachable or key_refused. With tools_unfinished: server_failed, tool_error, model_unavailable, run_bound or no_answer" } }, "required": [ @@ -7283,6 +7287,7 @@ "connection_pending", "connection_broken", "connection_not_allowed", + "connection_unavailable", "failing", "rate_limited", "unreachable", @@ -7630,6 +7635,7 @@ "connection_pending", "connection_broken", "connection_not_allowed", + "connection_unavailable", "failing", "rate_limited", "unreachable", @@ -7646,7 +7652,7 @@ "not_in_order", "baseline_check" ], - "description": "Why. With model_not_offered: provider_not_configured when its provider is not set up on this server while others are, model_not_allowed when it is outside the models the operator allows. With tool_not_offered: tool_server_not_configured, tool_not_allowed or tool_not_listed, or connection_pending, connection_broken or connection_not_allowed for a connection a person added. With tool_server_failed: failing, rate_limited, unreachable or key_refused. With tools_unfinished: server_failed, tool_error, model_unavailable, run_bound or no_answer" + "description": "Why. With model_not_offered: provider_not_configured when its provider is not set up on this server while others are, model_not_allowed when it is outside the models the operator allows. With tool_not_offered: tool_server_not_configured, tool_not_allowed or tool_not_listed, or connection_pending, connection_broken, connection_not_allowed or connection_unavailable for a connection a person added. With tool_server_failed: failing, rate_limited, unreachable or key_refused. With tools_unfinished: server_failed, tool_error, model_unavailable, run_bound or no_answer" } }, "required": [ @@ -7702,6 +7708,7 @@ "connection_pending", "connection_broken", "connection_not_allowed", + "connection_unavailable", "failing", "rate_limited", "unreachable", @@ -8096,6 +8103,7 @@ "connection_pending", "connection_broken", "connection_not_allowed", + "connection_unavailable", "failing", "rate_limited", "unreachable", @@ -9857,9 +9865,10 @@ "key_refused", "connection_pending", "connection_broken", - "connection_not_allowed" + "connection_not_allowed", + "connection_unavailable" ], - "description": "key_refused when the server did not accept the key this server gives it, which only whoever runs this server can put right; failing, rate_limited or unreachable when asking again later may work; connection_pending when nobody has approved the connection yet, connection_broken when its credential cannot be read, so it is removed and added again, and connection_not_allowed when a rule of whoever runs this server no longer allows it" + "description": "key_refused when the server did not accept the key this server gives it, which only whoever runs this server can put right; failing, rate_limited or unreachable when asking again later may work; connection_pending when nobody has approved the connection yet, connection_broken when its credential is gone from the secret store or is not the tokens of a consent, its consent was revoked or has ended, or its client is no longer registered, so it is removed and added again, connection_not_allowed when a rule of whoever runs this server no longer allows it, and connection_unavailable when its credential cannot be read just now, so asking again later may work" } }, "required": [ @@ -10429,9 +10438,10 @@ "key_refused", "connection_pending", "connection_broken", - "connection_not_allowed" + "connection_not_allowed", + "connection_unavailable" ], - "description": "Why it could not be asked: key_refused when it did not accept the key this server gives it, which only whoever runs this server can put right; failing, rate_limited or unreachable when asking again later may work; connection_not_allowed when a rule of whoever runs this server no longer allows the connection" + "description": "Why it could not be asked: key_refused when it did not accept the key this server gives it, which only whoever runs this server can put right; failing, rate_limited or unreachable when asking again later may work; connection_not_allowed when a rule of whoever runs this server no longer allows the connection; connection_unavailable when the credential of the connection cannot be read just now, so asking again later may work" } }, "required": [ @@ -12171,6 +12181,7 @@ "connection_pending", "connection_broken", "connection_not_allowed", + "connection_unavailable", "failing", "rate_limited", "unreachable", diff --git a/docs/engineering/reference/reasoning-format.md b/docs/engineering/reference/reasoning-format.md index 447a7a636..88b4c0700 100644 --- a/docs/engineering/reference/reasoning-format.md +++ b/docs/engineering/reference/reasoning-format.md @@ -95,7 +95,7 @@ A document is checked without calling a model, and the same document always give ### Tools -`tools` names the tools a run may call, each `server/tool` or `server/*`, from the servers in [`tool_servers`](../../self-host/configuration.md#tool-servers) that serve the playbook's org and playbook and the [connections](../../self-host/connections.md) of its org that serve it, a connection named by its name. A run checks them once its model is known to be offered, and before the model is called, so a run whose model is not offered reaches no server and starts no process: a server that is not configured for the playbook is `unavailable` of the kind `tool_not_offered` because `tool_server_not_configured`, a tool outside the server's `allowed` because `tool_not_allowed`, a tool the server does not list because `tool_not_listed`, and a connection that is not ready because `connection_pending`, `connection_broken` or `connection_not_allowed`; a server that cannot be used is `unavailable` of the kind `tool_server_failed`, because `unreachable`, `failing`, `rate_limited` or `key_refused`, the last when the server did not accept the key the runtime gives it, whose words say to check that key rather than to try again. `server/*` gives every tool the server lists that the server's `allowed` names, or every tool it lists when `allowed` is left out. `search` lists the servers that serve the playbook and the tools each would give `server/*`, asking each server as a run does, and scores the tools by `terms` where the servers offer many, so the names can be found rather than told ([Tool servers](../../reference/http.md#tool-servers)); `introspect` answers one tool whole, and `execute` calls one as a run would and answers what its model would see, so what a tool answers can be read before a function names it ([Calling a tool by hand](../../reference/http.md#calling-a-tool-by-hand)). +`tools` names the tools a run may call, each `server/tool` or `server/*`, from the servers in [`tool_servers`](../../self-host/configuration.md#tool-servers) that serve the playbook's org and playbook and the [connections](../../self-host/connections.md) of its org that serve it, a connection named by its name. A run checks them once its model is known to be offered, and before the model is called, so a run whose model is not offered reaches no server and starts no process: a server that is not configured for the playbook is `unavailable` of the kind `tool_not_offered` because `tool_server_not_configured`, a tool outside the server's `allowed` because `tool_not_allowed`, a tool the server does not list because `tool_not_listed`, and a connection that is not ready because `connection_pending`, `connection_broken`, `connection_not_allowed` or `connection_unavailable`; a server that cannot be used is `unavailable` of the kind `tool_server_failed`, because `unreachable`, `failing`, `rate_limited` or `key_refused`, the last when the server did not accept the key the runtime gives it, whose words say to check that key rather than to try again. `server/*` gives every tool the server lists that the server's `allowed` names, or every tool it lists when `allowed` is left out. `search` lists the servers that serve the playbook and the tools each would give `server/*`, asking each server as a run does, and scores the tools by `terms` where the servers offer many, so the names can be found rather than told ([Tool servers](../../reference/http.md#tool-servers)); `introspect` answers one tool whole, and `execute` calls one as a run would and answers what its model would see, so what a tool answers can be read before a function names it ([Calling a tool by hand](../../reference/http.md#calling-a-tool-by-hand)). The model receives each tool under a unique name derived from `server__tool`, for an MCP server and an HTTP API alike, and the server's name ends at the first `__`. Characters other than letters, digits and underscores become underscores; names longer than 64 characters or sharing a normalized name receive an 8-character hash. A numeric suffix resolves any remaining collision within the run's tool list. The description is cut to 4 KiB and the server's input schema is unchanged; what in it the provider of the run's model may refuse or not hold the tool's arguments to is added to the run's `warnings`: a root that is not an object, and for Anthropic models the bounds, `oneOf` and the formats they do not enforce or accept, such as `graph__search inputSchema#/properties/query/maxLength`. Tools are not sent as strict structured outputs, so the rules a JSON output schema has for open objects and optional properties do not apply to them. The model keeps the function's output format and may call tools, several in one step, before it answers, each call forwarded with the run id in the request's metadata under `auto.on/run_id`: diff --git a/docs/reference/http.md b/docs/reference/http.md index d774d50b0..03f19c617 100644 --- a/docs/reference/http.md +++ b/docs/reference/http.md @@ -886,17 +886,17 @@ Authorization: Bearer Each server has these fields: -| Field | Contents | -| ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `name` | The name a function writes before the slash | -| `connection` | For a connection a person added, its id; absent for a server of the configuration. A connection whose name the configuration now sets up is listed beside that server, unavailable | -| `provider` | The name of the [provider](#providers) a connection was added from, or an entry of the configuration names, or `null` for a server given by hand | -| `type` | `mcp` for a remote MCP server, `stdio` for an MCP server the runtime starts as a process, or `openapi` for an HTTP API described by an OpenAPI document | -| `record_content` | Whether the content of its calls is kept | -| `tool_count` | How many tools of the server the operator allows, of those `only` keeps | -| `unavailable` | Instead of `tool_count`: why the server could not be asked for its tools, in words cut to 1 KiB. The other servers are still searched | -| `because` | With `unavailable`: `key_refused` when the server did not accept the key the runtime gives it, which only the operator can fix; `failing`, `rate_limited` or `unreachable` when asking again later may work; for a connection, `connection_pending`, `connection_broken` or `connection_not_allowed`, as [below](#a-connection-that-is-not-ready) | -| `playbooks` | On the org route only: the playbooks of the org whose functions may use it, or `["*"]` for every playbook of the org | +| Field | Contents | +| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | The name a function writes before the slash | +| `connection` | For a connection a person added, its id; absent for a server of the configuration. A connection whose name the configuration now sets up is listed beside that server, unavailable | +| `provider` | The name of the [provider](#providers) a connection was added from, or an entry of the configuration names, or `null` for a server given by hand | +| `type` | `mcp` for a remote MCP server, `stdio` for an MCP server the runtime starts as a process, or `openapi` for an HTTP API described by an OpenAPI document | +| `record_content` | Whether the content of its calls is kept | +| `tool_count` | How many tools of the server the operator allows, of those `only` keeps | +| `unavailable` | Instead of `tool_count`: why the server could not be asked for its tools, in words cut to 1 KiB. The other servers are still searched | +| `because` | With `unavailable`: `key_refused` when the server did not accept the key the runtime gives it, which only the operator can fix; `failing`, `rate_limited` or `unreachable` when asking again later may work; for a connection, `connection_pending`, `connection_broken`, `connection_not_allowed` or `connection_unavailable`, as [below](#a-connection-that-is-not-ready) | +| `playbooks` | On the org route only: the playbooks of the org whose functions may use it, or `["*"]` for every playbook of the org | Each tool has these fields: @@ -1048,7 +1048,7 @@ Content-Type: application/json The response is `200` whenever the call was made, whatever the tool returned. Nothing is called when: -- the operator does not allow the tool, the server is not set up for the playbook, or the server does not list the tool: `unavailable` with kind `tool_not_offered` and `because` set to `tool_not_allowed`, `tool_server_not_configured` or `tool_not_listed`, or for a connection that is not ready `connection_pending`, `connection_broken` or `connection_not_allowed`; +- the operator does not allow the tool, the server is not set up for the playbook, or the server does not list the tool: `unavailable` with kind `tool_not_offered` and `because` set to `tool_not_allowed`, `tool_server_not_configured` or `tool_not_listed`, or for a connection that is not ready `connection_pending`, `connection_broken`, `connection_not_allowed` or `connection_unavailable`; - the server cannot be used: `unavailable` with kind `tool_server_failed`. A call is live: a tool its server does not mark read-only may change something. That is why `execute` is marked destructive over MCP while a tool server is configured or people may add connections. @@ -1145,8 +1145,9 @@ A connected connection is a tool server of its org: `search` lists it with `conn | `because` | The connection | | ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `connection_pending` | Nobody has approved it yet; the person who added it opens its link, or removes it and adds it again for a new link | -| `connection_broken` | Its credential cannot be read, and the runtime recorded it `broken`, with why in `broken`; the person removes it and adds it again | +| `connection_broken` | Its credential is gone from the secret store or is not the tokens of a consent, its consent was revoked or has ended, or its client is no longer registered; the runtime recorded it `broken`, with why in `broken`, and the person removes it and adds it again | | `connection_not_allowed` | The runtime does not use it, in the words of why: a rule of the operator no longer takes it, in the sentence `add_connection` would give; `connections.people_may_add` is `false`; its entry is no longer one the runtime takes; the variable its reference names is not set; the configuration now sets up a server of its name; or a request it would make goes to a host the rules do not allow | +| `connection_unavailable` | Its credential cannot be read just now, in the words of why: the runtime's secret store cannot unwrap it or does not answer, which whoever runs the runtime puts right. The connection stays connected, and a later call reads the credential again | ## Responses and errors diff --git a/docs/reference/mcp.md b/docs/reference/mcp.md index c8e8c4aa2..c7f1f7833 100644 --- a/docs/reference/mcp.md +++ b/docs/reference/mcp.md @@ -173,7 +173,7 @@ Four tools work with the tool servers of the playbook, those set up in the confi - `introspect` and `dry_run` change nothing and call no tool, but they ask the servers for their tool lists. So they are marked read-only and open-world, and a key that may read the playbook is offered them. - `execute` needs `playbook:write` whatever the tool does, so a key that may only read is not offered it. Each call is recorded in the playbook's history. - A call that names a server or a tool the playbook may not use is refused, and the refusal's words say what `search` shows. -- A connection is listed with `connection`, its id, and named by its name. One that is not ready is refused, and listed unavailable, with `because` `connection_pending` while nobody has approved it, `connection_broken` once its credential cannot be read, or `connection_not_allowed` when a rule of the operator no longer takes it. +- A connection is listed with `connection`, its id, and named by its name. One that is not ready is refused, and listed unavailable, with `because` `connection_pending` while nobody has approved it, `connection_broken` once its credential is gone from the secret store or is not the tokens of a consent, its consent was revoked or has ended, or its client is no longer registered, `connection_not_allowed` when a rule of the operator no longer takes it, or `connection_unavailable` while its credential cannot be read just now, which a later call may find put right. - Each server carries `provider`: the name of the provider a connection was added from or an entry of the configuration names, or `null` for a server given by hand. - On `/mcp` and the org endpoint, to a caller who may add a connection, `search` also lists in `providers` the [providers](http.md#providers) a person may connect that no server serves yet, each with its `name`, `title` and `description`, found by `terms` as tools are. A provider none of whose ways is offered, as while a value whoever runs Auto sets for it is not set, is left out. An agent asked for a service no server serves can then offer to connect one with `add_connection`. A key, a caller who may only read and a runtime where nobody may add get no `providers`, and neither does `search` with `server`. - When no server serves the playbook, the words of `search` say that whoever runs the server sets one up. Where people may add connections, they also say that a person adds one, or, where `search` lists a provider, that a person connects a provider it lists. diff --git a/docs/reference/reasoning-format.md b/docs/reference/reasoning-format.md index 883ea1357..6b1c222b6 100644 --- a/docs/reference/reasoning-format.md +++ b/docs/reference/reasoning-format.md @@ -195,18 +195,19 @@ Do not guess tool names. Ask the runtime: A run checks its tools after it confirms that its model is offered, and before it calls the model. A run whose model is not offered contacts no tool server. A tool that cannot be used ends the run as `unavailable`, and the model is never called: -| Kind | `because` | When | -| -------------------- | ---------------------------- | -------------------------------------------------------------------------------------- | -| `tool_not_offered` | `tool_server_not_configured` | The server is not configured for the playbook | -| `tool_not_offered` | `tool_not_allowed` | The tool is outside the server's `allowed` | -| `tool_not_offered` | `tool_not_listed` | The server does not list the tool | -| `tool_not_offered` | `connection_pending` | The server is a connection nobody has approved yet | -| `tool_not_offered` | `connection_broken` | The server is a connection whose credential cannot be read; remove it and add it again | -| `tool_not_offered` | `connection_not_allowed` | The server is a connection a rule of the operator no longer allows | -| `tool_server_failed` | `unreachable` | The server cannot be reached, or does not respond in time | -| `tool_server_failed` | `failing` | The server fails, or refuses to list its tools | -| `tool_server_failed` | `rate_limited` | The server asks the run to slow down | -| `tool_server_failed` | `key_refused` | The server did not accept the key the runtime gives it; check that key, do not retry | +| Kind | `because` | When | +| -------------------- | ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `tool_not_offered` | `tool_server_not_configured` | The server is not configured for the playbook | +| `tool_not_offered` | `tool_not_allowed` | The tool is outside the server's `allowed` | +| `tool_not_offered` | `tool_not_listed` | The server does not list the tool | +| `tool_not_offered` | `connection_pending` | The server is a connection nobody has approved yet | +| `tool_not_offered` | `connection_broken` | The server is a connection whose credential is gone from the secret store or is not the tokens of a consent, whose consent was revoked or has ended, or whose client is no longer registered; remove it and add it again | +| `tool_not_offered` | `connection_not_allowed` | The server is a connection a rule of the operator no longer allows | +| `tool_not_offered` | `connection_unavailable` | The server is a connection whose credential cannot be read just now; try again later | +| `tool_server_failed` | `unreachable` | The server cannot be reached, or does not respond in time | +| `tool_server_failed` | `failing` | The server fails, or refuses to list its tools | +| `tool_server_failed` | `rate_limited` | The server asks the run to slow down | +| `tool_server_failed` | `key_refused` | The server did not accept the key the runtime gives it; check that key, do not retry | ### What the model sees diff --git a/docs/self-host/connections.md b/docs/self-host/connections.md index 1c57c2aef..c1a449d2c 100644 --- a/docs/self-host/connections.md +++ b/docs/self-host/connections.md @@ -104,17 +104,18 @@ A connected connection is a tool server of its org, for every playbook of the or A connection that is not ready is refused, and `search` lists it unavailable with the same words: -| The connection | A function that names it is refused with | Its words say | -| -------------------------------------------------------------------------------------------------------------------- | ---------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | -| Nobody has approved it yet | `connection_pending` | The person who added it opens its link, or removes it and adds it again for a new link | -| Its credential cannot be read | `connection_broken` | Why, and that the person who added it removes it and adds it again | -| Its consent was revoked or has ended, or the client it was made with is no longer registered | `connection_broken` | Why, and that the person who added it removes it and adds it again | -| A rule of `auto-runtime.yaml` no longer takes it | `connection_not_allowed` | The rule, in the sentence `add_connection` would give | -| `people_may_add` is now `false` | `connection_not_allowed` | That this runtime uses no connection, since you set up its tool servers yourself | -| Its entry is no longer one Auto takes, as when a model provider you set up since has its name | `connection_not_allowed` | What `add_connection` would refuse in it | -| Its `reference` names a variable that is not set | `connection_not_allowed` | The variable, and that you set it and start Auto again | -| Its name is that of a tool server of the settings file, for any org, which a later edit added | `connection_not_allowed` | That the settings file sets up a tool server of that name, and that it is removed and added again under another name | -| Its document names a server, or a call meets a redirect or an authorization server, on a host the rules do not allow | `connection_not_allowed` | The rule | +| The connection | A function that names it is refused with | Its words say | +| -------------------------------------------------------------------------------------------------------------------- | ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Nobody has approved it yet | `connection_pending` | The person who added it opens its link, or removes it and adds it again for a new link | +| Its credential cannot be read just now, since Auto's secret store cannot unwrap it or does not answer | `connection_unavailable` | `The credential of the connection crm cannot be read now:` and the store's words. It stays connected, and you put the store right; Auto logs those words once for each cause | +| Its credential is no longer in Auto's secret store, or is not the tokens of a consent | `connection_broken` | Why, and that the person who added it removes it and adds it again | +| Its consent was revoked or has ended, or the client it was made with is no longer registered | `connection_broken` | Why, and that the person who added it removes it and adds it again | +| A rule of `auto-runtime.yaml` no longer takes it | `connection_not_allowed` | The rule, in the sentence `add_connection` would give | +| `people_may_add` is now `false` | `connection_not_allowed` | That this runtime uses no connection, since you set up its tool servers yourself | +| Its entry is no longer one Auto takes, as when a model provider you set up since has its name | `connection_not_allowed` | What `add_connection` would refuse in it | +| Its `reference` names a variable that is not set | `connection_not_allowed` | The variable, and that you set it and start Auto again | +| Its name is that of a tool server of the settings file, for any org, which a later edit added | `connection_not_allowed` | That the settings file sets up a tool server of that name, and that it is removed and added again under another name | +| Its document names a server, or a call meets a redirect or an authorization server, on a host the rules do not allow | `connection_not_allowed` | The rule | Each comes as `unavailable` of the kind `tool_not_offered`, before the function calls anything. A name that is neither a tool server of the settings file nor a connection of the org stays `tool_server_not_configured`. @@ -162,7 +163,7 @@ Auto seals a secret under its secret key, `SECRETS_KEY`, or the file `SECRETS_KE Every value of a credential is a secret Auto scrubs from what it records and answers, in each form it may take, as it scrubs the value of a reference of the settings file. A value in a header that is a scheme and a credential, such as `Bearer `, is scrubbed whole and its credential alone; where the credential names its `scheme`, the value is the token, scrubbed whole. -A secret is only as recoverable as the key it was sealed under. When the key is lost, Auto can no longer read the credentials sealed under it, so their connections are broken. Auto marks such a connection `broken` when it first needs the credential, with a sentence that says why; the person removes it and adds it again. +A secret is only as recoverable as the key it was sealed under. While Auto cannot read a credential, because the key it was sealed under is not the key Auto has or its store does not answer, a call that needs it is refused with `connection_unavailable` in the store's words and its connection stays connected, so giving Auto the key again mends it. When the key is lost for good, nothing can read the credentials sealed under it, and the person removes each such connection and adds it again. Auto marks a connection `broken` only when its credential is no longer in the store, or is not the tokens of a consent, with a sentence that says why. ## Set the rules in `auto-runtime.yaml` diff --git a/packages/api/README.md b/packages/api/README.md index 283892020..2d66d1f3c 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -41,7 +41,7 @@ An outcome that is not a success becomes a problem document. The problem documen | failed | `500` `internal`, saying nothing about the cause; its `instance` is `urn:uuid:`, the incident id under which the server logs the error | | cancelled | `499` `client_closed_request` when the client went away, `503` `unavailable` when the server is stopping | -A route refuses a caller its principal does not admit to the org of the path with `403` `forbidden`, "The caller does not belong to this org", before it reads the input, as the MCP endpoints do; the dispatcher's own check of the org still holds for every call. A `403` `forbidden` also carries `WWW-Authenticate: Bearer error="insufficient_scope"`, and, with sign-in on, the operation's permissions, its registration's `permissions`, any one of which permits it, as `scope` and `resource_metadata` with the root metadata, for a caller of another org as for a caller that lacks the permission; `insufficientScope(resource, permissions, resourcePath?)` of `src/authentication/challenges.ts` builds it. With sign-in off the challenge names no scope, since there is no authorization server to ask. A `503` `unavailable` carries `Retry-After: 5` when a retry of the same request may resolve it, so not for the kind `tools_unfinished`, a run that called tools that only read and could not finish, which the same request answers with `tools_called` and a new run may make, nor for the kinds `tool_not_offered` and `model_not_offered`, which only a change of the function or the configuration resolves, nor for a rejection whose `because` is `builds_of_the_day`, a prediction function that has had its builds of the day, `stalled`, a dataset stopped at an event, `not_finite`, a model that answers what it cannot, or `not_in_order`, a model whose low is above its likely value or whose high is below it, which no wait resolves (`notResolvedByWaiting`). A `410` `unanswered` carries none: the same request answers it again. +A route refuses a caller its principal does not admit to the org of the path with `403` `forbidden`, "The caller does not belong to this org", before it reads the input, as the MCP endpoints do; the dispatcher's own check of the org still holds for every call. A `403` `forbidden` also carries `WWW-Authenticate: Bearer error="insufficient_scope"`, and, with sign-in on, the operation's permissions, its registration's `permissions`, any one of which permits it, as `scope` and `resource_metadata` with the root metadata, for a caller of another org as for a caller that lacks the permission; `insufficientScope(resource, permissions, resourcePath?)` of `src/authentication/challenges.ts` builds it. With sign-in off the challenge names no scope, since there is no authorization server to ask. A `503` `unavailable` carries `Retry-After: 5` when a retry of the same request may resolve it, so not for the kind `tools_unfinished`, a run that called tools that only read and could not finish, which the same request answers with `tools_called` and a new run may make, nor for the kinds `tool_not_offered` and `model_not_offered`, which a change of the function or the configuration resolves, or, for a connection whose credential cannot be read just now (`connection_unavailable`), whoever runs the server putting the secret store right: a later try may then pass, but no hint of seconds says when, and one would invite a loop of retries, nor for a rejection whose `because` is `builds_of_the_day`, a prediction function that has had its builds of the day, `stalled`, a dataset stopped at an event, `not_finite`, a model that answers what it cannot, or `not_in_order`, a model whose low is above its likely value or whose high is below it, which no wait resolves (`notResolvedByWaiting`). A `410` `unanswered` carries none: the same request answers it again. ## The OpenAPI document diff --git a/packages/connections/README.md b/packages/connections/README.md index 90bf93b4a..a4ec19efa 100644 --- a/packages/connections/README.md +++ b/packages/connections/README.md @@ -45,7 +45,7 @@ There is no read model: each call folds the stream (`connectionsFold`, the fold - `approve` and `give` look the connection up, `not_found`, then the link: open while the connection is pending, its digest the token's and `expires_at` later than now, else `not_found`, `There is no open link of this token: it is unknown, used up or expired`. Where people sign in, the caller must be `added_by`, else `forbidden`; `makeConnectionOperations` hands the two whether a link binds a person, which it does where `access` is `people`. `approve` connects a connection of the credential `none` or `reference`, recording `connection_approved` and `connection_connected`, and refuses one that takes a value with `conflict`; `give` records the two with the secret's id, and refuses a connection that takes none with `conflict`. Both refuse a connection whose credential is `authorization` with `conflict`, `The connection drive is connected by signing in to its provider and allowing Auto, which its page starts`, and `give_credential` refuses it so before it keeps anything. - `start` and `finish` take a connection whose credential is `authorization` and refuse another kind with `conflict`, `The connection crm is not connected by signing in to its provider, so it has no consent to start or finish`. `start` checks the link as `approve` does and records `connection_consent_started`. `finish` checks the link's token and its binding but not its own `expires_at`, since the start's ten minutes govern: a connection with no open start, none recorded or one whose `expires_at` is not later than now, is `not_found`, `There is no open consent of the connection drive: none was started, or it was started more than ten minutes ago`, a state whose digest is not the start's is `invalid_input` at `/state`, `Expected the state the consent was started with`, and otherwise it records `connection_approved` and `connection_connected` in one decision, as `give` does. - `remove` records `connection_removed`, and nothing for a connection already removed. -- `break` records `connection_broken` with its `reason` for a connection that is connected, and nothing for one that is not, so a credential two calls find unreadable at once breaks its connection once. Only the source of [The calls](#the-calls) issues it, as the runtime, `by: runtime`. Every failure to record a `break`, a change that came first among them, is logged, `logError` of Effect with the cause, and the connection is refused broken all the same; the decider's `not_found` cannot arise, since the source reads the connection from the stream it records the break on and the fold never drops one. The source keeps nothing and the tool access keeps no broken answer, so the next use reads the credential again and records the break again. +- `break` records `connection_broken` with its `reason` for a connection that is connected, and nothing for one that is not, so a credential two calls find gone at once breaks its connection once. Only the source of [The calls](#the-calls) issues it, as the runtime, `by: runtime`. Every failure to record a `break`, a change that came first among them, is logged, `logError` of Effect with the cause, and the connection is refused broken all the same; the decider's `not_found` cannot arise, since the source reads the connection from the stream it records the break on and the fold never drops one. The source keeps nothing of a break and the tool access keeps no broken answer, so the next use reads the credential again and records the break again. ## The operations @@ -107,7 +107,7 @@ A client the consent client registers dynamically at an authorization server is 1. The policy of the moment is checked with the rules `add_connection` applies (`useRefusalOf`), and a connection a rule no longer takes is `connection_not_allowed` with the rule's sentence, its credential unread: `credential_kinds`, `variables` for a `reference`, and `allowed_hosts` over the entry's `url`, `document` and `base_url` and the credential's addresses, `wayAddresses`. With `people_may_add` `false` the runtime uses no connection, in words that speak of using rather than adding, `This runtime uses no connection: its operator sets up its tool servers itself`, while `list_connections` still answers each as it is. 2. The connection is checked with `entryIssuesOf`, the checks of `add_connection` that need no value, against the model providers of the moment, so an entry the runtime no longer takes, as one named as a model provider set up since, is `connection_not_allowed`, `The entry of the connection is not one this runtime takes any more: `, its credential unread. -3. The credential is read (`src/serving/given-credentials.ts`): none for `none`; for a `reference`, the value of its `variable`, and a variable the environment leaves unset is `connection_not_allowed`, saying whoever runs the runtime sets it and starts it again; for a `secret` or `client_credentials`, the store's row of its secret's id; for `authorization`, the row of its tokens, the `secret_id` of its `connection_connected`, read as a `secret`'s is and decoded with `keptTokensIn` of `@on.auto/tool-servers`, a value that is no tokens breaking the connection as unreadable, `The credential of the connection crm cannot be read, so the person who added it removes it and adds it again: its value is not the tokens of a consent the runtime keeps`, which places nothing in the entry and is handed to `@on.auto/tool-servers`, decoded, with the rest of the connection's consent once the entry is read, below. A row the store cannot read, `SecretKeyUnknown`, or one that is gone breaks the connection: the source issues `break`, records `connection_broken` with the sentence of why, `The credential of the connection crm cannot be read, so the person who added it removes it and adds it again: ` or `… is no longer in the secret store, so …`, and answers it `connection_broken` in it. Nothing un-breaks a connection; the person removes it and adds it again. +3. The credential is read (`src/serving/given-credentials.ts`): none for `none`; for a `reference`, the value of its `variable`, and a variable the environment leaves unset is `connection_not_allowed`, saying whoever runs the runtime sets it and starts it again; for a `secret` or `client_credentials`, the store's row of its secret's id; for `authorization`, the row of its tokens, the `secret_id` of its `connection_connected`, read as a `secret`'s is and decoded with `keptTokensIn` of `@on.auto/tool-servers`, a value that is no tokens breaking the connection, `The credential of the connection crm is not the tokens of a consent the runtime keeps, so the person who added it removes it and adds it again`, which places nothing in the entry and is handed to `@on.auto/tool-servers`, decoded, with the rest of the connection's consent once the entry is read, below. A row the store cannot read now, for any of its three failures, `secret_key_unknown`, `custodian_refused` or `keeper_unavailable`, fails the call that needed it and breaks nothing: the source answers it unready, `connection_unavailable`, `The credential of the connection crm cannot be read now: `, records nothing, and the connection stays connected, so its credential is read again at its next use and the call is made once the store can read it. The source logs that sentence as a warning, annotated with the org and the connection's id, once for each cause of each connection in the process, the cause being the store's words, so a call that meets the same cause logs nothing more, one that meets another logs it, and one after a read that succeeded logs it again; the source keeps the last cause logged of each connection for as long as it lives, and forgets it at a read that succeeds. Only a row that is gone, and a value that is not the tokens the connection expects, break the connection, since nothing mends them: the source issues `break`, records `connection_broken` with the sentence of why, `The credential of the connection crm is no longer in the secret store, so the person who added it removes it and adds it again` or the sentence of tokens above, and answers it `connection_broken` in it. Nothing un-breaks a connection; the person removes it and adds it again. 4. The credential is placed as the file places a reference (`src/serving/placed-credentials.ts`): the entry, every `$` in it written `$$` so that it is sent as it is written, with a `secret` or a `reference` as `${…}` in the header, the query parameter or the security scheme its `in` and `name` say, after its `scheme` and a space where it names one, as in `Bearer ${…}`, or `client_credentials` as the `auth` block with its issuer, client id and scope, and the client secret, or the private key, as `${…}` beside the algorithm, the issuer, the client id, the scope and the algorithm each written with `$$` for `$` as the entry is; a `reference` names its own variable, and a value given on the page stands under a variable of its own, in an environment of that one entry. `placedReference` is that placing of a named reference, for client credentials as for a `reference`, to which `placedCredential` gives the variable the value stands under and its environment. In a connection made from a provider, the header and query values are read as the provider wrote them, its references substituted from `providerValues` and scrubbed as every reference's value is, and the rest of the entry as it is written. In one of client credentials or of the kind `authorization`, each `{client_id}` the entry holds stays as the provider wrote it, and the client id of the record's credential, the one the person gave on the page or the client the connection was made with, which `connection_connected` recorded, is handed to `readEntry` as its context's `placedValues` (`clientIdPlacedFor`, the one placing of it), which places it in the header and query values once the entry as written is judged, as its text, so the client id is sent where the provider says and, kept in the open in the record, is scrubbed from nothing; the reading at add places the stand-in client id. `readEntry` of `@on.auto/tool-servers`, the reading of an entry of the file, then resolves it and makes it `ServerSettings`, its type the connection's, its org the org and its `provider` the connection's (`readPlaced`), with `substituted`, `credentialProblems`, `unreferencedSecurity`, `misplacedReferencesOfEntry` and `secretsOfEntry` as they are for the file: every value the reference resolves to is a secret, the whole value of a security scheme too, and a value in a header that is a scheme and a credential, as `Bearer `, a secret whole and its credential alone, so the scrubber catches it in every form `@on.auto/tool-servers` lists. This is `readAddition`, the reading `add_connection` makes with a value standing in for the person's, made once here with the real value; what it refuses is `connection_not_allowed` in the words of step 2, so a check that a value can fail refuses in its own words, as an empty value of a reference does. For `authorization` no credential is placed: the entry is read as it is written, its `{client_id}` placed as above, and its `ServerSettings` take as `auth` the third kind of credential of `@on.auto/tool-servers`, `consent`, which `consentAuthOf` composes: the issuer, the client the connection recorded and the scope it was granted, with the way, the server and the return address `start` asks with, `reaches`, the row step 3 read, and `ConnectionTokens`, the store of that row, whose `read` reads it as step 3 does, whose `replace` and `claim` are the secret store's on it, and whose `broken(ended)` records the break as step 3 records one, the one place a break is recorded, in `The consent of the connection drive was revoked or has ended, so the person who added it removes it and adds it again` or in the sentence `finish` has for a client no longer registered, `The client the connection drive was made with is no longer registered, so the person who added it removes it and adds it again`, or, for a connection of `auto` whose key Auto's consent service does not know, `Auto's consent service does not know the key of the connection drive, so its tokens cannot be renewed, and the person who added it removes it and adds it again`. Each runs in the context the connection was prepared in, so its clock and its log are the server's. The consent client serves and renews the tokens from there, as its README says. The answer carries `reaches`, which answers a URL whose host `allowed_hosts` does not take with the sentence of the rule, `hostRefusalOf`; the tool access checks every request of the connection with it, the server its document names where `base_url` is left out among them, and the request of a redirect. diff --git a/packages/connections/src/serving/broken-connections.test.ts b/packages/connections/src/serving/broken-connections.test.ts index 2f6c016be..589fa78ca 100644 --- a/packages/connections/src/serving/broken-connections.test.ts +++ b/packages/connections/src/serving/broken-connections.test.ts @@ -9,8 +9,6 @@ import { ada, harness, operationNamed, type Harness, fixtureServing } from '../t import { makeConnectionSource } from './connection-source.ts'; import type { ServingServices } from './serving-services.ts'; -const sealedElsewhere = 'The secret is sealed under the key k2, which is neither the current key nor the previous one'; - function sourceOf(h: Harness, changes: Partial = {}): ConnectionSource { return makeConnectionSource({ registry: h.ledger.service, @@ -66,31 +64,33 @@ function loggedLines() { return { lines, logging }; } -function unreadableIn(h: Harness) { - return { ...h.secrets, read: () => Effect.fail({ detail: sealedElsewhere }) }; +function goneFrom(h: Harness) { + return { ...h.secrets, read: () => Effect.undefined }; } -function unreadableCounted(h: Harness) { +function goneCounted(h: Harness) { const reads = { made: 0 }; const secrets = { ...h.secrets, read: () => Effect.suspend(() => { reads.made += 1; - return Effect.fail({ detail: sealedElsewhere }); + return Effect.undefined; }), }; return { reads, secrets }; } -const brokenReason = `The credential of the connection crm cannot be read, so the person who added it removes it and adds it again: ${sealedElsewhere}`; +const brokenReason = + 'The credential of the connection crm is no longer in the secret store, so the person who added it removes it and adds it again'; -describe('a connection whose credential the secret store cannot read', () => { +describe('a connection whose credential the secret store no longer holds', () => { it('is broken by the runtime with the sentence of why, and answered broken from then on', async () => { const h = harness(); const id = await connectedCrm(h); + await Effect.runPromise(h.secrets.forget('secret-1')); - const refused = await refusalOf(sourceOf(h, { secrets: unreadableIn(h) }), id); + const refused = await refusalOf(sourceOf(h), id); expect(refused).toEqual({ because: 'connection_broken', detail: brokenReason }); expect((await h.facts()).at(-1)).toMatchObject({ @@ -103,18 +103,6 @@ describe('a connection whose credential the secret store cannot read', () => { unready: { because: 'connection_broken', detail: brokenReason }, }); }); - - it('is broken when the secret store no longer holds its credential', async () => { - const h = harness(); - const id = await connectedCrm(h); - await Effect.runPromise(h.secrets.forget('secret-1')); - - expect(await refusalOf(sourceOf(h), id)).toEqual({ - because: 'connection_broken', - detail: - 'The credential of the connection crm is no longer in the secret store, so the person who added it removes it and adds it again', - }); - }); }); describe('a connection the rules of the moment no longer allow', () => { @@ -165,7 +153,7 @@ describe('a connection the runtime cannot record broken', () => { const unrecorded = { ...h.ledger.service, execute: () => Effect.die(new Error('The ledger is not reachable')) }; const { lines, logging } = loggedLines(); - const refused = await refusalOf(sourceOf(h, { secrets: unreadableIn(h), registry: unrecorded }), id, logging); + const refused = await refusalOf(sourceOf(h, { secrets: goneFrom(h), registry: unrecorded }), id, logging); expect(refused).toEqual({ because: 'connection_broken', detail: brokenReason }); expect(lines).toEqual([expect.stringContaining('could not record it broken')]); @@ -179,7 +167,7 @@ describe('a break the registry does not take', () => { const id = await connectedCrm(h); const concurrent = new Conflict({ detail: 'The registry changed meanwhile', kind: 'concurrent_change' }); const changed = { ...h.ledger.service, execute: () => Effect.fail(concurrent) }; - const { reads, secrets } = unreadableCounted(h); + const { reads, secrets } = goneCounted(h); const { lines, logging } = loggedLines(); const source = sourceOf(h, { secrets, registry: changed }); @@ -192,7 +180,7 @@ describe('a break the registry does not take', () => { it('logs nothing when the connection was removed meanwhile', async () => { const h = harness(); const id = await connectedCrm(h); - const listed = await connectionOf(sourceOf(h, { secrets: unreadableIn(h) }), id); + const listed = await connectionOf(sourceOf(h, { secrets: goneFrom(h) }), id); await h.call(operationNamed(h.operations, 'remove_connection'), ada, { id }); const { lines, logging } = loggedLines(); diff --git a/packages/connections/src/serving/connection-source.ts b/packages/connections/src/serving/connection-source.ts index 89c9e4a16..6c12dba05 100644 --- a/packages/connections/src/serving/connection-source.ts +++ b/packages/connections/src/serving/connection-source.ts @@ -14,7 +14,7 @@ import { reachesUnder, useRefusalOf } from '../policy/policy-rules.ts'; import { providingOf } from '../providers/provider-additions.ts'; import { connectionsFold, connectionsStreamOfOrg } from '../registry/registry-decider.ts'; import { newestFirst, type KeptConnection } from '../registry/registry.ts'; -import { consentAuthOf, givenOf, notAllowed } from './given-credentials.ts'; +import { consentAuthOf, givenOf, loggedCauses, notAllowed } from './given-credentials.ts'; import type { Serving, ServingServices } from './serving-services.ts'; import { servingWords } from './serving-words.ts'; @@ -77,13 +77,14 @@ function orgConnectionOf(serving: Serving): OrgConnection { } export function makeConnectionSource(services: ServingServices): ConnectionSource { + const unreadable = loggedCauses(); return { takesConnections: services.policy.peopleMayAdd, connectionsOf: (org) => Effect.map(services.registry.load(connectionsStreamOfOrg(org), connectionsFold), ({ state }) => newestFirst(state) .filter((kept) => kept.connection.status !== 'removed') - .map((kept) => orgConnectionOf({ org, kept, services })), + .map((kept) => orgConnectionOf({ org, kept, services, unreadable })), ), }; } diff --git a/packages/connections/src/serving/consent-serving.test.ts b/packages/connections/src/serving/consent-serving.test.ts index 1edd05bf0..b049140b8 100644 --- a/packages/connections/src/serving/consent-serving.test.ts +++ b/packages/connections/src/serving/consent-serving.test.ts @@ -40,7 +40,7 @@ const keyUnknown = "Auto's consent service does not know the key of the connection files, so its tokens cannot be renewed, and the person who added it removes it and adds it again"; const notTokens = - 'The credential of the connection files cannot be read, so the person who added it removes it and adds it again: its value is not the tokens of a consent the runtime keeps'; + 'The credential of the connection files is not the tokens of a consent the runtime keeps, so the person who added it removes it and adds it again'; const gone = 'The credential of the connection files is no longer in the secret store, so the person who added it removes it and adds it again'; diff --git a/packages/connections/src/serving/given-credentials.ts b/packages/connections/src/serving/given-credentials.ts index 9eb9098f6..5bb327047 100644 --- a/packages/connections/src/serving/given-credentials.ts +++ b/packages/connections/src/serving/given-credentials.ts @@ -13,7 +13,7 @@ import { consentAskedOf } from '../page/consent-asking.ts'; import { placesNoValue } from '../registry/credential.ts'; import { byTheRuntime, connectionsStreamOfOrg, registryDecider } from '../registry/registry-decider.ts'; import type { GivenValue } from './placed-credentials.ts'; -import type { Serving } from './serving-services.ts'; +import type { LoggedCauses, Serving } from './serving-services.ts'; import { servingWords } from './serving-words.ts'; export type Given = Effect.Effect; @@ -45,13 +45,48 @@ function brokenWith({ org, kept, services }: Serving, reason: string): Effect.Ef }); } +export function loggedCauses(): LoggedCauses { + const lastLogged = new Map(); + return { + isNew: (connection, cause) => { + const isNew = lastLogged.get(connection) !== cause; + lastLogged.set(connection, cause); + return isNew; + }, + forget: (connection) => { + lastLogged.delete(connection); + }, + }; +} + +function unavailableNow({ org, kept, unreadable }: Serving, cause: string): Effect.Effect { + const { id, name } = kept.connection; + const unready: ConnectionUnready = { + because: 'connection_unavailable', + detail: servingWords.unreadableNow(name, cause), + }; + const loggedOnce = Effect.suspend(() => + unreadable.isNew(id, cause) + ? Effect.logWarning(unready.detail).pipe(Effect.annotateLogs({ org, connection: id })) + : Effect.void, + ); + return Effect.andThen(loggedOnce, Effect.fail(unready)); +} + +function readNow({ kept, unreadable }: Serving, stored: KeptRow): Effect.Effect { + return Effect.sync(() => { + unreadable.forget(kept.connection.id); + return stored; + }); +} + function keptSecretOf(serving: Serving): Effect.Effect { const { name } = serving.kept.connection; return serving.services.secrets.read(required(serving.kept.secretId)).pipe( Effect.matchEffect({ - onFailure: ({ detail }) => brokenWith(serving, servingWords.unreadable(name, detail)), + onFailure: ({ detail }) => unavailableNow(serving, detail), onSuccess: (stored) => - stored === undefined ? brokenWith(serving, servingWords.gone(name)) : Effect.succeed(stored), + stored === undefined ? brokenWith(serving, servingWords.gone(name)) : readNow(serving, stored), }), ); } @@ -79,7 +114,7 @@ function heldTokensOf(serving: Serving): Effect.Effect Option.match(keptTokensIn(value), { - onNone: () => brokenWith(serving, servingWords.unreadable(name, servingWords.notTokens)), + onNone: () => brokenWith(serving, servingWords.notTokens(name)), onSome: (tokens) => Effect.succeed({ tokens, version, claimedUntil }), }), ); diff --git a/packages/connections/src/serving/serving-services.ts b/packages/connections/src/serving/serving-services.ts index eac391bd0..6da24acc0 100644 --- a/packages/connections/src/serving/serving-services.ts +++ b/packages/connections/src/serving/serving-services.ts @@ -18,8 +18,14 @@ export interface ServingServices { readonly providerValues: ReadonlyMap; } +export interface LoggedCauses { + readonly isNew: (connection: string, cause: string) => boolean; + readonly forget: (connection: string) => void; +} + export interface Serving { readonly org: string; readonly kept: KeptConnection; readonly services: ServingServices; + readonly unreadable: LoggedCauses; } diff --git a/packages/connections/src/serving/serving-words.ts b/packages/connections/src/serving/serving-words.ts index 135e5e62c..b6e42e818 100644 --- a/packages/connections/src/serving/serving-words.ts +++ b/packages/connections/src/serving/serving-words.ts @@ -5,15 +5,16 @@ const removeAndAdd = 'the person who added it removes it and adds it again'; export const servingWords = { pending: (name: string) => `Nobody has approved the connection ${name} yet; the person who added it opens its link, or removes it and adds it again for a new link`, - unreadable: (name: string, detail: string) => - `The credential of the connection ${name} cannot be read, so ${removeAndAdd}: ${detail}`, + unreadableNow: (name: string, detail: string) => + `The credential of the connection ${name} cannot be read now: ${detail}`, gone: (name: string) => `The credential of the connection ${name} is no longer in the secret store, so ${removeAndAdd}`, unset: (name: string, variable: string) => `The variable ${variable} that the connection ${name} sends is not set on this runtime; whoever runs it sets the variable and starts it again`, unrecorded: (name: string) => `The runtime found the credential of the connection ${name} unreadable but could not record it broken; it reads the credential again, and records the break again, at the next use`, - notTokens: 'its value is not the tokens of a consent the runtime keeps', + notTokens: (name: string) => + `The credential of the connection ${name} is not the tokens of a consent the runtime keeps, so ${removeAndAdd}`, consentEnded: (name: string) => `The consent of the connection ${name} was revoked or has ended, so ${removeAndAdd}`, clientGone: (name: string) => `The client the connection ${name} was made with is no longer registered, so ${removeAndAdd}`, diff --git a/packages/connections/src/serving/unreadable-credentials.test.ts b/packages/connections/src/serving/unreadable-credentials.test.ts new file mode 100644 index 000000000..15043147d --- /dev/null +++ b/packages/connections/src/serving/unreadable-credentials.test.ts @@ -0,0 +1,179 @@ +import { custodianRefusal, memoryRows, type RecordingCustodian } from '@on.auto/secrets/testing'; +import type { ConnectionSource, ConnectionUnready, OrgConnection } from '@on.auto/tool-servers'; +import { + describedAccess, + fakeApiKey, + patientTiming, + playbookToolCalls, + serveFakeMcp, + type FakeMcpServer, +} from '@on.auto/tool-servers/testing'; +import { DateTime, Effect, Fiber, Logger, Option, Redacted } from 'effect'; +import { TestClock } from 'effect/testing'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { added } from '../testing/additions.ts'; +import { harness } from '../testing/harness.ts'; +import { givenThrough, sealedIn, sourceLoggingInto } from '../testing/sealed-connections.ts'; + +const closing: (() => Promise)[] = []; + +afterEach(async () => { + await Promise.all(closing.splice(0).map((close) => close())); +}); + +const now = Effect.map(DateTime.now, DateTime.toEpochMillis); + +async function fakeServerAt(): Promise { + const fake = await serveFakeMcp({ bearer: fakeApiKey }); + closing.push(fake.close); + return fake; +} + +function graphAt({ url }: FakeMcpServer) { + return { + name: 'graph', + type: 'mcp', + entry: { url }, + credential: { kind: 'secret', name: 'Authorization', in: 'header', scheme: 'Bearer' }, + }; +} + +function loggedLines() { + const lines: string[] = []; + const logging = Logger.layer([ + Logger.map(Logger.formatJson, (line: string) => { + lines.push(line); + }), + ]); + return { lines, logging }; +} + +async function connectionOf(source: ConnectionSource, id: string): Promise { + return Option.getOrThrow( + Option.fromNullishOr((await Effect.runPromise(source.connectionsOf('acme'))).find((each) => each.id === id)), + ); +} + +function servedOf({ state }: OrgConnection): Effect.Effect { + return state.status === 'connected' ? state.server : Effect.fail(state.unready); +} + +async function refusalOf(source: ConnectionSource, id: string): Promise { + return Effect.runPromise(Effect.flip(servedOf(await connectionOf(source, id)))); +} + +const searched = { server: 'graph', tool: 'search', arguments: { query: 'acme' } }; + +function unreadableNow(detail: string): string { + return `The credential of the connection graph cannot be read now: ${detail}`; +} + +function refusedBy(custodian: RecordingCustodian): string { + return unreadableNow(`The custodian of the key ${custodian.kid} refused to unwrap a data key: ${custodianRefusal}`); +} + +function refusedCallBy(custodian: RecordingCustodian) { + return { + status: 'rejected', + reason: 'unavailable', + kind: 'tool_not_offered', + because: 'connection_unavailable', + detail: refusedBy(custodian), + }; +} + +function warningsIn(lines: readonly string[]): readonly string[] { + return lines.filter((line) => line.includes('"level":"WARN"')); +} + +describe('a connection whose credential the secret store cannot read now', () => { + it('fails a tool call in the words of the store, records nothing and stays connected, logs the cause once, and is called once the custodian answers again', async () => { + const fake = await fakeServerAt(); + const h = harness(); + const { custodian, store } = sealedIn(memoryRows({ now })); + const id = await givenThrough(h, store, graphAt(fake), fakeApiKey); + const pending = await added(h, { ...graphAt(fake), name: 'notes' }); + const { lines, logging } = loggedLines(); + const source = sourceLoggingInto(h, store, logging); + const { access } = await describedAccess({}, { timing: patientTiming, connections: source }); + closing.push(access.close); + const calls = playbookToolCalls(access); + + custodian.refuse(); + const refused = [await calls.call('execute', searched), await calls.call('execute', searched)]; + const listedWhileRefused = (await connectionOf(source, id)).state.status; + custodian.answer(); + const answered = await calls.call('execute', searched); + + expect(refused).toEqual([refusedCallBy(custodian), refusedCallBy(custodian)]); + expect(listedWhileRefused).toBe('connected'); + expect((await connectionOf(source, pending.id)).state.status).toBe('unready'); + expect(answered).toMatchObject({ status: 'succeeded' }); + expect((await h.facts()).map(({ type }) => type)).not.toContain('connection_broken'); + expect(warningsIn(lines)).toEqual([expect.stringContaining(`"message":"${refusedBy(custodian)}"`)]); + expect(warningsIn(lines)[0]).toContain(`"annotations":{"org":"acme","connection":"${id}"}`); + expect(lines.join('\n')).not.toContain(fakeApiKey); + }); +}); + +describe('the log of a credential that cannot be read now', () => { + it('holds each cause once, not one line a call, again when the cause changes, and again once a read succeeded between', async () => { + const fake = await fakeServerAt(); + const h = harness(); + const rows = memoryRows({ now }); + const { custodian, store } = sealedIn(rows); + const id = await givenThrough(h, store, graphAt(fake), fakeApiKey); + const secretId = String(rows.kept()[0]?.id); + const other = sealedIn(rows); + const { lines, logging } = loggedLines(); + const source = sourceLoggingInto(h, store, logging); + + custodian.refuse(); + const refusals = [await refusalOf(source, id), await refusalOf(source, id)]; + await Effect.runPromise(other.store.replace(secretId, 1, Redacted.make(fakeApiKey))); + refusals.push(await refusalOf(source, id), await refusalOf(source, id)); + custodian.answer(); + await Effect.runPromise(store.replace(secretId, 2, Redacted.make(fakeApiKey))); + await Effect.runPromise(servedOf(await connectionOf(source, id))); + custodian.refuse(); + refusals.push(await refusalOf(source, id)); + + const keyUnknown = unreadableNow( + `The secret is sealed under the key ${other.custodian.kid}, which is neither the current key nor the previous one`, + ); + expect(refusals).toEqual( + [refusedBy(custodian), refusedBy(custodian), keyUnknown, keyUnknown, refusedBy(custodian)].map((detail) => ({ + because: 'connection_unavailable', + detail, + })), + ); + expect(warningsIn(lines).map((line) => line.includes(keyUnknown))).toEqual([false, true, false]); + }); +}); + +describe('a connection whose custodian does not answer', () => { + it('is unavailable once the store has waited 30 seconds, and never broken', async () => { + const fake = await fakeServerAt(); + const h = harness(); + const { custodian, store } = sealedIn(memoryRows({ now })); + const id = await givenThrough(h, store, graphAt(fake), fakeApiKey); + const served = servedOf(await connectionOf(sourceLoggingInto(h, store, loggedLines().logging), id)); + + const refused = await Effect.runPromise( + Effect.gen(function* () { + const asked = custodian.stall(); + const serving = yield* Effect.forkChild(Effect.flip(served)); + yield* Effect.promise(() => asked); + yield* TestClock.adjust('30 seconds'); + return yield* Fiber.join(serving); + }).pipe(Effect.provide(TestClock.layer())), + ); + + expect(refused).toEqual({ + because: 'connection_unavailable', + detail: unreadableNow('The keeper of the secret, or the custodian of its key, did not answer within 30 seconds'), + }); + expect((await h.facts()).map(({ type }) => type)).not.toContain('connection_broken'); + }); +}); diff --git a/packages/connections/src/testing/sealed-connections.ts b/packages/connections/src/testing/sealed-connections.ts new file mode 100644 index 000000000..4cc287e53 --- /dev/null +++ b/packages/connections/src/testing/sealed-connections.ts @@ -0,0 +1,60 @@ +import { sealedKeeper, secretStoreOn, type SecretRows, type SecretStore } from '@on.auto/secrets'; +import { recordingCustodian, type RecordingCustodian } from '@on.auto/secrets/testing'; +import type { ConnectionSource, OrgConnection } from '@on.auto/tool-servers'; +import { Effect, type Layer } from 'effect'; + +import { makeConnectionOperations } from '../operations/connection-operations.ts'; +import { connectionPolicyOf } from '../policy/connection-policy.ts'; +import { makeConnectionSource } from '../serving/connection-source.ts'; +import { added } from './additions.ts'; +import { ada, fixtureServing, operationNamed, type Harness } from './harness.ts'; + +export interface Sealed { + readonly rows: SecretRows; + readonly custodian: RecordingCustodian; + readonly store: SecretStore; +} + +export function sealedIn(rows: SecretRows, custodian: RecordingCustodian = recordingCustodian()): Sealed { + return { rows, custodian, store: secretStoreOn(rows, sealedKeeper({ current: custodian }), { current: custodian }) }; +} + +export async function givenThrough(h: Harness, secrets: SecretStore, input: unknown, value: string): Promise { + const giving = makeConnectionOperations({ + ...fixtureServing, + secrets, + policy: connectionPolicyOf({}), + fileEntries: [], + modelProviders: [], + access: 'people', + consent: h.consent, + }); + const { id, token } = await added(h, input); + await h.call(operationNamed(giving, 'give_credential'), ada, { id, token, value }); + return id; +} + +function servedLoggingInto(connection: OrgConnection, logging: Layer.Layer): OrgConnection { + const { state } = connection; + return state.status === 'connected' + ? { ...connection, state: { status: 'connected', server: Effect.provide(state.server, logging) } } + : connection; +} + +export function sourceLoggingInto(h: Harness, secrets: SecretStore, logging: Layer.Layer): ConnectionSource { + const source = makeConnectionSource({ + ...fixtureServing, + registry: h.ledger.service, + secrets, + policy: connectionPolicyOf({}), + variables: new Map(), + modelProviders: [], + }); + return { + ...source, + connectionsOf: (org) => + Effect.map(source.connectionsOf(org), (connections) => + connections.map((connection) => servedLoggingInto(connection, logging)), + ), + }; +} diff --git a/packages/definitions/src/runs/run.ts b/packages/definitions/src/runs/run.ts index 1835fa0da..fb70b1eb6 100644 --- a/packages/definitions/src/runs/run.ts +++ b/packages/definitions/src/runs/run.ts @@ -41,7 +41,7 @@ export const RunRejectionSchema = Schema.Union([ because: Schema.optionalKey( RejectionBecauseSchema.annotate({ description: - 'Why. With model_not_offered: provider_not_configured when its provider is not set up on this server while others are, model_not_allowed when it is outside the models the operator allows. With tool_not_offered: tool_server_not_configured, tool_not_allowed or tool_not_listed, or connection_pending, connection_broken or connection_not_allowed for a connection a person added. With tool_server_failed: failing, rate_limited, unreachable or key_refused. With tools_unfinished: server_failed, tool_error, model_unavailable, run_bound or no_answer', + 'Why. With model_not_offered: provider_not_configured when its provider is not set up on this server while others are, model_not_allowed when it is outside the models the operator allows. With tool_not_offered: tool_server_not_configured, tool_not_allowed or tool_not_listed, or connection_pending, connection_broken, connection_not_allowed or connection_unavailable for a connection a person added. With tool_server_failed: failing, rate_limited, unreachable or key_refused. With tools_unfinished: server_failed, tool_error, model_unavailable, run_bound or no_answer', }), ), }), diff --git a/packages/operations/README.md b/packages/operations/README.md index 6edd9b1cf..9b76ed70c 100644 --- a/packages/operations/README.md +++ b/packages/operations/README.md @@ -76,18 +76,18 @@ A command sets `irreversible: true` when what it does cannot be undone, such as `Unavailable` and `Conflict` may carry a `kind` and with it a `because`, one of `RejectionBecauseSchema`. Both reach the rejected outcome, where `explanationOf` and `unsuccessfulWords` turn them into plain words, and the problem document, as its extension members `kind` and `because`. An operation's `plainLanguage` may give `remedies` by `because`, which `unsuccessfulWords` puts in place of the shared remedy for that operation alone. -| `kind` | What could not be used | `because` | -| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `model_not_offered` | a model the server does not offer | `provider_not_configured`, its provider is not set up while others are; `model_not_allowed`, it is outside what the operator allows | -| `tool_not_offered` | a tool the server does not offer | `tool_server_not_configured`, its server is not configured for the org and playbook; `tool_not_allowed`, it is outside what the operator allows; `tool_not_listed`, its server does not list it; `connection_pending`, `connection_broken` and `connection_not_allowed`, its server is a connection nobody has approved yet, whose credential cannot be read, or that the runtime does not use, for the reason its detail gives, which its words follow | -| `tool_server_failed` | a tool server, before any call was recorded | `failing`, it kept failing; `rate_limited`, it asked to wait longer than a run waits; `unreachable`, it could not be reached in time; `key_refused`, it did not accept the key the server gives it, whose words say to check that key rather than to try again | -| `tools_unfinished` | anything, after a tool call was recorded, where every tool the run called is one its server marks `readOnlyHint`, so that running it again is safe | `server_failed`, a tool server failed; `tool_error`, the tool answered an error; `model_unavailable`, the model could not be called; `run_bound`, the run ran out of time; `no_answer`, the model still called tools when it had to answer | -| `rebuilding` | the view a recall function keeps, which is still being built from the playbook's history | none; trying again later may succeed, and the kind has a problem type of its own, `kindsWithTypes` | -| `requests_full` | room for one more open request in the playbook | none; trying again once some are answered, expire or are cancelled may succeed | -| `builds_full` | room for one more build, of the function in a day or of the server at once | none, every build the server may score at once being scored; trying again once one has scored may succeed. `builds_of_the_day`, the function has had its builds of the day, which only a try after the time the details name resolves | -| `dataset_not_ready` | the dataset a prediction function reads, which is still being built from the playbook's history or stopped at an event | none; trying again once `get_definition` shows it live may succeed. `stalled`, its dataset stopped at an event its fold could not take, which no retry resolves: `get_definition` names the event, and a change of the data part folds a dataset of its own | -| `model_process_busy` | a model process of the server, every one of which is taken | none; trying again shortly may succeed | -| `model_process_failed` | the model process that ran a question, which ended or answered a number that is not finite before it answered, or that checked a build's baselines | none, the process ended; trying again may succeed. `not_finite`, the model answered something other than finite numbers, which it answers again however often it is asked. `not_in_order`, the model answered a low above its likely value or a high below it, which no series holds and which it answers again too. `baseline_check`, the process that checks a build's baselines at its start ended before it answered; asking for the build again may succeed | +| `kind` | What could not be used | `because` | +| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `model_not_offered` | a model the server does not offer | `provider_not_configured`, its provider is not set up while others are; `model_not_allowed`, it is outside what the operator allows | +| `tool_not_offered` | a tool the server does not offer | `tool_server_not_configured`, its server is not configured for the org and playbook; `tool_not_allowed`, it is outside what the operator allows; `tool_not_listed`, its server does not list it; `connection_pending`, `connection_broken`, `connection_not_allowed` and `connection_unavailable`, its server is a connection nobody has approved yet; one whose credential is gone from the secret store or is not the tokens of a consent, whose consent was revoked or has ended, or whose client is no longer registered; one the runtime does not use; or one whose credential cannot be read just now, so trying again later can work; for the reason its detail gives, which its words follow | +| `tool_server_failed` | a tool server, before any call was recorded | `failing`, it kept failing; `rate_limited`, it asked to wait longer than a run waits; `unreachable`, it could not be reached in time; `key_refused`, it did not accept the key the server gives it, whose words say to check that key rather than to try again | +| `tools_unfinished` | anything, after a tool call was recorded, where every tool the run called is one its server marks `readOnlyHint`, so that running it again is safe | `server_failed`, a tool server failed; `tool_error`, the tool answered an error; `model_unavailable`, the model could not be called; `run_bound`, the run ran out of time; `no_answer`, the model still called tools when it had to answer | +| `rebuilding` | the view a recall function keeps, which is still being built from the playbook's history | none; trying again later may succeed, and the kind has a problem type of its own, `kindsWithTypes` | +| `requests_full` | room for one more open request in the playbook | none; trying again once some are answered, expire or are cancelled may succeed | +| `builds_full` | room for one more build, of the function in a day or of the server at once | none, every build the server may score at once being scored; trying again once one has scored may succeed. `builds_of_the_day`, the function has had its builds of the day, which only a try after the time the details name resolves | +| `dataset_not_ready` | the dataset a prediction function reads, which is still being built from the playbook's history or stopped at an event | none; trying again once `get_definition` shows it live may succeed. `stalled`, its dataset stopped at an event its fold could not take, which no retry resolves: `get_definition` names the event, and a change of the data part folds a dataset of its own | +| `model_process_busy` | a model process of the server, every one of which is taken | none; trying again shortly may succeed | +| `model_process_failed` | the model process that ran a question, which ended or answered a number that is not finite before it answered, or that checked a build's baselines | none, the process ended; trying again may succeed. `not_finite`, the model answered something other than finite numbers, which it answers again however often it is asked. `not_in_order`, the model answered a low above its likely value or a high below it, which no series holds and which it answers again too. `baseline_check`, the process that checks a build's baselines at its start ended before it answered; asking for the build again may succeed | Plain words have bounds: an outcome takes at most `mostOutcomeCharacters`, 400, and the words of a refusal at most `mostRefusalCharacters`, 600. `withinCharacters(text, most)` keeps the whole sentences that fit and ends with "The rest is in the details below.", since the detail is in the structured result beside the words; a first sentence longer than the bound is cut at a word with an ellipsis. diff --git a/packages/operations/src/outcome/rejection-because.ts b/packages/operations/src/outcome/rejection-because.ts index 47c533c72..307a0d687 100644 --- a/packages/operations/src/outcome/rejection-because.ts +++ b/packages/operations/src/outcome/rejection-because.ts @@ -9,6 +9,7 @@ export const RejectionBecauseSchema = Schema.Literals([ 'connection_pending', 'connection_broken', 'connection_not_allowed', + 'connection_unavailable', 'failing', 'rate_limited', 'unreachable', diff --git a/packages/operations/src/plain-language/connection-explanations.ts b/packages/operations/src/plain-language/connection-explanations.ts index 39547cb46..4018a1bcf 100644 --- a/packages/operations/src/plain-language/connection-explanations.ts +++ b/packages/operations/src/plain-language/connection-explanations.ts @@ -4,7 +4,7 @@ import type { Explanation, Remedies } from './plain-language.ts'; type ConnectionBecause = Extract< RejectionBecause, - 'connection_pending' | 'connection_broken' | 'connection_not_allowed' + 'connection_pending' | 'connection_broken' | 'connection_not_allowed' | 'connection_unavailable' >; export const connectionExplanations: Readonly, Explanation>> = { @@ -19,6 +19,7 @@ export const connectionBecauses: Readonly> = { connection_broken: 'because the credential of the connection it names can no longer be read', connection_not_allowed: 'because the connection it names cannot be used on this server, for the reason the details give', + connection_unavailable: 'because the credential of the connection it names cannot be read just now', }; export const connectionRemedies: Remedies = { @@ -28,4 +29,6 @@ export const connectionRemedies: Remedies = { 'Trying again will not help: the person removes the connection and adds it again, with its credential.', connection_not_allowed: 'Trying again will not help until what the details say is put right, or it names another tool server, which search shows.', + connection_unavailable: + 'Nothing needs to change in the connection: trying again later can work, once whoever runs the server puts right what the details say.', }; diff --git a/packages/operations/src/plain-language/tool-explanations.test.ts b/packages/operations/src/plain-language/tool-explanations.test.ts index 6b8bdbdc9..3577d105b 100644 --- a/packages/operations/src/plain-language/tool-explanations.test.ts +++ b/packages/operations/src/plain-language/tool-explanations.test.ts @@ -46,6 +46,12 @@ const toolEndings: ReadonlyArray { return state; } +const echoing = ['---', 'model: anthropic/claude-sonnet-4-5', 'tools: [crm/echo]', '---', 'Echo a note.'].join('\n'); + +async function ranEchoing(server: ReasoningServer) { + await server.call('POST', `${alpha}/definitions/reasoning`, { body: { name: 'echo', source: echoing } }); + return server.call('POST', `${alpha}/definitions/reasoning/echo/run`, { body: { input: {} } }); +} + function executed(server: ReasoningServer) { return server.call('POST', `${alpha}/tool-servers/crm/tools/echo/execute`, { body: { arguments: {} } }); } -describe('a connection whose credential the secret store of a restarted runtime cannot read', () => { - it('is broken once it is next used, the fact recorded, and listed broken in the sentence of why', async () => { +async function sealedAgainUnder(fileName: string, id: string, key: Readonly, version: number) { + const store = await secretStoreInFile(fileName, key); + const secretId = Option.getOrThrow(Option.fromNullishOr(await secretIdOf(fileName, 'acme', id))); + const replaced = await Effect.runPromise(store.replace(secretId, version, Redacted.make(`Bearer ${crmKey}`))); + await store.close(); + return replaced; +} + +describe('a connection whose credential the secret store of a restarted runtime cannot read now', () => { + it('is refused unavailable in the words of the store and listed so, stays connected with nothing recorded, and is called once the credential can be read again', async () => { const { fileName, id } = await connectedBeforeARestart(); - const restarted = await servedOn(fileName, { SECRETS_KEY: anotherKey }); + await sealedAgainUnder(fileName, id, anotherKey, 1); + const restarted = await servedOn(fileName); const refused = await executed(restarted); + const ran = await ranEchoing(restarted); const searched = await restarted.call('GET', `${alpha}/tool-servers/search`); const read = await restarted.call('GET', `/v1/orgs/acme/connections/${id}`); + const sealedAgain = await sealedAgainUnder(fileName, id, secretsKey, 2); + const called = await executed(restarted); - expect(refused).toMatchObject({ status: 503, body: { kind: 'tool_not_offered', because: 'connection_broken' } }); + expect(refused).toMatchObject({ + status: 503, + body: { kind: 'tool_not_offered', because: 'connection_unavailable', detail: unreadableNow }, + }); + expect(refused.headers.get('retry-after')).toBeNull(); + expect(ran).toMatchObject({ + status: 503, + body: { + reason: 'unavailable', + kind: 'tool_not_offered', + because: 'connection_unavailable', + detail: unreadableNow, + }, + }); expect(searched.body).toMatchObject({ - servers: [{ name: 'crm', connection: id, because: 'connection_broken', unavailable: unreadable }], + servers: [{ name: 'crm', connection: id, because: 'connection_unavailable', unavailable: unreadableNow }], }); - expect(read.body).toMatchObject({ status: 'broken', broken: brokenAsSaid }); - expect((await connectionFactsIn(fileName)).at(-1)).toBe('connection_broken'); + expect(read.body).toMatchObject({ status: 'connected' }); + expect(sealedAgain).toEqual({ replaced: true, version: 3 }); + expect(called.status).toBe(200); + expect(await connectionFactsIn(fileName)).not.toContain('connection_broken'); }); }); diff --git a/packages/server/src/connections/providers-searched.test.ts b/packages/server/src/connections/providers-searched.test.ts index a2b134930..687680c0e 100644 --- a/packages/server/src/connections/providers-searched.test.ts +++ b/packages/server/src/connections/providers-searched.test.ts @@ -1,8 +1,9 @@ import { plainTextIn, withMcpSession } from '@on.auto/api/testing'; import type { Provider } from '@on.auto/providers'; +import { Effect } from 'effect'; import { afterEach, describe, expect, it, onTestFinished } from 'vitest'; -import { secretsKey } from '../testing/records/kept-credentials.ts'; +import { secretIdOf, secretsKey, secretStoreInFile } from '../testing/records/kept-credentials.ts'; import { temporaryLedger, type TemporaryLedger } from '../testing/records/temporary-ledger.ts'; import { addedConnection, @@ -48,7 +49,11 @@ function searchedAs(states: ReadonlyMap { + const store = await secretStoreInFile(fileName, secretsKey); + await Effect.runPromise(store.forget(required(await secretIdOf(fileName, 'acme', connection)))); + await store.close(); +} let ledger: TemporaryLedger | undefined; @@ -112,13 +117,14 @@ describe('a provider a connection of the org serves', () => { ledger = temporaryLedger(); const kept = { LEDGER_FILE: ledger.fileName }; const first = await served({ ...kept, SECRETS_KEY: secretsKey.toString('base64') }); - await connectedWith( + const lost = await connectedWith( first, { provider: nameOf(brokenOne), credential: byAToken }, - 'token-sealed-under-the-first-key', + 'token-lost-before-the-restart', ); await first.stop(); - const restarted = await served({ ...kept, SECRETS_KEY: anotherKey }); + await forgottenIn(ledger.fileName, lost); + const restarted = await served({ ...kept, SECRETS_KEY: secretsKey.toString('base64') }); await restarted.call('POST', '/v1/orgs/acme/playbooks', { body: { playbook: 'alpha', name: 'Alpha' } }); const pending = await addedConnection(restarted, { provider: nameOf(pendingOne), credential: byAToken }); diff --git a/packages/tool-servers/README.md b/packages/tool-servers/README.md index 94e218edf..066f425d1 100644 --- a/packages/tool-servers/README.md +++ b/packages/tool-servers/README.md @@ -59,7 +59,7 @@ Nothing is learned from a server when the settings are read: they are read witho - `listServersInOrg({ org, playbook }, named?)`: what `listServers` answers, as `servers`, and beside it `settings`, the fields of every server that serves the org, whatever `playbook` and `named` keep, the file's and the org's connections in every state, from the same reading of the org's servers and with no client opened, by which the org's `search` judges which providers a server serves already. - `close()`: ends every session and stops every process, those of connections among them, when the server stops. -`open` fails with `ToolNotOffered` when a reference names a server not configured for the run's org and playbook (`tool_server_not_configured`), a tool the operator does not allow (`tool_not_allowed`), a tool its server does not list (`tool_not_listed`), or a connection that is not ready (`connection_pending`, `connection_broken` or `connection_not_allowed`), and with `ToolServerFailed`, whose tag is `tool_server_failed`, when a server cannot be used (`unreachable`, `failing`, `rate_limited`, or `key_refused` when it answers HTTP 401 to the key the runtime gives it). Otherwise it connects to each named server, lists its tools once, and keeps that listing for the run: a `list_changed` notification changes nothing, and a call to a tool the server no longer has is a tool error. `server/*` offers every listed tool the operator allows. +`open` fails with `ToolNotOffered` when a reference names a server not configured for the run's org and playbook (`tool_server_not_configured`), a tool the operator does not allow (`tool_not_allowed`), a tool its server does not list (`tool_not_listed`), or a connection that is not ready (`connection_pending`, `connection_broken`, `connection_not_allowed`, or `connection_unavailable` for one whose credential cannot be read just now), and with `ToolServerFailed`, whose tag is `tool_server_failed`, when a server cannot be used (`unreachable`, `failing`, `rate_limited`, or `key_refused` when it answers HTTP 401 to the key the runtime gives it). Otherwise it connects to each named server, lists its tools once, and keeps that listing for the run: a `list_changed` notification changes nothing, and a call to a tool the server no longer has is a tool error. `server/*` offers every listed tool the operator allows. `RunTools` offers each tool, its description and input schema scrubbed of the secrets of every server as the listing's are, under its model-facing name, `server__tool` for a tool of either kind, with characters outside letters, digits and underscore mapped to underscores, cut to 64 characters with an 8-character hash when longer or when two names would collide. A numeric suffix resolves any remaining collision, so every tool offered in a run has a distinct name. The cut and the suffix take only from the end, and a server's name has no `__` and no `_` at its end once mapped, so every name splits at its first `__` into the server and the rest; a tool a run offers that is not a tool server's, such as the `json` tool a provider makes of a JSON output, holds no `__`, which keeps the two apart. Its description is cut to 4 KiB and the server's input schema is unchanged. It also says whether the calls have ended (`callsEnded`, `ended`, `ending`), whether the run called any tool (`calledAny`), whether every tool it called is one its server marks `readOnlyHint` (`calledOnlyReadOnly`), which decides whether a run that cannot finish ends `tools_unfinished` or `effect_unknown`, and the tools it used in words (`usedInWords`), and lets the run's servers go (`close`). @@ -96,7 +96,7 @@ Five operations at the playbook's scope, under `/tool-servers`, reach the server ### Searching the tools -`defineSearch(access)` makes `search`, `GET /tool-servers/search` relative to the playbook (`src/search`). Its input is `terms`, one string of up to 8 words of up to 64 characters separated by spaces, since a query string carries no list, `server` and `only`, `read` or `write`; it answers `{ servers, tools, scopes }`. Each server, sorted by name and a file's entry before a connection of its name, is `{ name, type, record_content, connection?, provider, tool_count }`, `connection` the id of a connection and absent for a file's entry, `provider` the name of the provider a connection was added from or a file's entry names, and `null` for every other, its count of the tools its entry allows that `only` keeps, or `{ name, type, record_content, connection?, provider, unavailable, because }`, `key_refused` when it did not accept the key the runtime gives it (HTTP 401), which only the operator can put right, `connection_pending`, `connection_broken` or `connection_not_allowed` for a connection that is not ready, its words its own, otherwise `failing`, `rate_limited` or `unreachable`; each tool is `{ server, tool, summary, read_only }`, the summary the first sentence of its description, at most 300 characters, and `read_only` whether its server marks it `readOnlyHint`; `scopes` names the servers searched. Without `terms` it answers every tool while the servers offer at most 100 together, in the order of the servers and of each server's list, and the servers alone, `tools` empty, when they offer more, with words that say to search by terms. With `terms`, each tool is scored (`scoreOf` in `src/search/search-scoring.ts`, the one scorer of a name, the names of arguments and a description, which ranks the providers of the org's search too): the terms and a tool's name, the names of the properties of its input schema and its description are split into words at anything but a letter or a digit and at case changes, so `listCustomers`, `list_customers` and `list.customers` hold the same two words, compared whatever their case; each word of the terms counts 3 where it is a word of the name, 2 where it is a word of a property's name and 1 where it is a word of the description, and the tools with a score are answered best first, those of one score by name and then by server, at most 100. `only: read` keeps the tools marked read-only, `only: write` the others, before the count and the rule of 100. `server` keeps only the server of that name, and asks no other; a name that no server serving the playbook has is `invalid_input` at `/server`, whose words say so and to call it without `server`. Its words name each server and its tools, twenty at most, or the tools found best first, and say which servers could not be asked, a connection that is not ready in its own words; a playbook with no server is told that whoever runs the server sets one up and, on a runtime whose access `takesConnections`, that a person adds one to the org as a connection, or, where the org's search lists a provider, that a person connects a provider it lists, as the give-tools guide says; the org's search, which `/mcp` and the org endpoint serve beside `add_connection`, names it, `with add_connection where your tools include it`, and the playbook's, which a playbook endpoint serves without it, does not. +`defineSearch(access)` makes `search`, `GET /tool-servers/search` relative to the playbook (`src/search`). Its input is `terms`, one string of up to 8 words of up to 64 characters separated by spaces, since a query string carries no list, `server` and `only`, `read` or `write`; it answers `{ servers, tools, scopes }`. Each server, sorted by name and a file's entry before a connection of its name, is `{ name, type, record_content, connection?, provider, tool_count }`, `connection` the id of a connection and absent for a file's entry, `provider` the name of the provider a connection was added from or a file's entry names, and `null` for every other, its count of the tools its entry allows that `only` keeps, or `{ name, type, record_content, connection?, provider, unavailable, because }`, `key_refused` when it did not accept the key the runtime gives it (HTTP 401), which only the operator can put right, `connection_pending`, `connection_broken`, `connection_not_allowed` or `connection_unavailable` for a connection that is not ready, its words its own, otherwise `failing`, `rate_limited` or `unreachable`; each tool is `{ server, tool, summary, read_only }`, the summary the first sentence of its description, at most 300 characters, and `read_only` whether its server marks it `readOnlyHint`; `scopes` names the servers searched. Without `terms` it answers every tool while the servers offer at most 100 together, in the order of the servers and of each server's list, and the servers alone, `tools` empty, when they offer more, with words that say to search by terms. With `terms`, each tool is scored (`scoreOf` in `src/search/search-scoring.ts`, the one scorer of a name, the names of arguments and a description, which ranks the providers of the org's search too): the terms and a tool's name, the names of the properties of its input schema and its description are split into words at anything but a letter or a digit and at case changes, so `listCustomers`, `list_customers` and `list.customers` hold the same two words, compared whatever their case; each word of the terms counts 3 where it is a word of the name, 2 where it is a word of a property's name and 1 where it is a word of the description, and the tools with a score are answered best first, those of one score by name and then by server, at most 100. `only: read` keeps the tools marked read-only, `only: write` the others, before the count and the rule of 100. `server` keeps only the server of that name, and asks no other; a name that no server serving the playbook has is `invalid_input` at `/server`, whose words say so and to call it without `server`. Its words name each server and its tools, twenty at most, or the tools found best first, and say which servers could not be asked, a connection that is not ready in its own words; a playbook with no server is told that whoever runs the server sets one up and, on a runtime whose access `takesConnections`, that a person adds one to the org as a connection, or, where the org's search lists a provider, that a person connects a provider it lists, as the give-tools guide says; the org's search, which `/mcp` and the org endpoint serve beside `add_connection`, names it, `with add_connection where your tools include it`, and the playbook's, which a playbook endpoint serves without it, does not. `defineSearchInOrg(access, lookUpPlaybook, offering?)` makes the same name at the org's scope, `GET /tool-servers/search` relative to the org, an org query permitted by `org:read` or `playbook:read`, as `list_playbooks` is, since a key that may read a playbook may see what serves it. Its input adds an optional `playbook`: without it, it searches every server that serves the org, by naming it in `org` or by `*`, each with `playbooks`, the playbooks it serves, as the listing gives them, `['*']` for an entry that leaves them out; with it, the servers that serve that playbook, as the playbook's own search has them, each with `playbooks` too, and the words of the playbook's search. A key limited to some playbooks is shown in `playbooks` only the playbooks it may access, and `['*']` as it is; the dispatcher refuses it the search of the whole org, since an org operation that takes `playbook` checks the caller's access to the playbook it is given, and none is given, with the detail `The caller may access only some playbooks of this org; name one of them in playbook`. Given a playbook, it first looks it up with `lookUpPlaybook`, a `PlaybookLookup` that needs `OrgReader` and fails with `NotFound`, since the org's playbooks are the registry's and the dispatcher looks up no playbook of an org operation: the server gives `foundPlaybook` of `@on.auto/playbooks`, so a playbook the org does not have is `not_found` in the words of the playbook's own route, and no server is asked. A name no server of the org, or of the playbook, has is `invalid_input` at `/server`, whose words name the org or the playbook. Over MCP, `/mcp` offers this operation in place of the playbook's, under the same name, its `playbook` its own and optional (see the api README's [playbook argument](../api/README.md#over-mcp)), the org endpoint offers it too, and a playbook endpoint offers the playbook's. @@ -158,7 +158,7 @@ Every way in reads the servers of an org the same way (`src/org-servers/org-acce Each operation reads the org's connections once: the source of `@on.auto/connections` replays the org's stream of connections from the ledger on every reading, as the playbook registry replays the org's playbooks on every status it answers, and keeps nothing between readings, so what an operation reads is what the ledger holds as it starts. A listing, a check of names, an opening and one call each read once; a delivery, a read or a telling reads once for its start and its call together, through `preparedCall`, and `introspect`, `validate` and `dry_run` once for their check and their listing, through `lookUp`. A connection added, removed or broken meanwhile is seen at the next operation. -A connection's link is made when a run, a listing or a call first takes it, and kept by its org and id while the connection stays connected (`src/org-servers/connection-links.ts`), in two steps. The source's `server` reads the credential and checks the rules once for each connection a process holds, and its answer, ready or refused, is kept with the link, but for `connection_broken`: the source records a break, so the next reading lists the connection broken, and where it could not record it the next use reads the credential again and records the break again; within one operation, a check, a start and a call, a connection is prepared once whatever its answer. Once it is ready, the values of its credential join the access's secrets. Then the link: an `openapi` connection's document is read and described with the bounds and refusals `describedToolServers` applies to a file's document, through the connection's fetch, its tools left out reported through `reportLeftOut` (`src/org-servers/connection-linking.ts`). The description fails with one of two tagged errors: `DocumentUnread` (`src/openapi/document-reading.ts`), for a document that could not be read, since its address did not answer in time, answered an error or answered a page that is neither JSON nor YAML, such as a proxy's error page, which is `failing` in its words and read again when the connection is next used; and `DocumentRefused` (`src/openapi/described-api.ts`), for a document refused for what it is, too large, nested too deep, or a description the bounds or the checks of its security and its servers refuse, which is `failing` in its words and kept refused for as long as the process runs, so it is read once. A file's entry takes the problems of either alike, since its documents are read once at start; and a document whose first server, where `base_url` is left out, is an address the rules do not allow is `connection_not_allowed` in the rule's words. The link is the `serverLink` of a file's entry, its settings the connection's with `connection` its id, so the bridge, the MCP client, the bounds, the scrubbing, the recording and the failures are one code for both sources. `named` takes the first step alone and connects nowhere. A preparation or a link that fails in a way the runtime did not foresee, such as a source that dies, is that server's alone: it is listed and refused as `failing` in the failure's words, scrubbed, beside the servers that answer, and is let go, so the next use prepares or links it again. A connection the source no longer lists as connected, because it was removed or broken, has its link stopped at the next reading of its org's servers, by a listing, an opening, a check or a call, and every link is stopped at `close()`, each settled whatever another's stopping does. A removed connection's name may be taken by a new connection, which has a new id and a new link. +A connection's link is made when a run, a listing or a call first takes it, and kept by its org and id while the connection stays connected (`src/org-servers/connection-links.ts`), in two steps. The source's `server` reads the credential and checks the rules once for each connection a process holds, and its answer, ready or refused, is kept with the link, but for `connection_broken` and `connection_unavailable`: for `connection_broken` the source records a break, so the next reading lists the connection broken, and where it could not record it the next use reads the credential again and records the break again; for `connection_unavailable`, a credential the source cannot read just now, the connection stays connected and the next use prepares it again, so it is served once its credential can be read; within one operation, a check, a start and a call, a connection is prepared once whatever its answer. Once it is ready, the values of its credential join the access's secrets. Then the link: an `openapi` connection's document is read and described with the bounds and refusals `describedToolServers` applies to a file's document, through the connection's fetch, its tools left out reported through `reportLeftOut` (`src/org-servers/connection-linking.ts`). The description fails with one of two tagged errors: `DocumentUnread` (`src/openapi/document-reading.ts`), for a document that could not be read, since its address did not answer in time, answered an error or answered a page that is neither JSON nor YAML, such as a proxy's error page, which is `failing` in its words and read again when the connection is next used; and `DocumentRefused` (`src/openapi/described-api.ts`), for a document refused for what it is, too large, nested too deep, or a description the bounds or the checks of its security and its servers refuse, which is `failing` in its words and kept refused for as long as the process runs, so it is read once. A file's entry takes the problems of either alike, since its documents are read once at start; and a document whose first server, where `base_url` is left out, is an address the rules do not allow is `connection_not_allowed` in the rule's words. The link is the `serverLink` of a file's entry, its settings the connection's with `connection` its id, so the bridge, the MCP client, the bounds, the scrubbing, the recording and the failures are one code for both sources. `named` takes the first step alone and connects nowhere. A preparation or a link that fails in a way the runtime did not foresee, such as a source that dies, is that server's alone: it is listed and refused as `failing` in the failure's words, scrubbed, beside the servers that answer, and is let go, so the next use prepares or links it again. A connection the source no longer lists as connected, because it was removed or broken, has its link stopped at the next reading of its org's servers, by a listing, an opening, a check or a call, and every link is stopped at `close()`, each settled whatever another's stopping does. A removed connection's name may be taken by a new connection, which has a new id and a new link. The values of a connection's credential, its `secrets` and its `auth` credential, join the access's one set of secrets once its server is prepared (`Secrets.keep`), before its start is recorded and before its link is made, and are scrubbed for as long as the access lives, as a reference's value is for the file; a token minted for a connection is held under its id, so two connections of one name in two orgs never replace each other's tokens. A connection connected by consent holds no secret in its settings, so `secretsOfServer` lists nothing of it: its access token and its refresh token are two sources of the set, `connection access token` and `connection refresh token`, each put with `replace` by its token source from the moment the source reads them and again at every renewal, so each source holds its token and the one it replaced, as a minted token's does, and no record, answer or line the runtime writes holds either. Every request of a connection's link goes through `fetchWithin` of `@on.auto/outbound`, a fetch that takes a string or a `URL` and the fields its caller gives: the URL is checked with the connection's `reaches`, the rules' answer to its host, and refused before it is sent with `HostRefused`, carrying the rule, and the request is sent with `redirect: 'manual'`, so the runtime's `fetch` follows no redirect. `failureOf` finds `HostRefused` among the causes a client wraps it in, through `causesOf` of `@on.auto/outbound`, the walk `codesOf` takes too, eight causes at most, and answers it as `connection_not_allowed` in the rule's words alone, so a request to a refused host, the authorization server's and a redirect's among them, is refused as the rules refuse it, never as a failing server. The bridge sends every call with `redirect: 'manual'` and only to its base URL's origin and path, and answers a redirect as an error result; the MCP client follows a redirect only within the origin of the request, keeping its method, and sends each hop through the connection's fetch, which checks it again; its token and metadata requests go through the same fetch. So no request of a connection reaches a host the rules do not allow, redirects included. @@ -247,7 +247,7 @@ The arguments, as `JSON.stringify` writes them, and the protocol answer are scru - `fakeStdioServerPath` runs the same tools over stdio under `node`, with `--chatter`, `--pad`, `--stdout`, `--linger-ms` and `--start-once` to make it talk, misbehave or fail to start again. Start it with `process.execPath` directly, never through a package manager: it imports only the MCP server package and the fake's tools, not `effect`, and answers its first message in about 90 ms on a laptop. A spawn is real time, so a test that starts it takes `stdioTestTimeoutMs`, 30 s, and its client `patientTiming`, and a file shares one process across the tests that only need it running. - Its tools answer text (`search`, `echo`, `graph.query.v2`), structured content (`profile`), non-text content (`photo`), a denial marked `isError` (`denied`), an error (`broken`), slowly (`sleep`), at length (`large`), the channels of a workspace (`list_channels`, `fakeChannels`), with its environment (`environment`), or by exiting (`exit`). Their annotations cover every case of a test: `search`, `profile`, `photo`, `denied`, `sleep`, `large`, `broken` and `list_channels` are marked read-only, `graph.query.v2` read-only and destructive at once, `exit` destructive, the long-named tool only additive (`destructiveHint: false`), `echo` and `verbose` have none, `environment` has annotations with a title and no hint, and `verbose` has a title of its own. - `openFakeToolRun`, `reportingAccess`, `noToolServers`, an access with no server, `recordingCallJournal`, `toolRun`, `controlledSignals` and `inTurn` open a run's tools against it, and `playbookToolCalls` dispatches the five operations over a memory ledger, optionally one that refuses every write, as `toolCaller`, a caller that may write the playbook, unless the call names another, such as `toolReader`, which may only read it. -- `reportingAccess` and `describedAccess` take `connections`, a source of connections, `consent`, the consent client of the access, `consentClientWith()` where it is left out, and `now`, the clock its token sources read. Inside the package, `fakeConnections()` (`src/testing/fake-connections.ts`) is one: it reads each connection it is given as the file reads an entry, from an entry with references and the environment that resolves them, serves it with the `provider` it is given, `null` where it is given none, and with the `auth` it is given in place of the one its entry reads, the tokens of a consent among them, and answers it connected, pending, broken by `breakOff`, broken when its credential is next read by `loseTheCredentialOf`, refused by a rule by `disallow`, broken without the break recorded by `loseTheCredentialUnrecordedOf`, dying once while it is prepared by `dieOnceWhilePreparing`, or removed, and counts the reads of credentials and of the org's connections. `pendingWords`, the sentence it answers for a pending connection, is exported for `@on.auto/connections`, whose tests hold it to the sentence of its own source. +- `reportingAccess` and `describedAccess` take `connections`, a source of connections, `consent`, the consent client of the access, `consentClientWith()` where it is left out, and `now`, the clock its token sources read. Inside the package, `fakeConnections()` (`src/testing/fake-connections.ts`) is one: it reads each connection it is given as the file reads an entry, from an entry with references and the environment that resolves them, serves it with the `provider` it is given, `null` where it is given none, and with the `auth` it is given in place of the one its entry reads, the tokens of a consent among them, and answers it connected, pending, broken by `breakOff`, broken when its credential is next read by `loseTheCredentialOf`, refused by a rule by `disallow`, broken without the break recorded by `loseTheCredentialUnrecordedOf`, `connection_unavailable` at each use, in the words it is given, by `withholdTheCredentialOf` until it is connected again, dying once while it is prepared by `dieOnceWhilePreparing`, or removed, and counts the reads of credentials and of the org's connections. `pendingWords`, the sentence it answers for a pending connection, is exported for `@on.auto/connections`, whose tests hold it to the sentence of its own source. - `toolAccessBehaviour(serverUnderTest, sourceUnderTest)` is the shared behaviour suite (`src/access-testing`). Over each of `serversUnderTest`, an MCP server keyed by a bearer, an MCP server of client credentials, an HTTP API keyed by a header, one keyed by a security scheme in its query, one whose document declares no key, sent as a query parameter of the entry (`declaresKey: false` of `serveFakeApi`), and one of client credentials, each described by its entry, its entry without the credential and where its credential goes, it proves the recording, the content kept, the credential scrubbed in every form a call carries it, raw, base64, query-encoded, percent-encoded, as JSON and as JSON within JSON, from the answer, from what is kept and from the start of a call recorded before any link is made, the request ids, the denials, the failures and the bounds. `behaviourKey` holds a character each encoding changes, so no form is the raw key again. The package runs it over the file's entries and over `fakeConnections` (`sources-under-test.ts`), and `@on.auto/connections` over its own source, for every kind of credential. `connection-behaviour.ts` proves over both types what a connection adds: the listing with its id, the opening and the check of names, one call, its id recorded, the refusals of one that is not ready in their words, a link stopped after removal, a name a new id reuses, and the file's entry winning a name. - `recordingTimer` stands in for the timer a link's options take, which bounds the end of a session: it records each wait and whether it was stopped, so a test of that deadline touches no global timer. `fetchWithDeletion` sends a session's `DELETE` to a handler of the test's own. diff --git a/packages/tool-servers/src/org-servers/connection-links.test.ts b/packages/tool-servers/src/org-servers/connection-links.test.ts index 22359af92..38ad39b50 100644 --- a/packages/tool-servers/src/org-servers/connection-links.test.ts +++ b/packages/tool-servers/src/org-servers/connection-links.test.ts @@ -204,3 +204,27 @@ describe('a connection whose credential cannot be read, and whose break could no expect([connections.credentialReads(), fake.seen()]).toEqual([2, []]); }); }); + +describe('a connection whose credential cannot be read now', () => { + it('is refused unavailable in the words of its source and still listed, and is prepared again at each use until it is served', async () => { + const fake = await fakeServer(); + const { access, connections } = connectedTo(fake); + const withheld = 'The credential of the connection graph cannot be read now: the key is disabled'; + connections.withholdTheCredentialOf('graph-id', withheld); + + const listed = await Effect.runPromise(access.listServers(alpha)); + const refused = await Effect.runPromise(access.callOnce(searchOf('graph'))); + connections.connect('acme', { + id: 'graph-id', + name: 'graph', + entry: { url: fake.url, headers: { Authorization: 'Bearer ${GRAPH_API_KEY}' } }, + environment: { GRAPH_API_KEY: fakeApiKey }, + }); + const called = await Effect.runPromise(access.callOnce(searchOf('graph'))); + + expect(listed).toMatchObject([{ name: 'graph', because: 'connection_unavailable', unavailable: withheld }]); + expect(refused).toMatchObject({ kind: 'unopened', refused: 'tool_not_offered', because: 'connection_unavailable' }); + expect(called).toMatchObject({ kind: 'answered', outcome: 'result' }); + expect(connections.credentialReads()).toBe(3); + }); +}); diff --git a/packages/tool-servers/src/org-servers/connection-links.ts b/packages/tool-servers/src/org-servers/connection-links.ts index 94f268ea0..89b74e277 100644 --- a/packages/tool-servers/src/org-servers/connection-links.ts +++ b/packages/tool-servers/src/org-servers/connection-links.ts @@ -1,7 +1,13 @@ import { Effect } from 'effect'; import { secretsOfServer, type Secrets } from '../bounds/secrets.ts'; -import type { ConnectionServer, ConnectionSource, ConnectionUnready, OrgConnection } from './connection-source.ts'; +import type { + ConnectionServer, + ConnectionSource, + ConnectionUnready, + OrgConnection, + UnreadyBecause, +} from './connection-source.ts'; import { unreadyServer, type Linked, type OrgServer, type Served } from './org-servers.ts'; import { serverFailed, type Unusable } from './server-refusals.ts'; @@ -78,9 +84,11 @@ function failedWith(settings: Served, secrets: Keeping): (error: unknown) => Unu }; } +const preparedAgainAtTheNextUse: ReadonlySet = new Set(['connection_broken', 'connection_unavailable']); + function isKeptForTheProcess(prepared: Prepared): boolean { if ('unready' in prepared) { - return prepared.unready.because !== 'connection_broken'; + return !preparedAgainAtTheNextUse.has(prepared.unready.because); } return 'server' in prepared; } diff --git a/packages/tool-servers/src/org-servers/connection-source.ts b/packages/tool-servers/src/org-servers/connection-source.ts index cf33f3d3a..3447bdb8d 100644 --- a/packages/tool-servers/src/org-servers/connection-source.ts +++ b/packages/tool-servers/src/org-servers/connection-source.ts @@ -13,7 +13,12 @@ export type ConnectionServing = Pick< 'name' | 'type' | 'orgs' | 'playbooks' | 'allowed' | 'record_content' | 'request_id' | 'provider' >; -export const unreadyReasons = ['connection_pending', 'connection_broken', 'connection_not_allowed'] as const; +export const unreadyReasons = [ + 'connection_pending', + 'connection_broken', + 'connection_not_allowed', + 'connection_unavailable', +] as const; const UnreadyBecauseSchema = Schema.Literals(unreadyReasons); diff --git a/packages/tool-servers/src/search/search-schemas.ts b/packages/tool-servers/src/search/search-schemas.ts index e26b58dbf..37bb3b53a 100644 --- a/packages/tool-servers/src/search/search-schemas.ts +++ b/packages/tool-servers/src/search/search-schemas.ts @@ -70,7 +70,7 @@ const unavailableFields = { unavailable: Schema.String.annotate({ description: 'Why the server could not be asked for its tools, in words' }), because: UnavailableBecauseSchema.annotate({ description: - 'key_refused when the server did not accept the key this server gives it, which only whoever runs this server can put right; failing, rate_limited or unreachable when asking again later may work; connection_pending when nobody has approved the connection yet, connection_broken when its credential cannot be read, so it is removed and added again, and connection_not_allowed when a rule of whoever runs this server no longer allows it', + 'key_refused when the server did not accept the key this server gives it, which only whoever runs this server can put right; failing, rate_limited or unreachable when asking again later may work; connection_pending when nobody has approved the connection yet, connection_broken when its credential is gone from the secret store or is not the tokens of a consent, its consent was revoked or has ended, or its client is no longer registered, so it is removed and added again, connection_not_allowed when a rule of whoever runs this server no longer allows it, and connection_unavailable when its credential cannot be read just now, so asking again later may work', }), }; diff --git a/packages/tool-servers/src/testing/fake-connections.ts b/packages/tool-servers/src/testing/fake-connections.ts index 77ad30a73..490a4ba24 100644 --- a/packages/tool-servers/src/testing/fake-connections.ts +++ b/packages/tool-servers/src/testing/fake-connections.ts @@ -27,6 +27,7 @@ type Standing = | { readonly status: 'broken'; readonly reason: string } | { readonly status: 'unreadable'; readonly reason: string } | { readonly status: 'unrecorded'; readonly reason: string } + | { readonly status: 'withheld'; readonly reason: string } | { readonly status: 'not_allowed'; readonly rule: string } | { readonly status: 'dying'; readonly reason: string }; @@ -42,6 +43,7 @@ export interface FakeConnections extends ConnectionSource { readonly breakOff: (id: string, reason: string) => void; readonly loseTheCredentialOf: (id: string, reason: string) => void; readonly loseTheCredentialUnrecordedOf: (id: string, reason: string) => void; + readonly withholdTheCredentialOf: (id: string, reason: string) => void; readonly disallow: (id: string, rule: string) => void; readonly dieOnceWhilePreparing: (id: string, reason: string) => void; readonly remove: (id: string) => void; @@ -85,6 +87,16 @@ function standingOf({ standings }: Held, id: string): Standing { return Option.getOrThrow(Option.fromNullishOr(standings.get(id))); } +function refusalAtUseOf(standing: Standing): ConnectionUnready | undefined { + if (standing.status === 'unreadable' || standing.status === 'unrecorded') { + return { because: 'connection_broken', detail: standing.reason }; + } + if (standing.status === 'withheld') { + return { because: 'connection_unavailable', detail: standing.reason }; + } + return standing.status === 'not_allowed' ? { because: 'connection_not_allowed', detail: standing.rule } : undefined; +} + function preparedOf(held: Held, { connection, settings }: Kept): Effect.Effect { return Effect.suspend((): Effect.Effect => { held.credentialRead(); @@ -96,12 +108,10 @@ function preparedOf(held: Held, { connection, settings }: Kept): Effect.Effect; + +function standingsSetIn({ standings }: Held): StandingsSet { + return { + breakOff: (id, reason) => { + standings.set(id, { status: 'broken', reason }); + }, + loseTheCredentialOf: (id, reason) => { + standings.set(id, { status: 'unreadable', reason }); + }, + loseTheCredentialUnrecordedOf: (id, reason) => { + standings.set(id, { status: 'unrecorded', reason }); + }, + withholdTheCredentialOf: (id, reason) => { + standings.set(id, { status: 'withheld', reason }); + }, + disallow: (id, rule) => { + standings.set(id, { status: 'not_allowed', rule }); + }, + dieOnceWhilePreparing: (id, reason) => { + standings.set(id, { status: 'dying', reason }); + }, + }; +} + export function fakeConnections(): FakeConnections { let credentialReads = 0; let orgReads = 0; @@ -142,21 +185,7 @@ export function fakeConnections(): FakeConnections { pend: (org, connection) => { keep(org, connection, { status: 'pending' }); }, - breakOff: (id, reason) => { - held.standings.set(id, { status: 'broken', reason }); - }, - loseTheCredentialOf: (id, reason) => { - held.standings.set(id, { status: 'unreadable', reason }); - }, - loseTheCredentialUnrecordedOf: (id, reason) => { - held.standings.set(id, { status: 'unrecorded', reason }); - }, - disallow: (id, rule) => { - held.standings.set(id, { status: 'not_allowed', rule }); - }, - dieOnceWhilePreparing: (id, reason) => { - held.standings.set(id, { status: 'dying', reason }); - }, + ...standingsSetIn(held), remove: (id) => { held.kept.delete(id); }, diff --git a/packages/tool-servers/src/tool-reads/dry-run-schema.ts b/packages/tool-servers/src/tool-reads/dry-run-schema.ts index 5b761dd72..151ddf5b8 100644 --- a/packages/tool-servers/src/tool-reads/dry-run-schema.ts +++ b/packages/tool-servers/src/tool-reads/dry-run-schema.ts @@ -10,7 +10,7 @@ const ReachedSchema = Schema.Struct({ because: Schema.optionalKey( UnavailableBecauseSchema.annotate({ description: - 'Why it could not be asked: key_refused when it did not accept the key this server gives it, which only whoever runs this server can put right; failing, rate_limited or unreachable when asking again later may work; connection_not_allowed when a rule of whoever runs this server no longer allows the connection', + 'Why it could not be asked: key_refused when it did not accept the key this server gives it, which only whoever runs this server can put right; failing, rate_limited or unreachable when asking again later may work; connection_not_allowed when a rule of whoever runs this server no longer allows the connection; connection_unavailable when the credential of the connection cannot be read just now, so asking again later may work', }), ), }); From ab5c89ae2d547253626c24076fd85517729afb73 Mon Sep 17 00:00:00 2001 From: Rami Date: Tue, 20 Oct 2026 14:31:56 +0100 Subject: [PATCH 3/3] feat(server): the key that signs a retry's state is the runtime's own, kept in its secret store The key that signs the state an MCP client sends back when it retries a call is no longer derived from the secrets key, which a custodian holding its key elsewhere leaves no process to derive from: the runtime keeps it for itself in its secret store and names it by a fact of its own. - src/database/own-keys.ts, beside ledger-stores.ts, which opens the store the key is read from, keeps the stream runtime/own-keys, of the runtime and of no org, as connections keep runtime/client-registrations: one fact, own_key_kept { purpose, secret_id }, by the runtime, folded by purpose, the latest last. ownKeysDecider refuses a second record for a purpose that has one, conflict taken. The one purpose is retry_state. - At start, once the store is open and rotated, the server folds the stream; where no key is recorded it keeps 32 random bytes in the store and records the fact under the version it read, and where another server recorded first it forgets its own row and takes that one. It reads the row and gives the API RetryStateKeys { current }, with no previous. - A row it cannot read stops the start as an OwnKeyUnavailable, in words that name the purpose, the key in force, the store's words and the two ways out: start with the key the ledger's secrets were sealed under, or give that key as SECRETS_KEY_PREVIOUS or SECRETS_KEY_FILE_PREVIOUS beside a new key. A row that is gone, one that holds something other than 32 bytes, and a key the store cannot keep stop it too; the store is closed first. The HKDF derivation and retryStateKeysOf of secrets-keys.ts go, with their tests. - @on.auto/connections exports byTheRuntime, which the fact is recorded by. - Tests: the decider's rules on their own; the same key at the next start with one fact and one row; the key another server recorded first taken and the server's own forgotten; the start stopped for a row under another key, a row gone, a value that is no key and a key not kept; the store closed on a failed start and left open on one that has the key; a retry of add_connection answered after a restart on the same ledger, across a rotation, and refused by a server of another ledger under the same SECRETS_KEY; two servers starting at once on one PostgreSQL ledger keeping one key in one fact and one row, skipped without a database; a server that cannot read the key not starting, in process and as a process; nothing of the key in its log over two starts. - The pages: the configuration rows of SECRETS_KEY and SECRETS_KEY_PREVIOUS no longer derive the key, the API README says it is the runtime's own and the same for every server sharing the ledger, the server README sets out runtime/own-keys beside runtime/client-registrations, and the security, production and connections pages say a key that cannot read it stops the start. Co-Authored-By: Claude Fable 5.1 --- docs/self-host/configuration.md | 4 +- docs/self-host/connections.md | 2 +- docs/self-host/production.md | 2 +- docs/self-host/security.md | 2 +- packages/api/README.md | 2 +- packages/connections/src/index.ts | 1 + packages/server/README.md | 6 + .../src/composition/composition-root.ts | 10 +- .../connections/connections-over-mcp.test.ts | 98 ++++++-- .../src/connections/served-connections.ts | 9 +- packages/server/src/database/ledger-stores.ts | 5 +- .../database/own-keys-on-postgresql.test.ts | 78 ++++++ packages/server/src/database/own-keys.test.ts | 233 ++++++++++++++++++ packages/server/src/database/own-keys.ts | 168 +++++++++++++ .../server/src/database/secrets-keys.test.ts | 28 +-- packages/server/src/database/secrets-keys.ts | 17 +- .../src/lifecycle/secrets-startup.test.ts | 39 ++- packages/server/src/logging/logging.ts | 2 +- .../entries/serve-with-waiting-tool.ts | 4 +- .../src/testing/records/kept-credentials.ts | 15 ++ 20 files changed, 636 insertions(+), 89 deletions(-) create mode 100644 packages/server/src/database/own-keys-on-postgresql.test.ts create mode 100644 packages/server/src/database/own-keys.test.ts create mode 100644 packages/server/src/database/own-keys.ts diff --git a/docs/self-host/configuration.md b/docs/self-host/configuration.md index 733ca76e6..46ee7d5c2 100644 --- a/docs/self-host/configuration.md +++ b/docs/self-host/configuration.md @@ -120,9 +120,9 @@ Every setting is an environment variable. The tables below group them by area. | `LEDGER_FILE` | `data/ledger.db`, which in the image is on the `/data` volume; `~/.auto-runtime/ledger.db` with `npx` | The SQLite file of the ledger, absolute or relative to the working directory. Its folder is created when missing | | `DATABASE_URL` | none | A PostgreSQL URL. When set, the ledger is kept in that database instead, and setting `LEDGER_FILE` too stops the server at start. Environment only, since it holds a password. See [Use PostgreSQL](postgresql.md) | | `DATABASE_CREDENTIALS` | `url` | How the server signs in to that database: `url`, with the password the URL holds, or `aws-iam`, with a token signed for each connection. See [Without a password, on Amazon RDS](postgresql.md#without-a-password-on-amazon-rds) | -| `SECRETS_KEY` | none | The key the secrets kept in the ledger's database are sealed under: the base64 of 32 random bytes, as `openssl rand -base64 32` prints it. Set it or `SECRETS_KEY_FILE`, not both; a ledger in PostgreSQL needs one of them. The key that signs the state an MCP client sends back when it retries a call is derived from it too, so every server that shares it answers that retry. Environment only. See [Secrets at rest](security.md#secrets-at-rest) | +| `SECRETS_KEY` | none | The key the secrets kept in the ledger's database are sealed under: the base64 of 32 random bytes, as `openssl rand -base64 32` prints it. Set it or `SECRETS_KEY_FILE`, not both; a ledger in PostgreSQL needs one of them. The keys the runtime keeps for itself are among those secrets, such as the one that signs the state an MCP client sends back when it retries a call, so a server whose key cannot read them does not start. Environment only. See [Secrets at rest](security.md#secrets-at-rest) | | `SECRETS_KEY_FILE` | `secrets.key` beside a SQLite ledger file, made at the first start | A file that holds the same key, such as a secret your platform mounts into the container. A file named here is read as the platform mounts it, and the file the server makes is its owner's alone. A file named here is never made; one that is missing or that the server cannot read, one that holds anything but the key, and a default file that others than its owner may read stop the start. Without either setting, a SQLite ledger keeps its key in the default file. See [Keep the secrets key](production.md#keep-the-secrets-key) | -| `SECRETS_KEY_PREVIOUS` | none | The key that `SECRETS_KEY` or `SECRETS_KEY_FILE` replaces, in the same form and only beside one of them. Set it or `SECRETS_KEY_FILE_PREVIOUS`, not both. At start the server unwraps with it the key of each secret it wrapped and wraps that again under the current key, after which it can go. While it is set, the server also answers the retry of an MCP call whose state a server signed under the key it replaces. See [Rotate the secrets key](production.md#rotate-the-secrets-key) | +| `SECRETS_KEY_PREVIOUS` | none | The key that `SECRETS_KEY` or `SECRETS_KEY_FILE` replaces, in the same form and only beside one of them. Set it or `SECRETS_KEY_FILE_PREVIOUS`, not both. At start the server unwraps with it the key of each secret it wrapped and wraps that again under the current key, after which it can go. See [Rotate the secrets key](production.md#rotate-the-secrets-key) | | `SECRETS_KEY_FILE_PREVIOUS` | none | A file that holds the key being replaced, read as a file `SECRETS_KEY_FILE` names is, and otherwise as `SECRETS_KEY_PREVIOUS` | | `CONTENT_STORE_URL` | none | An `s3://bucket/prefix` URL. When set, content is kept in that bucket instead of the ledger's database. Content kept before a change is not moved; see [Keep content on object storage](object-storage.md#switch-a-deployment-that-already-has-content) | diff --git a/docs/self-host/connections.md b/docs/self-host/connections.md index c1a449d2c..1b770680e 100644 --- a/docs/self-host/connections.md +++ b/docs/self-host/connections.md @@ -163,7 +163,7 @@ Auto seals a secret under its secret key, `SECRETS_KEY`, or the file `SECRETS_KE Every value of a credential is a secret Auto scrubs from what it records and answers, in each form it may take, as it scrubs the value of a reference of the settings file. A value in a header that is a scheme and a credential, such as `Bearer `, is scrubbed whole and its credential alone; where the credential names its `scheme`, the value is the token, scrubbed whole. -A secret is only as recoverable as the key it was sealed under. While Auto cannot read a credential, because the key it was sealed under is not the key Auto has or its store does not answer, a call that needs it is refused with `connection_unavailable` in the store's words and its connection stays connected, so giving Auto the key again mends it. When the key is lost for good, nothing can read the credentials sealed under it, and the person removes each such connection and adds it again. Auto marks a connection `broken` only when its credential is no longer in the store, or is not the tokens of a consent, with a sentence that says why. +A secret is only as recoverable as the key it was sealed under. While Auto cannot read a credential because its store does not answer or will not unwrap it, a call that needs it is refused with `connection_unavailable` in the store's words, and the connection stays connected; the call is made once the store can read it again. Auto keeps a key of its own beside the credentials, so it does not start with a key other than the one they were sealed under, unless that one is given as the previous key, and a ledger whose key is lost for good cannot be started again, as [Secrets at rest](security.md#secrets-at-rest) says. Auto marks a connection `broken` only when its credential is gone from the store or is not the tokens of a consent, its consent was revoked or has ended, or its client is no longer registered, with a sentence that says why. ## Set the rules in `auto-runtime.yaml` diff --git a/docs/self-host/production.md b/docs/self-host/production.md index 1dc8d18ef..01bfb6434 100644 --- a/docs/self-host/production.md +++ b/docs/self-host/production.md @@ -56,7 +56,7 @@ Restrict access to the database, and to its backups, accordingly. ## Keep the secrets key -The runtime seals the secrets it keeps in the ledger's database under the secrets key, as [Secrets at rest](security.md#secrets-at-rest) describes. A container takes it from your platform's secret store, as it does the other secrets: in `SECRETS_KEY`, or in a file the platform mounts into the container, whose path `SECRETS_KEY_FILE` names. A file a setting names is read as the platform mounts it, and the file the server makes beside a SQLite ledger is its owner's alone. Either way the key is on no disk the ledger is on and in no backup of it; a ledger in PostgreSQL cannot start without one of them, and every server that shares the database takes the same key. Make one with either command, and keep a copy where your other secrets are kept, since a secret sealed under a lost key cannot be read again: +The runtime seals the secrets it keeps in the ledger's database under the secrets key, as [Secrets at rest](security.md#secrets-at-rest) describes. A container takes it from your platform's secret store, as it does the other secrets: in `SECRETS_KEY`, or in a file the platform mounts into the container, whose path `SECRETS_KEY_FILE` names. A file a setting names is read as the platform mounts it, and the file the server makes beside a SQLite ledger is its owner's alone. Either way the key is on no disk the ledger is on and in no backup of it; a ledger in PostgreSQL cannot start without one of them, and every server that shares the database takes the same key. Make one with either command, and keep a copy where your other secrets are kept. A secret sealed under a lost key cannot be read again, and the runtime keeps a key of its own under it, so a server started with another key does not start on that ledger: ```bash openssl rand -base64 32 diff --git a/docs/self-host/security.md b/docs/self-host/security.md index c6ecaf836..f50e0a59f 100644 --- a/docs/self-host/security.md +++ b/docs/self-host/security.md @@ -191,7 +191,7 @@ A web page may call the API from the Studio host, `https://studio.on.auto` unles ## Secrets at rest -The runtime has a store for the secrets it must hold, such as the credentials of a connection, in the ledger's database, apart from the ledger's records and never in the clear. Each secret is a row of its own: its value is sealed as a JWE under a key made for that value alone, which encrypts it with AES-GCM (`A256GCM`), and beside it the row keeps that key wrapped by the secrets key with AES key wrap (`A256KW`), and the id of the secrets key that wrapped it. So neither the database nor a backup of it holds a secret anyone can read without the secrets key, a new secrets key wraps the key of each value again and leaves the values as they are, and the store hands a secret back only as a redacted value, which prints as ``, so none reaches a log by accident. The secrets key is the base64 of 32 random bytes, in `SECRETS_KEY` or in the file `SECRETS_KEY_FILE` names, such as a secret your platform mounts into the container, and never in both. With a ledger in SQLite and neither set, the server makes a key at its first start, in `secrets.key` beside the ledger file with the mode `0600`, and reads it at every later start; a file `SECRETS_KEY_FILE` names is read and never made. A file a setting names is read as the platform mounts it, and the file the server makes is its owner's alone. A key file a setting names that is missing, a key file the server cannot read or that holds anything but such a key, and a file the server made that others than its owner may read, stop the start, naming the setting and the file. The file beside the ledger keeps a copy of the database unreadable, but not a copy of the folder that holds both, so a container [takes the key](production.md#keep-the-secrets-key) from its platform's secret store. A ledger in PostgreSQL, which several runtimes may share, needs `SECRETS_KEY` or `SECRETS_KEY_FILE`, and the server stops at start without either. At start the server logs where the key is and its id, the first 16 hex characters of the key's SHA-256, and never the key. A secret sealed under a lost key cannot be read again by anyone, the runtime included, so keep a copy of the key with your other secrets, apart from the backups of the ledger; [Rotate the secrets key](production.md#rotate-the-secrets-key) replaces it, through `SECRETS_KEY_PREVIOUS` or `SECRETS_KEY_FILE_PREVIOUS`. +The runtime has a store for the secrets it must hold, such as the credentials of a connection, in the ledger's database, apart from the ledger's records and never in the clear. Each secret is a row of its own: its value is sealed as a JWE under a key made for that value alone, which encrypts it with AES-GCM (`A256GCM`), and beside it the row keeps that key wrapped by the secrets key with AES key wrap (`A256KW`), and the id of the secrets key that wrapped it. So neither the database nor a backup of it holds a secret anyone can read without the secrets key, a new secrets key wraps the key of each value again and leaves the values as they are, and the store hands a secret back only as a redacted value, which prints as ``, so none reaches a log by accident. The secrets key is the base64 of 32 random bytes, in `SECRETS_KEY` or in the file `SECRETS_KEY_FILE` names, such as a secret your platform mounts into the container, and never in both. With a ledger in SQLite and neither set, the server makes a key at its first start, in `secrets.key` beside the ledger file with the mode `0600`, and reads it at every later start; a file `SECRETS_KEY_FILE` names is read and never made. A file a setting names is read as the platform mounts it, and the file the server makes is its owner's alone. A key file a setting names that is missing, a key file the server cannot read or that holds anything but such a key, and a file the server made that others than its owner may read, stop the start, naming the setting and the file. The file beside the ledger keeps a copy of the database unreadable, but not a copy of the folder that holds both, so a container [takes the key](production.md#keep-the-secrets-key) from its platform's secret store. A ledger in PostgreSQL, which several runtimes may share, needs `SECRETS_KEY` or `SECRETS_KEY_FILE`, and the server stops at start without either. At start the server logs where the key is and its id, the first 16 hex characters of the key's SHA-256, and never the key. The store also keeps the keys the runtime keeps for itself, each named in the ledger by a record of the runtime's own: the first is the key that signs the state an MCP client sends back when it retries a call, 32 random bytes the first server of a ledger makes, so every server that shares the ledger answers a retry another asked for. A server whose secrets key cannot read that key does not start: it names the key it was given and why, and says to start it with the key the ledger's secrets were sealed under, or to give that key as `SECRETS_KEY_PREVIOUS` or `SECRETS_KEY_FILE_PREVIOUS` beside a new one. A secret sealed under a lost key cannot be read again by anyone, the runtime included, and a ledger whose key is lost for good can no longer be started, so keep a copy of the key with your other secrets, apart from the backups of the ledger; [Rotate the secrets key](production.md#rotate-the-secrets-key) replaces it, through `SECRETS_KEY_PREVIOUS` or `SECRETS_KEY_FILE_PREVIOUS`. ## The page at the root diff --git a/packages/api/README.md b/packages/api/README.md index 2d66d1f3c..99ace01d7 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -150,7 +150,7 @@ So invalid arguments are a tool result with `isError` and an `invalid_input` pro The instructions are composed from the tools the caller may use, registered definition types and available recipes. They explain connection scope, put what the person wants kept in a recall function rather than the agent's own notes where the server runs recall functions, require reading the format before writing a definition, name model and tool discovery where offered, say that `add_connection` adds a provider or a server by hand where it is listed, and direct an answer to an outstanding interaction back to the same run. Scoped connections and read-only callers receive only applicable instructions and recipes. `src/mcp/instructions.test.ts` checks these combinations and the 2,000-character limit, about what Claude Code shows of a server's instructions; startup refuses instructions that exceed the bound. Tool descriptions and format guides carry the details that do not fit in initialization. -**A link to open.** An operation whose definition gives `linkToOpen`, as `add_connection` of `@on.auto/connections` does with Studio's page that approves a connection, puts the page in its output, and its tool also asks the client to open it, with the message the definition gives, through the `url` form of elicitation, when the client declared that form in the capabilities its request carries (`src/tools/linked-answers.ts`). Only a request of the `2026-07-28` revision carries them: the handler serves an earlier revision statelessly, each request "by a fresh instance from the same factory", as the SDK documents its `legacy: 'stateless'`, so the capabilities a `2025-11-25` client declared in its `initialize` never reach the instance that answers its call, nor would the client's answer to the SDK's `elicitInput`, which takes the `url` form, reach the instance that asked; a client of that revision is answered at once, the page in the output. A tool that asks answers `input_required`, with one `url` elicitation and a `requestState` that `createRequestStateCodec` of the SDK signs with the current key of `retryStateKeys`, the HMAC keys `mcpRoutes` is given, and binds with the codec's `bind` to the caller and the org of the request: the tool's answer, with the tool, the caller and the org it was made for. The client opens the page, or the person declines, and retries the call with the state; the SDK verifies the state, and the tool answers what it kept for the same tool, caller and org without running the operation again, so a retry adds nothing a second time. The server derives `retryStateKeys` from its secrets keys with HKDF and a label of its purpose, the current from the key `SECRETS_KEY` or `SECRETS_KEY_FILE` gives and the previous from the one `SECRETS_KEY_PREVIOUS` or `SECRETS_KEY_FILE_PREVIOUS` gives, and verifies a state under the current key and then under the previous one, so every server of a deployment, which shares that key, answers a retry another server asked for, before a rotation of the key and across it, as `src/tools/linked-answers.test.ts` and the server's `src/connections/connections-over-mcp.test.ts` show across two servers; a state of another key, made for another caller or org, or older than its ten minutes, is refused by the SDK as `-32602`, `Invalid or expired requestState`, and what the first call did stands. +**A link to open.** An operation whose definition gives `linkToOpen`, as `add_connection` of `@on.auto/connections` does with Studio's page that approves a connection, puts the page in its output, and its tool also asks the client to open it, with the message the definition gives, through the `url` form of elicitation, when the client declared that form in the capabilities its request carries (`src/tools/linked-answers.ts`). Only a request of the `2026-07-28` revision carries them: the handler serves an earlier revision statelessly, each request "by a fresh instance from the same factory", as the SDK documents its `legacy: 'stateless'`, so the capabilities a `2025-11-25` client declared in its `initialize` never reach the instance that answers its call, nor would the client's answer to the SDK's `elicitInput`, which takes the `url` form, reach the instance that asked; a client of that revision is answered at once, the page in the output. A tool that asks answers `input_required`, with one `url` elicitation and a `requestState` that `createRequestStateCodec` of the SDK signs with the current key of `retryStateKeys`, the HMAC keys `mcpRoutes` is given, and binds with the codec's `bind` to the caller and the org of the request: the tool's answer, with the tool, the caller and the org it was made for. The client opens the page, or the person declines, and retries the call with the state; the SDK verifies the state, and the tool answers what it kept for the same tool, caller and org without running the operation again, so a retry adds nothing a second time. The retry state is signed under a key the runtime keeps for itself in its secret store: 32 random bytes the first server of a ledger makes and names by a fact of the runtime's own, `own_key_kept` in `runtime/own-keys`, as the server's README says, which the server gives `mcpRoutes` as the `current` of `retryStateKeys`, with no `previous`; so every server that shares the ledger signs and verifies under the same key, whichever of them asked, across a restart and across a rotation of the secrets key, which wraps that key again and leaves it as it is, as the server's `src/connections/connections-over-mcp.test.ts` shows. A state is verified under `current` and then under `previous`, which is kept for a later rotation of that key, as `src/tools/linked-answers.test.ts` shows; a state of another key, made for another caller or org, or older than its ten minutes, is refused by the SDK as `-32602`, `Invalid or expired requestState`, and what the first call did stands. **Guides.** Every endpoint serves `get_guide`, a read-only tool whose `guide` argument is the enum of the guides' names and whose answer is one guide's whole Markdown as text, and lists the same guides as resources, `guide://`, `text/markdown`, annotated for the assistant, which answer the same text. The server passes them as `shelf`, the `GuideShelf` `guideShelfOf(guides, recipes, definitionTypes)` builds once and checks against the bounds, from the guides, a `Guide` each with its `name`, `title`, `description` and `text`, the recipes and the definition types, and hands the same shelf to `openApiDocumentRoutes` and `guideRoutes`. A `Recipe` is a guide with the `arguments` its prompt takes, the `formatGuide` it embeds, the tools its steps `call` and the `request` it makes of the person's words. Each recipe whose steps call only tools the connection lists, the rule that names it in the instructions, is also a prompt, under its name and title, so a playbook endpoint offers no `first-playbook` and a key that may only read no prompt: `prompts/get` answers one user message, the request with the person's words filled in and then the recipe, followed by its format guide as an embedded resource. `guideShelfOf` refuses a shelf without the guide of each definition type and the format guide of each recipe, so no endpoint mounts without them. diff --git a/packages/connections/src/index.ts b/packages/connections/src/index.ts index 852c75a1d..e01824de0 100644 --- a/packages/connections/src/index.ts +++ b/packages/connections/src/index.ts @@ -14,6 +14,7 @@ export { export { registeredClientSecrets } from './registered-clients/registered-clients.ts'; export type { RuntimeAccess } from './policy/policy-rules.ts'; export { ConnectionSchema, type Connection } from './registry/connection.ts'; +export { byTheRuntime } from './registry/registry-decider.ts'; export { makeConnectionSource } from './serving/connection-source.ts'; export { makeClientRegistrations } from './registrations/client-registrations.ts'; export { makeProviderOffering } from './providers/provider-offering.ts'; diff --git a/packages/server/README.md b/packages/server/README.md index 8d28a05c1..6da5716be 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -2,6 +2,12 @@ The Node.js server of Auto Runtime: `src/main.ts` starts it on the composition root of `src/composition/composition-root.ts`, `dev.ts` restarts it on every save, and `auto-runtime.ts` is the command of the npm package `@on.auto/runtime`, which `npm/pack.ts` packs with the `README.md` in `npm/`; the `Dockerfile` builds its container image. +## The runtime's own keys + +The runtime keeps keys of its own in its secret store, beside the secrets it keeps for others, and names each by a fact in a stream of the runtime's and of no org, `runtime/own-keys`, as `@on.auto/connections` names the clients it registers dynamically in `runtime/client-registrations` (`src/database/own-keys.ts`, beside `ledger-stores.ts`, which opens, rotates and logs the store the key is read from right after). The one fact, `own_key_kept { purpose, secret_id }`, by the runtime, `by: runtime`, is folded by purpose, the latest last, and the decider, `ownKeysDecider`, refuses a second for a purpose that has one, `conflict` of the kind `taken`, `The runtime keeps a key for retry_state already`. The one purpose is `retry_state`, the key that signs the state an MCP client sends back when it retries a call, as [`@on.auto/api`](../api/README.md) says. + +At start, once the secret store is open and rotated, `retryStateKeysOf` folds the stream. Where no key is recorded for the purpose it keeps 32 random bytes in the store, as their base64, and records the fact, which the ledger appends under the version of the stream it read; where the record is refused because another server recorded one first, it forgets the row it kept and folds again, taking that one. It then reads the row and gives the API `retryStateKeys` with that key as `current` and no `previous`, so every server that shares the ledger signs under the same key, and a rotation of the secrets key wraps it again and leaves it as it is. A row it cannot read stops the start as an `OwnKeyUnavailable`, named as the server's other start failures are, in words that name the purpose, the key in force as the start's log names it, the store's words and the two ways out: `The key the runtime keeps for retry_state cannot be read under the key that SECRETS_KEY holds: . Start the server with the key the ledger's secrets were sealed under, or give that key as SECRETS_KEY_PREVIOUS or SECRETS_KEY_FILE_PREVIOUS beside a new key`. A row that is gone, `… is no longer in the secret store, so the server does not start`, one that holds something other than 32 bytes, and a key the store cannot keep at the first start, `… cannot be kept under the key that SECRETS_KEY holds, so the server does not start: `, stop it too; the store is closed before the start fails. A deliberate reset of the runtime's own keys is not offered yet. Nothing of the key is logged, and no key is derived from another. + ## Testing The servers the tests start in the process run on `patientWorkerPool` of `src/testing/servers/worker-pools.ts`, the server's worker pool whose checks at save may take 30 s, through `checkingWithin` of `@on.auto/definitions/testing`, and whose inputs may take 30 s, through `decidingWithin` of `@on.auto/workflow-engine/testing`, so a check or a decision that a busy machine slows is never refused by the 2 s a save allows or the 2,000 ms of an input; `scriptedWorkerPool` answers the outcomes a test scripts before it hands the rest to that pool. The refusal of a check at its deadline stays held by `src/computation/checks-over-http.test.ts` and `src/computation/checks-over-mcp.test.ts`, whose pools answer `stopped: deadline`, and the answer of a start whose input runs past its deadline by `src/workflows/workflow-deadlines.test.ts`, whose evaluations answer `exhausted: deadline`. The servers the tests spawn as processes keep both bounds. diff --git a/packages/server/src/composition/composition-root.ts b/packages/server/src/composition/composition-root.ts index 27a8e6440..24a433b00 100644 --- a/packages/server/src/composition/composition-root.ts +++ b/packages/server/src/composition/composition-root.ts @@ -25,12 +25,8 @@ export function compositionRootWith( services: applicationLayer(ledgerLayerFor(settings, database, wiring.projections.appends)), serve: async (runtime, ownOrigin) => { const { ledger, workflows, identityIssuer } = settings; - const { stores, consent, connections, source, offering, clientMetadata } = await storesAndConnectionsOf( - runtime, - settings, - database, - ownOrigin, - ); + const { stores, retryStateKeys, consent, connections, source, offering, clientMetadata } = + await storesAndConnectionsOf(runtime, settings, database, ownOrigin); const functions = await functionsServedBy(runtime, settings, database, { ...wiring, ...(clock === undefined ? {} : { now: clock.now }), @@ -43,7 +39,7 @@ export function compositionRootWith( ledger, workflows, identityIssuer, - retryStateKeys: stores.retryStateKeys, + retryStateKeys, ...functions.parts, orgOperations: [...functions.parts.orgOperations, ...connections], ...(clock === undefined ? {} : { clock }), diff --git a/packages/server/src/connections/connections-over-mcp.test.ts b/packages/server/src/connections/connections-over-mcp.test.ts index ac97d6542..db96717ca 100644 --- a/packages/server/src/connections/connections-over-mcp.test.ts +++ b/packages/server/src/connections/connections-over-mcp.test.ts @@ -6,10 +6,14 @@ import { withMcpSession, type LinkOpening, } from '@on.auto/api/testing'; +import { sha256Of } from '@on.auto/operations'; import { Schema } from 'effect'; import { afterAll, beforeAll, describe, expect, it, onTestFinished } from 'vitest'; -import { secretsKey } from '../testing/records/kept-credentials.ts'; +import { compositionRoot } from '../composition/composition-root.ts'; +import { startServer } from '../lifecycle/lifecycle.ts'; +import { ownKeyFactsIn, secretsKey } from '../testing/records/kept-credentials.ts'; +import { temporaryLedger } from '../testing/records/temporary-ledger.ts'; import { servingReasoning, type ReasoningServer } from '../testing/servers/reasoning-server.ts'; let server: ReasoningServer; @@ -85,49 +89,99 @@ describe('add_connection over MCP', () => { async function servedFor(environment: Readonly> = {}): Promise { const served = await servingReasoning([], environment); - onTestFinished(() => served.stop()); - return served; + const stopping: { done?: Promise } = {}; + const stop = (): Promise => { + stopping.done ??= served.stop(); + return stopping.done; + }; + onTestFinished(stop); + return { ...served, stop }; +} + +function aLedgerFile(): string { + const { fileName, remove } = temporaryLedger(); + onTestFinished(remove); + return fileName; } function onMcpOf({ origin }: ReasoningServer) { return { url: `${origin}/mcp`, headers: {} }; } -describe('the retry of add_connection', () => { - it('is answered by another server of the same SECRETS_KEY with the connection the first added', async () => { - const sharing = { SECRETS_KEY: secretsKey.toString('base64') }; - const [first, second] = [await servedFor(sharing), await servedFor(sharing)]; - const call = { name: 'add_connection', arguments: { ...notes, name: 'notes-across' } }; +function addingNotes(name: string) { + return { name: 'add_connection', arguments: { ...notes, name } }; +} + +const sharing = { SECRETS_KEY: secretsKey.toString('base64') }; +describe('the retry of add_connection', () => { + it('is answered after a restart by the next server on the same ledger, under the key the runtime keeps for itself, adding nothing a second time', async () => { + const fileName = aLedgerFile(); + const call = addingNotes('notes-across'); + const first = await servedFor({ ...sharing, LEDGER_FILE: fileName }); const state = await stateAskedBy(onMcpOf(first), call); - const retried = await retriedWith(onMcpOf(second), call, state); + await first.stop(); + const next = await servedFor({ ...sharing, LEDGER_FILE: fileName }); + + const retried = await retriedWith(onMcpOf(next), call, state); + const listed = await next.call('GET', '/v1/orgs/local/connections'); + await next.stop(); expect(retried.structuredContent).toMatchObject({ name: 'notes-across', status: 'pending', added_by: 'local' }); - expect((await second.call('GET', '/v1/orgs/local/connections')).body).toEqual({ connections: [] }); - expect((await first.call('GET', '/v1/orgs/local/connections')).body).toMatchObject({ - connections: [{ name: 'notes-across' }], - }); + expect(listed.body).toMatchObject({ connections: [{ name: 'notes-across' }] }); + expect(await ownKeyFactsIn(fileName)).toMatchObject([ + { type: 'own_key_kept', data: { purpose: 'retry_state' }, context: { by: 'runtime' } }, + ]); }); - it('is answered across a rotation, by a server whose SECRETS_KEY_PREVIOUS is the key the first had', async () => { - const rotatedKey = Buffer.alloc(32, 9).toString('base64'); - const first = await servedFor({ SECRETS_KEY: secretsKey.toString('base64') }); - const rotated = await servedFor({ SECRETS_KEY: rotatedKey, SECRETS_KEY_PREVIOUS: secretsKey.toString('base64') }); - const call = { name: 'add_connection', arguments: { ...notes, name: 'notes-rotated' } }; - + it('is answered across a rotation, by a server on the same ledger whose SECRETS_KEY_PREVIOUS is the key the first had', async () => { + const fileName = aLedgerFile(); + const call = addingNotes('notes-rotated'); + const first = await servedFor({ ...sharing, LEDGER_FILE: fileName }); const state = await stateAskedBy(onMcpOf(first), call); + await first.stop(); + const rotated = await servedFor({ + LEDGER_FILE: fileName, + SECRETS_KEY: Buffer.alloc(32, 9).toString('base64'), + SECRETS_KEY_PREVIOUS: secretsKey.toString('base64'), + }); expect((await retriedWith(onMcpOf(rotated), call, state)).structuredContent).toMatchObject({ name: 'notes-rotated', }); }); - it('is refused by a server of another secrets key', async () => { - const [first, other] = [await servedFor(), await servedFor()]; - const call = { name: 'add_connection', arguments: { ...notes, name: 'notes-elsewhere' } }; + it('is refused by a server of another ledger, though it shares the SECRETS_KEY of the first', async () => { + const [first, other] = [await servedFor(sharing), await servedFor(sharing)]; + const call = addingNotes('notes-elsewhere'); const state = await stateAskedBy(onMcpOf(first), call); await expect(retriedWith(onMcpOf(other), call, state)).rejects.toThrow('Invalid or expired requestState'); }); }); + +describe('a server whose secrets key cannot read the key the runtime keeps for itself', () => { + it('does not start, naming the purpose of the key and the words of the store', async () => { + const ledger = temporaryLedger(); + onTestFinished(ledger.remove); + const first = await servedFor({ ...sharing, LEDGER_FILE: ledger.fileName }); + await first.stop(); + const anotherKey = Buffer.alloc(32, 9); + + const started = startServer( + { + HOST: '127.0.0.1', + PORT: '0', + LEDGER_FILE: ledger.fileName, + PREDICTION_CACHE_DIR: ledger.cacheDirectory, + SECRETS_KEY: anotherKey.toString('base64'), + }, + compositionRoot, + ); + + await expect(started).rejects.toThrow( + `The key the runtime keeps for retry_state cannot be read under the key that SECRETS_KEY holds: The secret is sealed under the key ${sha256Of(secretsKey).slice(0, 16)}, which is neither the current key nor the previous one. Start the server with the key the ledger's secrets were sealed under, or give that key as SECRETS_KEY_PREVIOUS or SECRETS_KEY_FILE_PREVIOUS beside a new key`, + ); + }); +}); diff --git a/packages/server/src/connections/served-connections.ts b/packages/server/src/connections/served-connections.ts index 3a7579c32..7dbf04ddb 100644 --- a/packages/server/src/connections/served-connections.ts +++ b/packages/server/src/connections/served-connections.ts @@ -1,4 +1,4 @@ -import { clientMetadataRoutes, type AppRuntime, type RegisterRoutes } from '@on.auto/api'; +import { clientMetadataRoutes, type AppRuntime, type RegisterRoutes, type RetryStateKeys } from '@on.auto/api'; import { makeConnectionOperations, makeConnectionSource, @@ -9,16 +9,17 @@ import { } from '@on.auto/connections'; import { accessModeOf } from '@on.auto/identity'; import type { DispatcherServices } from '@on.auto/operations'; -import type { SecretStore } from '@on.auto/secrets'; import type { DatabaseSettings } from '@on.auto/workflow-host'; import { runtimeLedger } from '../composition/runtime-ledger.ts'; import { consentClientOf, servedClientMetadataDocumentOf } from '../composition/served-consent.ts'; import { ledgerStoresOpenedBy, type LedgerStores } from '../database/ledger-stores.ts'; +import { retryStateKeysOf } from '../database/own-keys.ts'; import type { Settings } from '../settings/settings.ts'; export interface StoresAndConnections { readonly stores: LedgerStores; + readonly retryStateKeys: RetryStateKeys; readonly consent: ReturnType; readonly connections: ConnectionOperations; readonly source: ReturnType; @@ -50,7 +51,7 @@ function accessOf({ host, signIn, apiKeys, accessMode }: ServingSettings): Runti } interface Composed { - readonly secrets: SecretStore; + readonly secrets: LedgerStores['secrets']; readonly consent: ReturnType; readonly pages: ConnectionServices['pages']; } @@ -93,11 +94,13 @@ export async function storesAndConnectionsOf( ownOrigin: () => string, ): Promise { const stores = await ledgerStoresOpenedBy(runtime.context, settings, database); + const retryStateKeys = await retryStateKeysOf(runtime, stores.secrets, settings.secrets.current); const consent = consentClientOf(runtime, settings, stores.secrets); const pages = { studio: settings.studio, runtime: ownOrigin }; const services = connectionServicesOf(settings, { secrets: stores.secrets, consent, pages }); return { stores, + retryStateKeys, consent, connections: makeConnectionOperations(services), source: sourceServedBy(runtime, settings, { secrets: stores.secrets, pages }), diff --git a/packages/server/src/database/ledger-stores.ts b/packages/server/src/database/ledger-stores.ts index 79728c7f1..dfa458747 100644 --- a/packages/server/src/database/ledger-stores.ts +++ b/packages/server/src/database/ledger-stores.ts @@ -1,4 +1,3 @@ -import type { RetryStateKeys } from '@on.auto/api'; import type { SecretStore } from '@on.auto/secrets'; import type { DatabaseSettings } from '@on.auto/workflow-host'; import { Effect, type Context } from 'effect'; @@ -8,11 +7,10 @@ import { logContentStore, logLedger, logSecretStore } from '../logging/logging.t import { InvalidSettingsError } from '../settings/invalid-settings-error.ts'; import type { Settings } from '../settings/settings.ts'; import { custodiansOf, secretRowsOn, secretStoreOver } from './secret-store.ts'; -import { retryStateKeysOf, secretsKeysOf, type SecretKeys } from './secrets-keys.ts'; +import { secretsKeysOf, type SecretKeys } from './secrets-keys.ts'; export interface LedgerStores { readonly secrets: SecretStore; - readonly retryStateKeys: RetryStateKeys; readonly withSecretsClosed: (served: Served) => Served; } @@ -62,7 +60,6 @@ export async function ledgerStoresOpenedBy( const store = await run(openedAndRotated(database, keys, secrets)); return { secrets: store, - retryStateKeys: retryStateKeysOf(keys), withSecretsClosed: ({ routes, stopWork }) => ({ routes, stopWork: async () => { diff --git a/packages/server/src/database/own-keys-on-postgresql.test.ts b/packages/server/src/database/own-keys-on-postgresql.test.ts new file mode 100644 index 000000000..fee541f0d --- /dev/null +++ b/packages/server/src/database/own-keys-on-postgresql.test.ts @@ -0,0 +1,78 @@ +import { randomUUID } from 'node:crypto'; + +import { retriedWith, stateAskedBy } from '@on.auto/api/testing'; +import { Client } from 'pg'; +import { describe, expect, it, onTestFinished } from 'vitest'; + +import { servingReasoning, type ReasoningServer } from '../testing/servers/reasoning-server.ts'; +import { ownKeysStream } from './own-keys.ts'; + +const server = process.env['LEDGER_TEST_POSTGRESQL_URL'] ?? ''; + +const skipped = server === ''; + +const secretsKey = Buffer.alloc(32, 7).toString('base64'); + +const notice = skipped ? ', skipped: set LEDGER_TEST_POSTGRESQL_URL to the URL of a PostgreSQL server to run it' : ''; + +async function queried( + url: string, + statement: string, + values: readonly string[] = [], +): Promise>[]> { + const client = new Client({ connectionString: url }); + await client.connect(); + try { + const { rows } = await client.query>>(statement, [...values]); + return rows; + } finally { + await client.end(); + } +} + +async function aDatabaseOfItsOwn(): Promise { + const name = `own_keys_${randomUUID().replaceAll('-', '')}`; + await queried(server, `CREATE DATABASE ${name}`); + onTestFinished(async () => { + await queried(server, `DROP DATABASE ${name} WITH (FORCE)`); + }); + const database = new URL(server); + database.pathname = `/${name}`; + return database.href; +} + +async function servedOn(database: string): Promise { + const served = await servingReasoning([], { LEDGER_FILE: '', DATABASE_URL: database, SECRETS_KEY: secretsKey }); + onTestFinished(() => served.stop()); + return served; +} + +function onMcpOf({ origin }: ReasoningServer) { + return { url: `${origin}/mcp`, headers: {} }; +} + +const adding = { + name: 'add_connection', + arguments: { + name: 'notes-raced', + type: 'mcp', + entry: { url: 'https://notes.example.com/mcp' }, + credential: { kind: 'none' }, + }, +}; + +describe.skipIf(skipped)(`Two servers that start at once on one ledger in PostgreSQL${notice}`, () => { + it('keep one key that signs the state of a retry between them, recorded once and kept in one row, so a retry one asked for is answered by the other', async () => { + const database = await aDatabaseOfItsOwn(); + const [first, second] = await Promise.all([servedOn(database), servedOn(database)]); + + const state = await stateAskedBy(onMcpOf(first), adding); + const retried = await retriedWith(onMcpOf(second), adding, state); + + expect(retried.structuredContent).toMatchObject({ name: 'notes-raced', status: 'pending' }); + expect( + await queried(database, 'SELECT message_type FROM emt_messages WHERE stream_id = $1', [ownKeysStream]), + ).toEqual([{ message_type: 'own_key_kept' }]); + expect(await queried(database, 'SELECT count(*)::int AS rows FROM kept_secrets')).toEqual([{ rows: 1 }]); + }); +}); diff --git a/packages/server/src/database/own-keys.test.ts b/packages/server/src/database/own-keys.test.ts new file mode 100644 index 000000000..700525aac --- /dev/null +++ b/packages/server/src/database/own-keys.test.ts @@ -0,0 +1,233 @@ +import { makeAppRuntime } from '@on.auto/api'; +import { ledgerLayer } from '@on.auto/ledger/sqlite3'; +import { Conflict, type StreamReader, type StreamWriter } from '@on.auto/operations'; +import { memoryLedger, type MemoryLedger } from '@on.auto/operations/testing'; +import { sealedKeeper, secretStoreOn, type SecretRows, type SecretStore } from '@on.auto/secrets'; +import { custodianRefusal, memoryRows, recordingCustodian, type RecordingCustodian } from '@on.auto/secrets/testing'; +import { DateTime, Effect, Redacted, Result } from 'effect'; +import { describe, expect, it, onTestFinished } from 'vitest'; + +import { applicationLayer } from '../composition/application-layer.ts'; +import { + ownKeysDecider, + ownKeysStream, + retryStateKeysKeptBy, + retryStateKeysOf, + type OwnKeyEvent, + type OwnKeyKeeping, +} from './own-keys.ts'; + +const now = Effect.map(DateTime.now, DateTime.toEpochMillis); + +const at = '2026-10-20T09:00:00.000Z'; + +const keeping: OwnKeyKeeping = { purpose: 'retry_state', secret_id: 'secret-1', by: 'runtime', at }; + +type Started = Parameters; + +const fromTheEnvironment: Started[1] = { + from: 'environment', + setting: 'SECRETS_KEY', + key: Redacted.make(new Uint8Array(32)), +}; + +function started(services: Started[0]) { + return retryStateKeysKeptBy(services, fromTheEnvironment); +} + +const waysOut = + "Start the server with the key the ledger's secrets were sealed under, or give that key as SECRETS_KEY_PREVIOUS or SECRETS_KEY_FILE_PREVIOUS beside a new key"; + +function storeOver(rows: SecretRows, custodian: RecordingCustodian): SecretStore { + return secretStoreOn(rows, sealedKeeper({ current: custodian }), { current: custodian }); +} + +function bytesOf(key: Redacted.Redacted | undefined): readonly number[] { + return key === undefined ? [] : Array.from(Redacted.value(key)); +} + +const collecting = { + ...ownKeysDecider, + initialState: [], + evolve: (facts: readonly unknown[], fact: unknown) => [...facts, fact], +}; + +async function factsIn(ledger: MemoryLedger): Promise { + return (await Effect.runPromise(ledger.service.load(ownKeysStream, collecting))).state; +} + +function recordedFirstBy(ledger: MemoryLedger, otherStart: Effect.Effect): StreamReader & StreamWriter { + const raced = { yet: false }; + const racedOnce = Effect.suspend(() => { + const first = !raced.yet; + raced.yet = true; + return first ? otherStart : Effect.void; + }); + return { + ...ledger.service, + execute: (stream, decider, command, lineage) => + Effect.andThen(racedOnce, ledger.service.execute(stream, decider, command, lineage)), + }; +} + +function failureOf(start: Effect.Effect): Promise { + return Effect.runPromise(Effect.map(Effect.flip(start), String)); +} + +describe('the decider of the keys the runtime keeps for itself', () => { + it('records a key for a purpose that has none, by whoever keeps it, and refuses a second for a purpose that has one', () => { + const kept = ownKeysDecider.evolve(ownKeysDecider.initialState, { + type: 'own_key_kept', + data: { purpose: 'retry_state', secret_id: 'secret-1' }, + context: { by: 'runtime', at }, + }); + + expect(ownKeysDecider.decide(keeping, ownKeysDecider.initialState)).toEqual( + Result.succeed([{ type: 'own_key_kept', data: { purpose: 'retry_state', secret_id: 'secret-1' } }]), + ); + expect(ownKeysDecider.decide({ ...keeping, secret_id: 'secret-2' }, kept)).toEqual( + Result.fail(new Conflict({ kind: 'taken', detail: 'The runtime keeps a key for retry_state already' })), + ); + expect(ownKeysDecider.context(keeping, kept)).toEqual({ by: 'runtime', at }); + }); + + it('folds the keys by purpose, the latest last', () => { + const facts: readonly OwnKeyEvent[] = ['secret-1', 'secret-2'].map((secretId) => ({ + type: 'own_key_kept', + data: { purpose: 'retry_state', secret_id: secretId }, + context: { by: 'runtime', at }, + })); + + const folded = facts.reduce((keys, fact) => ownKeysDecider.evolve(keys, fact), ownKeysDecider.initialState); + + expect([...folded]).toEqual([['retry_state', 'secret-2']]); + }); +}); + +describe('the key that signs the state of a retry', () => { + it('is 32 random bytes kept in the secret store at the first start and recorded by the runtime, and the same at the next, with one fact and one row', async () => { + const ledger = memoryLedger(); + const rows = memoryRows({ now }); + const services = { registry: ledger.service, secrets: storeOver(rows, recordingCustodian()) }; + + const first = await Effect.runPromise(started(services)); + const next = await Effect.runPromise(started(services)); + + expect(bytesOf(first.current)).toHaveLength(32); + expect(bytesOf(next.current)).toEqual(bytesOf(first.current)); + expect([first.previous, next.previous]).toEqual([undefined, undefined]); + expect(await factsIn(ledger)).toMatchObject([ + { + type: 'own_key_kept', + data: { purpose: 'retry_state', secret_id: rows.kept()[0]?.id }, + context: { by: 'runtime' }, + }, + ]); + expect(rows.kept()).toHaveLength(1); + }); + + it('is the key another server recorded first, while the one this server kept meanwhile is forgotten', async () => { + const ledger = memoryLedger(); + const rows = memoryRows({ now }); + const custodian = recordingCustodian(); + const other = { registry: ledger.service, secrets: storeOver(rows, custodian) }; + const registry = recordedFirstBy(ledger, Effect.orDie(started(other))); + + const taken = await Effect.runPromise(started({ registry, secrets: storeOver(rows, custodian) })); + const theirs = await Effect.runPromise(started(other)); + + expect(bytesOf(taken.current)).toEqual(bytesOf(theirs.current)); + expect(await factsIn(ledger)).toHaveLength(1); + expect(rows.kept()).toHaveLength(1); + }); +}); + +describe('a start that cannot have the key that signs the state of a retry', () => { + it('stops, naming the purpose and the words of the store, when the row is sealed under a key it does not have', async () => { + const ledger = memoryLedger(); + const rows = memoryRows({ now }); + const sealing = recordingCustodian(); + await Effect.runPromise(started({ registry: ledger.service, secrets: storeOver(rows, sealing) })); + + const refused = await failureOf( + started({ registry: ledger.service, secrets: storeOver(rows, recordingCustodian()) }), + ); + + expect(refused).toBe( + `OwnKeyUnavailable: The key the runtime keeps for retry_state cannot be read under the key that SECRETS_KEY holds: The secret is sealed under the key ${sealing.kid}, which is neither the current key nor the previous one. ${waysOut}`, + ); + expect(await factsIn(ledger)).toHaveLength(1); + }); +}); + +describe('a start whose key that signs the state of a retry is gone or kept nowhere', () => { + it('stops when the row is gone, or holds something other than 32 bytes, and keeps no second key', async () => { + const [gone, other] = [memoryLedger(), memoryLedger()]; + const rows = memoryRows({ now }); + const store = storeOver(rows, recordingCustodian()); + await Effect.runPromise(started({ registry: gone.service, secrets: store })); + await Effect.runPromise(store.forget(String(rows.kept()[0]?.id))); + const notAKey = await Effect.runPromise(store.keep(Redacted.make('not a key'))); + await Effect.runPromise(other.service.execute(ownKeysStream, ownKeysDecider, { ...keeping, secret_id: notAKey })); + + const refusals = [ + await failureOf(started({ registry: gone.service, secrets: store })), + await failureOf(started({ registry: other.service, secrets: store })), + ]; + + expect(refusals).toEqual([ + 'OwnKeyUnavailable: The key the runtime keeps for retry_state is no longer in the secret store, so the server does not start', + 'OwnKeyUnavailable: The key the runtime keeps for retry_state holds something other than 32 bytes, so the server does not start', + ]); + expect(rows.kept()).toHaveLength(1); + }); + + it('stops at the first start when the secret store cannot keep it, and records nothing', async () => { + const ledger = memoryLedger(); + const custodian = recordingCustodian(); + custodian.refuse(); + + const refused = await failureOf( + started({ registry: ledger.service, secrets: storeOver(memoryRows({ now }), custodian) }), + ); + + expect(refused).toBe( + `OwnKeyUnavailable: The key the runtime keeps for retry_state cannot be kept under the key that SECRETS_KEY holds, so the server does not start: The custodian of the key ${custodian.kid} refused to wrap a data key: ${custodianRefusal}`, + ); + expect(await factsIn(ledger)).toEqual([]); + }); +}); + +async function aRuntime() { + const runtime = await makeAppRuntime(applicationLayer(ledgerLayer({ fileName: ':memory:' }))); + onTestFinished(() => runtime.dispose()); + return runtime; +} + +function closingCounted(store: SecretStore) { + const closes = { made: 0 }; + const counted: SecretStore = { + ...store, + close: () => { + closes.made += 1; + return store.close(); + }, + }; + return { closes, counted }; +} + +describe('the start of a server that reads the key that signs the state of a retry', () => { + it('closes its secret store before it fails, and leaves it open once it has the key', async () => { + const refusing = recordingCustodian(); + refusing.refuse(); + const failing = closingCounted(storeOver(memoryRows({ now }), refusing)); + const answering = closingCounted(storeOver(memoryRows({ now }), recordingCustodian())); + + const failed = retryStateKeysOf(await aRuntime(), failing.counted, fromTheEnvironment); + await expect(failed).rejects.toThrow('cannot be kept under the key that SECRETS_KEY holds'); + const kept = await retryStateKeysOf(await aRuntime(), answering.counted, fromTheEnvironment); + + expect(failing.closes.made).toBe(1); + expect([bytesOf(kept.current).length, answering.closes.made]).toEqual([32, 0]); + }); +}); diff --git a/packages/server/src/database/own-keys.ts b/packages/server/src/database/own-keys.ts new file mode 100644 index 000000000..62870d390 --- /dev/null +++ b/packages/server/src/database/own-keys.ts @@ -0,0 +1,168 @@ +import { randomBytes } from 'node:crypto'; + +import type { AppRuntime, RetryStateKeys } from '@on.auto/api'; +import { byTheRuntime } from '@on.auto/connections'; +import { + ActedContextSchema, + Conflict, + factOf, + type Context, + type Decided, + type Decider, + type DispatcherServices, + type StreamReader, + type StreamWriter, +} from '@on.auto/operations'; +import { secretKeyBytes, type SecretStore } from '@on.auto/secrets'; +import { Data, DateTime, Effect, Option, Redacted, Result, Schema } from 'effect'; + +import { runtimeLedger } from '../composition/runtime-ledger.ts'; +import { keyHeldBy } from '../logging/logging.ts'; +import { secretsKeyOfBase64, type SecretsKeySource } from '../settings/secrets-settings.ts'; + +export const ownKeysStream = 'runtime/own-keys'; + +const OwnKeyPurposeSchema = Schema.Literals(['retry_state']); + +type OwnKeyPurpose = typeof OwnKeyPurposeSchema.Type; + +const OwnKeyEventSchema = Schema.Union([ + factOf('own_key_kept', Schema.Struct({ purpose: OwnKeyPurposeSchema, secret_id: Schema.String }), ActedContextSchema), +]); + +export type OwnKeyEvent = typeof OwnKeyEventSchema.Type; + +export interface OwnKeyKeeping { + readonly purpose: OwnKeyPurpose; + readonly secret_id: string; + readonly by: string; + readonly at: string; +} + +type OwnKeys = ReadonlyMap; + +const noOwnKeys: OwnKeys = new Map(); + +function evolveOwnKeys(keys: OwnKeys, { data }: OwnKeyEvent): OwnKeys { + const kept = new Map(keys); + kept.delete(data.purpose); + return kept.set(data.purpose, data.secret_id); +} + +export const ownKeysDecider: Decider = { + initialState: noOwnKeys, + evolve: evolveOwnKeys, + decide: ({ purpose, secret_id }, keys): Result.Result[], Conflict> => + keys.has(purpose) + ? Result.fail(new Conflict({ kind: 'taken', detail: `The runtime keeps a key for ${purpose} already` })) + : Result.succeed([{ type: 'own_key_kept', data: { purpose, secret_id } }]), + context: ({ by, at }): Context => ({ by, at }), + eventSchema: OwnKeyEventSchema, +}; + +class OwnKeyUnavailable extends Data.TaggedError('OwnKeyUnavailable')<{ readonly message: string }> {} + +interface Worded { + readonly detail: string; +} + +interface Asked { + readonly purpose: OwnKeyPurpose; + readonly keyFrom: SecretsKeySource; +} + +function unreadableUnder({ purpose, keyFrom }: Asked, words: string): OwnKeyUnavailable { + return new OwnKeyUnavailable({ + message: `The key the runtime keeps for ${purpose} cannot be read under the key ${keyHeldBy(keyFrom)}: ${words}. Start the server with the key the ledger's secrets were sealed under, or give that key as SECRETS_KEY_PREVIOUS or SECRETS_KEY_FILE_PREVIOUS beside a new key`, + }); +} + +function unkeptUnder({ purpose, keyFrom }: Asked, words: string): OwnKeyUnavailable { + return new OwnKeyUnavailable({ + message: `The key the runtime keeps for ${purpose} cannot be kept under the key ${keyHeldBy(keyFrom)}, so the server does not start: ${words}`, + }); +} + +function unusable({ purpose }: Asked, why: string): OwnKeyUnavailable { + return new OwnKeyUnavailable({ + message: `The key the runtime keeps for ${purpose} ${why}, so the server does not start`, + }); +} + +interface OwnKeyServices { + readonly registry: StreamReader & StreamWriter; + readonly secrets: Pick; +} + +function aKeyInBase64(): Redacted.Redacted { + return Redacted.make(randomBytes(secretKeyBytes).toString('base64')); +} + +function keptAndRecorded( + { registry, secrets }: OwnKeyServices, + asked: Asked, +): Effect.Effect { + return Effect.gen(function* () { + const secretId = yield* Effect.mapError(secrets.keep(aKeyInBase64()), (failure: Worded) => + unkeptUnder(asked, failure.detail), + ); + const at = DateTime.formatIso(yield* DateTime.now); + const keeping = { purpose: asked.purpose, secret_id: secretId, by: byTheRuntime, at }; + return yield* registry.execute(ownKeysStream, ownKeysDecider, keeping).pipe( + Effect.as(secretId), + Effect.catch(() => Effect.andThen(secrets.forget(secretId), Effect.undefined)), + ); + }); +} + +function ownKeyIdOf(services: OwnKeyServices, asked: Asked): Effect.Effect { + return Effect.gen(function* () { + const { state } = yield* services.registry.load(ownKeysStream, ownKeysDecider); + const recorded = state.get(asked.purpose) ?? (yield* keptAndRecorded(services, asked)); + return recorded ?? (yield* ownKeyIdOf(services, asked)); + }); +} + +function ownKeyIn( + { secrets }: OwnKeyServices, + asked: Asked, + secretId: string, +): Effect.Effect, OwnKeyUnavailable> { + return secrets.read(secretId).pipe( + Effect.mapError((failure: Worded) => unreadableUnder(asked, failure.detail)), + Effect.flatMap((stored) => + stored === undefined + ? Effect.fail(unusable(asked, 'is no longer in the secret store')) + : Option.match(Option.fromNullishOr(secretsKeyOfBase64(Redacted.value(stored.value))), { + onNone: () => Effect.fail(unusable(asked, `holds something other than ${secretKeyBytes} bytes`)), + onSome: Effect.succeed, + }), + ), + ); +} + +export function retryStateKeysKeptBy( + services: OwnKeyServices, + keyFrom: SecretsKeySource, +): Effect.Effect { + const asked: Asked = { purpose: 'retry_state', keyFrom }; + return Effect.gen(function* () { + const secretId = yield* ownKeyIdOf(services, asked); + return { current: yield* ownKeyIn(services, asked, secretId) }; + }); +} + +export async function retryStateKeysOf( + runtime: AppRuntime, + secrets: SecretStore, + keyFrom: SecretsKeySource, +): Promise { + try { + return await Effect.runPromiseWith(runtime.context)( + retryStateKeysKeptBy({ registry: runtimeLedger(runtime), secrets }, keyFrom), + ); + } catch (failure) { + await secrets.close(); + throw failure; + } +} diff --git a/packages/server/src/database/secrets-keys.test.ts b/packages/server/src/database/secrets-keys.test.ts index 1baf50a80..ede3b4ec8 100644 --- a/packages/server/src/database/secrets-keys.test.ts +++ b/packages/server/src/database/secrets-keys.test.ts @@ -6,7 +6,7 @@ import { Redacted } from 'effect'; import { describe, expect, it, onTestFinished } from 'vitest'; import type { SecretsKeyGiven, SecretsKeySource } from '../settings/secrets-settings.ts'; -import { retryStateKeysOf, secretsKeysOf } from './secrets-keys.ts'; +import { secretsKeysOf } from './secrets-keys.ts'; function aFolder(): string { const folder = mkdtempSync(join(tmpdir(), 'auto-runtime-secrets-')); @@ -140,29 +140,3 @@ describe('the secrets keys of the other settings', () => { function bytesOf(key: Redacted.Redacted | undefined): readonly number[] { return key === undefined ? [] : Array.from(Redacted.value(key)); } - -function derivedOf(bytes: Readonly): readonly number[] { - return bytesOf(retryStateKeysOf({ current: Redacted.make(Uint8Array.from(bytes)) }).current); -} - -describe('the key that signs the state of a retry', () => { - it('is derived from the current secrets key, the same of the same key, and never the key itself', () => { - const key = new Uint8Array(32).fill(7); - - const [first, again, ofAnother] = [derivedOf(key), derivedOf(key), derivedOf(new Uint8Array(32).fill(9))]; - - expect(first).toHaveLength(32); - expect(again).toEqual(first); - expect(ofAnother).not.toEqual(first); - expect(first).not.toEqual(Array.from(key)); - }); - - it('is derived from the previous secrets key too, as a server that still has it as its current derives it', () => { - const [previous, current] = [new Uint8Array(32).fill(7), new Uint8Array(32).fill(9)]; - - const rotating = retryStateKeysOf({ current: Redacted.make(current), previous: Redacted.make(previous) }); - - expect([bytesOf(rotating.current), bytesOf(rotating.previous)]).toEqual([derivedOf(current), derivedOf(previous)]); - expect(bytesOf(retryStateKeysOf({ current: Redacted.make(current) }).previous)).toEqual([]); - }); -}); diff --git a/packages/server/src/database/secrets-keys.ts b/packages/server/src/database/secrets-keys.ts index b83760a38..eda8f369a 100644 --- a/packages/server/src/database/secrets-keys.ts +++ b/packages/server/src/database/secrets-keys.ts @@ -1,8 +1,7 @@ -import { hkdfSync, randomBytes } from 'node:crypto'; +import { randomBytes } from 'node:crypto'; import { accessSync, constants, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; -import type { RetryStateKeys } from '@on.auto/api'; import { secretKeyBytes } from '@on.auto/secrets'; import { Redacted } from 'effect'; @@ -88,20 +87,6 @@ function keyOf(source: SecretsKeySource): Redacted.Redacted { return source.from === 'file' ? keyInFile(source) : Redacted.make(Uint8Array.from(randomBytes(secretKeyBytes))); } -const retryStatePurpose = 'auto-runtime: the key that signs the state of a retry of an MCP tool'; - -function retryStateKeyOf(key: Redacted.Redacted): Redacted.Redacted { - const derived = hkdfSync('sha256', Redacted.value(key), new Uint8Array(), retryStatePurpose, secretKeyBytes); - return Redacted.make(new Uint8Array(derived)); -} - -export function retryStateKeysOf({ current, previous }: SecretKeys): RetryStateKeys { - return { - current: retryStateKeyOf(current), - previous: previous === undefined ? undefined : retryStateKeyOf(previous), - }; -} - export function secretsKeysOf({ current, previous }: SecretsSettings): SecretKeys { return { current: keyOf(current), previous: previous === undefined ? undefined : keyOf(previous) }; } diff --git a/packages/server/src/lifecycle/secrets-startup.test.ts b/packages/server/src/lifecycle/secrets-startup.test.ts index a366c6fec..8c315c8fd 100644 --- a/packages/server/src/lifecycle/secrets-startup.test.ts +++ b/packages/server/src/lifecycle/secrets-startup.test.ts @@ -8,7 +8,7 @@ import { describe, expect, it, onTestFinished } from 'vitest'; import { spawnServer } from '../testing/processes/spawned-server.ts'; import { messagesOf } from '../testing/processes/stderr-lines.ts'; -import { secretStoreInFile } from '../testing/records/kept-credentials.ts'; +import { ownKeyFactsIn, secretStoreInFile } from '../testing/records/kept-credentials.ts'; import { temporaryLedger } from '../testing/records/temporary-ledger.ts'; const mainModule = fileURLToPath(new URL('../main.ts', import.meta.url)); @@ -173,6 +173,43 @@ describe('a server whose secrets key cannot be had', () => { }); }); +async function ownKeyIn(fileName: string): Promise { + const [kept] = await ownKeyFactsIn(fileName); + const store = await secretStoreInFile(fileName, currentKey); + const read = await Effect.runPromise(store.read(String(kept?.data.secret_id))); + await store.close(); + return read === undefined ? '' : Redacted.value(read.value); +} + +describe('the key the runtime keeps for itself as the server starts', () => { + it('is kept in the secret store at the first start, read again at the next, recorded once, and in no line of the log', async () => { + const fileName = aLedgerFile(); + const current = { LEDGER_FILE: fileName, SECRETS_KEY: currentKey.toString('base64') }; + + const first = await startedWith(current); + const next = await startedWith(current); + const ownKey = await ownKeyIn(fileName); + + expect(Buffer.from(ownKey, 'base64')).toHaveLength(32); + expect(await ownKeyFactsIn(fileName)).toHaveLength(1); + expect(secretsIn(`${first}${next}`, [ownKey, Buffer.from(ownKey, 'base64').toString('hex')])).toEqual([]); + }); + + it('stops the process at start when SECRETS_KEY cannot read it, naming its purpose and the words of the store and never a key', async () => { + const fileName = aLedgerFile(); + await startedWith({ LEDGER_FILE: fileName, SECRETS_KEY: currentKey.toString('base64') }); + const ownKey = await ownKeyIn(fileName); + + const stderr = await refusedStart({ LEDGER_FILE: fileName, SECRETS_KEY: previousKey.toString('base64') }); + + expect(lastLinesOf(stderr)).toEqual([ + `Auto Runtime could not start: OwnKeyUnavailable: The key the runtime keeps for retry_state cannot be read under the key that SECRETS_KEY holds: The secret is sealed under the key ${kidOf(currentKey)}, which is neither the current key nor the previous one. Start the server with the key the ledger's secrets were sealed under, or give that key as SECRETS_KEY_PREVIOUS or SECRETS_KEY_FILE_PREVIOUS beside a new key`, + '', + ]); + expect(secretsIn(stderr, [ownKey])).toEqual([]); + }); +}); + describe('a server given its secrets keys in more places than one', () => { it('stops the process at start for two previous keys, naming both settings and never a key', async () => { const fileName = aLedgerFile(); diff --git a/packages/server/src/logging/logging.ts b/packages/server/src/logging/logging.ts index 983bf518c..d698e98fd 100644 --- a/packages/server/src/logging/logging.ts +++ b/packages/server/src/logging/logging.ts @@ -149,7 +149,7 @@ export interface SecretStoreReport { readonly keyFrom: SecretsKeySource; } -function keyHeldBy(source: SecretsKeySource): string { +export function keyHeldBy(source: SecretsKeySource): string { if (source.from === 'environment') { return `that ${source.setting} holds`; } diff --git a/packages/server/src/testing/entries/serve-with-waiting-tool.ts b/packages/server/src/testing/entries/serve-with-waiting-tool.ts index c5fd7b0ed..353757245 100644 --- a/packages/server/src/testing/entries/serve-with-waiting-tool.ts +++ b/packages/server/src/testing/entries/serve-with-waiting-tool.ts @@ -3,7 +3,7 @@ import { playbookOperations } from '@on.auto/playbooks'; import { compositionRoot } from '../../composition/composition-root.ts'; import { routesServing } from '../../composition/served-routes.ts'; -import { retryStateKeysOf, secretsKeysOf } from '../../database/secrets-keys.ts'; +import { secretsKeysOf } from '../../database/secrets-keys.ts'; import { servedBy } from '../../lifecycle/lifecycle.ts'; import { exitOnStartupFailure, runServer } from '../../lifecycle/run-server.ts'; import { stopInsistedBy, stopRequestedBy } from '../../lifecycle/stop-request.ts'; @@ -20,7 +20,7 @@ await runServer( servedBy( routesServing([...playbookOperations, waitForeverAnnouncedIn(process.env)], { identityIssuer: settings.identityIssuer, - retryStateKeys: retryStateKeysOf(secretsKeysOf({ current: { from: 'start' }, previous: undefined })), + retryStateKeys: secretsKeysOf({ current: { from: 'start' }, previous: undefined }), })(runtime), ), }), diff --git a/packages/server/src/testing/records/kept-credentials.ts b/packages/server/src/testing/records/kept-credentials.ts index 90d329112..01070960b 100644 --- a/packages/server/src/testing/records/kept-credentials.ts +++ b/packages/server/src/testing/records/kept-credentials.ts @@ -4,6 +4,7 @@ import { Ledger } from '@on.auto/operations'; import type { SecretStore } from '@on.auto/secrets'; import { Effect, ManagedRuntime, Redacted } from 'effect'; +import { ownKeysDecider, ownKeysStream, type OwnKeyEvent } from '../../database/own-keys.ts'; import { custodiansOf, secretStoreOver } from '../../database/secret-store.ts'; export const secretsKey = Buffer.alloc(32, 7); @@ -24,6 +25,20 @@ export async function secretIdOf(fileName: string, org: string, connection: stri return state.get(connection)?.secretId; } +export async function ownKeyFactsIn(fileName: string): Promise { + const runtime = ManagedRuntime.make(ledgerLayer({ fileName })); + const collecting = { + ...ownKeysDecider, + initialState: [], + evolve: (facts: readonly OwnKeyEvent[], fact: OwnKeyEvent) => [...facts, fact], + }; + const { state } = await runtime.runPromise( + Effect.flatMap(Ledger, (ledger) => ledger.load(ownKeysStream, collecting)), + ); + await runtime.dispose(); + return state; +} + export interface KeptSecret { readonly value: string | undefined; readonly endsWith: string | undefined;