From 10c399666c60862ac5634795e316a2380e5e00f6 Mon Sep 17 00:00:00 2001 From: Rami Date: Tue, 20 Oct 2026 20:24:16 +0100 Subject: [PATCH 1/4] feat(connections): a secret store that cannot answer now fails the call in its words The paths a refusing or silent custodian reached still died, read a failure as nothing or revoked what the runtime had been given; each now fails the call in the store's words and keeps what it can. - ClientRegistrations.latestAt, named and keep fail with { _tag, detail }, the store's tag and words: makeClientRegistrations no longer reads a row it cannot read now as no registration, nor dies on a keep. The consent client answers both as registration_unavailable: a start and a finish as unavailable, "The client the runtime registered for the connection drive cannot be kept or read now: ", a renewal as a failed call that breaks nothing, and a removal by logging that the tokens were not revoked. - A registration made at the authorization server that the store cannot keep now is logged at WARN by its issuer and client id, never its secret, held in memory by the consent client, and kept at the next start at that issuer before anything is read or registered, so no second client is registered there while the process runs. One the store refuses, secret_refused, is never held, since it will never be kept: it is logged as staying registered there unused, and the start fails as registration_refused, conflict in the refusal's words. - SecretKeeping's failures carry their tag. A renewal whose replacement the store cannot make now revokes nothing, holds the renewed tokens in memory and fails the call in the store's words; while its claim lasts it serves the access token it holds, and the next renewal claims the row again and keeps those tokens without asking the server, or takes the value another process kept. A value the store refuses, secret_refused, still revokes. The renewals at the vendor and through Auto's consent service keep through the same replace. - start answers unavailable where its verifier cannot be kept; finish where the verifier cannot be read, redeeming nothing, and where the tokens cannot be kept once the code is redeemed, revoking them and saying the person starts again, since the code is spent; give_credential where the value cannot be kept, leaving the link open. A removal logs tokens it cannot read now and goes on. - The log of a credential that cannot be read now keys its cause by the failure's tag and kid, not by its words. - valuesInTheClear of @on.auto/secrets/testing refuses a value past its bound as a SecretRefused. - Tests over a recording custodian told to refuse: registrations read and kept with the store's tag, a start, a finish before and after the code is redeemed, give_credential, two removals, renewals at the vendor and through the service, a row another process replaced before the next claim, a row that cannot be read now, and the log of each cause; and a registration the store refuses, registered again at the next start and answered as conflict. - The connections and tool-servers READMEs, the HTTP reference's responses of a connection, and the descriptions of give_credential and finish, with docs/assets/openapi.json written again. Co-Authored-By: Claude Fable 5.1 --- docs/assets/openapi.json | 4 +- docs/reference/http.md | 4 +- packages/connections/README.md | 12 +- .../src/operations/connection-services.ts | 15 +- .../src/operations/remove-connection.test.ts | 55 +++- .../connections/src/page/consent-asking.ts | 13 +- .../connections/src/page/consent-finish.ts | 25 +- .../connections/src/page/consent-start.ts | 15 +- .../connections/src/page/consent-words.ts | 34 ++- .../connections/src/page/give-credential.ts | 7 +- packages/connections/src/page/given-fields.ts | 16 +- .../src/page/store-unavailable.test.ts | 177 +++++++++++++ .../client-registrations.test.ts | 52 +++- .../src/registrations/client-registrations.ts | 13 +- .../src/serving/consent-serving.test.ts | 5 +- .../src/serving/given-credentials.ts | 16 +- .../serving/unreadable-credentials.test.ts | 28 +- packages/connections/src/testing/consents.ts | 4 +- .../src/testing/sealed-connections.ts | 34 ++- .../src/testing/sealed-consents.ts | 38 +++ packages/secrets/src/testing/index.ts | 1 - .../src/testing/values-in-the-clear.test.ts | 8 +- .../src/testing/values-in-the-clear.ts | 22 +- packages/tool-servers/README.md | 6 +- .../src/consent-testing/given-consents.ts | 42 ++- .../src/consent-testing/memory-tokens.ts | 44 +++- .../src/consent-testing/sealed-tokens.ts | 40 +++ .../src/consent-testing/served-consents.ts | 3 +- .../src/consent-tokens/held-in-memory.ts | 18 ++ .../src/consent-tokens/renewal-unkept.test.ts | 242 ++++++++++++++++++ .../src/consent-tokens/renewal-words.ts | 10 + .../src/consent-tokens/renewed-tokens.ts | 60 ++++- .../src/consent/acting-clients.ts | 28 +- .../src/consent/consent-client.ts | 34 ++- .../src/consent/consent-clients.test.ts | 103 +++++++- .../src/consent/dynamic-registration.ts | 105 +++++++- packages/tool-servers/src/index.ts | 3 + packages/tool-servers/src/testing/index.ts | 1 + 38 files changed, 1182 insertions(+), 155 deletions(-) create mode 100644 packages/connections/src/page/store-unavailable.test.ts create mode 100644 packages/connections/src/testing/sealed-consents.ts create mode 100644 packages/tool-servers/src/consent-testing/sealed-tokens.ts create mode 100644 packages/tool-servers/src/consent-tokens/held-in-memory.ts create mode 100644 packages/tool-servers/src/consent-tokens/renewal-unkept.test.ts diff --git a/docs/assets/openapi.json b/docs/assets/openapi.json index a203d00bc..1c3879098 100644 --- a/docs/assets/openapi.json +++ b/docs/assets/openapi.json @@ -3733,7 +3733,7 @@ "post": { "operationId": "give_credential", "summary": "Give credential", - "description": "Gives the credential of a connection through its open link, which approves the connection and uses the link up, and answers it connected. The runtime keeps the value sealed in its secret store, and no answer, record or history of the org ever holds it. On a runtime that signs people in, only the person who added the connection may give it.", + "description": "Gives the credential of a connection through its open link, which approves the connection and uses the link up, and answers it connected. The runtime keeps the value sealed in its secret store, and no answer, record or history of the org ever holds it. Where the secret store cannot keep it just now, it answers unavailable in the words of the store, keeps nothing and leaves the link open, so the same credential is given again. On a runtime that signs people in, only the person who added the connection may give it.", "tags": [ "connections" ], @@ -4311,7 +4311,7 @@ "post": { "operationId": "finish", "summary": "Finish consent", - "description": "Finishes the consent of a connection with the code and the state its authorization server sent the person back with, which connects it, and answers it connected. A consent begun through Auto's consent service is finished with the handle and the state the service sent the person back with instead. The runtime keeps the tokens sealed in its secret store, and no answer, record or history of the org ever holds them. On a runtime that signs people in, only the person who added the connection may finish it.", + "description": "Finishes the consent of a connection with the code and the state its authorization server sent the person back with, which connects it, and answers it connected. A consent begun through Auto's consent service is finished with the handle and the state the service sent the person back with instead. The runtime keeps the tokens sealed in its secret store, and no answer, record or history of the org ever holds them. Where the secret store cannot read the verifier just now, it answers unavailable and the same code may be tried again; where it cannot keep the tokens once the code is redeemed, it answers unavailable, revokes them, and the person starts again, since the code is spent. On a runtime that signs people in, only the person who added the connection may finish it.", "tags": [ "connections" ], diff --git a/docs/reference/http.md b/docs/reference/http.md index 03f19c617..feb7b5374 100644 --- a/docs/reference/http.md +++ b/docs/reference/http.md @@ -1088,7 +1088,7 @@ Every operation answers the connection: `id`, a UUIDv7; `name`; `type`, `mcp` or - `approve` connects a connection whose credential is `none` or `reference` and uses its link up. `give_credential` keeps the value sealed in the runtime's secret store, connects the connection and uses its link up: giving the credential is its approval. - A connection is added by a person: on a runtime that signs people in, `add_connection` refuses a key, and a runtime of API keys alone, which signs nobody in, refuses every connection, since no person could approve it. - On a runtime that signs people in, `approve`, `give_credential`, `start` and `finish` need the session of the person who added the connection. In local mode the token alone binds. -- `remove_connection` removes a connection for good, and the runtime forgets its credential, and what a consent started for it keeps, having first revoked at the provider the tokens of a connection connected by consent. Removing a removed connection succeeds and records nothing. Its name may then be given to a new connection. +- `remove_connection` removes a connection for good, and the runtime forgets its credential, and what a consent started for it keeps, having first revoked at the provider the tokens of a connection connected by consent; tokens it cannot read or revoke now are logged and left to end on their own, and the removal goes on. Removing a removed connection succeeds and records nothing. Its name may then be given to a new connection. | Case | Response | | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------ | @@ -1112,6 +1112,8 @@ Every operation answers the connection: `id`, a UUIDv7; `name`; `type`, `mcp` or | `finish` with no consent started, or one started more than ten minutes ago | `not_found` | | `finish` with a state the consent was not started with, at `/state`, an issuer that is not the server's, at `/iss`, or a code the server refuses, at `/code` | `invalid_input` | | `start` or `finish` where the authorization server does not answer | `unavailable` | +| `give_credential`, `start` or `finish` where the runtime's secret store cannot keep or read now what it needs, in the store's words; a `finish` whose code was redeemed then says the person starts again, its tokens revoked | `unavailable` | +| `start` whose client, registered by the runtime at the authorization server, is one its secret store refuses to keep, in the store's words | `conflict` | | `approve` of a connection whose credential is to be given, or `give_credential` of one that takes none | `conflict` | ### Providers diff --git a/packages/connections/README.md b/packages/connections/README.md index a4ec19efa..a8fbdf71b 100644 --- a/packages/connections/README.md +++ b/packages/connections/README.md @@ -69,10 +69,10 @@ There is no read model: each call folds the stream (`connectionsFold`, the fold - `list_providers` (`src/providers`) answers the providers the policy could allow here, sorted by name, each with `name`, `title`, `description`, `website`, `type`, its ways, `credentials`, as `{ kind, in?, name?, scheme?, issuer?, scope?, description?, offered, because? }`, and the placeholders a person gives, `placeholders`, as `{ name, description, pattern?, example?, required, choices? }`. A way is listed where `credential_kinds` takes its kind and the hosts it reaches, `hostAlternativesOf` of `@on.auto/providers`, could pass `allowed_hosts`: a fixed host a pattern takes; a host with a placeholder where every host is allowed, where an allowed host has its shape, the placeholder standing for letters, digits and hyphens, or where a pattern `*.` and a host ends the part of the host after its last placeholder; none where `allowed_hosts` is empty; and, of a placeholder whose choices set the host, the hosts of any one choice. A way is `offered: false` while a variable its provider's references name is not set, with `because` saying whoever runs the runtime sets it up first, and a way of the kind `authorization` is `offered: false` where the consent client has none of the client sources it lists, `sourcesOf` of what `hadSources` of `ConsentClient` answers for the base of the Studio the page opens, `studioBaseFor`, once the clients of Auto's consent service are listed where the client takes that source and the base is one the service sends people back to, judged with the way's issuer as the provider writes it, with `because` `The provider drive takes consent through a client whoever runs Auto registers, which is not set up here`; `add_connection` refuses such a way at `/credential/kind` in the same words, judged with the issuer resolved. A provider with no way listed is left out. `may_add` says whether this caller may add a connection here, and `refusal`, where it may not, the first sentence that refuses, in this order: the sentence a caller without `org:write` is refused with (`permissionRefusalOf` of `@on.auto/operations`), which the dispatcher checks before any handler runs, then the adder check `add_connection` begins with, then `people_may_add`; it is `null` otherwise. Nothing of an entry is secret, so the list is answered whole. - `makeProviderOffering(services)` (`src/providers/provider-offering.ts`) is the `ProviderOffering` of `@on.auto/tool-servers`, which the server gives the org's `search`, an effect for a caller: to a caller for whom `may_add` would be `true`, by the same judgement, `refusalToAdd`, it answers the providers `list_providers` lists with at least one way `offered`, each by `name`, `title` and `description`, reading `offered` from the listing and judging nothing again, so a provider none of whose ways is offered is left out of `search` while the page lists it with its ways marked, and to any other caller nothing, so `search` lists the providers a person may connect only to a caller who may add one. - `read_link` matches the token as `approve` does, never binding a person, and answers the record and `link`, `{ expires_at }`, the time `connection_link_issued` holds and the match checks, so the page can show the connection and how long the person has, and `consent`, `{ started_at, expires_at }`, while a start is open. For a connection of the kind `authorization` it answers `client_source` too, the source a start would take now, `sourceOf` of what `hadSources` of the consent client answers for the base of the Studio the link opens, the one rule the start takes its source by, `registered` where whoever runs the runtime registered a client at the issuer, else `auto` where Auto's consent service serves it, then the rest, so the page can say before the person goes that Auto's client is used; it leaves the field out where the runtime has no source the way takes, and for a connection of another kind asks the consent client nothing. The start's own `client_source` is the source it took, which differs only where a metadata document the start meant to take is one the server does not take. `approve` and `give_credential` answer the record alone, since they use the link up. -- `give_credential` checks the link and the fields the credential's kind takes, `value` alone for a `secret`, `client_secret` alone or `private_key` with `algorithm` for `client_credentials`, else `invalid_input`. Client credentials take `client_id` beside them where the credential holds none, and refuse it where it holds one, each at `/client_id`: `Expected client_id: the connection's credential holds no client id, so it is given here with the secret or the key` and `Expected no client_id: the connection's credential holds its client id already`. A client id given here is read as `add_connection` reads the entry, with the value standing in for the secret (`clientIdReadingIssuesOf`), and one that looks like a credential is refused at `/client_id` before anything is kept, so no connection is connected with a client id its use would refuse: `Looks like a credential, which a client id never is: the connection's record keeps the client id in the open, so the secret or the key goes in its own field, where it is sealed`. A `secret` whose credential names a `scheme` refuses a value that starts with the scheme and a space, in any case, at `/value`, `Paste the token alone, without Bearer before it: the runtime writes Bearer itself`. It keeps exactly the value in the secret store, a value the store refuses being `invalid_input` at its pointer with the store's sentence, and records the credential's id, and the client id given on the page. A decision that fails after the value was kept, as when the connection was removed meanwhile, forgets it. -- `start` (`src/page/consent-start.ts`) takes `{ token }` and asks the consent client to start (`src/page/consent-asking.ts`) with the base of the Studio the link opens, `studioBaseFor` of `src/links/connect-page.ts`, exactly as the link opens it, which the client judges and sends to Auto's consent service where it starts through it: the connection's server, `{ type: mcp, url }` or `{ type: openapi }`, its credential's way, the return address, `returnAddressOf` of `src/links/connect-page.ts`, written beside `connectPageOf` as the page `connect-return` under the base the link's page takes, `studioBaseOf`, beside the page `connect`, since Studio's build names its files relative to the page: `/studio/connect-return` where the runtime serves Studio, since a page on the Studio host cannot reach a runtime on a loopback address, and the Studio host's `/connect-return` otherwise, `STUDIO_ORIGIN` followed by it, so the person returns to the Studio they started from; a runtime has one return address at a time, the one its connect link opens, and a client is registered with both only where it serves several runtimes, some serving Studio and some not, the rule's `reaches`, `reachesUnder` of `src/policy/policy-rules.ts`, which the serving of a connection takes too, and a state of 32 random bytes in base64url, `issuedToken` of `src/links/link-token.ts`, the link's token's own. It keeps the verifier the client made in the secret store, records the start with the state's digest and ten minutes of its own, `tenMinutesOn`, and answers `{ authorize: { url, expires_at } }`, `201`; a start that cannot be recorded, whether the record fails or dies, forgets the verifier it kept, and a start that replaces one forgets the verifier of the one before. What the client refuses is answered in the words of `src/page/consent-words.ts`: another issuer, another resource, no metadata, no S256 challenge and no client the runtime has as `conflict`, `The server of the connection drive names https://elsewhere.example.com as its authorization server, where its provider names https://accounts.drive.example.com, so it cannot be connected as it was added`, or, where the metadata of the authorization server names another issuer, `The authorization server of the connection drive names https://elsewhere.example.com as its issuer in its own metadata, where its provider names https://accounts.drive.example.com, so it cannot be connected as it was added`, a host the rules refuse as `forbidden` in the rule's words, and a server that does not answer as `unavailable`. The start records the digest of the state the client answers, its own for a client of the runtime's and the service's for `auto`, and answers `client_source`, the source of the client it was started with, beside `authorize`. A Studio neither hosted nor on the person's computer leaves Auto's client out, so a later source the way lists serves the start; where none remains, the start is refused before anything is sent, `conflict`, `The connection drive takes consent through Auto's client only where Studio is Auto's or on this computer, so whoever runs this runtime registers a client`. A start Auto's consent service refuses, since Auto has no client at the issuer or the service does not send people back to the Studio the link opens, is `conflict` too, `Auto's consent service did not start the consent of the connection drive: Auto has no client at its authorization server, or the service does not send people back to the Studio its link opens, so whoever runs this runtime registers a client`. -- `finish` (`src/page/consent-finish.ts`) takes one of two shapes, a union of the input schema, `{ token, state, code, iss? }` for a start with a client of the runtime's, `registered`, `metadata_document` or `dynamic`, and `{ token, state, handle }` for a start of `auto`, whose handle Auto's consent service sends the person back with; the start's client decides, and a finish of the other shape is `invalid_input` at the field it gave, `/code`, `Expected handle: the consent of the connection drive began through Auto's consent service, which answers a handle, not a code`, or `/handle`, `Expected code: the consent of the connection drive began with a client of this runtime's, which answers a code, not a handle`; a handle the service refuses, taken already, ended or answering another start, is `invalid_input` at `/handle`, `Expected a handle Auto's consent service takes; it refused this one`, and leaves the start open as a refused code does. Otherwise it finds the open start as the decider does, forgetting the verifier of one whose ten minutes are over, reads the verifier, `not_found` where its row is gone, and asks the consent client to redeem the code with the client the start recorded. It keeps the tokens as one sealed value in a new row of the secret store, forgets the verifier's row, and records the approval and the connection with the client the consent client answers it redeemed with, the one the start recorded, or for `auto` the one the service's handoff names, answering the connected record. An issuer that is not the server's is `invalid_input` at `/iss`, `Expected the issuer of the authorization server of the connection, https://accounts.drive.example.com`, a code the server refuses at `/code`, `Expected a code the authorization server takes; it refused this one`, and the start stays open; a client no longer registered is `conflict`, `The client the connection drive was made with is no longer registered, so the person who added it removes it and adds it again`, a grant of no scope to a scope asked, which the client refuses after revoking the tokens, is `conflict`, `The authorization server of the connection drive granted none of the scopes asked, so the connection is not connected; start again and allow them`, and a grant that is not a scope, refused so too, is `conflict`, `The authorization server of the connection drive granted a scope written with characters a scope does not hold, so it cannot be connected as it was added`; an empty grant where no scope was asked records none. Where the tokens exist and the finish is refused after, a value the store refuses, `conflict` in the store's words, or a decision that fails, as when the connection was removed meanwhile, it revokes the tokens through the client and forgets their row before it answers; the steps from keeping the tokens to the record run under `Effect.onError`, so a step that dies or is interrupted revokes and forgets as one that fails does. -- `remove_connection` forgets the connection's secret once its removal is appended, and the verifier of a start it holds, and forgetting twice is forgetting once. A connection of the kind `authorization` whose row of tokens the store still holds has them revoked first, before the row is forgotten: it reads the row and asks the consent client to revoke them, `revoke` of `ConsentClient`, with the client the connection recorded, bounded as a call is, at the revocation endpoint the metadata or the way names. A revocation that does not happen is logged at WARN in words of its own for each case (`unrevokedWordsOf` of `src/page/consent-words.ts`), and the removal goes on: refused, `The authorization server of the connection drive refused to revoke its tokens, answering HTTP 401, so they end on their own`; not answered, `The authorization server of the connection drive did not answer the revocation of its tokens, so they end on their own`, or, for a connection of `auto`, `Auto's consent service did not answer the revocation of the tokens of the connection drive, so they end on their own`; a key Auto's consent service does not know, `Auto's consent service does not know the key of the connection drive, so its tokens were not revoked and end on their own`; a host the rules refuse, `The tokens of the connection drive were not revoked, so they end on their own: `; and a client gone or metadata that changed, `The tokens of the connection drive could not be revoked with the client and at the authorization server it was made with, so they end on their own`. None holds a word of the server's or a token; a connection that holds no tokens, pending or removed already, revokes nothing. +- `give_credential` checks the link and the fields the credential's kind takes, `value` alone for a `secret`, `client_secret` alone or `private_key` with `algorithm` for `client_credentials`, else `invalid_input`. Client credentials take `client_id` beside them where the credential holds none, and refuse it where it holds one, each at `/client_id`: `Expected client_id: the connection's credential holds no client id, so it is given here with the secret or the key` and `Expected no client_id: the connection's credential holds its client id already`. A client id given here is read as `add_connection` reads the entry, with the value standing in for the secret (`clientIdReadingIssuesOf`), and one that looks like a credential is refused at `/client_id` before anything is kept, so no connection is connected with a client id its use would refuse: `Looks like a credential, which a client id never is: the connection's record keeps the client id in the open, so the secret or the key goes in its own field, where it is sealed`. A `secret` whose credential names a `scheme` refuses a value that starts with the scheme and a space, in any case, at `/value`, `Paste the token alone, without Bearer before it: the runtime writes Bearer itself`. It keeps exactly the value in the secret store, a value the store refuses, `secret_refused`, being `invalid_input` at its pointer with the store's sentence, and a store that cannot keep it now, its custodian refusing or not answering, `unavailable`, `The credential of the connection crm cannot be kept now, so the connection is not connected and its link stays open: `, keeping nothing, so the same credential is given again through the same link; it records the credential's id, and the client id given on the page. A decision that fails after the value was kept, as when the connection was removed meanwhile, forgets it. +- `start` (`src/page/consent-start.ts`) takes `{ token }` and asks the consent client to start (`src/page/consent-asking.ts`) with the base of the Studio the link opens, `studioBaseFor` of `src/links/connect-page.ts`, exactly as the link opens it, which the client judges and sends to Auto's consent service where it starts through it: the connection's server, `{ type: mcp, url }` or `{ type: openapi }`, its credential's way, the return address, `returnAddressOf` of `src/links/connect-page.ts`, written beside `connectPageOf` as the page `connect-return` under the base the link's page takes, `studioBaseOf`, beside the page `connect`, since Studio's build names its files relative to the page: `/studio/connect-return` where the runtime serves Studio, since a page on the Studio host cannot reach a runtime on a loopback address, and the Studio host's `/connect-return` otherwise, `STUDIO_ORIGIN` followed by it, so the person returns to the Studio they started from; a runtime has one return address at a time, the one its connect link opens, and a client is registered with both only where it serves several runtimes, some serving Studio and some not, the rule's `reaches`, `reachesUnder` of `src/policy/policy-rules.ts`, which the serving of a connection takes too, and a state of 32 random bytes in base64url, `issuedToken` of `src/links/link-token.ts`, the link's token's own. It keeps the verifier the client made in the secret store, or answers `unavailable` where the store cannot keep it now, `The consent of the connection drive cannot be started now, since the runtime cannot keep its verifier: `, recording nothing, records the start with the state's digest and ten minutes of its own, `tenMinutesOn`, and answers `{ authorize: { url, expires_at } }`, `201`; a start that cannot be recorded, whether the record fails or dies, forgets the verifier it kept, and a start that replaces one forgets the verifier of the one before. What the client refuses is answered in the words of `src/page/consent-words.ts`: another issuer, another resource, no metadata, no S256 challenge and no client the runtime has as `conflict`, `The server of the connection drive names https://elsewhere.example.com as its authorization server, where its provider names https://accounts.drive.example.com, so it cannot be connected as it was added`, or, where the metadata of the authorization server names another issuer, `The authorization server of the connection drive names https://elsewhere.example.com as its issuer in its own metadata, where its provider names https://accounts.drive.example.com, so it cannot be connected as it was added`, a host the rules refuse as `forbidden` in the rule's words, a server that does not answer as `unavailable`, and a client the runtime registered dynamically that the store cannot keep or read now as `unavailable` too, `The client the runtime registered for the connection drive cannot be kept or read now: `, or one it refuses to keep as `conflict`, `The client the runtime registered for the connection drive cannot be kept: `. The start records the digest of the state the client answers, its own for a client of the runtime's and the service's for `auto`, and answers `client_source`, the source of the client it was started with, beside `authorize`. A Studio neither hosted nor on the person's computer leaves Auto's client out, so a later source the way lists serves the start; where none remains, the start is refused before anything is sent, `conflict`, `The connection drive takes consent through Auto's client only where Studio is Auto's or on this computer, so whoever runs this runtime registers a client`. A start Auto's consent service refuses, since Auto has no client at the issuer or the service does not send people back to the Studio the link opens, is `conflict` too, `Auto's consent service did not start the consent of the connection drive: Auto has no client at its authorization server, or the service does not send people back to the Studio its link opens, so whoever runs this runtime registers a client`. +- `finish` (`src/page/consent-finish.ts`) takes one of two shapes, a union of the input schema, `{ token, state, code, iss? }` for a start with a client of the runtime's, `registered`, `metadata_document` or `dynamic`, and `{ token, state, handle }` for a start of `auto`, whose handle Auto's consent service sends the person back with; the start's client decides, and a finish of the other shape is `invalid_input` at the field it gave, `/code`, `Expected handle: the consent of the connection drive began through Auto's consent service, which answers a handle, not a code`, or `/handle`, `Expected code: the consent of the connection drive began with a client of this runtime's, which answers a code, not a handle`; a handle the service refuses, taken already, ended or answering another start, is `invalid_input` at `/handle`, `Expected a handle Auto's consent service takes; it refused this one`, and leaves the start open as a refused code does. Otherwise it finds the open start as the decider does, forgetting the verifier of one whose ten minutes are over, reads the verifier, `not_found` where its row is gone and `unavailable` where the store cannot read it now, `The consent of the connection drive cannot be finished now, since the runtime cannot read its verifier; the same code may be tried again shortly: `, redeeming nothing, and asks the consent client to redeem the code with the client the start recorded. It keeps the tokens as one sealed value in a new row of the secret store, forgets the verifier's row, and records the approval and the connection with the client the consent client answers it redeemed with, the one the start recorded, or for `auto` the one the service's handoff names, answering the connected record. An issuer that is not the server's is `invalid_input` at `/iss`, `Expected the issuer of the authorization server of the connection, https://accounts.drive.example.com`, a code the server refuses at `/code`, `Expected a code the authorization server takes; it refused this one`, and the start stays open; a client no longer registered is `conflict`, `The client the connection drive was made with is no longer registered, so the person who added it removes it and adds it again`, a grant of no scope to a scope asked, which the client refuses after revoking the tokens, is `conflict`, `The authorization server of the connection drive granted none of the scopes asked, so the connection is not connected; start again and allow them`, and a grant that is not a scope, refused so too, is `conflict`, `The authorization server of the connection drive granted a scope written with characters a scope does not hold, so it cannot be connected as it was added`; an empty grant where no scope was asked records none. Where the tokens exist and the finish is refused after, a value the store refuses, `conflict` in the store's words, a store that cannot keep them now, `unavailable`, `The tokens of the connection drive cannot be kept now, and the code they were given for is spent, so the person starts the consent again: `, or a decision that fails, as when the connection was removed meanwhile, it revokes the tokens through the client and forgets their row before it answers; the steps from keeping the tokens to the record run under `Effect.onError`, so a step that dies or is interrupted revokes and forgets as one that fails does. +- `remove_connection` forgets the connection's secret once its removal is appended, and the verifier of a start it holds, and forgetting twice is forgetting once. A connection of the kind `authorization` whose row of tokens the store still holds has them revoked first, before the row is forgotten: it reads the row and asks the consent client to revoke them, `revoke` of `ConsentClient`, with the client the connection recorded, bounded as a call is, at the revocation endpoint the metadata or the way names. A revocation that does not happen is logged at WARN in words of its own for each case (`unrevokedWordsOf` of `src/page/consent-words.ts`), and the removal goes on: refused, `The authorization server of the connection drive refused to revoke its tokens, answering HTTP 401, so they end on their own`; not answered, `The authorization server of the connection drive did not answer the revocation of its tokens, so they end on their own`, or, for a connection of `auto`, `Auto's consent service did not answer the revocation of the tokens of the connection drive, so they end on their own`; a key Auto's consent service does not know, `Auto's consent service does not know the key of the connection drive, so its tokens were not revoked and end on their own`; a host the rules refuse, `The tokens of the connection drive were not revoked, so they end on their own: `; a row the store cannot read now, `The tokens of the connection drive cannot be read now, so they were not revoked and end on their own: `, and a dynamic registration of the client it cannot read now, `The tokens of the connection drive were not revoked, since the client they were given to cannot be read now, so they end on their own: `; and a client gone or metadata that changed, `The tokens of the connection drive could not be revoked with the client and at the authorization server it was made with, so they end on their own`. None holds a word of the server's or a token; a connection that holds no tokens, pending or removed already, revokes nothing. - The five operations of the page are permitted by every permission, since the token binds them and, where people sign in, the person: their caller must still belong to the org of the route. ## The ending of a credential @@ -89,7 +89,7 @@ There is no read model: each call folds the stream (`connectionsFold`, the fold ### The clients the runtime registered itself -A client the consent client registers dynamically at an authorization server is kept by `makeClientRegistrations({ registry, secrets })` (`src/registrations/client-registrations.ts`), the `ClientRegistrations` of `@on.auto/tool-servers`: the registration, sealed in the secret store, and its fact, `client_registered`, `{ issuer, client_id, secret_id }`, by the runtime, `by: runtime`, in a stream of the runtime's own and of no org, `runtime/client-registrations`, read and written by the full stream name as the source of the calls reads the org's stream. The fold keeps the registrations by issuer and client id, the latest last, so `latestAt(issuer)` answers the registration a later connection at the issuer is made with, and `named(issuer, client_id)` the one a connection recorded, whichever came after it. A registration whose row the store no longer keeps or cannot read is none. +A client the consent client registers dynamically at an authorization server is kept by `makeClientRegistrations({ registry, secrets })` (`src/registrations/client-registrations.ts`), the `ClientRegistrations` of `@on.auto/tool-servers`: the registration, sealed in the secret store, and its fact, `client_registered`, `{ issuer, client_id, secret_id }`, by the runtime, `by: runtime`, in a stream of the runtime's own and of no org, `runtime/client-registrations`, read and written by the full stream name as the source of the calls reads the org's stream. The fold keeps the registrations by issuer and client id, the latest last, so `latestAt(issuer)` answers the registration a later connection at the issuer is made with, and `named(issuer, client_id)` the one a connection recorded, whichever came after it. A registration whose row the store no longer keeps is none; one the store cannot read now fails with the store's tag and words, `{ _tag, detail }`, and so does a registration it cannot keep now or refuses, which records no fact, so the consent client answers both as its README says rather than registering again over a client it cannot read. ### The providers the setting adds @@ -107,7 +107,7 @@ A client the consent client registers dynamically at an authorization server is 1. The policy of the moment is checked with the rules `add_connection` applies (`useRefusalOf`), and a connection a rule no longer takes is `connection_not_allowed` with the rule's sentence, its credential unread: `credential_kinds`, `variables` for a `reference`, and `allowed_hosts` over the entry's `url`, `document` and `base_url` and the credential's addresses, `wayAddresses`. With `people_may_add` `false` the runtime uses no connection, in words that speak of using rather than adding, `This runtime uses no connection: its operator sets up its tool servers itself`, while `list_connections` still answers each as it is. 2. The connection is checked with `entryIssuesOf`, the checks of `add_connection` that need no value, against the model providers of the moment, so an entry the runtime no longer takes, as one named as a model provider set up since, is `connection_not_allowed`, `The entry of the connection is not one this runtime takes any more: `, its credential unread. -3. The credential is read (`src/serving/given-credentials.ts`): none for `none`; for a `reference`, the value of its `variable`, and a variable the environment leaves unset is `connection_not_allowed`, saying whoever runs the runtime sets it and starts it again; for a `secret` or `client_credentials`, the store's row of its secret's id; for `authorization`, the row of its tokens, the `secret_id` of its `connection_connected`, read as a `secret`'s is and decoded with `keptTokensIn` of `@on.auto/tool-servers`, a value that is no tokens breaking the connection, `The credential of the connection crm is not the tokens of a consent the runtime keeps, so the person who added it removes it and adds it again`, which places nothing in the entry and is handed to `@on.auto/tool-servers`, decoded, with the rest of the connection's consent once the entry is read, below. A row the store cannot read now, for any of its three failures, `secret_key_unknown`, `custodian_refused` or `keeper_unavailable`, fails the call that needed it and breaks nothing: the source answers it unready, `connection_unavailable`, `The credential of the connection crm cannot be read now: `, records nothing, and the connection stays connected, so its credential is read again at its next use and the call is made once the store can read it. The source logs that sentence as a warning, annotated with the org and the connection's id, once for each cause of each connection in the process, the cause being the store's words, so a call that meets the same cause logs nothing more, one that meets another logs it, and one after a read that succeeded logs it again; the source keeps the last cause logged of each connection for as long as it lives, and forgets it at a read that succeeds. Only a row that is gone, and a value that is not the tokens the connection expects, break the connection, since nothing mends them: the source issues `break`, records `connection_broken` with the sentence of why, `The credential of the connection crm is no longer in the secret store, so the person who added it removes it and adds it again` or the sentence of tokens above, and answers it `connection_broken` in it. Nothing un-breaks a connection; the person removes it and adds it again. +3. The credential is read (`src/serving/given-credentials.ts`): none for `none`; for a `reference`, the value of its `variable`, and a variable the environment leaves unset is `connection_not_allowed`, saying whoever runs the runtime sets it and starts it again; for a `secret` or `client_credentials`, the store's row of its secret's id; for `authorization`, the row of its tokens, the `secret_id` of its `connection_connected`, read as a `secret`'s is and decoded with `keptTokensIn` of `@on.auto/tool-servers`, a value that is no tokens breaking the connection, `The credential of the connection crm is not the tokens of a consent the runtime keeps, so the person who added it removes it and adds it again`, which places nothing in the entry and is handed to `@on.auto/tool-servers`, decoded, with the rest of the connection's consent once the entry is read, below. A row the store cannot read now, for any of its three failures, `secret_key_unknown`, `custodian_refused` or `keeper_unavailable`, fails the call that needed it and breaks nothing: the source answers it unready, `connection_unavailable`, `The credential of the connection crm cannot be read now: `, records nothing, and the connection stays connected, so its credential is read again at its next use and the call is made once the store can read it. The source logs that sentence as a warning, annotated with the org and the connection's id, once for each cause of each connection in the process, the cause being the failure's tag and the kid it names and not its words, so a call that meets the same cause logs nothing more, whatever the custodian says this time, one that meets another logs it, and one after a read that succeeded logs it again; the source keeps the last cause logged of each connection for as long as it lives, and forgets it at a read that succeeds. Only a row that is gone, and a value that is not the tokens the connection expects, break the connection, since nothing mends them: the source issues `break`, records `connection_broken` with the sentence of why, `The credential of the connection crm is no longer in the secret store, so the person who added it removes it and adds it again` or the sentence of tokens above, and answers it `connection_broken` in it. Nothing un-breaks a connection; the person removes it and adds it again. 4. The credential is placed as the file places a reference (`src/serving/placed-credentials.ts`): the entry, every `$` in it written `$$` so that it is sent as it is written, with a `secret` or a `reference` as `${…}` in the header, the query parameter or the security scheme its `in` and `name` say, after its `scheme` and a space where it names one, as in `Bearer ${…}`, or `client_credentials` as the `auth` block with its issuer, client id and scope, and the client secret, or the private key, as `${…}` beside the algorithm, the issuer, the client id, the scope and the algorithm each written with `$$` for `$` as the entry is; a `reference` names its own variable, and a value given on the page stands under a variable of its own, in an environment of that one entry. `placedReference` is that placing of a named reference, for client credentials as for a `reference`, to which `placedCredential` gives the variable the value stands under and its environment. In a connection made from a provider, the header and query values are read as the provider wrote them, its references substituted from `providerValues` and scrubbed as every reference's value is, and the rest of the entry as it is written. In one of client credentials or of the kind `authorization`, each `{client_id}` the entry holds stays as the provider wrote it, and the client id of the record's credential, the one the person gave on the page or the client the connection was made with, which `connection_connected` recorded, is handed to `readEntry` as its context's `placedValues` (`clientIdPlacedFor`, the one placing of it), which places it in the header and query values once the entry as written is judged, as its text, so the client id is sent where the provider says and, kept in the open in the record, is scrubbed from nothing; the reading at add places the stand-in client id. `readEntry` of `@on.auto/tool-servers`, the reading of an entry of the file, then resolves it and makes it `ServerSettings`, its type the connection's, its org the org and its `provider` the connection's (`readPlaced`), with `substituted`, `credentialProblems`, `unreferencedSecurity`, `misplacedReferencesOfEntry` and `secretsOfEntry` as they are for the file: every value the reference resolves to is a secret, the whole value of a security scheme too, and a value in a header that is a scheme and a credential, as `Bearer `, a secret whole and its credential alone, so the scrubber catches it in every form `@on.auto/tool-servers` lists. This is `readAddition`, the reading `add_connection` makes with a value standing in for the person's, made once here with the real value; what it refuses is `connection_not_allowed` in the words of step 2, so a check that a value can fail refuses in its own words, as an empty value of a reference does. For `authorization` no credential is placed: the entry is read as it is written, its `{client_id}` placed as above, and its `ServerSettings` take as `auth` the third kind of credential of `@on.auto/tool-servers`, `consent`, which `consentAuthOf` composes: the issuer, the client the connection recorded and the scope it was granted, with the way, the server and the return address `start` asks with, `reaches`, the row step 3 read, and `ConnectionTokens`, the store of that row, whose `read` reads it as step 3 does, whose `replace` and `claim` are the secret store's on it, and whose `broken(ended)` records the break as step 3 records one, the one place a break is recorded, in `The consent of the connection drive was revoked or has ended, so the person who added it removes it and adds it again` or in the sentence `finish` has for a client no longer registered, `The client the connection drive was made with is no longer registered, so the person who added it removes it and adds it again`, or, for a connection of `auto` whose key Auto's consent service does not know, `Auto's consent service does not know the key of the connection drive, so its tokens cannot be renewed, and the person who added it removes it and adds it again`. Each runs in the context the connection was prepared in, so its clock and its log are the server's. The consent client serves and renews the tokens from there, as its README says. The answer carries `reaches`, which answers a URL whose host `allowed_hosts` does not take with the sentence of the rule, `hostRefusalOf`; the tool access checks every request of the connection with it, the server its document names where `base_url` is left out among them, and the request of a redirect. diff --git a/packages/connections/src/operations/connection-services.ts b/packages/connections/src/operations/connection-services.ts index a9876fddf..9bbb2003a 100644 --- a/packages/connections/src/operations/connection-services.ts +++ b/packages/connections/src/operations/connection-services.ts @@ -1,5 +1,12 @@ import type { ProviderList } from '@on.auto/providers'; -import type { ConsentClient, KeptRow, TokensClaimed, TokensReplaced } from '@on.auto/tool-servers'; +import type { + ConsentClient, + KeptRow, + SecretKeepingFailure, + SecretReadingFailure, + TokensClaimed, + TokensReplaced, +} from '@on.auto/tool-servers'; import type { Effect, Redacted } from 'effect'; import type { ConnectPages } from '../links/connect-page.ts'; @@ -7,13 +14,13 @@ import type { ConnectionPolicy } from '../policy/connection-policy.ts'; import type { RuntimeAccess } from '../policy/policy-rules.ts'; export interface SecretKeeping { - readonly keep: (value: Redacted.Redacted, endsWith?: string) => Effect.Effect; - readonly read: (id: string) => Effect.Effect; + readonly keep: (value: Redacted.Redacted, endsWith?: string) => Effect.Effect; + readonly read: (id: string) => Effect.Effect; readonly replace: ( id: string, version: number, value: Redacted.Redacted, - ) => Effect.Effect; + ) => Effect.Effect; readonly claim: (id: string, version: number, lastingMs: number) => Effect.Effect; readonly endingsOf: (ids: readonly string[]) => Effect.Effect>; readonly forget: (id: string) => Effect.Effect; diff --git a/packages/connections/src/operations/remove-connection.test.ts b/packages/connections/src/operations/remove-connection.test.ts index dbff4dbcf..3db65a4c4 100644 --- a/packages/connections/src/operations/remove-connection.test.ts +++ b/packages/connections/src/operations/remove-connection.test.ts @@ -1,11 +1,14 @@ +import { custodianRefusal } from '@on.auto/secrets/testing'; import type { ConsentClientOptions } from '@on.auto/tool-servers'; -import { admitted, hostRefusedAt, registeredAt, type FakeMcpServer } from '@on.auto/tool-servers/testing'; +import { admitted, consentedAt, hostRefusedAt, registeredAt, type FakeMcpServer } from '@on.auto/tool-servers/testing'; import { Schema } from 'effect'; import { afterEach, describe, expect, it } from 'vitest'; import { added, crm, notes } from '../testing/additions.ts'; import { consentedOf, consentFake, consentHarnessAt, finishing } from '../testing/consents.ts'; import { acmeReader, ada, harness, operationNamed, type Harness } from '../testing/harness.ts'; +import { sealedInMemory } from '../testing/sealed-connections.ts'; +import { calledThrough, sealedConsentAt, type SealedHarness } from '../testing/sealed-consents.ts'; const closing: (() => Promise)[] = []; @@ -181,3 +184,53 @@ describe('a removal whose revocation cannot be made', () => { ); }); }); + +const StartedSchema = Schema.Struct({ output: Schema.Struct({ authorize: Schema.Struct({ url: Schema.String }) }) }); + +async function connectedThrough(sealed: SealedHarness): Promise { + const { id, token } = await added(sealed.h, { provider: 'files' }); + const started = Schema.decodeUnknownSync(StartedSchema)(await calledThrough(sealed, 'start', { id, token })); + await calledThrough(sealed, 'finish', { id, token, ...(await consentedAt(started.output.authorize.url)) }); + return id; +} + +describe('a removal whose tokens the secret store cannot read now', () => { + it('logs in the words of the store that they were not revoked, and goes on to forget their row', async () => { + const fake = await consentServed(); + const sealed = sealedConsentAt(fake); + const id = await connectedThrough(sealed); + + sealed.custodian.refuse(); + const removed = await calledThrough(sealed, 'remove_connection', { id }); + + expect(removed).toMatchObject({ status: 'succeeded', output: { status: 'removed' } }); + expect(sealed.h.logged().join('\n')).toContain( + `The tokens of the connection files cannot be read now, so they were not revoked and end on their own: The custodian of the key ${sealed.custodian.kid} refused to unwrap a data key: ${custodianRefusal}`, + ); + expect(revocationsAt(fake)).toBe(0); + expect(sealed.rows.kept()).toEqual([]); + }); +}); + +describe('a removal whose client registration the secret store cannot read now', () => { + it('logs in the words of the store that the tokens were not revoked, and goes on to forget their row', async () => { + const fake = await consentFake({ dynamicRegistration: true }); + closing.push(fake.close); + const registrations = sealedInMemory(); + const sealed = sealedConsentAt(fake, { + way: { clients: ['dynamic'] }, + consenting: { registeredClients: [] }, + registrations: registrations.store, + }); + const id = await connectedThrough(sealed); + + registrations.custodian.refuse(); + const removed = await calledThrough(sealed, 'remove_connection', { id }); + + expect(removed).toMatchObject({ status: 'succeeded', output: { status: 'removed' } }); + expect(sealed.h.logged().join('\n')).toContain( + `The tokens of the connection files were not revoked, since the client they were given to cannot be read now, so they end on their own: The custodian of the key ${registrations.custodian.kid} refused to unwrap a data key: ${custodianRefusal}`, + ); + expect(sealed.rows.kept()).toEqual([]); + }); +}); diff --git a/packages/connections/src/page/consent-asking.ts b/packages/connections/src/page/consent-asking.ts index 347297478..0aa430d0f 100644 --- a/packages/connections/src/page/consent-asking.ts +++ b/packages/connections/src/page/consent-asking.ts @@ -1,5 +1,5 @@ import { keptTokensIn, type ConsentServer, type StartAsked } from '@on.auto/tool-servers'; -import { Effect, Function, Option } from 'effect'; +import { Effect, Option } from 'effect'; import { returnAddressOf } from '../links/connect-page.ts'; import type { ConnectionServices } from '../operations/connection-services.ts'; @@ -13,7 +13,7 @@ import { type KeptWithLink, } from '../registry/registry-lookup.ts'; import type { KeptConnection } from '../registry/registry.ts'; -import { unrevokedWordsOf, type ConsentNamed } from './consent-words.ts'; +import { unavailableWords, unrevokedWordsOf, type ConsentNamed } from './consent-words.ts'; export type ConsentAsked = Omit; @@ -84,15 +84,18 @@ export function tokensRevokedOf( return Effect.void; } const client = { client_id: required(credential.client_id), client_source: required(credential.client_source) }; + const named = namedOf(connection); return services.secrets.read(secretId).pipe( - Effect.orElseSucceed(Function.constUndefined), Effect.map((row) => Option.flatMap(Option.fromNullishOr(row), ({ value }) => keptTokensIn(value))), Effect.flatMap( Option.match({ onNone: () => Effect.void, - onSome: (tokens) => services.consent.revoke({ ...consentAskedOf(connection, services), client, tokens }), + onSome: (tokens) => + services.consent + .revoke({ ...consentAskedOf(connection, services), client, tokens }) + .pipe(Effect.catch((failure) => Effect.logWarning(unrevokedWordsOf(named, failure, client.client_source)))), }), ), - Effect.catch((failure) => Effect.logWarning(unrevokedWordsOf(namedOf(connection), failure, client.client_source))), + Effect.catch(({ detail }) => Effect.logWarning(unavailableWords.tokensUnread(named.name, detail))), ); } diff --git a/packages/connections/src/page/consent-finish.ts b/packages/connections/src/page/consent-finish.ts index e29e063d3..6be0bbbb8 100644 --- a/packages/connections/src/page/consent-finish.ts +++ b/packages/connections/src/page/consent-finish.ts @@ -1,4 +1,4 @@ -import { allPermissions, Conflict, defineHttpOperation, type NotFound } from '@on.auto/operations'; +import { allPermissions, defineHttpOperation, Unavailable, type NotFound } from '@on.auto/operations'; import { Effect, Schema } from 'effect'; import { tokenSha256Of } from '../links/link-token.ts'; @@ -14,7 +14,14 @@ import { type ConsentServices, type FinishedConsent, } from './consent-asking.ts'; -import { finishRefusalOf, wrongShapeOf, type ConsentRefused, type Returned } from './consent-words.ts'; +import { + finishRefusalOf, + tokensUnkeptOf, + unavailableWords, + wrongShapeOf, + type ConsentRefused, + type Returned, +} from './consent-words.ts'; const GivenField = Schema.String.check(Schema.isMinLength(1)); @@ -69,14 +76,20 @@ function redeemedOf( ): Effect.Effect { const { consent, secrets } = services; const { client_id: clientId, client_source: clientSource, verifier_secret_id: verifierSecretId } = kept.consent; + const named = namedOf(kept.connection); return Effect.gen(function* () { - const verifier = yield* Effect.orDie(secrets.read(verifierSecretId)); + const verifier = yield* secrets + .read(verifierSecretId) + .pipe( + Effect.mapError( + ({ detail }) => new Unavailable({ detail: unavailableWords.verifierUnread(named.name, detail) }), + ), + ); if (verifier === undefined) { return yield* Effect.fail(noOpenConsent(kept.connection.name)); } const asked = consentAskedOf(kept.connection, services); const client = { client_id: clientId, client_source: clientSource }; - const named = namedOf(kept.connection); return yield* consent .finish({ ...asked, client, ...returnedFieldsOf(finishing), verifier: verifier.value }) .pipe(Effect.mapError((refusal) => finishRefusalOf(named, refusal, returnedOf(finishing)))); @@ -98,8 +111,9 @@ function recordedWith(finished: FinishedConsent, secretId: string, { use, stateS } function connectedWith(finished: FinishedConsent, recording: Recording, { secrets }: ConsentServices) { + const named = namedOf(recording.kept.connection); return Effect.gen(function* () { - const secretId = yield* Effect.mapError(secrets.keep(finished.tokens), ({ detail }) => new Conflict({ detail })); + const secretId = yield* Effect.mapError(secrets.keep(finished.tokens), (failure) => tokensUnkeptOf(named, failure)); return yield* Effect.andThen( secrets.forget(recording.kept.consent.verifier_secret_id), recordedWith(finished, secretId, recording), @@ -116,6 +130,7 @@ export function defineFinishConsent(services: ConsentServices) { 'Finishes the consent of a connection with the code and the state its authorization server sent the person back with, which connects it, and answers it connected.', "A consent begun through Auto's consent service is finished with the handle and the state the service sent the person back with instead.", 'The runtime keeps the tokens sealed in its secret store, and no answer, record or history of the org ever holds them.', + 'Where the secret store cannot read the verifier just now, it answers unavailable and the same code may be tried again; where it cannot keep the tokens once the code is redeemed, it answers unavailable, revokes them, and the person starts again, since the code is spent.', 'On a runtime that signs people in, only the person who added the connection may finish it.', ].join(' '), route: { method: 'POST', path: '/connections/{id}/consent/code' }, diff --git a/packages/connections/src/page/consent-start.ts b/packages/connections/src/page/consent-start.ts index f00bf6278..fa521e76a 100644 --- a/packages/connections/src/page/consent-start.ts +++ b/packages/connections/src/page/consent-start.ts @@ -1,4 +1,4 @@ -import { allPermissions, defineHttpOperation } from '@on.auto/operations'; +import { allPermissions, defineHttpOperation, Unavailable } from '@on.auto/operations'; import { Effect, Schema } from 'effect'; import { studioBaseFor } from '../links/connect-page.ts'; @@ -7,7 +7,7 @@ import { ClientSourceField, ExpiresAtField } from '../operations/connection-fiel import { actedNow, loadRegistry, recordInRegistry } from '../operations/registry-access.ts'; import { startableConnectionOf } from '../registry/registry-decisions.ts'; import { consentAskedOf, forgottenVerifierOf, namedOf, type ConsentServices } from './consent-asking.ts'; -import { consentRefusalOf } from './consent-words.ts'; +import { consentRefusalOf, unavailableWords } from './consent-words.ts'; import { LinkSchema } from './link-operations.ts'; const StartedSchema = Schema.Struct({ @@ -46,11 +46,18 @@ export function defineStartConsent(services: ConsentServices) { const use = { connection: id, token_sha256: tokenSha256Of(token), bindsPerson }; const kept = yield* Effect.fromResult(startableConnectionOf(yield* loadRegistry, use, yield* actedNow)); const asked = consentAskedOf(kept.connection, services); + const named = namedOf(kept.connection); const state = issuedToken(); const started = yield* consent .start({ ...asked, state: state.token, studioBase: studioBaseFor(services.pages) }) - .pipe(Effect.mapError((refusal) => consentRefusalOf(namedOf(kept.connection), refusal))); - const verifierSecretId = yield* Effect.orDie(secrets.keep(started.verifier)); + .pipe(Effect.mapError((refusal) => consentRefusalOf(named, refusal))); + const verifierSecretId = yield* secrets + .keep(started.verifier) + .pipe( + Effect.mapError( + ({ detail }) => new Unavailable({ detail: unavailableWords.verifierUnkept(named.name, detail) }), + ), + ); const expiresAt = yield* tenMinutesOn; yield* recordInRegistry({ type: 'start', diff --git a/packages/connections/src/page/consent-words.ts b/packages/connections/src/page/consent-words.ts index a432f1baf..52b9b03e0 100644 --- a/packages/connections/src/page/consent-words.ts +++ b/packages/connections/src/page/consent-words.ts @@ -1,5 +1,5 @@ import { Conflict, Forbidden, InvalidInput, Unavailable } from '@on.auto/operations'; -import type { ClientSource, ConsentRefusal, RevocationFailure } from '@on.auto/tool-servers'; +import type { ClientSource, ConsentRefusal, RevocationFailure, SecretKeepingFailure } from '@on.auto/tool-servers'; import { servingWords } from '../serving/serving-words.ts'; @@ -31,6 +31,21 @@ type Unnamed = Exclude + `The client the runtime registered for the connection ${name} cannot be kept or read now: ${detail}`, + verifierUnkept: (name: string, detail: string) => + `The consent of the connection ${name} cannot be started now, since the runtime cannot keep its verifier: ${detail}`, + verifierUnread: (name: string, detail: string) => + `The consent of the connection ${name} cannot be finished now, since the runtime cannot read its verifier; the same code may be tried again shortly: ${detail}`, + tokensUnkept: (name: string, detail: string) => + `The tokens of the connection ${name} cannot be kept now, and the code they were given for is spent, so the person starts the consent again: ${detail}`, + credentialUnkept: (name: string, detail: string) => + `The credential of the connection ${name} cannot be kept now, so the connection is not connected and its link stays open: ${detail}`, + tokensUnread: (name: string, detail: string) => + `The tokens of the connection ${name} cannot be read now, so they were not revoked and end on their own: ${detail}`, +} as const; + const conflictsOf: Readonly string>> = { no_metadata: ({ name, issuer }) => `The authorization server of the connection ${name}, ${issuer}, publishes no metadata, ${notAsAdded}`, @@ -84,6 +99,14 @@ export function consentRefusalOf(named: ConsentNamed, refusal: ConsentRefusal): detail: `The authorization server of the connection ${named.name} did not answer as an authorization server does; try again in a moment`, }); } + if (refusal.refused === 'registration_unavailable') { + return new Unavailable({ detail: unavailableWords.registration(named.name, refusal.detail) }); + } + if (refusal.refused === 'registration_refused') { + return new Conflict({ + detail: `The client the runtime registered for the connection ${named.name} cannot be kept: ${refusal.detail}`, + }); + } if (refusal.refused === 'iss') { return refusedAt('/iss', `Expected the issuer of the authorization server of the connection, ${named.issuer}`); } @@ -92,6 +115,12 @@ export function consentRefusalOf(named: ConsentNamed, refusal: ConsentRefusal): : conflictOf(named, refusal); } +export function tokensUnkeptOf({ name }: ConsentNamed, { _tag, detail }: SecretKeepingFailure): Conflict | Unavailable { + return _tag === 'secret_refused' + ? new Conflict({ detail }) + : new Unavailable({ detail: unavailableWords.tokensUnkept(name, detail) }); +} + const throughAuto = (name: string) => `Expected handle: the consent of the connection ${name} began through Auto's consent service, which answers a handle, not a code`; @@ -129,6 +158,9 @@ export function unrevokedWordsOf({ name }: ConsentNamed, failure: RevocationFail if (failure.refused === 'unknown_key') { return `Auto's consent service does not know the key of the connection ${name}, so its tokens were not revoked and end on their own`; } + if (failure.refused === 'registration_unavailable') { + return `The tokens of the connection ${name} were not revoked, since the client they were given to cannot be read now, so they end on their own: ${failure.detail}`; + } return failure.refused === 'unanswered' ? unansweredRevocationOf(name, source) : `The tokens of the connection ${name} could not be revoked with the client and at the authorization server it was made with, so they end on their own`; diff --git a/packages/connections/src/page/give-credential.ts b/packages/connections/src/page/give-credential.ts index d61c066a7..45f386fba 100644 --- a/packages/connections/src/page/give-credential.ts +++ b/packages/connections/src/page/give-credential.ts @@ -7,7 +7,7 @@ import { actedNow, loadRegistry, recordInRegistry } from '../operations/registry import { providingOf } from '../providers/provider-additions.ts'; import { ConnectionSchema, connectionShown } from '../registry/connection.ts'; import { linkedConnectionOf } from '../registry/registry-lookup.ts'; -import { GivingSchema, givenOf, refused } from './given-fields.ts'; +import { GivingSchema, givenOf, unkeptRefusalOf } from './given-fields.ts'; export function defineGiveCredential({ secrets, @@ -24,6 +24,7 @@ export function defineGiveCredential({ description: [ 'Gives the credential of a connection through its open link, which approves the connection and uses the link up, and answers it connected.', 'The runtime keeps the value sealed in its secret store, and no answer, record or history of the org ever holds it.', + 'Where the secret store cannot keep it just now, it answers unavailable in the words of the store, keeps nothing and leaves the link open, so the same credential is given again.', 'On a runtime that signs people in, only the person who added the connection may give it.', ].join(' '), route: { method: 'POST', path: '/connections/{id}/credential' }, @@ -31,7 +32,7 @@ export function defineGiveCredential({ permittedBy: allPermissions, inputSchema: GivingSchema, answer: { kind: 'json', outputSchema: ConnectionSchema }, - reasons: ['not_found', 'forbidden', 'conflict', 'invalid_input'], + reasons: ['not_found', 'forbidden', 'conflict', 'invalid_input', 'unavailable'], handle: Effect.fnUntraced(function* (giving) { const use = { connection: giving.id, token_sha256: tokenSha256Of(giving.token), bindsPerson }; const kept = yield* Effect.fromResult(linkedConnectionOf(yield* loadRegistry, use, yield* actedNow)); @@ -41,7 +42,7 @@ export function defineGiveCredential({ const given = yield* Effect.fromResult(givenOf(kept.connection, giving, checking)); const secretId = yield* secrets .keep(Redacted.make(given.value), given.endsWith) - .pipe(Effect.mapError(({ detail }) => refused([{ pointer: given.pointer, detail }]))); + .pipe(Effect.mapError((failure) => unkeptRefusalOf(kept.connection.name, given.pointer, failure))); const algorithm = given.algorithm === undefined ? {} : { algorithm: given.algorithm }; const client = given.clientId === undefined ? {} : { client_id: given.clientId }; const connected = yield* recordInRegistry({ diff --git a/packages/connections/src/page/given-fields.ts b/packages/connections/src/page/given-fields.ts index 121ecf2d3..5249e71c1 100644 --- a/packages/connections/src/page/given-fields.ts +++ b/packages/connections/src/page/given-fields.ts @@ -1,4 +1,5 @@ -import { InvalidInput, type Conflict, type Issue } from '@on.auto/operations'; +import { InvalidInput, Unavailable, type Conflict, type Issue } from '@on.auto/operations'; +import type { SecretKeepingFailure } from '@on.auto/tool-servers'; import { Result, Schema } from 'effect'; import { clientIdReadingIssuesOf, type EntryChecking } from '../entries/entry-checks.ts'; @@ -6,6 +7,7 @@ import { givenFieldsOf, wordsOfThePage, type Given } from '../entries/given-valu import { ConnectionIdField, TokenField } from '../operations/connection-fields.ts'; import type { RecordedConnection } from '../registry/connection.ts'; import { connectedByConsent, takesNoCredential } from '../registry/registry-decisions.ts'; +import { unavailableWords } from './consent-words.ts'; const GivenField = Schema.String.check(Schema.isMinLength(1)); @@ -31,10 +33,20 @@ export const GivingSchema = Schema.Struct({ type Giving = typeof GivingSchema.Type; -export function refused(issues: readonly Issue[]): InvalidInput { +function refused(issues: readonly Issue[]): InvalidInput { return new InvalidInput({ detail: 'The credential cannot be taken as it is given', issues }); } +export function unkeptRefusalOf( + name: string, + pointer: string, + { _tag, detail }: SecretKeepingFailure, +): InvalidInput | Unavailable { + return _tag === 'secret_refused' + ? refused([{ pointer, detail }]) + : new Unavailable({ detail: unavailableWords.credentialUnkept(name, detail) }); +} + function withAReadableClientId( connection: RecordedConnection, given: Given, diff --git a/packages/connections/src/page/store-unavailable.test.ts b/packages/connections/src/page/store-unavailable.test.ts new file mode 100644 index 000000000..c58265fa6 --- /dev/null +++ b/packages/connections/src/page/store-unavailable.test.ts @@ -0,0 +1,177 @@ +import { SecretRefused, type SecretStore } from '@on.auto/secrets'; +import { custodianRefusal } from '@on.auto/secrets/testing'; +import { consentedAt, gatedRequests, type FakeMcpServer } from '@on.auto/tool-servers/testing'; +import { Effect, Schema } from 'effect'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { added, crm } from '../testing/additions.ts'; +import { consentFake } from '../testing/consents.ts'; +import { ada, harness, operationNamed } from '../testing/harness.ts'; +import { operationsOver, sealedInMemory } from '../testing/sealed-connections.ts'; +import { calledThrough, sealedConsentAt, type SealedHarness } from '../testing/sealed-consents.ts'; + +const closing: (() => Promise)[] = []; + +afterEach(async () => { + await Promise.all(closing.splice(0).map((close) => close())); +}); + +async function served(changes: Parameters[0] = {}): Promise { + const fake = await consentFake(changes); + closing.push(fake.close); + return fake; +} + +const StartedSchema = Schema.Struct({ output: Schema.Struct({ authorize: Schema.Struct({ url: Schema.String }) }) }); + +interface Link { + readonly id: string; + readonly token: string; +} + +function linkOf({ id, token }: Link): Link { + return { id, token }; +} + +async function consentedThrough(sealed: SealedHarness, link: Link) { + const started = Schema.decodeUnknownSync(StartedSchema)(await calledThrough(sealed, 'start', linkOf(link))); + return { ...linkOf(link), ...(await consentedAt(started.output.authorize.url)) }; +} + +function refusedTo({ custodian }: SealedHarness, act: string): string { + return `The custodian of the key ${custodian.kid} refused to ${act} a data key: ${custodianRefusal}`; +} + +async function typesOfTheFacts({ h }: SealedHarness): Promise { + return (await h.facts()).map(({ type }) => type); +} + +describe('a start whose verifier the secret store cannot keep now', () => { + it('answers unavailable in the words of the store, records no start and keeps nothing', async () => { + const sealed = sealedConsentAt(await served()); + const link = await added(sealed.h, { provider: 'files' }); + + sealed.custodian.refuse(); + const refused = await calledThrough(sealed, 'start', linkOf(link)); + + expect(refused).toMatchObject({ + status: 'rejected', + reason: 'unavailable', + detail: `The consent of the connection files cannot be started now, since the runtime cannot keep its verifier: ${refusedTo(sealed, 'wrap')}`, + }); + expect(await typesOfTheFacts(sealed)).not.toContain('connection_consent_started'); + expect(sealed.rows.kept()).toEqual([]); + }); +}); + +describe('a start whose client registration the secret store cannot read now', () => { + it('answers unavailable in the words of the store, before the person is sent anywhere', async () => { + const fake = await served({ dynamicRegistration: true }); + const sealed = sealedConsentAt(fake, { way: { clients: ['dynamic'] }, consenting: { registeredClients: [] } }); + const link = await added(sealed.h, { provider: 'files' }); + await calledThrough(sealed, 'start', linkOf(link)); + + sealed.custodian.refuse(); + const refused = await calledThrough(sealed, 'start', linkOf(link)); + + expect(refused).toMatchObject({ + reason: 'unavailable', + detail: `The client the runtime registered for the connection files cannot be kept or read now: ${refusedTo(sealed, 'unwrap')}`, + }); + expect(fake.heard().filter(({ path }) => path === '/register')).toHaveLength(1); + }); +}); + +function refusingToKeep(store: SecretStore): SecretStore { + const tooLarge = new SecretRefused({ because: 'too_large', detail: 'A secret to keep holds more than 8 KiB' }); + return { ...store, keep: () => Effect.fail(tooLarge) }; +} + +describe('a start whose client registration the secret store refuses to keep', () => { + it('answers conflict in the words of the refusal, since the registration will never be kept', async () => { + const fake = await served({ dynamicRegistration: true }); + const sealed = sealedConsentAt(fake, { + way: { clients: ['dynamic'] }, + consenting: { registeredClients: [] }, + registrations: refusingToKeep(sealedInMemory().store), + }); + const link = await added(sealed.h, { provider: 'files' }); + + const refused = await calledThrough(sealed, 'start', linkOf(link)); + + expect(refused).toMatchObject({ + status: 'rejected', + reason: 'conflict', + detail: + 'The client the runtime registered for the connection files cannot be kept: A secret to keep holds more than 8 KiB', + }); + }); +}); + +describe('a finish whose verifier the secret store cannot read now', () => { + it('answers unavailable in the words of the store and redeems nothing, so the same code finishes it once the store reads it again', async () => { + const fake = await served(); + const sealed = sealedConsentAt(fake); + const finishing = await consentedThrough(sealed, await added(sealed.h, { provider: 'files' })); + + sealed.custodian.refuse(); + const refused = await calledThrough(sealed, 'finish', finishing); + const redeemedBefore = fake.tokenRequests(); + sealed.custodian.answer(); + const finished = await calledThrough(sealed, 'finish', finishing); + + expect(refused).toMatchObject({ + reason: 'unavailable', + detail: `The consent of the connection files cannot be finished now, since the runtime cannot read its verifier; the same code may be tried again shortly: ${refusedTo(sealed, 'unwrap')}`, + }); + expect(redeemedBefore).toBe(0); + expect(finished).toMatchObject({ status: 'succeeded', output: { status: 'connected' } }); + }); +}); + +describe('a finish whose tokens the secret store cannot keep now, once the code is redeemed', () => { + it('answers unavailable in the words of the store, says the person starts again, revokes the tokens and connects nothing', async () => { + const fake = await served(); + const gate = gatedRequests(); + const sealed = sealedConsentAt(fake, { consenting: { fetch: gate.fetch } }); + const finishing = await consentedThrough(sealed, await added(sealed.h, { provider: 'files' })); + + const refusing = calledThrough(sealed, 'finish', finishing); + await vi.waitFor(() => { + expect(gate.holding()).toBe(1); + }); + sealed.custodian.refuse(); + gate.release(); + const refused = await refusing; + + expect(refused).toMatchObject({ + reason: 'unavailable', + detail: `The tokens of the connection files cannot be kept now, and the code they were given for is spent, so the person starts the consent again: ${refusedTo(sealed, 'wrap')}`, + }); + expect(fake.heard().filter(({ path }) => path === '/revoke')).toHaveLength(1); + expect(await typesOfTheFacts(sealed)).not.toContain('connection_connected'); + expect(sealed.rows.kept()).toHaveLength(1); + }); +}); + +describe('a credential the secret store cannot keep now', () => { + it('answers unavailable in the words of the store, keeps nothing and leaves the link open for the same credential', async () => { + const h = harness(); + const { custodian, rows, store } = sealedInMemory(); + const giving = operationNamed(operationsOver(h, store), 'give_credential'); + const { id, token } = await added(h, crm); + + custodian.refuse(); + const refused = await h.call(giving, ada, { id, token, value: 'crm-api-key' }); + const keptWhileRefused = rows.kept().length; + custodian.answer(); + const given = await h.call(giving, ada, { id, token, value: 'crm-api-key' }); + + expect(refused).toMatchObject({ + reason: 'unavailable', + detail: `The credential of the connection crm cannot be kept now, so the connection is not connected and its link stays open: The custodian of the key ${custodian.kid} refused to wrap a data key: ${custodianRefusal}`, + }); + expect(keptWhileRefused).toBe(0); + expect(given).toMatchObject({ status: 'succeeded', output: { status: 'connected' } }); + }); +}); diff --git a/packages/connections/src/registrations/client-registrations.test.ts b/packages/connections/src/registrations/client-registrations.test.ts index 44f4b92b5..8e6b2bf9e 100644 --- a/packages/connections/src/registrations/client-registrations.test.ts +++ b/packages/connections/src/registrations/client-registrations.test.ts @@ -1,10 +1,12 @@ import { memoryLedger } from '@on.auto/operations/testing'; +import { custodianRefusal } from '@on.auto/secrets/testing'; import type { ClientRegistrations } from '@on.auto/tool-servers'; import { Effect, Redacted } from 'effect'; import { describe, expect, it } from 'vitest'; import { keptSecrets } from '../testing/harness-services.ts'; -import { makeClientRegistrations } from './client-registrations.ts'; +import { sealedInMemory } from '../testing/sealed-connections.ts'; +import { clientRegistrationsStream, makeClientRegistrations } from './client-registrations.ts'; const accounts = 'https://accounts.example.com'; @@ -54,20 +56,46 @@ describe('the clients the runtime registered itself', () => { expect([clientIdOf(latest), registrationTextOf(latest)]).toEqual(['one', '{"client_id":"one"}']); }); - it('are none whose row the secret store no longer keeps or cannot read', async () => { + it('are none whose row the secret store no longer keeps', async () => { const secrets = keptSecrets({}); - const ledger = memoryLedger(); - const registrations = makeClientRegistrations({ registry: ledger.service, secrets }); - const unreadable = makeClientRegistrations({ - registry: ledger.service, - secrets: { ...secrets, read: () => Effect.fail({ detail: 'The key of the secret is unknown' }) }, - }); + const registrations = makeClientRegistrations({ registry: memoryLedger().service, secrets }); await kept(registrations, [accounts, 'one'], [accounts, 'two']); await Effect.runPromise(secrets.forget('secret-1')); - expect([ - await Effect.runPromise(registrations.named(accounts, 'one')), - await Effect.runPromise(unreadable.named(accounts, 'two')), - ]).toEqual([undefined, undefined]); + expect(await Effect.runPromise(registrations.named(accounts, 'one'))).toBeUndefined(); + }); +}); + +describe('the clients the runtime registered itself, in a secret store that cannot answer now', () => { + it('fail in the words of the store where it cannot read their row now, by issuer and by issuer and id alike', async () => { + const { custodian, store } = sealedInMemory(); + const registrations = makeClientRegistrations({ registry: memoryLedger().service, secrets: store }); + await kept(registrations, [accounts, 'one']); + + custodian.refuse(); + const failures = await Effect.runPromise( + Effect.all([Effect.flip(registrations.latestAt(accounts)), Effect.flip(registrations.named(accounts, 'one'))]), + ); + + const words = `The custodian of the key ${custodian.kid} refused to unwrap a data key: ${custodianRefusal}`; + expect(failures).toMatchObject([ + { _tag: 'custodian_refused', detail: words }, + { _tag: 'custodian_refused', detail: words }, + ]); + }); + + it('fail in the words of the store where it cannot keep a registration now, and record nothing', async () => { + const { custodian, store } = sealedInMemory(); + const ledger = memoryLedger(); + const registrations = makeClientRegistrations({ registry: ledger.service, secrets: store }); + + custodian.refuse(); + const failure = await Effect.runPromise(Effect.flip(registrations.keep(accounts, registrationOf('one')))); + + expect(failure).toMatchObject({ + _tag: 'custodian_refused', + detail: `The custodian of the key ${custodian.kid} refused to wrap a data key: ${custodianRefusal}`, + }); + expect(ledger.streamNames()).not.toContain(clientRegistrationsStream); }); }); diff --git a/packages/connections/src/registrations/client-registrations.ts b/packages/connections/src/registrations/client-registrations.ts index e3bde9d02..b5813a6be 100644 --- a/packages/connections/src/registrations/client-registrations.ts +++ b/packages/connections/src/registrations/client-registrations.ts @@ -7,8 +7,8 @@ import { type StreamReader, type StreamWriter, } from '@on.auto/operations'; -import type { ClientRegistrations, KeptRegistration } from '@on.auto/tool-servers'; -import { DateTime, Effect, Function, Result, Schema } from 'effect'; +import type { ClientRegistrations, KeptRegistration, RegistrationUnavailable } from '@on.auto/tool-servers'; +import { DateTime, Effect, Result, Schema } from 'effect'; import type { SecretKeeping } from '../operations/connection-services.ts'; import type { CommandMetadata } from '../registry/connection-commands.ts'; @@ -63,10 +63,9 @@ function keptOf( { secrets }: RegistrationServices, clientId: string, secretId: string, -): Effect.Effect { - return secrets.read(secretId).pipe( - Effect.map((stored) => (stored === undefined ? undefined : { clientId, registration: stored.value })), - Effect.orElseSucceed(Function.constUndefined), +): Effect.Effect { + return Effect.map(secrets.read(secretId), (stored) => + stored === undefined ? undefined : { clientId, registration: stored.value }, ); } @@ -91,7 +90,7 @@ export function makeClientRegistrations(services: RegistrationServices): ClientR }), keep: (issuer, { clientId, registration }) => Effect.gen(function* () { - const secretId = yield* Effect.orDie(services.secrets.keep(registration)); + const secretId = yield* services.secrets.keep(registration); const at = DateTime.formatIso(yield* DateTime.now); const registering = { issuer, client_id: clientId, secret_id: secretId, by: byTheRuntime, at }; yield* Effect.orDie(services.registry.execute(clientRegistrationsStream, registrationsDecider, registering)); diff --git a/packages/connections/src/serving/consent-serving.test.ts b/packages/connections/src/serving/consent-serving.test.ts index b049140b8..f9474906e 100644 --- a/packages/connections/src/serving/consent-serving.test.ts +++ b/packages/connections/src/serving/consent-serving.test.ts @@ -217,7 +217,10 @@ describe('the store of the tokens of a connection connected by consent', () => { const refused = await Effect.runPromise(Effect.flip(tokens.replace(2, Redacted.make('x'.repeat(4097))))); expect(claims).toEqual([{ claimed: true, until: whileClaimed.claimedUntil }, { claimed: false }]); - expect(refused).toEqual({ detail: 'A secret to keep holds more than 4096 characters' }); + expect(refused).toMatchObject({ + _tag: 'secret_refused', + detail: 'A secret to keep holds more than 4096 characters', + }); expect([replaced, moved]).toEqual([{ replaced: true, version: 2 }, { replaced: false }]); expect(read).toEqual({ tokens: renewed, version: 2, claimedUntil: undefined }); }); diff --git a/packages/connections/src/serving/given-credentials.ts b/packages/connections/src/serving/given-credentials.ts index 5bb327047..44cb97b98 100644 --- a/packages/connections/src/serving/given-credentials.ts +++ b/packages/connections/src/serving/given-credentials.ts @@ -5,6 +5,7 @@ import { type ConsentAuth, type HeldTokens, type KeptRow, + type SecretReadingFailure, type TokensEnded, } from '@on.auto/tool-servers'; import { DateTime, Effect, Option, type Context } from 'effect'; @@ -59,14 +60,21 @@ export function loggedCauses(): LoggedCauses { }; } -function unavailableNow({ org, kept, unreadable }: Serving, cause: string): Effect.Effect { +function causeOf({ _tag, kid = '' }: SecretReadingFailure): string { + return `${_tag} ${kid}`; +} + +function unavailableNow( + { org, kept, unreadable }: Serving, + failure: SecretReadingFailure, +): Effect.Effect { const { id, name } = kept.connection; const unready: ConnectionUnready = { because: 'connection_unavailable', - detail: servingWords.unreadableNow(name, cause), + detail: servingWords.unreadableNow(name, failure.detail), }; const loggedOnce = Effect.suspend(() => - unreadable.isNew(id, cause) + unreadable.isNew(id, causeOf(failure)) ? Effect.logWarning(unready.detail).pipe(Effect.annotateLogs({ org, connection: id })) : Effect.void, ); @@ -84,7 +92,7 @@ function keptSecretOf(serving: Serving): Effect.Effect unavailableNow(serving, detail), + onFailure: (failure) => unavailableNow(serving, failure), onSuccess: (stored) => stored === undefined ? brokenWith(serving, servingWords.gone(name)) : readNow(serving, stored), }), diff --git a/packages/connections/src/serving/unreadable-credentials.test.ts b/packages/connections/src/serving/unreadable-credentials.test.ts index 15043147d..717549e26 100644 --- a/packages/connections/src/serving/unreadable-credentials.test.ts +++ b/packages/connections/src/serving/unreadable-credentials.test.ts @@ -117,8 +117,14 @@ describe('a connection whose credential the secret store cannot read now', () => }); }); +async function wrappedAgainBadly(rows: ReturnType, kid: string): Promise { + const [read] = await Effect.runPromise(rows.heldBy({ keptBy: 'ledger', kid }, 1)); + const held = Option.getOrThrow(Option.fromNullishOr(read)); + await Effect.runPromise(rows.reheld(held, { ...held.holding, wrappedKey: 'not a wrapping' })); +} + describe('the log of a credential that cannot be read now', () => { - it('holds each cause once, not one line a call, again when the cause changes, and again once a read succeeded between', async () => { + it('holds each cause once, by the failure and the key it names and not by its words, again when the cause changes, and again once a read succeeded between', async () => { const fake = await fakeServerAt(); const h = harness(); const rows = memoryRows({ now }); @@ -131,9 +137,11 @@ describe('the log of a credential that cannot be read now', () => { custodian.refuse(); const refusals = [await refusalOf(source, id), await refusalOf(source, id)]; + custodian.answer(); + await wrappedAgainBadly(rows, custodian.kid); + refusals.push(await refusalOf(source, id)); await Effect.runPromise(other.store.replace(secretId, 1, Redacted.make(fakeApiKey))); refusals.push(await refusalOf(source, id), await refusalOf(source, id)); - custodian.answer(); await Effect.runPromise(store.replace(secretId, 2, Redacted.make(fakeApiKey))); await Effect.runPromise(servedOf(await connectionOf(source, id))); custodian.refuse(); @@ -142,13 +150,19 @@ describe('the log of a credential that cannot be read now', () => { const keyUnknown = unreadableNow( `The secret is sealed under the key ${other.custodian.kid}, which is neither the current key nor the previous one`, ); + const unopened = unreadableNow( + `The custodian of the key ${custodian.kid} refused to unwrap a data key: its wrapped key does not open under this key`, + ); expect(refusals).toEqual( - [refusedBy(custodian), refusedBy(custodian), keyUnknown, keyUnknown, refusedBy(custodian)].map((detail) => ({ - because: 'connection_unavailable', - detail, - })), + [refusedBy(custodian), refusedBy(custodian), unopened, keyUnknown, keyUnknown, refusedBy(custodian)].map( + (detail) => ({ because: 'connection_unavailable', detail }), + ), ); - expect(warningsIn(lines).map((line) => line.includes(keyUnknown))).toEqual([false, true, false]); + expect(warningsIn(lines).map((line) => [line.includes(keyUnknown), line.includes(unopened)])).toEqual([ + [false, false], + [true, false], + [false, false], + ]); }); }); diff --git a/packages/connections/src/testing/consents.ts b/packages/connections/src/testing/consents.ts index c7127ba0c..c87203178 100644 --- a/packages/connections/src/testing/consents.ts +++ b/packages/connections/src/testing/consents.ts @@ -29,9 +29,9 @@ export function consentFake(changes: Partial = {}): Promise< return servedForConsent({ redirectUris: [studioReturn], ...changes }); } -const filesWay = { scope: 'files.read', parameters: { access_type: 'offline' } }; +export const filesWay = { scope: 'files.read', parameters: { access_type: 'offline' } }; -function filesAt(fake: FakeMcpServer, way: Readonly> = filesWay) { +export function filesAt(fake: FakeMcpServer, way: Readonly> = filesWay) { return providerOf({ name: 'files', title: 'Files', diff --git a/packages/connections/src/testing/sealed-connections.ts b/packages/connections/src/testing/sealed-connections.ts index 4cc287e53..af5491ef4 100644 --- a/packages/connections/src/testing/sealed-connections.ts +++ b/packages/connections/src/testing/sealed-connections.ts @@ -1,26 +1,38 @@ import { sealedKeeper, secretStoreOn, type SecretRows, type SecretStore } from '@on.auto/secrets'; -import { recordingCustodian, type RecordingCustodian } from '@on.auto/secrets/testing'; +import { memoryRows, recordingCustodian, type MemoryRows, type RecordingCustodian } from '@on.auto/secrets/testing'; import type { ConnectionSource, OrgConnection } from '@on.auto/tool-servers'; -import { Effect, type Layer } from 'effect'; +import { DateTime, Effect, type Layer } from 'effect'; -import { makeConnectionOperations } from '../operations/connection-operations.ts'; +import { makeConnectionOperations, type ConnectionOperations } from '../operations/connection-operations.ts'; +import type { ConnectionServices } from '../operations/connection-services.ts'; import { connectionPolicyOf } from '../policy/connection-policy.ts'; import { makeConnectionSource } from '../serving/connection-source.ts'; import { added } from './additions.ts'; import { ada, fixtureServing, operationNamed, type Harness } from './harness.ts'; -export interface Sealed { - readonly rows: SecretRows; +export interface Sealed { + readonly rows: Rows; readonly custodian: RecordingCustodian; readonly store: SecretStore; } -export function sealedIn(rows: SecretRows, custodian: RecordingCustodian = recordingCustodian()): Sealed { +export function sealedIn( + rows: Rows, + custodian: RecordingCustodian = recordingCustodian(), +): Sealed { return { rows, custodian, store: secretStoreOn(rows, sealedKeeper({ current: custodian }), { current: custodian }) }; } -export async function givenThrough(h: Harness, secrets: SecretStore, input: unknown, value: string): Promise { - const giving = makeConnectionOperations({ +export function sealedInMemory(): Sealed { + return sealedIn(memoryRows({ now: Effect.map(DateTime.now, DateTime.toEpochMillis) })); +} + +export function operationsOver( + h: Harness, + secrets: SecretStore, + services: Partial = {}, +): ConnectionOperations { + return makeConnectionOperations({ ...fixtureServing, secrets, policy: connectionPolicyOf({}), @@ -28,9 +40,13 @@ export async function givenThrough(h: Harness, secrets: SecretStore, input: unkn modelProviders: [], access: 'people', consent: h.consent, + ...services, }); +} + +export async function givenThrough(h: Harness, secrets: SecretStore, input: unknown, value: string): Promise { const { id, token } = await added(h, input); - await h.call(operationNamed(giving, 'give_credential'), ada, { id, token, value }); + await h.call(operationNamed(operationsOver(h, secrets), 'give_credential'), ada, { id, token, value }); return id; } diff --git a/packages/connections/src/testing/sealed-consents.ts b/packages/connections/src/testing/sealed-consents.ts new file mode 100644 index 000000000..ac72956f3 --- /dev/null +++ b/packages/connections/src/testing/sealed-consents.ts @@ -0,0 +1,38 @@ +import { providerListOf } from '@on.auto/providers/testing'; +import type { SecretStore } from '@on.auto/secrets'; +import type { MemoryRows } from '@on.auto/secrets/testing'; +import type { ConsentClientOptions } from '@on.auto/tool-servers'; +import { registeredAt, type FakeMcpServer } from '@on.auto/tool-servers/testing'; + +import type { ConnectionOperations } from '../operations/connection-operations.ts'; +import { consentHarnessAt, filesAt, filesWay } from './consents.ts'; +import { harnessConsentOf } from './harness-services.ts'; +import { ada, operationNamed, type Harness } from './harness.ts'; +import { operationsOver, sealedInMemory, type Sealed } from './sealed-connections.ts'; + +export interface SealedHarness extends Sealed { + readonly h: Harness; + readonly operations: ConnectionOperations; +} + +export interface SealedConsenting { + readonly way?: Readonly>; + readonly consenting?: Partial; + readonly registrations?: SecretStore; +} + +export function sealedConsentAt( + fake: FakeMcpServer, + { way = filesWay, consenting = {}, registrations }: SealedConsenting = {}, +): SealedHarness { + const clients = { registeredClients: [registeredAt(fake.origin)], ...consenting }; + const h = consentHarnessAt(fake, { way, consenting: clients }); + const sealed = sealedInMemory(); + const consent = harnessConsentOf(h.ledger.service, registrations ?? sealed.store, clients); + const operations = operationsOver(h, sealed.store, { providers: providerListOf(filesAt(fake, way)), consent }); + return { ...sealed, h, operations }; +} + +export function calledThrough({ h, operations }: SealedHarness, name: string, input: unknown) { + return h.call(operationNamed(operations, name), ada, input); +} diff --git a/packages/secrets/src/testing/index.ts b/packages/secrets/src/testing/index.ts index 5982bebbb..4fe988dc8 100644 --- a/packages/secrets/src/testing/index.ts +++ b/packages/secrets/src/testing/index.ts @@ -6,7 +6,6 @@ export { storeBehaviour } from './store-behaviour.ts'; export { valuesInTheClear, type ValueInTheClear, - type ValueRefused, type ValuesInTheClear, type ValuesInTheClearOptions, } from './values-in-the-clear.ts'; diff --git a/packages/secrets/src/testing/values-in-the-clear.test.ts b/packages/secrets/src/testing/values-in-the-clear.test.ts index 3aa72f55c..8f23a7b73 100644 --- a/packages/secrets/src/testing/values-in-the-clear.test.ts +++ b/packages/secrets/src/testing/values-in-the-clear.test.ts @@ -1,6 +1,7 @@ import { Effect, Redacted } from 'effect'; import { describe, expect, it } from 'vitest'; +import { SecretRefused } from '../store/secret-refused.ts'; import { valuesInTheClear, type ValueInTheClear } from './values-in-the-clear.ts'; function valuesAt(now: () => number) { @@ -11,10 +12,13 @@ function revealed(read: ValueInTheClear | undefined) { return read === undefined ? read : { ...read, value: Redacted.value(read.value) }; } -const refusedAsTooLarge = { detail: 'A secret to keep holds more than 16 characters' }; +const refusedAsTooLarge = new SecretRefused({ + because: 'too_large', + detail: 'A secret to keep holds more than 16 characters', +}); describe('the values a test keeps in the clear in memory', () => { - it('keep a value with its ending under the id they are given, refuse one past their bound, and forget a row', async () => { + it('keep a value with its ending under the id they are given, refuse one past their bound as the store refuses it, and forget a row', async () => { const values = valuesAt(() => 0); await Effect.runPromise(values.keep('secret-1', Redacted.make('first-tokens'), 'kens')); diff --git a/packages/secrets/src/testing/values-in-the-clear.ts b/packages/secrets/src/testing/values-in-the-clear.ts index 6aea97745..6c6fd8934 100644 --- a/packages/secrets/src/testing/values-in-the-clear.ts +++ b/packages/secrets/src/testing/values-in-the-clear.ts @@ -1,12 +1,9 @@ import { Effect, Redacted } from 'effect'; import type { Claim, Holding, Replacement } from '../rows/secret-rows.ts'; +import { SecretRefused } from '../store/secret-refused.ts'; import { memoryRows, type MemoryRowsOptions } from './memory-rows.ts'; -export interface ValueRefused { - readonly detail: string; -} - export interface ValueInTheClear { readonly value: Redacted.Redacted; readonly version: number; @@ -14,9 +11,13 @@ export interface ValueInTheClear { } export interface ValuesInTheClear { - readonly keep: (id: string, value: Redacted.Redacted, endsWith?: string) => Effect.Effect; + readonly keep: (id: string, value: Redacted.Redacted, endsWith?: string) => Effect.Effect; readonly read: (id: string) => Effect.Effect; - readonly replace: (id: string, version: number, value: Redacted.Redacted) => Effect.Effect; + readonly replace: ( + id: string, + version: number, + value: Redacted.Redacted, + ) => Effect.Effect; readonly claim: (id: string, version: number, lastingMs: number) => Effect.Effect; readonly endingsOf: (ids: readonly string[]) => Effect.Effect>; readonly forget: (id: string) => Effect.Effect; @@ -35,11 +36,16 @@ export function valuesInTheClear({ now, mostCharacters }: ValuesInTheClearOption const heldOr = ( value: Redacted.Redacted, held: (holding: Holding) => Effect.Effect, - ): Effect.Effect => + ): Effect.Effect => Effect.suspend(() => { const text = Redacted.value(value); return text.length > mostCharacters - ? Effect.fail({ detail: `A secret to keep holds more than ${mostCharacters} characters` }) + ? Effect.fail( + new SecretRefused({ + because: 'too_large', + detail: `A secret to keep holds more than ${mostCharacters} characters`, + }), + ) : held({ keptBy: keptInMemory, held: text }); }); return { diff --git a/packages/tool-servers/README.md b/packages/tool-servers/README.md index 066f425d1..b9aa368f1 100644 --- a/packages/tool-servers/README.md +++ b/packages/tool-servers/README.md @@ -177,9 +177,9 @@ With an `auth` block, tokens come from the MCP client's client-credentials or pr `makeConsentClient({ fetch, registeredClients, publicUrl, registrations, autoClients?, requestMs?, requestTimeout?, renewalSpreadMs? })` (`src/consent/consent-client.ts`) is the runtime acting as a client at an authorization server, for the consent a person gives there to a connection made from a provider's way `authorization`. The server makes it with its outbound fetch, the clients whoever runs the runtime registered, `PUBLIC_URL` and `autoClients` of `packages/server/src/consent/consent-settings.ts`, `AUTO_CLIENTS` and `CONSENT_SERVICE_URL` with the return a Studio base asks, `loopback` for a base on `localhost`, `127.0.0.1` or `[::1]`, `hosted` for the Studio host's own, `elsewhere` otherwise, and gives it to `@on.auto/connections`, whose page operations start and finish a consent through it. The server makes one, in `packages/server/src/composition/served-consent.ts`, and gives that one to `@on.auto/connections` and to its tool access, `consent` of `makeToolAccess`, since the single flight of its registrations lives in it. Nothing of it reaches the network when it is made: its answers load `src/consent-through-auto/consents-by-source.ts`, which asks the authorization server or Auto's consent service by the client's source, and with it `src/consent/consent-requests.ts`, `src/consent-tokens/token-requests.ts` and the MCP client's discovery, `startAuthorization`, `exchangeAuthorization` and `refreshAuthorization`, through a dynamic import when a consent first starts or finishes, a connection's tokens are first served or a removal first revokes, and the file that makes it imports the library's types alone. It never goes through the library's `auth()`, whose fallbacks would start a new authorization from a background renewal. The return address it is given with a start, a finish and the tokens of a connection is the caller's, which `@on.auto/connections` makes the page `connect-return` of the Studio the person reaches, under the runtime's own `/studio/` where it serves Studio and under the Studio host otherwise; a runtime has one return address at a time, the one its connect link opens, and a registered client is registered with both only where it serves several runtimes, some serving Studio and some not. It answers: - `hadSources(studioBase)`: an effect that answers, for the Studio base a link opens, `sourcesOf(way)`, the client sources the way lists in `clients`, `['registered']` where it lists none, that the runtime has, in the runtime's order, `registered`, `auto`, `metadata_document` then `dynamic` as `clientSources` lists them, whatever order the way gives (`src/consent/client-sources.ts`), and `sourceOf(way)`, the first of them, `sourceTakenOf`, the one rule by which a start takes its source and by which `@on.auto/connections` says on a link which source a start would take, the start alone passing on from a metadata document to the next source once the server's metadata says it takes none; it asks nothing but the clients of Auto's consent service, and those only where the client was made with that source on and the base is one the service sends people back to, `auto` being left out of the sources had otherwise (see [Consent through Auto's consent service](#consent-through-autos-consent-service)). A way lists the three sources of `wayClientSources`, never `auto`, which a way that takes `registered` takes as well, since it is a client Auto registered: `registered` where a registered client names the way's issuer, the same issuer a trailing slash aside, as the MCP client compares issuers (`isSameIssuer`), or names it as a template, `https://login.example.net/{tenant}/v2.0`, that the issuer is with a value other than `/`, `?` and `#` in each placeholder; `metadata_document` where `PUBLIC_URL` is https, its address `{PUBLIC_URL}/oauth/client.json`; and `dynamic` always. The document at that address is `clientMetadataDocumentOf(publicUrl, returnAddress)`, `{ address, document }`, or nothing where `PUBLIC_URL` is not https: the metadata the runtime registers its client with, `client_name` `Auto Runtime`, `redirect_uris` the return address, `grant_types` `authorization_code` and `refresh_token`, `response_types` `code` and `application_type` `web`, with `client_id` its own address and `token_endpoint_auth_method` `none`, as a Client ID Metadata Document is; the path is written once, in `src/consent/client-sources.ts`, and the server makes the document from the `PUBLIC_URL` it signs people in at, which it makes this client with, and the return address it makes the connections with, and gives it `@on.auto/api`, which answers it at the path of its address; a runtime in local mode, whatever its `PUBLIC_URL`, signs nobody in and serves none. `@on.auto/connections` judges with it whether a provider's way is offered and whether an addition takes it. -- `start({ server, way, returnAddress, reaches, state, studioBase })`: the authorization request a person is sent to, `{ url, state, verifier, client }`, `state` the one it was given, for every source but `auto`, whose start the next section sets out. It reads the server's metadata as the way says (`src/consent/server-metadata.ts`): for an `mcp` server its protected-resource metadata, through `discoverOAuthProtectedResourceMetadata`, where the server publishes it, whose `authorization_servers` must name the way's issuer, else `another_issuer` by the `server`, and whose `resource` must be the server's own address or one above it on its path, as the library's `checkResourceAllowed` compares them, else `another_resource`, and is the `resource` the request sends; where the server answers 404 there is none, and the request sends no `resource`. Then the authorization server's metadata: built from the way's `authorization_endpoint`, `token_endpoint` and `revocation_endpoint` where it gives them, with `response_types_supported: [code]` and `code_challenge_methods_supported: [S256]` and nothing read, else discovered at the issuer by `discoverAuthorizationServerMetadata`, whose check that the metadata names the issuer it was asked at refuses another as `another_issuer` by the `authorization_server` and whose finding nothing is `no_metadata`. Metadata that takes no `code` or lists no `S256` method is `no_challenge`, and an endpoint it names, the authorization, token, revocation and registration endpoints, whose host `reaches` refuses is `host` with the rule's words, before anything is chosen. The client is the first source of `sourcesOf` the server takes (`src/consent/acting-clients.ts`): the registered client at the issuer, with its secret, its key, whose assertion `createPrivateKeyJwtAuth` signs, or neither; the metadata document, its client id and return address those of the document, where the metadata says `client_id_metadata_document_supported`; or a registration made at the metadata's `registration_endpoint` (`src/consent/dynamic-registration.ts`), a public request of the runtime's own since the library's `registerClient` is deprecated, which sends the metadata the document holds but its client id and its method, whose answer counts only where it is a success that names a client id `ClientIdSchema` takes, else `unanswered`, kept through `registrations` and taken again for every later start at the issuer, `registrations.latestAt`, until the `client_secret_expires_at` it gave passes. The read of the latest and the registration are one flight for each issuer within the process, so a second start at the same issuer waits for the first's, and two starts never register twice. None of them is `no_client`. `startAuthorization` makes the verifier and the challenge and writes the client id, the return address as `redirect_uri`, `state`, the scope asked and the `resource`; the scope asked is the way's `scope`, with `offline_access` added where the metadata was discovered, its `scopes_supported` lists it and the scope lacks it, as the library's `auth()` adds it, and never where the way gives its endpoints or no scope; the way's `parameters` are set after, so its `prompt` replaces the one the library appends for `offline_access`. `verifier` is a `Redacted`, and `client` is `{ client_id, client_source }`, which the connection records. +- `start({ server, way, returnAddress, reaches, state, studioBase })`: the authorization request a person is sent to, `{ url, state, verifier, client }`, `state` the one it was given, for every source but `auto`, whose start the next section sets out. It reads the server's metadata as the way says (`src/consent/server-metadata.ts`): for an `mcp` server its protected-resource metadata, through `discoverOAuthProtectedResourceMetadata`, where the server publishes it, whose `authorization_servers` must name the way's issuer, else `another_issuer` by the `server`, and whose `resource` must be the server's own address or one above it on its path, as the library's `checkResourceAllowed` compares them, else `another_resource`, and is the `resource` the request sends; where the server answers 404 there is none, and the request sends no `resource`. Then the authorization server's metadata: built from the way's `authorization_endpoint`, `token_endpoint` and `revocation_endpoint` where it gives them, with `response_types_supported: [code]` and `code_challenge_methods_supported: [S256]` and nothing read, else discovered at the issuer by `discoverAuthorizationServerMetadata`, whose check that the metadata names the issuer it was asked at refuses another as `another_issuer` by the `authorization_server` and whose finding nothing is `no_metadata`. Metadata that takes no `code` or lists no `S256` method is `no_challenge`, and an endpoint it names, the authorization, token, revocation and registration endpoints, whose host `reaches` refuses is `host` with the rule's words, before anything is chosen. The client is the first source of `sourcesOf` the server takes (`src/consent/acting-clients.ts`): the registered client at the issuer, with its secret, its key, whose assertion `createPrivateKeyJwtAuth` signs, or neither; the metadata document, its client id and return address those of the document, where the metadata says `client_id_metadata_document_supported`; or a registration made at the metadata's `registration_endpoint` (`src/consent/dynamic-registration.ts`), a public request of the runtime's own since the library's `registerClient` is deprecated, which sends the metadata the document holds but its client id and its method, whose answer counts only where it is a success that names a client id `ClientIdSchema` takes, else `unanswered`, kept through `registrations` and taken again for every later start at the issuer, `registrations.latestAt`, until the `client_secret_expires_at` it gave passes. `registrations` fails with the store's tag and words, `{ _tag, detail }`: a latest the store cannot read now is `registration_unavailable`, `{ detail }` in the store's words, and so is a registration made that the store cannot keep now, which is logged at WARN by its issuer and its client id and never its secret, `The client registered at https://accounts.drive.example.com as client-1 cannot be kept now, so this process keeps it at its next consent there: `: the process holds it in memory, and the next start at the issuer keeps it before it reads or registers anything, so no second client is registered while it lasts, and a process that stops first leaves it registered at the server and unused. A registration the store refuses, `secret_refused`, is never held, since it will never be kept: the start fails as `registration_refused` in the refusal's words, and the WARN line says so, `The client registered at https://accounts.drive.example.com as client-1 cannot be kept, so it stays registered there unused: `. A finish, a renewal and a removal that cannot read the client a connection recorded are `registration_unavailable` alike. The read of the latest and the registration are one flight for each issuer within the process, so a second start at the same issuer waits for the first's, and two starts never register twice. None of them is `no_client`. `startAuthorization` makes the verifier and the challenge and writes the client id, the return address as `redirect_uri`, `state`, the scope asked and the `resource`; the scope asked is the way's `scope`, with `offline_access` added where the metadata was discovered, its `scopes_supported` lists it and the scope lacks it, as the library's `auth()` adds it, and never where the way gives its endpoints or no scope; the way's `parameters` are set after, so its `prompt` replaces the one the library appends for `offline_access`. `verifier` is a `Redacted`, and `client` is `{ client_id, client_source }`, which the connection records. - `finish({ server, way, returnAddress, reaches, client, code, iss, verifier })`, or `finish({ …, client, handle, verifier })` for a start of `auto` (see the next section): reads the metadata again as `start` does, takes the client the start recorded and no other, the registered client at the issuer with that client id, the metadata document while `PUBLIC_URL` names it, or the registration `registrations.named` finds by issuer and client id, else `client_gone`, and redeems the code with `exchangeAuthorization`, with the verifier, the return address and the `resource`. The library compares `iss` with the issuer of the metadata, and refuses one that differs, or none where the metadata says the server sends it, before the code is sent, as `iss`; a code the token endpoint refuses as `invalid_grant` is `code`. It answers `{ tokens, scope, revoked, client }`: `client`, the client it redeemed with, the one the start recorded, or for `auto` the one the handoff names, which the connection records; `tokens`, the one value the connection keeps, `{ access_token, refresh_token?, expires_at?, scope?, token_type }` as JSON in a `Redacted`, `expires_at` the time `expires_in` ends on Effect's `Clock`, left out where that time is past what a date holds; `scope`, the scope the server granted, split on white space and on commas, which GitHub and Slack write between scopes, and joined by single spaces (`src/consent/scopes.ts`), or the scope asked where it says none, as RFC 6749 has it, where a grant of no scope is `no_scope` when a scope was asked and no scope when none was, and a grant that is not scopes as RFC 6749 writes them, `isScope`, is `malformed_scope`; and `revoked`, an effect that revokes the refresh token, or the access token where there is none, at the metadata's revocation endpoint with the same client (`src/consent/revocation.ts`): an assertion for one with a key, and otherwise the method the library's `selectClientAuthMethod` chooses from the metadata's `revocation_endpoint_auth_methods_supported`, or its `token_endpoint_auth_methods_supported` where it lists none, HTTP Basic, the secret in the form or the client id alone; it does nothing where the metadata names no revocation endpoint. `revoked` never fails: a revocation that fails leaves the token to end on its own. It is made as soon as the code is redeemed, and every step after the exchange runs under it, so a refusal, a defect or an interruption there revokes the tokens before `finish` answers. -- `tokenSource(served, serving)`: the token source of a connection connected by consent (`src/consent-tokens/renewed-tokens.ts`), which `tokensOf` asks for an `auth` of the kind `consent`. `served` is that credential: the way, the server, the return address, `reaches`, the client the connection recorded, `held`, the row of its tokens as the source read it, decoded, its `tokens`, its `version` and its claim, which a store decodes from the value with `keptTokensIn`, so the source never reads a value it cannot serve and holds its tokens among the secrets as it is made, and `tokens`, the store of that row, `ConnectionTokens`, with `read`, `replace(version, value)`, `claim(version, lastingMs)` and `broken(ended)`, which `@on.auto/connections` implements over the secret store. The value is the JSON `finish` keeps, `{ access_token, refresh_token?, expires_at?, scope?, token_type, connection_key? }`, the key alone of a consent through Auto's consent service (`src/consent-tokens/kept-tokens.ts`). Made, it holds both tokens among the secrets and serves the access token until its renewal time, `renewalTimeOf` from the time it read them to `expires_at`, less the spread of the process, `renewalSpreadMs` of the options, up to five seconds drawn once when the client is made and never more than half the lead, so processes sharing a ledger do not all renew in the same second and a token of a short life is never renewed at every use; a token without an expiry, and one without a refresh token, is served until the server answers 401. At the renewal time, or on a 401, it reads the row again: where its version moved, another process renewed, and it takes the value instead, renewing it in turn only where that value is itself past its renewal time; where another process holds a claim of the row, it waits on its timer for the claim to end and reads again, and gives up as `requestMs` ends, `The tokens of the connection drive are being renewed by another process of the runtime, which did not finish within 30000 ms; the next call tries again`; else it reads the metadata as `start` does, takes the client the connection recorded as `finish` does, claims the row for the lease, the bound of the request it then sends, `requestMs`, and fifteen seconds more for the write of the claim, the write of the replacement and the gap between the clocks of the servers, forty-five seconds where the bound is the default thirty (`leaseOf` of `src/consent-tokens/token-requests.ts`, the one rule of it), so the refresh is cut off before the claim ends and a second claim never sends the same refresh token while the first is in flight or being kept, refreshes with `refreshAuthorization`, the client and the `resource` `finish` sends, keeping the refresh token where the server sends none, as the library does, and replaces the value under the version it read, which ends the claim. Within a process one renewal is in flight for a connection, whatever asks for it. A replacement that matches no row reads the row again: where it is gone, as when the connection was removed meanwhile, the source revokes what it got and fails in the words the row is read with; where its version moved, it takes the value the row holds and revokes nothing, since for a server that does not rotate its refresh tokens the value it got carries the refresh token the row's value lives by. One the store refuses revokes what it got and fails in the store's words. A refresh refused as `invalid_grant` reads the row again: where it still holds the value refused, the source breaks the connection, `broken('consent_ended')`, and where it moved, takes the new value; a 401 with no refresh token breaks it so too, and a client no longer registered breaks it with `broken('client_gone')`, before anything is sent. Any other failure fails the call in its own words, scrubbed as every failure is, and leaves the connection connected: a host the rules refuse in the rule's words, as `HostRefused`, a server that does not answer, `The authorization server of the connection drive did not answer as an authorization server does, so its tokens were not renewed; the next call tries again`, metadata that no longer takes the connection, `The authorization server of the connection drive no longer answers as it did when the connection was made, so its tokens cannot be renewed`, and a refresh the server refuses otherwise in the words of the library. A source remembers the claim of a refresh of its own that failed, refused by the server otherwise or by the store: until that claim ends, it serves the access token it holds while it is before its `expires_at`, and otherwise fails at once in the words of that failure, a 401 among the otherwise, and never waits on its own claim nor says another process holds it; once the claim ends, the next renewal claims the row again. A second 401 after a renewal is a failure, as for client credentials. +- `tokenSource(served, serving)`: the token source of a connection connected by consent (`src/consent-tokens/renewed-tokens.ts`), which `tokensOf` asks for an `auth` of the kind `consent`. `served` is that credential: the way, the server, the return address, `reaches`, the client the connection recorded, `held`, the row of its tokens as the source read it, decoded, its `tokens`, its `version` and its claim, which a store decodes from the value with `keptTokensIn`, so the source never reads a value it cannot serve and holds its tokens among the secrets as it is made, and `tokens`, the store of that row, `ConnectionTokens`, with `read`, `replace(version, value)`, `claim(version, lastingMs)` and `broken(ended)`, which `@on.auto/connections` implements over the secret store. The value is the JSON `finish` keeps, `{ access_token, refresh_token?, expires_at?, scope?, token_type, connection_key? }`, the key alone of a consent through Auto's consent service (`src/consent-tokens/kept-tokens.ts`). Made, it holds both tokens among the secrets and serves the access token until its renewal time, `renewalTimeOf` from the time it read them to `expires_at`, less the spread of the process, `renewalSpreadMs` of the options, up to five seconds drawn once when the client is made and never more than half the lead, so processes sharing a ledger do not all renew in the same second and a token of a short life is never renewed at every use; a token without an expiry, and one without a refresh token, is served until the server answers 401. At the renewal time, or on a 401, it reads the row again: where its version moved, another process renewed, and it takes the value instead, renewing it in turn only where that value is itself past its renewal time; where another process holds a claim of the row, it waits on its timer for the claim to end and reads again, and gives up as `requestMs` ends, `The tokens of the connection drive are being renewed by another process of the runtime, which did not finish within 30000 ms; the next call tries again`; else it reads the metadata as `start` does, takes the client the connection recorded as `finish` does, claims the row for the lease, the bound of the request it then sends, `requestMs`, and fifteen seconds more for the write of the claim, the write of the replacement and the gap between the clocks of the servers, forty-five seconds where the bound is the default thirty (`leaseOf` of `src/consent-tokens/token-requests.ts`, the one rule of it), so the refresh is cut off before the claim ends and a second claim never sends the same refresh token while the first is in flight or being kept, refreshes with `refreshAuthorization`, the client and the `resource` `finish` sends, keeping the refresh token where the server sends none, as the library does, and replaces the value under the version it read, which ends the claim. Within a process one renewal is in flight for a connection, whatever asks for it. A replacement that matches no row reads the row again: where it is gone, as when the connection was removed meanwhile, the source revokes what it got and fails in the words the row is read with; where its version moved, it takes the value the row holds and revokes nothing, since for a server that does not rotate its refresh tokens the value it got carries the refresh token the row's value lives by. A value the store refuses, tagged `secret_refused`, revokes what it got and fails in the store's words. A replacement the store cannot make now, its custodian refusing or not answering, revokes nothing: the source keeps what it got in memory and fails the call in the store's words, and while its claim lasts serves the access token it holds as a failed claim does, below; once the claim ends, the next renewal claims the row again and keeps what it got there without asking the server again, or, where another process replaced the row meanwhile, takes that value and leaves what it got. The replacement's failure is told apart by its tag, for the renewals at the vendor and through Auto's consent service alike, since both keep through `replace`. A refresh refused as `invalid_grant` reads the row again: where it still holds the value refused, the source breaks the connection, `broken('consent_ended')`, and where it moved, takes the new value; a 401 with no refresh token breaks it so too, and a client no longer registered breaks it with `broken('client_gone')`, before anything is sent. Any other failure fails the call in its own words, scrubbed as every failure is, and leaves the connection connected: a host the rules refuse in the rule's words, as `HostRefused`, a server that does not answer, `The authorization server of the connection drive did not answer as an authorization server does, so its tokens were not renewed; the next call tries again`, metadata that no longer takes the connection, `The authorization server of the connection drive no longer answers as it did when the connection was made, so its tokens cannot be renewed`, a client registration the store cannot read now, `The client the connection drive was made with cannot be read now, so its tokens were not renewed; the next call tries again: `, and a refresh the server refuses otherwise in the words of the library. A source remembers the claim of a refresh of its own that failed, refused by the server otherwise or by the store: until that claim ends, it serves the access token it holds while it is before its `expires_at`, and otherwise fails at once in the words of that failure, a 401 among the otherwise, and never waits on its own claim nor says another process holds it; once the claim ends, the next renewal claims the row again. A second 401 after a renewal is a failure, as for client credentials. - `revoke(asked)`: revokes the tokens of a connection, `asked.tokens` the value of its row, with the client it recorded, through the code `finish`'s `revoked` takes (`src/consent/revocation.ts`), at the revocation endpoint the metadata or the way names, and nothing where neither names one. It fails where the revocation did not happen, with the refusal of the metadata or of the client, `client_gone` among them, `not_revoked` with the `status` the endpoint answered, `unknown_key` for a connection of `auto` whose key Auto's consent service does not know, or, for a request that failed, `host` with the rule or `unanswered`, as `refusalOf` maps a failed request of a start or a finish, so a refusal carries no word of the server's and no token, which `@on.auto/connections` logs in words of its own as `remove_connection` goes on to forget the row. ### Consent through Auto's consent service @@ -242,7 +242,7 @@ The arguments, as `JSON.stringify` writes them, and the protocol answer are scru - `serveFakeMcp({ bearer?, client?, requestIdHeader?, issuesSessionIds?, annotated?, hints?, chat?, data? })` serves it over Streamable HTTP on a loopback port the system picks, with a bearer check or a fake authorization server, sessions, or none with `issuesSessionIds: false`, its tools listed without their annotations with `annotated: false`, its tools' annotations replaced by `hints`, and ways to answer the next requests with a status (`answerNextWith`, for a 429 with `Retry-After`, or `answerNextOf` for the next of one method alone, such as `tools/call`), answer the next tool call after messages that are not JSON-RPC (`answerNextCallAfterNoise`), forget every session (a 404), revoke tokens (a 401), remove a tool and send `list_changed`. With `chat: true` it also holds a chat, `chat`, whose `post_message` posts a message and answers where it landed and whose `thread_replies` lists the messages of a conversation or of one thread after a point, oldest first, and a test replies as a person with `chat.reply`. With `data: true` it also holds tools whose answers a call reads: `thread`, structured content beside a summary text, the replies of `threadReplies`; `pages`, one page and the cursor of the next; `blocks`, two text blocks of JSON; `strict`, which refuses an argument that is not a whole number with -32602; and `promised`, which declares an output schema of a count and answers `brokenPromise`, a count that is not a number. `holdNextOf` holds the next request of one method unanswered, such as a listing that never comes. A request its handler fails on is answered with a 500 and recorded in `failures()`, beside the requests in `seen()` and the method, the path and the `User-Agent` of every request it heard in `heard()`, instead of rejecting where no test awaits it. - `serveFakeApi({ key?, client?, version?, requestIdHeader? })` serves a fake HTTP API on a loopback port, with its OpenAPI document, `fakeApiDocument('3.0' | '3.1', serverUrl)`, at `documentUrl` and its operations under `baseUrl`: `search`, `echo`, `get_profiles_id`, `updateProfile`, `removeProfile`, `rows`, `denied`, `photo` and `notes` (`fakeApiToolNames`), and three operations no tool is offered for, `sessionState`, which needs the header `X-Session` and is offered to an entry whose `headers` give it, answering the value it was sent, one that takes an object in its query and one whose body is XML. The 3.0 document writes `nullable` and a boolean `exclusiveMinimum` where the 3.1 one writes their 2020-12 forms, so the tools derived from either are the same. It takes a key in `X-Api-Key` or in the query as `api_key`, or with `client` a token from a fake authorization server on another loopback origin, `issuing.issuer`, whose metadata it publishes by OpenID Connect discovery alone with `openIdConnect: true`; `answerNextWith({ status, headers?, body? }, times?)` answers the next calls so, `seen()` holds every request but the document's, `heard()` the method, the path and the `User-Agent` of every request it and its authorization server heard, the document's among them, and `documentReads()` counts the reads of the document. `describedAccess(servers, options)` is `reportingAccess` for entries of any type, reading their documents first. - The fake authorization server both fakes take as `client`, a `ClientRegistration` (`src/authorization-testing`), grants client credentials to a client that is not public. Beside that it grants the authorization code with PKCE and `S256` at `/authorize`, sending the person back at once to one of the client's `redirectUris` with the code, the state and its issuer, or with the error of a consent it cannot grant, and answering a client or a redirect URI it does not know on its own page; it redeems a code once, for its client, its redirect URI and its verifier, for an access token and a refresh token, and refreshes them, keeping the refresh token unless `rotatesRefreshTokens` asks it to rotate one at each refresh, when a rotated token used again ends its whole grant. It revokes at `/revoke` an access token alone, or a refresh token with every token of its grant, for the client it was issued to. Its clients authenticate in one way for every grant and for its revocation endpoint, by a secret in the Basic header or the form, by an assertion its `publicKey` verifies, or, public, by their id alone (`fake-clients.ts`); with `clientMetadataDocuments` it takes a client by the metadata document its id addresses, and with `dynamicRegistration` it registers one dynamically at `/register`. Its metadata says each of these, and leaves out the two it does not offer. For a consent it also lists the challenge methods `challengeMethods` names in place of `S256`, grants `grantedScope` whatever scope it was asked, gives no refresh token with `issuesRefreshTokens: false`, publishes no metadata with `publishesMetadata: false`, answering 404 at its addresses, lists `scopesSupported` as its `scopes_supported`, takes a client secret in the form alone with `takesBasicSecrets: false` and lists no methods for its revocation endpoint with `listsRevocationMethods: false`, refuses a registration with `refusesRegistration`, answers a registration with the client id `registeredClientId` and ends its secret `registrationSecretSeconds` from now, names `protectedResource` as the resource it protects in place of its own address, lists `protectedScopes` as the `scopes_supported` of its protected-resource metadata, and names its endpoints on `endpointHost` in place of its own host. Its tests drive every branch through `fetch` (`fake-grants.test.ts`, `fake-clients.test.ts`), and the tests of the consent client the options a consent takes. -- `src/consent-testing` holds what the tests of a consent share: `servedForConsent`, a fake MCP server whose authorization server knows the client `auto-runtime` with a secret and the return address `returnAddress`; `registeredAt(issuer)`, that client as registered there; `consentClientWith(options)`, a consent client over the global `fetch`, with no registered client, no public address and `memoryRegistrations()`, registrations kept in memory, unless the options say otherwise; `startAskedAt(fake)`, the start of a consent at the fake; `consentedAt(url)`, the code, the state and the issuer the fake sends the person back with; `finishAskedOf(client, asked)`, the finish of the consent a client starts; and `tokensKeptBy(finished)`, the tokens a finish answers, decoded. For the life of the tokens it holds `memoryTokens(value, { now, mostCharacters? })`, a store of a connection's tokens, `ConnectionTokens`, over one value of `valuesInTheClear` of `@on.auto/secrets/testing`, values held in the clear over the rows kept in memory under the rule of the secret store's lease, which `@on.auto/connections` keeps its harness's secrets in too, whose claims are judged by `now`, which counts the leases it is asked to claim, `claims()`, and the breaks, `ends()`, refuses a replacement past `mostCharacters`, dies on a first value past it, and `forget`s its row as a removal does, failing in `goneWords` and `brokenWords(ended)`; `servedAt(fake, client, tokens, way?)`, the credential a source composes over such a store; `connectedByConsent(fake, way?, options?)`, a consent the fake finished under Effect's test clock, and `rowOf`, `servedOf` and `servedTokensOf(served, now, consent, secrets?)`, the row, the credential and the token source over them; `clockAt(moment)`, a clock a test sets; `manualTimer()`, a timer that records each wait and fires when the test says; `consentClientAt(fake, options?)`, a consent client with the fake's registered client and no spread; `hostRefusedAt(path, rule)`, a fetch that refuses the requests to a path as the rules refuse a host; and `gatedRequests(path?)`, a fetch that holds the requests to a path, the token endpoint unless the test names another, until the test releases them, so a test drives two processes over one row and never waits on a clock. +- `src/consent-testing` holds what the tests of a consent share: `servedForConsent`, a fake MCP server whose authorization server knows the client `auto-runtime` with a secret and the return address `returnAddress`; `registeredAt(issuer)`, that client as registered there; `consentClientWith(options)`, a consent client over the global `fetch`, with no registered client, no public address and `memoryRegistrations(custodian?)`, registrations kept in memory, each in a store sealed by a recording custodian of `@on.auto/secrets/testing`, `custodian`, which a test tells to refuse, unless the options say otherwise; `startAskedAt(fake)`, the start of a consent at the fake; `consentedAt(url)`, the code, the state and the issuer the fake sends the person back with; `finishAskedOf(client, asked)`, the finish of the consent a client starts; and `tokensKeptBy(finished)`, the tokens a finish answers, decoded. For the life of the tokens it holds `memoryTokens(value, { now, mostCharacters? })`, a store of a connection's tokens, `ConnectionTokens`, over one value of `valuesInTheClear` of `@on.auto/secrets/testing`, values held in the clear over the rows kept in memory under the rule of the secret store's lease, which `@on.auto/connections` keeps its harness's secrets in too, whose claims are judged by `now`, which counts the leases it is asked to claim, `claims()`, and the breaks, `ends()`, refuses a replacement past `mostCharacters`, dies on a first value past it, and `forget`s its row as a removal does, failing in `goneWords` and `brokenWords(ended)`; `sealedTokens(value, now)`, the same over a store sealed by a recording custodian, `custodian`, which a test tells to refuse, its read failing as the connections' source fails it, `connection_unavailable` in the store's words; `servedAt(fake, client, tokens, way?)`, the credential a source composes over such a store; `connectedByConsent(fake, way?, options?)`, a consent the fake finished under Effect's test clock, and `rowOf`, `servedOf` and `servedTokensOf(served, now, consent, secrets?)`, the row, the credential and the token source over them; `clockAt(moment)`, a clock a test sets; `manualTimer()`, a timer that records each wait and fires when the test says; `consentClientAt(fake, options?)`, a consent client with the fake's registered client and no spread; `hostRefusedAt(path, rule)`, a fetch that refuses the requests to a path as the rules refuse a host; and `gatedRequests(path?)`, a fetch that holds the requests to a path, the token endpoint unless the test names another, until the test releases them, so a test drives two processes over one row and never waits on a clock. - `serveFakeConsentService({ vendor, now })` (`src/consent-testing/fake-consent-service.ts`, its records and its calls of the vendor in `fake-consent-store.ts`) serves Auto's consent service on a loopback port over the fake authorization server of `vendor`, as the contract the runtime calls holds it: it lists its one client, `auto-runtime`, at the vendor's issuer; starts a consent for a return `hosted` on its own Studio base, `studioBase`, or on a loopback base, `http`, ending in `/`, with a state, a PKCE pair and later a handle and a connection key of 32 random bytes each, refusing another issuer or base with a `400`; describes a state and redeems its code at the vendor with the client's secret for a person signed in, once, answering a call without a session `401` with `WWW-Authenticate: Bearer`; hands the tokens off once, within ten minutes of the redeem, to the verifier the start was challenged with, refusing another as `wrong_verifier`; refreshes and revokes at the vendor for a connection key it handed off, refusing another key as `unknown_key` and a body it does not take with a `400`; answers `code_refused` and `invalid_grant` only where the vendor answered `400` with that error, any other failure of the vendor's as a `502` without a refusal, and a revocation the vendor did not make as `502` `not_revoked` with the vendor's status as `vendor_status`; ends a start and a handoff ten minutes on by `now`; records every call, `calls()`, with its body and its `Authorization`; and is told to refuse the next start, redeem, refresh or revocation, `refuseNext`, or to answer nothing more, `stall`. `src/consent-testing/auto-consents.ts` holds the steps a test takes: `servedThroughAuto(changes?)`, a vendor and the service over it on one test clock; `autoClientsOf(service, kind?)`, the source on with the return kind given; `consentClientThrough(serving, options?)`, a client with the source on; `handleOf(service, url)`, the consent at the vendor and the redeem the hosted Studio makes; `handedOffAskedOf(client, asked, service)`, the finish of the start a client makes; `connectedThroughAuto(serving, way?)`, a connection the service handed off; and `serviceCallsAt(service)`, the runtime's calls of the contract at it. - `fakeStdioServerPath` runs the same tools over stdio under `node`, with `--chatter`, `--pad`, `--stdout`, `--linger-ms` and `--start-once` to make it talk, misbehave or fail to start again. Start it with `process.execPath` directly, never through a package manager: it imports only the MCP server package and the fake's tools, not `effect`, and answers its first message in about 90 ms on a laptop. A spawn is real time, so a test that starts it takes `stdioTestTimeoutMs`, 30 s, and its client `patientTiming`, and a file shares one process across the tests that only need it running. - Its tools answer text (`search`, `echo`, `graph.query.v2`), structured content (`profile`), non-text content (`photo`), a denial marked `isError` (`denied`), an error (`broken`), slowly (`sleep`), at length (`large`), the channels of a workspace (`list_channels`, `fakeChannels`), with its environment (`environment`), or by exiting (`exit`). Their annotations cover every case of a test: `search`, `profile`, `photo`, `denied`, `sleep`, `large`, `broken` and `list_channels` are marked read-only, `graph.query.v2` read-only and destructive at once, `exit` destructive, the long-named tool only additive (`destructiveHint: false`), `echo` and `verbose` have none, `environment` has annotations with a title and no hint, and `verbose` has a title of its own. diff --git a/packages/tool-servers/src/consent-testing/given-consents.ts b/packages/tool-servers/src/consent-testing/given-consents.ts index 47a357e5a..8b4bea850 100644 --- a/packages/tool-servers/src/consent-testing/given-consents.ts +++ b/packages/tool-servers/src/consent-testing/given-consents.ts @@ -1,4 +1,6 @@ -import { Effect, Option } from 'effect'; +import { sealedKeeper, secretStoreOn } from '@on.auto/secrets'; +import { memoryRows, recordingCustodian, type RecordingCustodian } from '@on.auto/secrets/testing'; +import { DateTime, Effect, Option } from 'effect'; import { keptTokensIn } from '../consent-tokens/kept-tokens.ts'; import type { @@ -6,7 +8,6 @@ import type { ConsentClient, FinishAsked, FinishedConsent, - KeptRegistration, StartAsked, } from '../consent/consent-client.ts'; @@ -18,6 +19,13 @@ export interface Consented { export interface MemoryRegistrations extends ClientRegistrations { readonly kept: () => readonly (readonly [string, string])[]; + readonly custodian: RecordingCustodian; +} + +interface KeptAt { + readonly issuer: string; + readonly clientId: string; + readonly secretId: string; } export async function consentedAt(url: string): Promise { @@ -41,18 +49,26 @@ export async function finishAskedOf(client: ConsentClient, asked: StartAsked): P return { server, way, returnAddress, reaches, client: start.client, code, iss, verifier: start.verifier }; } -export function memoryRegistrations(): MemoryRegistrations { - const kept: (readonly [string, KeptRegistration])[] = []; - const at = (issuer: string): readonly KeptRegistration[] => - kept.filter(([keptAt]) => keptAt === issuer).map(([, registration]) => registration); +export function memoryRegistrations(custodian: RecordingCustodian = recordingCustodian()): MemoryRegistrations { + const rows = memoryRows({ now: Effect.map(DateTime.now, DateTime.toEpochMillis) }); + const store = secretStoreOn(rows, sealedKeeper({ current: custodian }), { current: custodian }); + const kept: KeptAt[] = []; + const readOf = (found: KeptAt | undefined) => + found === undefined + ? Effect.undefined + : Effect.map(store.read(found.secretId), (stored) => ({ + clientId: found.clientId, + registration: Option.getOrThrow(Option.fromNullishOr(stored)).value, + })); + const at = (issuer: string): readonly KeptAt[] => kept.filter((each) => each.issuer === issuer); return { - latestAt: (issuer) => Effect.sync(() => at(issuer).at(-1)), - named: (issuer, clientId) => - Effect.sync(() => at(issuer).find((registration) => registration.clientId === clientId)), - keep: (issuer, registration) => - Effect.sync(() => { - kept.push([issuer, registration]); + latestAt: (issuer) => Effect.suspend(() => readOf(at(issuer).at(-1))), + named: (issuer, clientId) => Effect.suspend(() => readOf(at(issuer).find((each) => each.clientId === clientId))), + keep: (issuer, { clientId, registration }) => + Effect.map(store.keep(registration), (secretId) => { + kept.push({ issuer, clientId, secretId }); }), - kept: () => kept.map(([issuer, { clientId }]): readonly [string, string] => [issuer, clientId]), + kept: () => kept.map(({ issuer, clientId }): readonly [string, string] => [issuer, clientId]), + custodian, }; } diff --git a/packages/tool-servers/src/consent-testing/memory-tokens.ts b/packages/tool-servers/src/consent-testing/memory-tokens.ts index 42269442a..b185a9dfa 100644 --- a/packages/tool-servers/src/consent-testing/memory-tokens.ts +++ b/packages/tool-servers/src/consent-testing/memory-tokens.ts @@ -8,9 +8,11 @@ import type { ConnectionTokens, ConsentServed, ConsentWay, + HeldTokens, TokensEnded, } from '../consent/consent-client.ts'; import type { FakeMcpServer } from '../mcp-testing/fake-mcp-server.ts'; +import type { ConnectionUnready } from '../org-servers/connection-source.ts'; import { everyHost, returnAddress } from './consent-fakes.ts'; export interface MemoryTokens extends ConnectionTokens { @@ -34,35 +36,49 @@ export const goneWords = 'The tokens of the connection are no longer in the secr const row = 'secret-1'; -function decoded({ value }: ValueInTheClear): KeptTokens { +function decoded({ value }: Pick): KeptTokens { return Option.getOrThrow(keptTokensIn(value)); } +export function heldTokensOf(kept: ValueInTheClear | undefined): Effect.Effect { + return kept === undefined + ? Effect.fail({ because: 'connection_broken', detail: goneWords }) + : Effect.succeed({ tokens: decoded(kept), version: kept.version, claimedUntil: kept.claimedUntil }); +} + +export interface RecordedBreaks extends Pick { + readonly ends: () => readonly TokensEnded[]; +} + +export function recordedBreaks(): RecordedBreaks { + const ends: TokensEnded[] = []; + return { + broken: (ended) => + Effect.suspend(() => { + ends.push(ended); + return Effect.fail({ because: 'connection_broken', detail: brokenWords(ended) }); + }), + ends: () => [...ends], + }; +} + export function memoryTokens(value: Redacted.Redacted, { now, mostCharacters = 8192 }: MemoryTokensOptions) { const rows = valuesInTheClear({ now: Effect.sync(now), mostCharacters }); Effect.runSync(Effect.orDie(rows.keep(row, value))); const claims: number[] = []; - const ends: TokensEnded[] = []; + const { broken, ends } = recordedBreaks(); const tokens: MemoryTokens = { - read: Effect.flatMap(rows.read(row), (kept) => - kept === undefined - ? Effect.fail({ because: 'connection_broken', detail: goneWords }) - : Effect.succeed({ tokens: decoded(kept), version: kept.version, claimedUntil: kept.claimedUntil }), - ), + read: Effect.flatMap(rows.read(row), heldTokensOf), replace: (version, replacing) => rows.replace(row, version, replacing), claim: (version, lastingMs) => { claims.push(lastingMs); return rows.claim(row, version, lastingMs); }, - broken: (ended) => - Effect.suspend(() => { - ends.push(ended); - return Effect.fail({ because: 'connection_broken', detail: brokenWords(ended) }); - }), + broken, kept: () => Option.getOrUndefined(Option.map(Option.fromNullishOr(Effect.runSync(rows.read(row))), decoded)), version: () => Effect.runSync(rows.read(row))?.version, claims: () => [...claims], - ends: () => [...ends], + ends, forget: () => { Effect.runSync(rows.forget(row)); }, @@ -73,7 +89,7 @@ export function memoryTokens(value: Redacted.Redacted, { now, mostCharacters = 8 export function servedAt( fake: FakeMcpServer, client: ChosenClient, - tokens: MemoryTokens, + tokens: ConnectionTokens, way: Partial = {}, ): Effect.Effect { return Effect.map(Effect.orDie(tokens.read), (held) => ({ diff --git a/packages/tool-servers/src/consent-testing/sealed-tokens.ts b/packages/tool-servers/src/consent-testing/sealed-tokens.ts new file mode 100644 index 000000000..4a55f32ab --- /dev/null +++ b/packages/tool-servers/src/consent-testing/sealed-tokens.ts @@ -0,0 +1,40 @@ +import { sealedKeeper, secretStoreOn } from '@on.auto/secrets'; +import { memoryRows, recordingCustodian, type RecordingCustodian } from '@on.auto/secrets/testing'; +import { Effect, type Redacted } from 'effect'; + +import type { ConnectionTokens, TokensEnded } from '../consent/consent-client.ts'; +import type { ConnectionUnready } from '../org-servers/connection-source.ts'; +import { heldTokensOf, recordedBreaks } from './memory-tokens.ts'; + +export interface SealedTokens extends ConnectionTokens { + readonly custodian: RecordingCustodian; + readonly version: () => number | undefined; + readonly claims: () => readonly number[]; + readonly ends: () => readonly TokensEnded[]; +} + +function unreadableNow({ detail }: { readonly detail: string }): ConnectionUnready { + return { because: 'connection_unavailable', detail }; +} + +export async function sealedTokens(value: Redacted.Redacted, now: () => number): Promise { + const custodian = recordingCustodian(); + const rows = memoryRows({ now: Effect.sync(now) }); + const store = secretStoreOn(rows, sealedKeeper({ current: custodian }), { current: custodian }); + const id = await Effect.runPromise(Effect.orDie(store.keep(value))); + const claims: number[] = []; + const { broken, ends } = recordedBreaks(); + return { + read: Effect.flatMap(Effect.mapError(store.read(id), unreadableNow), heldTokensOf), + replace: (version, replacing) => store.replace(id, version, replacing), + claim: (version, lastingMs) => { + claims.push(lastingMs); + return store.claim(id, version, lastingMs); + }, + broken, + custodian, + version: () => rows.kept().map((row) => row.version)[0], + claims: () => [...claims], + ends, + }; +} diff --git a/packages/tool-servers/src/consent-testing/served-consents.ts b/packages/tool-servers/src/consent-testing/served-consents.ts index aaaea5eda..372e11271 100644 --- a/packages/tool-servers/src/consent-testing/served-consents.ts +++ b/packages/tool-servers/src/consent-testing/served-consents.ts @@ -6,6 +6,7 @@ import type { TokenSource } from '../clients/token-source.ts'; import type { KeptTokens } from '../consent-tokens/kept-tokens.ts'; import type { ChosenClient, + ConnectionTokens, ConsentClient, ConsentClientOptions, ConsentServed, @@ -74,7 +75,7 @@ export function rowOf({ tokens }: ConsentConnected, now: () => number, mostChara export function servedOf( { fake, client }: ConsentConnected, - row: MemoryTokens, + row: ConnectionTokens, way: Partial = {}, ): ConsentServed { return Effect.runSync(servedAt(fake, client, row, way)); diff --git a/packages/tool-servers/src/consent-tokens/held-in-memory.ts b/packages/tool-servers/src/consent-tokens/held-in-memory.ts new file mode 100644 index 000000000..f0cfe28dc --- /dev/null +++ b/packages/tool-servers/src/consent-tokens/held-in-memory.ts @@ -0,0 +1,18 @@ +export interface HeldInMemory { + readonly held: () => A | undefined; + readonly hold: (value: A) => void; + readonly forget: () => void; +} + +export function heldInMemory(): HeldInMemory { + const memory: { value?: A } = {}; + return { + held: () => memory.value, + hold: (value) => { + memory.value = value; + }, + forget: () => { + delete memory.value; + }, + }; +} diff --git a/packages/tool-servers/src/consent-tokens/renewal-unkept.test.ts b/packages/tool-servers/src/consent-tokens/renewal-unkept.test.ts new file mode 100644 index 000000000..04641ebea --- /dev/null +++ b/packages/tool-servers/src/consent-tokens/renewal-unkept.test.ts @@ -0,0 +1,242 @@ +import { custodianRefusal } from '@on.auto/secrets/testing'; +import { Effect } from 'effect'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { admitted } from '../authorization-testing/authorization-requests.ts'; +import type { ConsentClient, ConsentServed } from '../consent/consent-client.ts'; +import type { FakeMcpServer } from '../mcp-testing/fake-mcp-server.ts'; +import { + clockAt, + connectedByConsent, + connectedThroughAuto, + consentClientAt, + consentClientThrough, + gatedRequests, + memoryRegistrations, + rowOf, + sealedTokens, + servedAt, + servedForConsent, + servedOf, + servedThroughAuto, + servedTokensOf, + type ConsentConnected, + type GatedRequests, + type ManualClock, + type SealedTokens, +} from '../testing/index.ts'; +import { until } from '../testing/until.ts'; +import { renewalWords } from './renewal-words.ts'; + +const closing: (() => Promise)[] = []; + +afterEach(async () => { + await Promise.all(closing.splice(0).map((close) => close())); +}); + +const renewalTime = 3_540_000; + +const afterTheClaim = renewalTime + 45_000; + +interface Sealed { + readonly connected: ConsentConnected; + readonly clock: ManualClock; + readonly row: SealedTokens; + readonly served: ConsentServed; +} + +async function sealedFor(connected: ConsentConnected, clock: ManualClock): Promise { + const row = await sealedTokens(connected.tokens, clock.now); + const served = await Effect.runPromise(servedAt(connected.fake, connected.client, row)); + return { connected, clock, row, served }; +} + +function unwrittenBy({ row }: Sealed): Error { + return new Error(`The custodian of the key ${row.custodian.kid} refused to wrap a data key: ${custodianRefusal}`); +} + +interface Refused { + readonly failed: unknown; + readonly renewed: string; +} + +async function renewedWhileTheStoreRefuses( + { clock, row, served }: Sealed, + consent: ConsentClient, + gate: GatedRequests, +): Promise['source'] }> { + const { source } = servedTokensOf(served, clock.now, consent); + clock.setTo(renewalTime); + const renewing = source.token().catch((error: unknown) => error); + await until(() => gate.holding() === 1); + row.custodian.refuse(); + gate.release(); + const failed = await renewing; + const [renewed = ''] = gate.answered(); + return { failed, renewed, source }; +} + +function claimedAfterAnotherKept(sealed: Sealed, other: ConsentConnected) { + const anotherKeptFirst = Effect.orDie(sealed.row.replace(1, other.tokens)); + const claims = { made: 0 }; + const row: SealedTokens = { + ...sealed.row, + claim: (version, lastingMs) => + Effect.suspend(() => { + claims.made += 1; + const claimed = sealed.row.claim(version, lastingMs); + return claims.made === 2 ? Effect.andThen(anotherKeptFirst, claimed) : claimed; + }), + }; + return { row, claimsMade: () => claims.made }; +} + +function revocationsAt(fake: FakeMcpServer): number { + return fake.heard().filter(({ path }) => path === '/revoke').length; +} + +describe('renewed tokens the secret store cannot keep now', () => { + it('are kept in memory and revoked nowhere, the call failing in the words of the store, and are kept under the next claim without asking the server again', async () => { + const fake = await servedForConsent(); + closing.push(fake.close); + const sealed = await sealedFor(await connectedByConsent(fake), clockAt(0)); + const gate = gatedRequests(); + + const { failed, renewed, source } = await renewedWhileTheStoreRefuses( + sealed, + consentClientAt(fake, { fetch: gate.fetch }), + gate, + ); + const whileClaimed = await source.token(); + sealed.row.custodian.answer(); + sealed.clock.setTo(afterTheClaim); + const keptLater = await source.token(); + + expect(failed).toEqual(unwrittenBy(sealed)); + expect(renewed).not.toBe(sealed.connected.kept.access_token); + expect(await admitted(fake, renewed)).toBe(true); + expect(revocationsAt(fake)).toBe(0); + expect(whileClaimed).toBe(sealed.connected.kept.access_token); + expect(keptLater).toBe(renewed); + expect(sealed.row.version()).toBe(2); + expect(sealed.row.claims()).toEqual([45_000, 45_000]); + expect(sealed.row.ends()).toEqual([]); + expect(fake.tokenRequests()).toBe(2); + }); +}); + +describe('renewed tokens the secret store cannot keep now, through Auto’s consent service', () => { + it('are kept in memory alike, and the service is asked to revoke nothing', async () => { + const serving = await servedThroughAuto(); + closing.push(serving.close); + const sealed = await sealedFor(await connectedThroughAuto(serving), serving.clock); + const gate = gatedRequests('/tokens/refresh'); + + const { failed, source } = await renewedWhileTheStoreRefuses( + sealed, + consentClientThrough(serving, { fetch: gate.fetch }), + gate, + ); + sealed.row.custodian.answer(); + serving.clock.setTo(afterTheClaim); + const keptLater = await source.token(); + + expect(failed).toEqual(unwrittenBy(sealed)); + expect(keptLater).not.toBe(sealed.connected.kept.access_token); + expect(await admitted(serving.fake, keptLater)).toBe(true); + expect(sealed.row.version()).toBe(2); + expect(serving.service.calls().map(({ path }) => path)).toEqual([ + '/clients', + '/consents', + expect.stringMatching(/^\/consents\/[\w-]+\/redeem$/u), + expect.stringMatching(/^\/handoffs\/[\w-]+$/u), + '/tokens/refresh', + ]); + }); +}); + +describe('renewed tokens the secret store could not keep, before their next claim', () => { + it('are left where another process keeps tokens of its own first, whose tokens are served instead', async () => { + const fake = await servedForConsent(); + closing.push(fake.close); + const sealed = await sealedFor(await connectedByConsent(fake), clockAt(0)); + const other = await connectedByConsent(fake); + const { row, claimsMade } = claimedAfterAnotherKept(sealed, other); + const served = await Effect.runPromise(servedAt(fake, sealed.connected.client, row)); + const gate = gatedRequests(); + + const { source } = await renewedWhileTheStoreRefuses( + { ...sealed, row, served }, + consentClientAt(fake, { fetch: gate.fetch }), + gate, + ); + sealed.row.custodian.answer(); + sealed.clock.setTo(afterTheClaim); + const servedLater = await source.token(); + + expect(servedLater).toBe(other.kept.access_token); + expect(sealed.row.version()).toBe(2); + expect(claimsMade()).toBe(2); + expect(fake.tokenRequests()).toBe(3); + }); +}); + +describe('a renewal whose row the secret store cannot read now', () => { + it('fails the call in the words of the store, breaks nothing and asks the server nothing', async () => { + const fake = await servedForConsent(); + closing.push(fake.close); + const sealed = await sealedFor(await connectedByConsent(fake), clockAt(0)); + const { source } = servedTokensOf(sealed.served, sealed.clock.now, consentClientAt(fake)); + + sealed.row.custodian.refuse(); + sealed.clock.setTo(renewalTime); + const failed = await source.token().catch((error: unknown) => error); + + expect(failed).toEqual( + new Error( + `The custodian of the key ${sealed.row.custodian.kid} refused to unwrap a data key: ${custodianRefusal}`, + ), + ); + expect([sealed.row.version(), sealed.row.claims(), sealed.row.ends(), fake.tokenRequests()]).toEqual([ + 1, + [], + [], + 1, + ]); + }); +}); + +describe('a renewal whose client registration the secret store cannot read now', () => { + it('fails the call in the words of the store and breaks nothing, and renews once the store reads it again', async () => { + const fake = await servedForConsent({ dynamicRegistration: true }); + closing.push(fake.close); + const registrations = memoryRegistrations(); + const dynamic = { registrations, registeredClients: [] }; + const connected = await connectedByConsent(fake, { clients: ['dynamic'] }, dynamic); + const clock = clockAt(0); + const row = rowOf(connected, clock.now); + const { source } = servedTokensOf( + servedOf(connected, row, { clients: ['dynamic'] }), + clock.now, + consentClientAt(fake, dynamic), + ); + + registrations.custodian.refuse(); + clock.setTo(renewalTime); + const failed = await source.token().catch((error: unknown) => error); + registrations.custodian.answer(); + const renewed = await source.token(); + + expect(failed).toEqual( + new Error( + renewalWords.registrationUnreadable( + 'files', + `The custodian of the key ${registrations.custodian.kid} refused to unwrap a data key: ${custodianRefusal}`, + ), + ), + ); + expect(row.ends()).toEqual([]); + expect(renewed).not.toBe(connected.kept.access_token); + expect(row.version()).toBe(2); + }); +}); diff --git a/packages/tool-servers/src/consent-tokens/renewal-words.ts b/packages/tool-servers/src/consent-tokens/renewal-words.ts index c8e473701..e6cfdfc22 100644 --- a/packages/tool-servers/src/consent-tokens/renewal-words.ts +++ b/packages/tool-servers/src/consent-tokens/renewal-words.ts @@ -1,3 +1,5 @@ +import type { ConsentRefusal } from '../consent/consent-client.ts'; + export const renewalWords = { unanswered: (name: string) => `The authorization server of the connection ${name} did not answer as an authorization server does, so its tokens were not renewed; the next call tries again`, @@ -5,6 +7,14 @@ export const renewalWords = { `Auto's consent service did not answer for the connection ${name}, so its tokens were not renewed; the next call tries again`, unrenewable: (name: string) => `The authorization server of the connection ${name} no longer answers as it did when the connection was made, so its tokens cannot be renewed`, + registrationUnreadable: (name: string, detail: string) => + `The client the connection ${name} was made with cannot be read now, so its tokens were not renewed; the next call tries again: ${detail}`, unfinished: (name: string, waitMs: number) => `The tokens of the connection ${name} are being renewed by another process of the runtime, which did not finish within ${waitMs} ms; the next call tries again`, + unrenewedBy: (refusal: ConsentRefusal, name: string): string => { + if (refusal.refused === 'registration_unavailable') { + return renewalWords.registrationUnreadable(name, refusal.detail); + } + return refusal.refused === 'unanswered' ? renewalWords.unanswered(name) : renewalWords.unrenewable(name); + }, } as const; diff --git a/packages/tool-servers/src/consent-tokens/renewed-tokens.ts b/packages/tool-servers/src/consent-tokens/renewed-tokens.ts index cc46852c9..f6ac41107 100644 --- a/packages/tool-servers/src/consent-tokens/renewed-tokens.ts +++ b/packages/tool-servers/src/consent-tokens/renewed-tokens.ts @@ -6,10 +6,12 @@ import type { ConsentRefusal, ConsentServed, HeldTokens, + SecretKeepingFailure, TokenServing, TokensEnded, } from '../consent/consent-client.ts'; import type { ConnectionUnready } from '../org-servers/connection-source.ts'; +import { heldInMemory, type HeldInMemory } from './held-in-memory.ts'; import { heldAmongSecrets, keptValueOf, type KeptTokens } from './kept-tokens.ts'; import { endingOf } from './refresh-endings.ts'; import { renewalWords } from './renewal-words.ts'; @@ -43,10 +45,17 @@ interface Held { readonly endsAt: number; } +interface Unkept { + readonly tokens: KeptTokens; + readonly version: number; + readonly refreshing: Refreshing; +} + interface Renewing extends Sourced { readonly renewal: Renewal; readonly refused: boolean; readonly deadline: number; + readonly unkept: HeldInMemory; } interface Claimed { @@ -114,12 +123,10 @@ function refusedWith(refusal: ConsentRefusal, renewing: Renewing): Effect.Effect if (refusal.refused === 'client_gone') { return ended(renewing, 'client_gone'); } - if (refusal.refused === 'host') { - return Effect.fail(new HostRefused({ rule: refusal.rule })); - } - const { name } = renewing.serving; return Effect.fail( - new Error(refusal.refused === 'unanswered' ? renewalWords.unanswered(name) : renewalWords.unrenewable(name)), + refusal.refused === 'host' + ? new HostRefused({ rule: refusal.rule }) + : new Error(renewalWords.unrenewedBy(refusal, renewing.serving.name)), ); } @@ -142,10 +149,27 @@ function movedFrom(row: HeldTokens, renewing: Renewing): Effect.Effect { + const { held, refreshing } = claimed; + const unkept: Effect.Effect = + _tag === 'secret_refused' + ? refreshing.revoked(renewed) + : Effect.sync(() => { + renewing.unkept.hold({ tokens: renewed, version: held.version, refreshing }); + }); + return Effect.andThen(unkept, failedUnder(claimed, new Error(detail))); +} + function keptIn(claimed: Claimed, renewed: KeptTokens, renewing: Renewing): Effect.Effect { const { held, refreshing } = claimed; return renewing.served.tokens.replace(held.version, keptValueOf(renewed)).pipe( - Effect.catch(({ detail }) => Effect.andThen(refreshing.revoked(renewed), failedUnder(claimed, new Error(detail)))), + Effect.catch((failure) => unkeptUnder(claimed, renewed, renewing, failure)), + Effect.tap(() => Effect.sync(renewing.unkept.forget)), Effect.flatMap((replaced) => replaced.replaced ? Effect.succeed(heldOf(renewed, replaced.version, renewing)) @@ -157,6 +181,23 @@ function keptIn(claimed: Claimed, renewed: KeptTokens, renewing: Renewing): Effe ); } +function keptAgain(held: Held, unkept: Unkept, renewing: Renewing): Effect.Effect { + return Effect.flatMap(renewing.served.tokens.claim(held.version, renewing.renewal.leaseMs), (claim) => + claim.claimed + ? keptIn( + { held, refreshToken: '', refreshing: unkept.refreshing, until: Date.parse(claim.until) }, + unkept.tokens, + renewing, + ) + : renewedFrom(held, renewing), + ); +} + +function unkeptOf(held: Held, { unkept }: Renewing): Unkept | undefined { + const waiting = unkept.held(); + return waiting?.version === held.version ? waiting : undefined; +} + function exchangedFor(claimed: Claimed, renewing: Renewing): Effect.Effect { const { held, refreshToken, refreshing } = claimed; return refreshing.refreshed(refreshToken, held.tokens, renewing.serving.now()).pipe( @@ -172,6 +213,10 @@ function exchangedFor(claimed: Claimed, renewing: Renewing): Effect.Effect { return Effect.gen(function* () { + const unkept = unkeptOf(held, renewing); + if (unkept !== undefined) { + return yield* keptAgain(held, unkept, renewing); + } const refreshToken = held.tokens.refresh_token; if (refreshToken === undefined) { return yield* ended(renewing, 'consent_ended'); @@ -210,6 +255,7 @@ export function renewedTokens(sourced: Sourced, renewal: Renewal): TokenSource { renewing: undefined, failed: undefined, }; + const unkept = heldInMemory(); const renewedAccess = (refused: boolean): Effect.Effect => Effect.suspend(() => { const { failed, held } = holding; @@ -217,7 +263,7 @@ export function renewedTokens(sourced: Sourced, renewal: Renewal): TokenSource { return underItsFailedClaim(failed, held, refused, serving); } holding.failed = undefined; - const renewing: Renewing = { ...sourced, renewal, refused, deadline: serving.now() + renewal.waitMs }; + const renewing: Renewing = { ...sourced, renewal, refused, deadline: serving.now() + renewal.waitMs, unkept }; return renewedFrom(held, renewing).pipe( Effect.map((renewed) => { holding.held = renewed; diff --git a/packages/tool-servers/src/consent/acting-clients.ts b/packages/tool-servers/src/consent/acting-clients.ts index dd0759126..beb7b0cfe 100644 --- a/packages/tool-servers/src/consent/acting-clients.ts +++ b/packages/tool-servers/src/consent/acting-clients.ts @@ -23,7 +23,14 @@ import type { FinishAsked, KeptRegistration, } from './consent-client.ts'; -import { registrationFieldsOf, registrationFor, singleFlight, type Registering } from './dynamic-registration.ts'; +import { + registrationFieldsOf, + registrationFor, + registrationUnavailable, + singleFlight, + unkeptRegistrations, + type RegistrationKeeping, +} from './dynamic-registration.ts'; import type { AuthorizationServer } from './server-metadata.ts'; export interface ClientIdentity { @@ -59,9 +66,7 @@ interface ClientChoice { readonly recordedFor: (recorded: RecordedClient) => Effect.Effect; } -interface Choosing extends ClientChoosing { - readonly registeredOnce: Registering; -} +type Choosing = ClientChoosing & RegistrationKeeping; const noClient: ConsentRefusal = { refused: 'no_client' }; @@ -119,7 +124,7 @@ function dynamicActing(kept: KeptRegistration, returnAddress: string): ActingCli function dynamicallyRegistered( { asked, server, fetch }: Taking, - { registrations, registeredOnce }: Choosing, + choosing: Choosing, ): Effect.Effect { const endpoint = server.registration_endpoint; if (endpoint === undefined) { @@ -127,9 +132,7 @@ function dynamicallyRegistered( } const { returnAddress } = asked; const registration = { issuer: asked.way.issuer, endpoint, returnAddress, fetch }; - return Effect.map(registrationFor(registration, registrations, registeredOnce), (kept) => - dynamicActing(kept, returnAddress), - ); + return Effect.map(registrationFor(registration, choosing), (kept) => dynamicActing(kept, returnAddress)); } function takenFrom( @@ -177,8 +180,11 @@ function recordedOf({ way, client, returnAddress }: RecordedClient, choosing: Ch function recordedFor(recorded: RecordedClient, choosing: Choosing): Effect.Effect { const { way, client, returnAddress } = recorded; if (client.client_source === 'dynamic') { - return Effect.flatMap(choosing.registrations.named(way.issuer, client.client_id), (kept) => - kept === undefined ? Effect.fail(clientGone) : Effect.succeed(dynamicActing(kept, returnAddress)), + return choosing.registrations.named(way.issuer, client.client_id).pipe( + Effect.mapError(registrationUnavailable), + Effect.flatMap((kept) => + kept === undefined ? Effect.fail(clientGone) : Effect.succeed(dynamicActing(kept, returnAddress)), + ), ); } const acting = recordedOf(recorded, choosing); @@ -186,7 +192,7 @@ function recordedFor(recorded: RecordedClient, choosing: Choosing): Effect.Effec } function clientChoice(choosing: ClientChoosing): ClientChoice { - const choosingOnce: Choosing = { ...choosing, registeredOnce: singleFlight() }; + const choosingOnce: Choosing = { ...choosing, registeredOnce: singleFlight(), unkept: unkeptRegistrations() }; return { chosenAt: (taking) => chosenAt(taking, choosingOnce), recordedFor: (recorded) => recordedFor(recorded, choosingOnce), diff --git a/packages/tool-servers/src/consent/consent-client.ts b/packages/tool-servers/src/consent/consent-client.ts index d1f46711e..7ea200a21 100644 --- a/packages/tool-servers/src/consent/consent-client.ts +++ b/packages/tool-servers/src/consent/consent-client.ts @@ -95,7 +95,9 @@ export type ConsentRefusal = | { readonly refused: 'studio_elsewhere' } | { readonly refused: 'start_refused' } | { readonly refused: 'no_scope' } - | { readonly refused: 'malformed_scope' }; + | { readonly refused: 'malformed_scope' } + | { readonly refused: 'registration_unavailable'; readonly detail: string } + | { readonly refused: 'registration_refused'; readonly detail: string }; export interface KeptRow { readonly value: Redacted.Redacted; @@ -113,12 +115,25 @@ export type TokensClaimed = { readonly claimed: true; readonly until: string } | export type TokensEnded = 'consent_ended' | 'client_gone' | 'key_unknown'; +export interface SecretKeepingFailure { + readonly _tag: 'secret_refused' | 'custodian_refused' | 'keeper_unavailable'; + readonly detail: string; +} + +export interface SecretReadingFailure { + readonly _tag: 'secret_key_unknown' | 'custodian_refused' | 'keeper_unavailable'; + readonly kid?: string | undefined; + readonly detail: string; +} + +export interface RegistrationUnavailable { + readonly _tag: SecretKeepingFailure['_tag'] | SecretReadingFailure['_tag']; + readonly detail: string; +} + export interface ConnectionTokens { readonly read: Effect.Effect; - readonly replace: ( - version: number, - value: Redacted.Redacted, - ) => Effect.Effect; + readonly replace: (version: number, value: Redacted.Redacted) => Effect.Effect; readonly claim: (version: number, lastingMs: number) => Effect.Effect; readonly broken: (ended: TokensEnded) => Effect.Effect; } @@ -151,9 +166,12 @@ export interface KeptRegistration { } export interface ClientRegistrations { - readonly latestAt: (issuer: string) => Effect.Effect; - readonly named: (issuer: string, clientId: string) => Effect.Effect; - readonly keep: (issuer: string, kept: KeptRegistration) => Effect.Effect; + readonly latestAt: (issuer: string) => Effect.Effect; + readonly named: ( + issuer: string, + clientId: string, + ) => Effect.Effect; + readonly keep: (issuer: string, kept: KeptRegistration) => Effect.Effect; } export interface ConsentClientOptions extends Omit { diff --git a/packages/tool-servers/src/consent/consent-clients.test.ts b/packages/tool-servers/src/consent/consent-clients.test.ts index 4e37eca46..796e657bc 100644 --- a/packages/tool-servers/src/consent/consent-clients.test.ts +++ b/packages/tool-servers/src/consent/consent-clients.test.ts @@ -1,4 +1,5 @@ -import { Effect } from 'effect'; +import { custodianRefusal } from '@on.auto/secrets/testing'; +import { Effect, Function, Logger } from 'effect'; import { afterEach, describe, expect, it } from 'vitest'; import { @@ -10,7 +11,7 @@ import { type FakeMcpServer, } from '../testing/index.ts'; import type { WayClientSource } from './client-sources.ts'; -import type { ConsentClient, StartAsked } from './consent-client.ts'; +import type { ClientRegistrations, ConsentClient, RegistrationUnavailable, StartAsked } from './consent-client.ts'; const closing: (() => Promise)[] = []; @@ -112,6 +113,104 @@ describe('a dynamic registration the runtime does not keep', () => { }); }); +function startedLogging(client: ConsentClient, asked: StartAsked, log: (line: string) => void) { + const logging = Logger.layer([Logger.map(Logger.formatJson, log)]); + return Effect.runPromise(Effect.result(client.start(asked)).pipe(Effect.provide(logging))); +} + +describe('a dynamic registration the secret store cannot read now', () => { + it('fails the start in the words of the store, and registers nothing', async () => { + const fake = await served({ dynamicRegistration: true }); + const registrations = memoryRegistrations(); + const client = consentClientWith({ registrations }); + const asked = startAskedAt(fake, { clients: ['dynamic'] }); + await started(client, asked); + + registrations.custodian.refuse(); + const refusal = await refused(consentClientWith({ registrations }), asked); + + expect(refusal).toEqual({ + refused: 'registration_unavailable', + detail: `The custodian of the key ${registrations.custodian.kid} refused to unwrap a data key: ${custodianRefusal}`, + }); + expect(fake.heard().filter(({ path }) => path === '/register')).toHaveLength(1); + }); +}); + +describe('a dynamic registration the secret store cannot keep now', () => { + it('fails the start in the words of the store, logs its client, and is kept at the next start without registering again', async () => { + const fake = await served({ dynamicRegistration: true }); + const registrations = memoryRegistrations(); + const client = consentClientWith({ registrations }); + const asked = startAskedAt(fake, { clients: ['dynamic'] }); + const lines: string[] = []; + + registrations.custodian.refuse(); + const failed = await startedLogging(client, asked, (line) => { + lines.push(line); + }); + registrations.custodian.answer(); + const later = await started(client, asked); + + const words = `The custodian of the key ${registrations.custodian.kid} refused to wrap a data key: ${custodianRefusal}`; + expect(failed).toMatchObject({ failure: { refused: 'registration_unavailable', detail: words } }); + expect(lines).toEqual([ + expect.stringContaining( + `"message":"The client registered at ${fake.origin} as registered-1 cannot be kept now, so this process keeps it at its next consent there: ${words}"`, + ), + ]); + expect(lines.join('\n')).not.toContain('client_secret'); + expect(later.client).toEqual({ client_id: 'registered-1', client_source: 'dynamic' }); + expect(registrations.kept()).toEqual([[fake.origin, 'registered-1']]); + expect(fake.heard().filter(({ path }) => path === '/register')).toHaveLength(1); + }); + + it('registers again once the client it holds has ended before the store could keep it', async () => { + const fake = await served({ dynamicRegistration: true, registrationSecretSeconds: -60 }); + const registrations = memoryRegistrations(); + const client = consentClientWith({ registrations }); + const asked = startAskedAt(fake, { clients: ['dynamic'] }); + + registrations.custodian.refuse(); + await startedLogging(client, asked, Function.constVoid); + registrations.custodian.answer(); + const later = await started(client, asked); + + expect(later.client.client_id).toBe('registered-2'); + expect(registrations.kept()).toEqual([[fake.origin, 'registered-2']]); + }); +}); + +const tooLarge: RegistrationUnavailable = { _tag: 'secret_refused', detail: 'A secret to keep holds more than 8 KiB' }; + +describe('a dynamic registration the secret store refuses to keep', () => { + it("fails the start in the refusal's words, logs the client as left unkept, and is not held, so the next start registers again", async () => { + const fake = await served({ dynamicRegistration: true }); + const registrations: ClientRegistrations = { ...memoryRegistrations(), keep: () => Effect.fail(tooLarge) }; + const client = consentClientWith({ registrations }); + const asked = startAskedAt(fake, { clients: ['dynamic'] }); + const lines: string[] = []; + const logged = (line: string) => { + lines.push(line); + }; + + const first = await startedLogging(client, asked, logged); + const second = await startedLogging(client, asked, logged); + + expect([first, second]).toMatchObject([ + { failure: { refused: 'registration_refused', detail: tooLarge.detail } }, + { failure: { refused: 'registration_refused', detail: tooLarge.detail } }, + ]); + expect(lines).toEqual([ + expect.stringContaining( + `"message":"The client registered at ${fake.origin} as registered-1 cannot be kept, so it stays registered there unused: ${tooLarge.detail}"`, + ), + expect.stringContaining(`as registered-2 cannot be kept, so it stays registered there unused`), + ]); + expect(fake.heard().filter(({ path }) => path === '/register')).toHaveLength(2); + }); +}); + describe('a start at a server that takes no client the runtime has', () => { it('is refused', async () => { const fake = await served(); diff --git a/packages/tool-servers/src/consent/dynamic-registration.ts b/packages/tool-servers/src/consent/dynamic-registration.ts index 3c5abddc7..67deb90a3 100644 --- a/packages/tool-servers/src/consent/dynamic-registration.ts +++ b/packages/tool-servers/src/consent/dynamic-registration.ts @@ -2,7 +2,12 @@ import type { FetchLike } from '@modelcontextprotocol/client'; import { Clock, Deferred, Effect, Option, Redacted, Schema } from 'effect'; import { ClientIdSchema, clientMetadataOf } from './client-sources.ts'; -import type { ClientRegistrations, ConsentRefusal, KeptRegistration } from './consent-client.ts'; +import type { + ClientRegistrations, + ConsentRefusal, + KeptRegistration, + RegistrationUnavailable, +} from './consent-client.ts'; import { refusalOf, unanswered } from './consent-failures.ts'; const RegistrationFieldsSchema = Schema.Struct({ @@ -31,6 +36,18 @@ export type Registering = ( register: Effect.Effect, ) => Effect.Effect; +export interface UnkeptRegistrations { + readonly heldAt: (issuer: string) => KeptRegistration | undefined; + readonly hold: (issuer: string, kept: KeptRegistration) => void; + readonly release: (issuer: string) => void; +} + +export interface RegistrationKeeping { + readonly registrations: ClientRegistrations; + readonly registeredOnce: Registering; + readonly unkept: UnkeptRegistrations; +} + export interface Registration { readonly issuer: string; readonly endpoint: string; @@ -38,6 +55,12 @@ export interface Registration { readonly fetch: FetchLike; } +const unkeptWords = (issuer: string, clientId: string, detail: string) => + `The client registered at ${issuer} as ${clientId} cannot be kept now, so this process keeps it at its next consent there: ${detail}`; + +const refusedWords = (issuer: string, clientId: string, detail: string) => + `The client registered at ${issuer} as ${clientId} cannot be kept, so it stays registered there unused: ${detail}`; + export function registrationFieldsOf({ registration }: KeptRegistration): RegistrationFields { return decodeRegistration(Redacted.value(registration)); } @@ -90,22 +113,86 @@ export function singleFlight(): Registering { }); } +export function unkeptRegistrations(): UnkeptRegistrations { + const held = new Map(); + return { + heldAt: (issuer) => held.get(issuer), + hold: (issuer, kept) => { + held.set(issuer, kept); + }, + release: (issuer) => { + held.delete(issuer); + }, + }; +} + +export function registrationUnavailable({ detail }: RegistrationUnavailable): ConsentRefusal { + return { refused: 'registration_unavailable', detail }; +} + +function heldUntilKept( + issuer: string, + kept: KeptRegistration, + unkept: UnkeptRegistrations, + failure: RegistrationUnavailable, +): Effect.Effect { + return Effect.sync(() => { + unkept.hold(issuer, kept); + }).pipe( + Effect.andThen(Effect.logWarning(unkeptWords(issuer, kept.clientId, failure.detail))), + Effect.andThen(Effect.fail(registrationUnavailable(failure))), + ); +} + +function leftUnkept( + issuer: string, + kept: KeptRegistration, + { detail }: RegistrationUnavailable, +): Effect.Effect { + const refusal: ConsentRefusal = { refused: 'registration_refused', detail }; + return Effect.andThen(Effect.logWarning(refusedWords(issuer, kept.clientId, detail)), Effect.fail(refusal)); +} + +function isRefused({ _tag }: RegistrationUnavailable): boolean { + return _tag === 'secret_refused'; +} + +function keptAt( + issuer: string, + kept: KeptRegistration, + { registrations, unkept }: RegistrationKeeping, +): Effect.Effect { + return registrations.keep(issuer, kept).pipe( + Effect.matchEffect({ + onFailure: (failure) => + isRefused(failure) ? leftUnkept(issuer, kept, failure) : heldUntilKept(issuer, kept, unkept, failure), + onSuccess: () => + Effect.sync(() => { + unkept.release(issuer); + return kept; + }), + }), + ); +} + export function registrationFor( registration: Registration, - registrations: ClientRegistrations, - registeredOnce: Registering, + keeping: RegistrationKeeping, ): Effect.Effect { const { issuer } = registration; - return registeredOnce( + return keeping.registeredOnce( issuer, Effect.gen(function* () { - const latest = yield* registrations.latestAt(issuer); - if (latest !== undefined && isLasting(latest, yield* Clock.currentTimeMillis)) { + const now = yield* Clock.currentTimeMillis; + const waiting = keeping.unkept.heldAt(issuer); + if (waiting !== undefined && isLasting(waiting, now)) { + return yield* keptAt(issuer, waiting, keeping); + } + const latest = yield* Effect.mapError(keeping.registrations.latestAt(issuer), registrationUnavailable); + if (latest !== undefined && isLasting(latest, now)) { return latest; } - const kept = yield* answeredRegistration(registration); - yield* registrations.keep(issuer, kept); - return kept; + return yield* keptAt(issuer, yield* answeredRegistration(registration), keeping); }), ); } diff --git a/packages/tool-servers/src/index.ts b/packages/tool-servers/src/index.ts index 84f850fab..9483f95f4 100644 --- a/packages/tool-servers/src/index.ts +++ b/packages/tool-servers/src/index.ts @@ -97,8 +97,11 @@ export { type HeldTokens, type KeptRow, type KeptRegistration, + type RegistrationUnavailable, type RevocationFailure, type RevokeAsked, + type SecretKeepingFailure, + type SecretReadingFailure, type StartAsked, type StartedConsent, type TokensClaimed, diff --git a/packages/tool-servers/src/testing/index.ts b/packages/tool-servers/src/testing/index.ts index f29e52f61..9ecf2a249 100644 --- a/packages/tool-servers/src/testing/index.ts +++ b/packages/tool-servers/src/testing/index.ts @@ -101,6 +101,7 @@ export { type ManualTimer, type MemoryTokens, } from '../consent-testing/memory-tokens.ts'; +export { sealedTokens, type SealedTokens } from '../consent-testing/sealed-tokens.ts'; export { gatedRequests, hostRefusedAt, type GatedRequests } from '../consent-testing/gated-requests.ts'; export { clockAt, From 13e13d0f8ffdcd648e89ae484abdfd507e2552c9 Mon Sep 17 00:00:00 2001 From: Rami Date: Tue, 20 Oct 2026 20:24:17 +0100 Subject: [PATCH 2/4] feat(secrets): a custodian that holds its key in AWS KMS, its fake, its probe and a 10 second bound A data key can now be wrapped by a key that never leaves AWS KMS: each value keeps its own data key, KMS wraps it at each keep and unwraps it at each read, and each use is an entry in the account's trail under a context that names the purpose. - @on.auto/secrets/aws, a subpath of its own over @aws-sdk/client-kms pinned to 3.1149.0, the version of the other AWS clients, so that only a setting that names a key in KMS loads the KMS client. kmsCustodian({ keyId, region, proxy, credentials, endpoint?, waits? }) makes the custodian as an effect: for the ARN of an alias it asks DescribeKey once, within the store's bound, and pins the key the alias names, so an alias pointed at another key makes the custodian of another key; for the ARN of a key it asks nothing. A DescribeKey refused or not answered fails the making in the service's words, with the alias as its kid. - The custodian wraps with Encrypt under the pinned key's ARN, the data key as Plaintext and the context { 'on.auto/runtime': 'secret data key' }, answering the ciphertext blob as base64, and unwraps with Decrypt under the same key and context. Its kid is the key's id, the last path segment of the key's ARN. - Its client takes the settings of awsClientSettingsOf(proxy) as the S3 client does, and waits 5 seconds to connect and the store's 10 for an answer. A refusal that is the caller's fault, DisabledException, KMSInvalidStateException, AccessDeniedException, NotFoundException, InvalidCiphertextException or IncorrectKeyException, is custodian_refused in the service's words with the kid. A request not answered, a throttled one, which is anything isThrottlingError of @smithy/core/retry judges so, ThrottlingException and LimitExceededException among them, a fault of the service or a credential not found is keeper_unavailable with the kid, "The custodian of the key did not answer a request to wrap a data key: ", so the log of each cause tells custodians apart. The account of every ARN is taken out of those words. - The store's bound of a hold and a reveal is 10 seconds, the one rule in answer-bound.ts, shorter than the fifteen seconds a renewal's claim keeps for its write. - custodianProbed(custodian) wraps and unwraps one data key within that bound and zeroes both copies, for the server to probe a custodian at start. - serveFakeKms({ keys, aliases? }) of @on.auto/secrets/testing answers Encrypt, Decrypt and DescribeKey of the KMS JSON protocol under X-Amz-Target, with a blob sealed under a key of its own, which holds no plaintext in the clear and opens only under its key and context; it is told a key is disabled, pending deletion, denied or stalling, made to answer an error such as a throttling one, points an alias at another key, and records each call with its key id. - storeBehaviour takes the custodians it runs over, local ones by default, and recordingCustodian wraps any custodian; the suite proves no row holds a data key in the clear. - Tests: the custodian's kid, its Encrypt and Decrypt with their key and context; an alias resolved once and its key pinned, a custodian of another key once the alias names another, an alias unknown, denied or silent; each refusal in the words of KMS without the account or a credential; ThrottlingException, LimitExceededException and an HTTP 429 as keeper_unavailable with the kid; a KMS that stalls or cannot be reached given up on within 100 ms waits; storeBehaviour over keys in the fake; the probe answering, refused and silent at 10 seconds of TestClock; the fake's blobs, contexts, descriptions and refusals; and storeBehaviour, an alias pointed at another key, an unknown key and the context against a real server of the KMS API when SECRETS_TEST_KMS_URL names one, skipped otherwise, which passes against moto's server mode. - The README sets out the custodian of a key in KMS and of an alias, the bound and its words, the fake and the server suite, and that only the aws subpath loads any part of the AWS SDK. Co-Authored-By: Claude Fable 5.1 --- .../serving/unreadable-credentials.test.ts | 6 +- packages/secrets/README.md | 34 +- packages/secrets/package.json | 4 + packages/secrets/src/aws/index.ts | 9 + .../secrets/src/aws/kms-custodian.test.ts | 277 ++++++++++++++++ packages/secrets/src/aws/kms-custodian.ts | 173 ++++++++++ .../secrets/src/aws/kms-on-a-server.test.ts | 132 ++++++++ packages/secrets/src/aws/kms-store.test.ts | 36 +++ .../src/custodians/custodian-probe.test.ts | 47 +++ .../secrets/src/custodians/custodian-probe.ts | 16 + packages/secrets/src/index.ts | 1 + .../secrets/src/keepers/keeper-unavailable.ts | 10 + packages/secrets/src/store/answer-bound.ts | 4 +- .../secrets/src/testing/failure-behaviour.ts | 38 +-- packages/secrets/src/testing/fake-kms.test.ts | 130 ++++++++ packages/secrets/src/testing/fake-kms.ts | 296 ++++++++++++++++++ packages/secrets/src/testing/index.ts | 9 + .../secrets/src/testing/lease-behaviour.ts | 35 +-- .../secrets/src/testing/memory-rows.test.ts | 20 +- .../src/testing/recording-custodian.ts | 3 +- packages/secrets/src/testing/secrets-entry.ts | 43 +-- .../secrets/src/testing/store-behaviour.ts | 128 ++++---- pnpm-lock.yaml | 24 ++ 23 files changed, 1325 insertions(+), 150 deletions(-) create mode 100644 packages/secrets/src/aws/index.ts create mode 100644 packages/secrets/src/aws/kms-custodian.test.ts create mode 100644 packages/secrets/src/aws/kms-custodian.ts create mode 100644 packages/secrets/src/aws/kms-on-a-server.test.ts create mode 100644 packages/secrets/src/aws/kms-store.test.ts create mode 100644 packages/secrets/src/custodians/custodian-probe.test.ts create mode 100644 packages/secrets/src/custodians/custodian-probe.ts create mode 100644 packages/secrets/src/testing/fake-kms.test.ts create mode 100644 packages/secrets/src/testing/fake-kms.ts diff --git a/packages/connections/src/serving/unreadable-credentials.test.ts b/packages/connections/src/serving/unreadable-credentials.test.ts index 717549e26..7e00385c3 100644 --- a/packages/connections/src/serving/unreadable-credentials.test.ts +++ b/packages/connections/src/serving/unreadable-credentials.test.ts @@ -167,7 +167,7 @@ describe('the log of a credential that cannot be read now', () => { }); describe('a connection whose custodian does not answer', () => { - it('is unavailable once the store has waited 30 seconds, and never broken', async () => { + it('is unavailable once the store has waited 10 seconds, and never broken', async () => { const fake = await fakeServerAt(); const h = harness(); const { custodian, store } = sealedIn(memoryRows({ now })); @@ -179,14 +179,14 @@ describe('a connection whose custodian does not answer', () => { const asked = custodian.stall(); const serving = yield* Effect.forkChild(Effect.flip(served)); yield* Effect.promise(() => asked); - yield* TestClock.adjust('30 seconds'); + yield* TestClock.adjust('10 seconds'); return yield* Fiber.join(serving); }).pipe(Effect.provide(TestClock.layer())), ); expect(refused).toEqual({ because: 'connection_unavailable', - detail: unreadableNow('The keeper of the secret, or the custodian of its key, did not answer within 30 seconds'), + detail: unreadableNow('The keeper of the secret, or the custodian of its key, did not answer within 10 seconds'), }); expect((await h.facts()).map(({ type }) => type)).not.toContain('connection_broken'); }); diff --git a/packages/secrets/README.md b/packages/secrets/README.md index 3108d5c87..b03506c32 100644 --- a/packages/secrets/README.md +++ b/packages/secrets/README.md @@ -1,6 +1,6 @@ # @on.auto/secrets -The secret store of the runtime: what it must hold to act for someone, such as a key a person pasted, a client secret whoever runs it registered, a registration it made for itself or the tokens a person's consent gave it. The server opens the store at start and gives it to the connections of [`@on.auto/connections`](../connections/README.md), which keep in it the credential a person gives, through their port `SecretKeeping`. Behind the store are three seams: the **rows**, which keep the bookkeeping of each secret and which the ledger implements over SQLite and PostgreSQL, as its README's [Kept secrets](../ledger/README.md#kept-secrets) says; the **keeper**, which holds each value; and the **custodian**, which holds the key that wraps each value's own data key. This package holds the store, the three ports, the one custodian and the one keeper there are, so the ledger knows no key and the connections know no ledger. +The secret store of the runtime: what it must hold to act for someone, such as a key a person pasted, a client secret whoever runs it registered, a registration it made for itself or the tokens a person's consent gave it. The server opens the store at start and gives it to the connections of [`@on.auto/connections`](../connections/README.md), which keep in it the credential a person gives, through their port `SecretKeeping`. Behind the store are three seams: the **rows**, which keep the bookkeeping of each secret and which the ledger implements over SQLite and PostgreSQL, as its README's [Kept secrets](../ledger/README.md#kept-secrets) says; the **keeper**, which holds each value; and the **custodian**, which holds the key that wraps each value's own data key. This package holds the store, the three ports, the custodian of a key in hand and the custodian of a key in AWS KMS, and the one keeper there is, so the ledger knows no key, the connections know no ledger, and only the `aws` subpath loads any part of the AWS SDK, which a server whose settings name no key in KMS never imports. ## Words @@ -9,7 +9,7 @@ The secret store of the runtime: what it must hold to act for someone, such as a - **A holding**: how a value is held, `{ keptBy, held, wrappedKey?, kid? }`: the keeper that holds it, what the keeper handed back, and for a sealed value the wrapped data key and the custodian's kid. - **The keeper**: where a value rests. The one there is keeps it in the row itself, sealed, under the name `ledger`. - **Sealed**: encrypted under a data key made for that value alone, the data key wrapped by the custodian's key and kept beside the value. -- **The custodian**: who holds the key that wraps data keys; the one there is holds a key in hand. +- **The custodian**: who holds the key that wraps data keys: a key in hand, or a key in AWS KMS, which never leaves it. - **The kid**: the id of the custodian's key, written on each sealed row and in the log, never the key. - **Rotation**: a new custodian named beside the previous one; at start every data key the previous one wrapped is wrapped again by the current one, and no value is touched. @@ -20,9 +20,12 @@ The secret store of the runtime: what it must hold to act for someone, such as a - the ports, each an interface of functions: `SecretRows`, with `Holding`, `SecretRow`, `HeldRow`, `Holder` and the answers `Replacement` and `Claim`; `Custodian`, `{ kid, wrap(dataKey), unwrap(wrapped) }`, each failing with a `KeepingFailure`, and `Custodians`, `{ current, previous? }`; and `Keeper`, `{ name, hold(id, value), reveal(holding), release(holding) }`, whose `hold` fails with a `KeepingFailure` and whose `reveal` with a `ReadingFailure`; - `secretStoreOn(rows, keeper, custodians)`, the `SecretStore` over them, its refusal of a value, `SecretRefused`, and its three failures, `SecretKeyUnknown`, `CustodianRefused` and `KeeperUnavailable`, with `KeepingFailure`, the last two, and `ReadingFailure`, all three; - `localCustodian(key)`, the custodian of a key in hand, and `secretKeyBytes`, the length of such a key, 32; -- `sealedKeeper(custodians)`, the keeper that seals a value into its row. +- `sealedKeeper(custodians)`, the keeper that seals a value into its row; +- `custodianProbed(custodian)`, the probe the server makes of a custodian at start: one data key made, wrapped and unwrapped within the store's bound, both copies then filled with zeros, failing as the custodian does. -The server composes them in `packages/server/src/database/secret-store.ts`: the rows for its database, a `localCustodian` of the current key and one of the previous key where there is one, the `sealedKeeper` of both, and the store over the three. +`@on.auto/secrets/aws` (`src/aws/index.ts`) is a subpath of its own, as `@on.auto/outbound/aws` is, so that only a server whose settings name a key in KMS loads it, and with it `@aws-sdk/client-kms`: it exports `kmsCustodian(options)`, `kidOfKey(keyId)`, `encryptionContext` and `kmsWaits`, below. + +The server composes them in `packages/server/src/database/secret-store.ts`: the rows for its database, a custodian of the current key and one of the previous key where there is one, a `localCustodian` for a key in hand and a `kmsCustodian` for a key in KMS, the `sealedKeeper` of both, and the store over the three. ## The row @@ -50,6 +53,8 @@ A value the `ledger` keeper holds is sealed by `jose`, the library that verifies `localCustodian(key)` takes a key of 32 bytes as a `Redacted` byte array, and a key of another length stops it, naming the length and never the bytes: `A key that seals secrets is 32 bytes, and this one is 16`. Its `kid` is the first 16 hex characters of the key's SHA-256, `sha256Of` of `@on.auto/operations`, so no one names a key and one key always has one kid. `wrap(dataKey)` answers one JWE in its compact form, `{ alg: 'A256KW', enc: 'A256GCM' }`, of the data key under the key, `jose`'s envelope of AES key wrap around a content key of its own; `unwrap(wrapped)` answers the data key, taking those two algorithms alone and no compression, so a wrapping under another key, or of another algorithm under this one, is refused as a `CustodianRefused` in its own words, `The custodian of the key refused to unwrap a data key: its wrapped key does not open under this key`. Under the kid it names, that means a wrapping changed outside the runtime. +`kmsCustodian({ keyId, region, proxy, credentials, endpoint?, waits? })` makes the custodian of a key in AWS KMS, `keyId` the ARN of a symmetric key or of an alias, as an effect: for the ARN of a key it asks nothing, and for the ARN of an alias it asks `DescribeKey` once, within the store's bound, and takes the ARN of the key the alias names from the answer, so the custodian is pinned to that key and never to the alias. An alias pointed at another key therefore makes the custodian of another key at the next start, under which the rows the key before wrapped are wrapped again once its ARN is the previous custodian's. A `DescribeKey` KMS refuses fails the making as a `CustodianRefused`, `The custodian of the alias alias/auto-runtime refused to describe the key it names: `, and one it does not answer as a `KeeperUnavailable`, `… did not answer a request to describe the key it names: …`, each with the alias as its kid. The custodian's `kid` is the key's id, the last path segment of the key's ARN, `kidOfKey`, so the kid a row keeps names the key and no account. `wrap(dataKey)` is `Encrypt` with that key's ARN as `KeyId`, the data key as `Plaintext` and the encryption context `{ 'on.auto/runtime': 'secret data key' }`, `encryptionContext`, which names the purpose in the account's trail and binds the wrapping to it, and answers the ciphertext blob as base64; `unwrap(wrapped)` is `Decrypt` with the same key and context, so KMS opens the blob under that key and that context alone, and answers the data key. The client is an `@aws-sdk/client-kms` of the region given, with the settings of `awsClientSettingsOf(proxy)` of [`@on.auto/outbound/aws`](../outbound/README.md#aws-clients), and the server gives it the credentials of `awsCredentialChain(proxy)`, as the object store's are; `endpoint` points it elsewhere than the region's KMS, which the tests do. Every request is bounded as the S3 client's are, by the client's own waits, `kmsWaits`, 5 seconds to connect and 10 seconds for an answer, the store's bound, after which it fails as a `TimeoutError`, and the store's bound holds the whole of it. An answer KMS refuses as the caller's fault, `DisabledException`, `KMSInvalidStateException` for a key pending deletion, `AccessDeniedException`, `NotFoundException`, `InvalidCiphertextException` and `IncorrectKeyException` among them, is a `CustodianRefused` in the service's words with the kid, `The custodian of the key 1234abcd-12ab-34cd-56ef-1234567890ab refused to unwrap a data key: DisabledException: key/1234abcd-12ab-34cd-56ef-1234567890ab is disabled.`, the account of every ARN in those words left out; a request that is not answered, a throttled one, which is `ThrottlingException`, `LimitExceededException` or anything else `isThrottlingError` of the SDK's own retry, `@smithy/core/retry`, judges throttling, a fault of the service and a credential that cannot be found are a `KeeperUnavailable` with the kid, `The custodian of the key did not answer a request to wrap a data key: `, so the log of each cause tells the custodians apart. Nothing of a data key, of a credential or of the account reaches those words. KMS's own rotation of a key's material needs nothing of the runtime, since KMS decrypts under the material that encrypted. + The store takes `custodians: { current, previous? }`: the current one wraps every data key, and a read asks the one its row's kid names. A previous custodian with the current one's kid is no previous at all. Each read unwraps its data key again and nothing keeps one unwrapped, since a row is read when a connection is first used in a process and when its tokens are renewed. ## The keeper @@ -62,7 +67,7 @@ A value the store cannot read now fails the call that asked for it, and nothing - `SecretKeyUnknown`, `secret_key_unknown`, `{ kid, detail }`: the row is sealed under a kid that is neither the current custodian's nor the previous one's, `The secret is sealed under the key , which is neither the current key nor the previous one`. - `CustodianRefused`, `custodian_refused`, `{ kid, detail }`: the custodian of the kid failed to wrap or unwrap a data key, and says why, `The custodian of the key refused to unwrap a data key: `. -- `KeeperUnavailable`, `keeper_unavailable`, `{ detail }`: the keeper, or the custodian it asks, did not answer within the bound the store sets, 30 seconds through Effect's `timeoutOrElse`, `The keeper of the secret, or the custodian of its key, did not answer within 30 seconds`. +- `KeeperUnavailable`, `keeper_unavailable`, `{ kid?, detail }`: the keeper, or the custodian it asks, did not answer, the kid naming the custodian where it can tell, or the store gave up at the bound it sets, 10 seconds through Effect's `timeoutOrElse` (`src/store/answer-bound.ts`, the one rule of it), shorter than the fifteen seconds a renewal's claim keeps for its write in `@on.auto/tool-servers`, so a write the store cannot make now fails while the claim still holds, `The keeper of the secret, or the custodian of its key, did not answer within 10 seconds`. A read fails with any of the three, a `ReadingFailure`; a keep and a replacement with the last two, a `KeepingFailure`, and keep nothing then, so a value is kept or replaced only when its keeper and its custodian answered. A custodian or a keeper answers the last two itself where it can tell, and the store answers `KeeperUnavailable` for one that is still silent at the bound. @@ -86,12 +91,23 @@ A secret is only as recoverable as its key. The store keeps no copy of a key and - `memoryRows({ now })`, rows kept in memory under the rule of the lease, whose claims and times are read from `now`, and which fail as a defect for an id they keep already, as the ledger's table does: the one copy of the lease beside the ledger's SQL. `kept()` answers each row by its columns, in the order of its ids. - `valuesInTheClear({ now, mostCharacters })`, values a test holds in the clear over `memoryRows`, `{ keptBy: 'memory', held }`, under ids it gives: it keeps, reads, replaces, claims, answers endings and forgets as the rows do, refuses a value to keep or a replacement past its bound, `A secret to keep holds more than characters`, and answers what it holds, `values()` and `endings()`. The tool-servers package's `memoryTokens` and the connections package's harness keep their values in it. -- `recordingCustodian(key?)`, a custodian of a key in hand, a random one unless it is given one, that counts its wraps and unwraps, `wraps()` and `unwraps()`, and holds the data keys it wrapped and unwrapped, `dataKeys()`, so a test sees them filled with zeros. Told to `refuse()`, it refuses both as a `CustodianRefused` in the words `custodianRefusal`, `the key is disabled`; told to `stall()`, it answers neither, and the promise `stall()` answers settles once it is first asked; told to `answer()`, it answers again. +- `recordingCustodian(custodian?)`, the custodian it is given, one of a random key in hand unless it is given one, that counts its wraps and unwraps, `wraps()` and `unwraps()`, and holds the data keys it wrapped and unwrapped, `dataKeys()`, so a test sees them filled with zeros. Told to `refuse()`, it refuses both as a `CustodianRefused` in the words `custodianRefusal`, `the key is disabled`; told to `stall()`, it answers neither, and the promise `stall()` answers settles once it is first asked; told to `answer()`, it answers again. - `rowsBehaviour(entry)`, every claim on a `SecretRows`: a holding kept, with and without a wrapped key and a kid, read back and forgotten twice as once, no second row under an id kept, the endings, a replacement at the version read, after which the next version may be claimed at once, and none at another or of a row that is gone, a claim a read answers while it is open and none while one is open, of a moved version or of a row that is gone, the rows a keeper holds under a kid, in order and at most as many as asked, a row held again only while its version and its holding are the ones read, its version, claim, ending and time kept, and a sealed row replaced or held again with the holding of another keeper, its wrapped key and kid then empty. -- `storeBehaviour(entry)`, what the store adds over any rows, the claims of the rows themselves being `rowsBehaviour`'s: a value kept under a UUIDv7 with its ending and read back redacted, sealed as a `dir` JWE under a data key of its own that the row's `wrapped_key` holds under the row's `kid`, a wrap at each keep and an unwrap at each read with each data key then filled with zeros, a forget twice as once, through which the keeper releases, the empty and the oversized value refused before anything is wrapped, as a replacement, a replacement at the next version under a new data key, and a rotation over more than one batch that wraps again the data keys of the previous key's rows, touches no value, keeps each row's version and claim and leaves a stray kid, and none without a previous key. Its failures are a suite of their own in `src/testing/failure-behaviour.ts`: a value under another key not read, `secret_key_unknown` with that key's kid; a value and a replacement a refusing custodian does not wrap, `custodian_refused`, the row as it was; and a value a refusing custodian does not unwrap, `custodian_refused`, and one a stalled custodian leaves unanswered, `keeper_unavailable` at 30 seconds of `TestClock` and not a millisecond before, read again once the custodian answers, its row as it was and no failure naming the value. +- `serveFakeKms({ keys, aliases? })` (`src/testing/fake-kms.ts`), an HTTP server on a loopback port answering the KMS JSON protocol for `Encrypt`, `Decrypt` and `DescribeKey` under `X-Amz-Target`, for the key ARNs it is given and the alias ARNs that name them: its blob is the key, the context and the plaintext sealed under AES-GCM by a key of its own, so it never holds a plaintext in the clear and opens only under the key and the context it was made with, `IncorrectKeyException` and `InvalidCiphertextException` otherwise, and a key it was not given is `NotFoundException`. `tell(keyId, state)` makes a key `disabled`, `pending_deletion`, `denied`, which KMS answers for a key unknown to the caller or one it may not use, `stalling`, which answers nothing, or `enabled` again; `failWith(keyId, { type, status })` makes a key answer that error, such as `ThrottlingException`, until it is told a state; `point(aliasArn, keyArn)` points an alias at another key; `DescribeKey` answers the key's id and ARN for a key or an alias that names it, enabled, disabled or pending deletion, as KMS does, and `NotFoundException` for an alias or a key it does not know; `calls()` answers each call with its key id, and the context of an `Encrypt` or a `Decrypt`. +- `storeBehaviour(entry, custodians?)`, what the store adds over any rows and any custodians, local ones unless it is given others, the claims of the rows themselves being `rowsBehaviour`'s: a value kept under a UUIDv7 with its ending and read back redacted, sealed as a `dir` JWE under a data key of its own that the row's `wrapped_key` holds under the row's `kid` and never in the clear, a wrap at each keep and an unwrap at each read with each data key then filled with zeros, a forget twice as once, through which the keeper releases, the empty and the oversized value refused before anything is wrapped, as a replacement, a replacement at the next version under a new data key, and a rotation over more than one batch that wraps again the data keys of the previous key's rows, touches no value, keeps each row's version and claim and leaves a stray kid, and none without a previous key. Its failures are a suite of their own in `src/testing/failure-behaviour.ts`: a value under another key not read, `secret_key_unknown` with that key's kid; a value and a replacement a refusing custodian does not wrap, `custodian_refused`, the row as it was; and a value a refusing custodian does not unwrap, `custodian_refused`, and one a stalled custodian leaves unanswered, `keeper_unavailable` at 10 seconds of `TestClock` and not a millisecond before, read again once the custodian answers, its row as it was and no failure naming the value. + +An entry, `SecretsEntry`, is `{ aDatabase, rowsOn, rowsIn }`: a fresh database, the rows on one, and the rows of one as its table holds them, by their columns in the order of their ids. The custodians a suite takes, `{ aCustodian }`, answer a custodian of a key of its own at each call. `src/testing/memory-rows.test.ts` runs both suites over `memoryRows`, and the ledger runs them over SQLite and PostgreSQL. `src/aws/kms-store.test.ts` runs `storeBehaviour` over keys in the fake KMS, and `src/aws/kms-custodian.test.ts` holds what the KMS custodian adds: its kid, the `Encrypt` and the `Decrypt` it sends with their key and context, a blob that holds no data key in the clear, an alias resolved once by `DescribeKey` and its key pinned, a custodian of another key once the alias is pointed at another, an alias KMS does not know or will not describe, its refusals in the words of KMS without the account, `ThrottlingException`, `LimitExceededException` and a `TooManyRequestsException` of HTTP 429 as a keeper that does not answer, with the kid, and a KMS that stalls or cannot be reached, given up on within waits of 100 ms on the real clock, since the SDK's request handler keeps its own timers. + +The same suite runs against a real server of the KMS API when `SECRETS_TEST_KMS_URL` names one with its credentials, `http://access:secret@host:port`, `src/aws/kms-on-a-server.test.ts`, which makes a key of its own through `CreateKey` for each custodian it asks for, and adds an alias made with `CreateAlias`, whose custodian is pinned to its key and reads, once `UpdateAlias` points the alias at another key and the key before is the previous custodian, what the key before wrapped, a key the server does not know, refused as `NotFoundException` in the server's words, a data key the server opens under the purpose's context and under no other, and nothing written to `console.warn` by the SDK; without the variable it is skipped, and its title says how to run it: + +```bash +SECRETS_TEST_KMS_URL=http://kms-test:kms-test-secret@127.0.0.1:19760 pnpm --filter @on.auto/secrets test +``` + +It passes against moto's server mode, `motoserver/moto`, which answers `Encrypt` and `Decrypt` with their context but not the state of a key, so a key disabled, pending deletion or denied is the fake's to show. -An entry, `SecretsEntry`, is `{ aDatabase, rowsOn, rowsIn }`: a fresh database, the rows on one, and the rows of one as its table holds them, by their columns in the order of their ids. `src/testing/memory-rows.test.ts` runs both suites over `memoryRows`, and the ledger runs them over SQLite and PostgreSQL. `src/store/secret-store.test.ts` adds what only a keeper or rows of a test's own show: the id the store hands its keeper and the holding it releases, a replacement made between a rotation's read of a row and its write, which the rotation leaves as it is, the data key a rotation unwrapped, filled with zeros once wrapped again, and a rotation whose previous custodian refuses, which fails with that refusal and holds nothing again. +`src/store/secret-store.test.ts` adds what only a keeper or rows of a test's own show: the id the store hands its keeper and the holding it releases, a replacement made between a rotation's read of a row and its write, which the rotation leaves as it is, the data key a rotation unwrapped, filled with zeros once wrapped again, and a rotation whose previous custodian refuses, which fails with that refusal and holds nothing again. ## Source -`src/rows` holds the port of the rows, `src/custodians` the port of the custodian, its refusal and the custodian of a key in hand, `src/keepers` the port of the keeper, the ledger's keeper, its data keys, the unknown kid and the keeper that does not answer, `src/store` the store, the bound of an answer, its rotation and its refusal, and `src/testing` the rows in memory, the values in the clear, the recording custodian, the suites and the entry of the tests. +`src/rows` holds the port of the rows, `src/custodians` the port of the custodian, its refusal, its probe and the custodian of a key in hand, `src/aws` the custodian of a key in KMS and its tests, `src/keepers` the port of the keeper, the ledger's keeper, its data keys, the unknown kid and the keeper that does not answer, `src/store` the store, the bound of an answer, its rotation and its refusal, and `src/testing` the rows in memory, the values in the clear, the recording custodian, the fake KMS, the suites and the entry of the tests. diff --git a/packages/secrets/package.json b/packages/secrets/package.json index 99fed1d4e..b29095e5f 100644 --- a/packages/secrets/package.json +++ b/packages/secrets/package.json @@ -6,6 +6,7 @@ "type": "module", "exports": { ".": "./src/index.ts", + "./aws": "./src/aws/index.ts", "./testing": "./src/testing/index.ts" }, "scripts": { @@ -14,7 +15,10 @@ "test": "vitest run --coverage" }, "dependencies": { + "@aws-sdk/client-kms": "3.1149.0", "@on.auto/operations": "workspace:*", + "@on.auto/outbound": "workspace:*", + "@smithy/core": "3.35.2", "effect": "catalog:", "jose": "catalog:" }, diff --git a/packages/secrets/src/aws/index.ts b/packages/secrets/src/aws/index.ts new file mode 100644 index 000000000..14fb97237 --- /dev/null +++ b/packages/secrets/src/aws/index.ts @@ -0,0 +1,9 @@ +export { + encryptionContext, + kidOfKey, + kmsCustodian, + kmsWaits, + type KmsCredentials, + type KmsCustodianOptions, + type KmsWaits, +} from './kms-custodian.ts'; diff --git a/packages/secrets/src/aws/kms-custodian.test.ts b/packages/secrets/src/aws/kms-custodian.test.ts new file mode 100644 index 000000000..e364ffbbc --- /dev/null +++ b/packages/secrets/src/aws/kms-custodian.test.ts @@ -0,0 +1,277 @@ +import { randomUUID } from 'node:crypto'; +import { once } from 'node:events'; +import { createServer } from 'node:http'; + +import { Effect, Redacted, Schema } from 'effect'; +import { afterAll, describe, expect, it } from 'vitest'; + +import type { Custodian } from '../custodians/custodian.ts'; +import type { KeepingFailure } from '../keepers/keeper.ts'; +import { serveFakeKms, type FakeKeyState } from '../testing/fake-kms.ts'; +import { encryptionContext, kidOfKey, kmsCustodian, type KmsWaits } from './kms-custodian.ts'; + +const account = '111122223333'; + +function aKeyArn(): string { + return `arn:aws:kms:eu-west-1:${account}:key/${randomUUID()}`; +} + +function anAliasArn(name: string): string { + return `arn:aws:kms:eu-west-1:${account}:alias/${name}`; +} + +const [key, otherKey, unknownKey, aliasedKey, nextKey] = [aKeyArn(), aKeyArn(), aKeyArn(), aKeyArn(), aKeyArn()]; + +const [alias, movingAlias, unknownAlias] = [ + anAliasArn('auto-runtime'), + anAliasArn('team/auto-runtime'), + anAliasArn('unknown'), +]; + +const fake = await serveFakeKms({ + keys: [key, otherKey, aliasedKey, nextKey], + aliases: { [alias]: aliasedKey, [movingAlias]: aliasedKey }, +}); + +afterAll(fake.close); + +const credentials = { accessKeyId: 'an-access-key', secretAccessKey: 'a-secret-access-key' }; + +const noProxy = { enabled: false, environment: {} }; + +function madeOf(keyId: string, endpoint = fake.url, waits?: KmsWaits): Effect.Effect { + return kmsCustodian({ keyId, region: 'eu-west-1', proxy: noProxy, credentials, endpoint, waits }); +} + +function custodianOf(keyId: string, endpoint = fake.url, waits?: KmsWaits): Promise { + return Effect.runPromise(madeOf(keyId, endpoint, waits)); +} + +function aDataKey(): Redacted.Redacted { + return Redacted.make(crypto.getRandomValues(new Uint8Array(32))); +} + +function failureOf(effect: Effect.Effect): Promise { + return Effect.runPromise(Effect.flip(effect)); +} + +const decodeAddress = Schema.decodeUnknownSync(Schema.Struct({ port: Schema.Number })); + +async function aClosedPort(): Promise { + const server = createServer().listen(0, '127.0.0.1'); + await once(server, 'listening'); + const { port } = decodeAddress(server.address()); + server.close(); + await once(server, 'close'); + return `http://127.0.0.1:${String(port)}`; +} + +function idOf(arn: string): string { + return arn.slice(arn.lastIndexOf('/') + 1); +} + +const waitsOf100Ms: KmsWaits = { connectionMs: 100, requestMs: 100 }; + +describe('the custodian of a key in KMS', () => { + it('is named by the id of the key its ARN names, and is made without reaching anything', async () => { + const made = await Effect.runPromise( + kmsCustodian({ keyId: key, region: 'eu-west-1', proxy: noProxy, credentials }), + ); + + expect([made.kid, kidOfKey(key), kidOfKey(idOf(key))]).toEqual([idOf(key), idOf(key), idOf(key)]); + }); + + it('wraps a data key with Encrypt under its key and the context that names the purpose, as base64 that holds no data key, and unwraps it with Decrypt under the same key and context', async () => { + const custodian = await custodianOf(key); + const dataKey = aDataKey(); + const before = fake.calls().length; + + const wrapped = await Effect.runPromise(custodian.wrap(dataKey)); + const unwrapped = await Effect.runPromise(custodian.unwrap(wrapped)); + + const bytes = Buffer.from(Redacted.value(dataKey)); + expect(Buffer.from(Redacted.value(unwrapped)).equals(bytes)).toBe(true); + expect(Buffer.from(wrapped, 'base64').toString('base64')).toBe(wrapped); + expect([wrapped.includes(bytes.toString('base64')), Buffer.from(wrapped, 'base64').includes(bytes)]).toEqual([ + false, + false, + ]); + expect(fake.calls().slice(before)).toEqual([ + { operation: 'Encrypt', keyId: key, context: { 'on.auto/runtime': 'secret data key' } }, + { operation: 'Decrypt', keyId: key, context: encryptionContext }, + ]); + }); +}); + +describe('the custodian of a key that an alias names', () => { + it('asks DescribeKey once as it is made, is named by the id of the key the alias names, and pins that key, never the alias, on Encrypt and Decrypt', async () => { + const before = fake.calls().length; + + const custodian = await custodianOf(alias); + const wrapped = await Effect.runPromise(custodian.wrap(aDataKey())); + await Effect.runPromise(custodian.unwrap(wrapped)); + + expect(custodian.kid).toBe(idOf(aliasedKey)); + expect(fake.calls().slice(before)).toEqual([ + { operation: 'DescribeKey', keyId: alias }, + { operation: 'Encrypt', keyId: aliasedKey, context: encryptionContext }, + { operation: 'Decrypt', keyId: aliasedKey, context: encryptionContext }, + ]); + }); + + it('is a custodian of another key once the alias is pointed at another, and what the key it named before wrapped opens under that key alone', async () => { + const before = await custodianOf(movingAlias); + const wrapped = await Effect.runPromise(before.wrap(aDataKey())); + + fake.point(movingAlias, nextKey); + const after = await custodianOf(movingAlias); + const refused = await failureOf(after.unwrap(wrapped)); + const opened = await Effect.runPromise((await custodianOf(aliasedKey)).unwrap(wrapped)); + + expect([before.kid, after.kid]).toEqual([idOf(aliasedKey), idOf(nextKey)]); + expect(refused).toMatchObject({ _tag: 'custodian_refused', kid: idOf(nextKey) }); + expect(Redacted.value(opened)).toHaveLength(32); + }); +}); + +describe('the custodian of an alias KMS does not describe', () => { + it('is not made under an alias KMS does not know, or whose key the identity may not describe, in the words of KMS without the account', async () => { + const unknown = await failureOf(madeOf(unknownAlias)); + fake.tell(aliasedKey, 'denied'); + const denied = await failureOf(madeOf(alias)); + fake.tell(aliasedKey, 'enabled'); + + expect([unknown, denied]).toMatchObject([ + { + _tag: 'custodian_refused', + kid: 'alias/unknown', + detail: + 'The custodian of the alias alias/unknown refused to describe the key it names: NotFoundException: Alias alias/unknown is not found.', + }, + { + _tag: 'custodian_refused', + kid: 'alias/auto-runtime', + detail: `The custodian of the alias alias/auto-runtime refused to describe the key it names: AccessDeniedException: User: user/runtime is not authorized to perform: kms:DescribeKey on resource: key/${idOf(aliasedKey)} because no identity-based policy allows the kms:DescribeKey action`, + }, + ]); + expect(JSON.stringify([unknown, denied])).not.toContain(account); + }); + + it('is not made where KMS does not answer DescribeKey within the waits of its client, as a keeper that does not answer, naming the alias', async () => { + fake.tell(aliasedKey, 'stalling'); + + const stalled = await failureOf(madeOf(alias, fake.url, waitsOf100Ms)); + fake.tell(aliasedKey, 'enabled'); + + expect(stalled).toMatchObject({ _tag: 'keeper_unavailable', kid: 'alias/auto-runtime' }); + expect(stalled.detail).toMatch( + /^The custodian of the alias alias\/auto-runtime did not answer a request to describe the key it names: TimeoutError/u, + ); + }); +}); + +const refusals: readonly (readonly [FakeKeyState, string])[] = [ + ['disabled', `DisabledException: key/${idOf(key)} is disabled.`], + ['pending_deletion', `KMSInvalidStateException: key/${idOf(key)} is pending deletion.`], + [ + 'denied', + `AccessDeniedException: User: user/runtime is not authorized to perform: kms:Encrypt on resource: key/${idOf(key)} because no identity-based policy allows the kms:Encrypt action`, + ], +]; + +describe('the custodian of a key KMS will not use', () => { + it.each(refusals)( + 'refuses to wrap under a key %s in the words of KMS, naming its kid and never the account or a credential', + async (state, words) => { + const custodian = await custodianOf(key); + fake.tell(key, state); + + const refused = await failureOf(custodian.wrap(aDataKey())); + fake.tell(key, 'enabled'); + + expect(refused).toMatchObject({ + _tag: 'custodian_refused', + kid: idOf(key), + detail: `The custodian of the key ${idOf(key)} refused to wrap a data key: ${words}`, + }); + expect(JSON.stringify(refused)).not.toMatch(new RegExp(`${account}|${credentials.secretAccessKey}`, 'u')); + }, + ); + + it('refuses a key KMS does not know, and unwraps nothing it did not wrap under that key and context', async () => { + const wrapped = await Effect.runPromise((await custodianOf(otherKey)).wrap(aDataKey())); + const custodian = await custodianOf(key); + + const [unknown, ofAnother, unmade] = [ + await failureOf((await custodianOf(unknownKey)).wrap(aDataKey())), + await failureOf(custodian.unwrap(wrapped)), + await failureOf(custodian.unwrap(Buffer.from('not a ciphertext').toString('base64'))), + ]; + + expect([unknown, ofAnother, unmade]).toMatchObject([ + { + _tag: 'custodian_refused', + detail: `The custodian of the key ${idOf(unknownKey)} refused to wrap a data key: NotFoundException: Key 'key/${idOf(unknownKey)}' does not exist`, + }, + { + _tag: 'custodian_refused', + detail: `The custodian of the key ${idOf(key)} refused to unwrap a data key: IncorrectKeyException: The key in the request is not the key that encrypted the ciphertext`, + }, + { + _tag: 'custodian_refused', + detail: `The custodian of the key ${idOf(key)} refused to unwrap a data key: InvalidCiphertextException: The ciphertext is not one this KMS made`, + }, + ]); + }); +}); + +const throttlings: readonly (readonly [string, number])[] = [ + ['ThrottlingException', 400], + ['LimitExceededException', 400], + ['TooManyRequestsException', 429], +]; + +describe('the custodian of a key in a KMS that throttles it', () => { + it.each(throttlings)( + 'does not answer now, for %s answered with HTTP %i, as a keeper that does not answer, naming its kid', + async (type, status) => { + const custodian = await custodianOf(key); + fake.failWith(key, { type, status }); + + const throttled = await failureOf(custodian.wrap(aDataKey())); + fake.tell(key, 'enabled'); + + expect(throttled).toMatchObject({ + _tag: 'keeper_unavailable', + kid: idOf(key), + detail: `The custodian of the key ${idOf(key)} did not answer a request to wrap a data key: ${type}: Rate exceeded`, + }); + }, + ); +}); + +describe('the custodian of a key in a KMS that does not answer', () => { + it('gives up within the waits of its client, where KMS stalls and where it cannot be reached, as a keeper that does not answer, naming its kid', async () => { + const stalling = await custodianOf(key, fake.url, waitsOf100Ms); + const unreachable = await custodianOf(key, await aClosedPort(), waitsOf100Ms); + fake.tell(key, 'stalling'); + const started = performance.now(); + + const stalled = await failureOf(stalling.wrap(aDataKey())); + const unreached = await failureOf(unreachable.unwrap('d3JhcHBlZA==')); + fake.tell(key, 'enabled'); + + expect([stalled, unreached]).toMatchObject([ + { _tag: 'keeper_unavailable', kid: idOf(key) }, + { _tag: 'keeper_unavailable', kid: idOf(key) }, + ]); + expect(stalled.detail).toMatch( + new RegExp( + `^The custodian of the key ${idOf(key)} did not answer a request to wrap a data key: TimeoutError`, + 'u', + ), + ); + expect(unreached.detail).toMatch(/did not answer a request to unwrap a data key: .*ECONNREFUSED/u); + expect(performance.now() - started).toBeLessThan(20_000); + }); +}); diff --git a/packages/secrets/src/aws/kms-custodian.ts b/packages/secrets/src/aws/kms-custodian.ts new file mode 100644 index 000000000..66ba2eb74 --- /dev/null +++ b/packages/secrets/src/aws/kms-custodian.ts @@ -0,0 +1,173 @@ +import { + DecryptCommand, + DescribeKeyCommand, + EncryptCommand, + KMSClient, + KMSServiceException, + type KMSClientConfig, +} from '@aws-sdk/client-kms'; +import type { ProxySettings } from '@on.auto/outbound'; +import { awsClientSettingsOf } from '@on.auto/outbound/aws'; +import { isThrottlingError } from '@smithy/core/retry'; +import { Effect, Option, Redacted, Schema } from 'effect'; + +import { CustodianRefused, custodianRefused, type CustodianAct } from '../custodians/custodian-refused.ts'; +import type { Custodian } from '../custodians/custodian.ts'; +import { custodianUnanswered, KeeperUnavailable } from '../keepers/keeper-unavailable.ts'; +import type { KeepingFailure } from '../keepers/keeper.ts'; +import { answerBoundMs, answeredWithinItsBound } from '../store/answer-bound.ts'; + +export interface KmsWaits { + readonly connectionMs: number; + readonly requestMs: number; +} + +export type KmsCredentials = NonNullable; + +export interface KmsCustodianOptions { + readonly keyId: string; + readonly region: string; + readonly proxy: ProxySettings; + readonly credentials: KmsCredentials; + readonly endpoint?: string | undefined; + readonly waits?: KmsWaits | undefined; +} + +export const kmsWaits: KmsWaits = { connectionMs: 5000, requestMs: answerBoundMs }; + +export const encryptionContext: Readonly> = { 'on.auto/runtime': 'secret data key' }; + +const keyResource = 'key/'; + +const aliasResource = 'alias/'; + +const accountOfAnArn = /arn:aws[\w-]*:[\w-]+:[\w-]*:\d{12}:/gu; + +function resourceOf(arn: string): string { + return arn.slice(arn.lastIndexOf(':') + 1); +} + +export function kidOfKey(keyArn: string): string { + const resource = resourceOf(keyArn); + return resource.startsWith(keyResource) ? resource.slice(keyResource.length) : resource; +} + +function wordsOf(failure: unknown): string { + return String(failure).replaceAll(accountOfAnArn, ''); +} + +function isRefusal(failure: unknown): boolean { + if (!(failure instanceof KMSServiceException) || failure.$fault !== 'client') { + return false; + } + const { name, message, $metadata } = failure; + return !isThrottlingError({ name, message, $metadata }); +} + +function failureOf(kid: string, act: CustodianAct, failure: unknown): KeepingFailure { + return isRefusal(failure) + ? custodianRefused(kid, act, wordsOf(failure)) + : custodianUnanswered(kid, act, wordsOf(failure)); +} + +function aliasFailureOf(alias: string, failure: unknown): KeepingFailure { + const words = wordsOf(failure); + return isRefusal(failure) + ? new CustodianRefused({ + kid: alias, + detail: `The custodian of the alias ${alias} refused to describe the key it names: ${words}`, + }) + : new KeeperUnavailable({ + kid: alias, + detail: `The custodian of the alias ${alias} did not answer a request to describe the key it names: ${words}`, + }); +} + +function kmsClientOf({ region, proxy, credentials, endpoint, waits = kmsWaits }: KmsCustodianOptions): KMSClient { + const { requestHandler } = awsClientSettingsOf(proxy); + return new KMSClient({ + requestHandler: { + ...requestHandler, + connectionTimeout: waits.connectionMs, + requestTimeout: waits.requestMs, + throwOnRequestTimeout: true, + }, + credentials, + region, + ...(endpoint === undefined ? {} : { endpoint }), + }); +} + +interface KmsRequests { + readonly encrypted: ( + keyArn: string, + dataKey: Redacted.Redacted, + abortSignal: AbortSignal, + ) => Promise; + readonly decrypted: ( + keyArn: string, + wrapped: string, + abortSignal: AbortSignal, + ) => Promise>; + readonly described: (aliasArn: string, abortSignal: AbortSignal) => Promise; +} + +const decodeDescribed = Schema.decodeUnknownSync(Schema.Struct({ KeyMetadata: Schema.Struct({ Arn: Schema.String }) })); + +function requestsOf(options: KmsCustodianOptions): KmsRequests { + const client = kmsClientOf(options); + return { + encrypted: async (keyArn, dataKey, abortSignal) => { + const { CiphertextBlob } = await client.send( + new EncryptCommand({ + KeyId: keyArn, + Plaintext: Redacted.value(dataKey), + EncryptionContext: { ...encryptionContext }, + }), + { abortSignal }, + ); + return Buffer.from(Option.getOrThrow(Option.fromNullishOr(CiphertextBlob))).toString('base64'); + }, + decrypted: async (keyArn, wrapped, abortSignal) => { + const { Plaintext } = await client.send( + new DecryptCommand({ + KeyId: keyArn, + CiphertextBlob: Buffer.from(wrapped, 'base64'), + EncryptionContext: { ...encryptionContext }, + }), + { abortSignal }, + ); + return Redacted.make(Option.getOrThrow(Option.fromNullishOr(Plaintext))); + }, + described: async (aliasArn, abortSignal) => + decodeDescribed(await client.send(new DescribeKeyCommand({ KeyId: aliasArn }), { abortSignal })).KeyMetadata.Arn, + }; +} + +function custodianPinnedTo(requests: KmsRequests, keyArn: string): Custodian { + const kid = kidOfKey(keyArn); + const asked = (act: CustodianAct, request: (signal: AbortSignal) => Promise) => + Effect.tryPromise({ try: request, catch: (failure) => failureOf(kid, act, failure) }); + return { + kid, + wrap: (dataKey) => asked('wrap', (abortSignal) => requests.encrypted(keyArn, dataKey, abortSignal)), + unwrap: (wrapped) => asked('unwrap', (abortSignal) => requests.decrypted(keyArn, wrapped, abortSignal)), + }; +} + +function keyNamedBy(requests: KmsRequests, aliasArn: string): Effect.Effect { + return answeredWithinItsBound( + Effect.tryPromise({ + try: (abortSignal) => requests.described(aliasArn, abortSignal), + catch: (failure) => aliasFailureOf(resourceOf(aliasArn), failure), + }), + ); +} + +export function kmsCustodian(options: KmsCustodianOptions): Effect.Effect { + const { keyId } = options; + const requests = requestsOf(options); + return resourceOf(keyId).startsWith(aliasResource) + ? Effect.map(keyNamedBy(requests, keyId), (keyArn) => custodianPinnedTo(requests, keyArn)) + : Effect.succeed(custodianPinnedTo(requests, keyId)); +} diff --git a/packages/secrets/src/aws/kms-on-a-server.test.ts b/packages/secrets/src/aws/kms-on-a-server.test.ts new file mode 100644 index 000000000..547eda687 --- /dev/null +++ b/packages/secrets/src/aws/kms-on-a-server.test.ts @@ -0,0 +1,132 @@ +import { + CreateAliasCommand, + CreateKeyCommand, + DecryptCommand, + KMSClient, + UpdateAliasCommand, +} from '@aws-sdk/client-kms'; +import { Effect, Option, Redacted } from 'effect'; +import { describe, expect, it, vi } from 'vitest'; + +import type { Custodian } from '../custodians/custodian.ts'; +import { inMemoryEntry, kept, readBack, storeWith, type TestCustodians } from '../testing/secrets-entry.ts'; +import { storeBehaviour } from '../testing/store-behaviour.ts'; +import { kmsCustodian } from './kms-custodian.ts'; + +const server = process.env['SECRETS_TEST_KMS_URL'] ?? ''; + +const skipped = server === ''; + +const notice = skipped + ? ', skipped: set SECRETS_TEST_KMS_URL to the URL of a server that answers the KMS API, with its credentials, such as http://access:secret@127.0.0.1:19760, to run it' + : ''; + +const region = 'us-east-1'; + +interface ServerKms { + readonly client: () => KMSClient; + readonly custodianOf: (keyId: string) => Promise; +} + +function serverKmsOf(url: string): ServerKms { + const { origin, username, password } = new URL(url); + const credentials = { accessKeyId: decodeURIComponent(username), secretAccessKey: decodeURIComponent(password) }; + const proxy = { enabled: false, environment: {} }; + const client = new KMSClient({ endpoint: origin, region, credentials }); + return { + client: () => client, + custodianOf: (keyId) => Effect.runPromise(kmsCustodian({ keyId, region, proxy, credentials, endpoint: origin })), + }; +} + +interface MadeKey { + readonly arn: string; + readonly id: string; +} + +async function aKeyMadeOn({ client }: ServerKms): Promise { + const { KeyMetadata } = await client().send(new CreateKeyCommand({ Description: 'A key of the tests of secrets' })); + const made = Option.getOrThrow(Option.fromNullishOr(KeyMetadata)); + return { arn: String(made.Arn), id: String(made.KeyId) }; +} + +function aDataKey(): Redacted.Redacted { + return Redacted.make(crypto.getRandomValues(new Uint8Array(32))); +} + +function onTheServer(): TestCustodians { + return { + aCustodian: async () => { + const kms = serverKmsOf(server); + return kms.custodianOf((await aKeyMadeOn(kms)).arn); + }, + }; +} + +describe.skipIf(skipped)( + `The custodian of an alias in a server that answers the KMS API${notice}`, + { timeout: 120_000 }, + () => { + it('resolves an alias to the key it names, and reads under the next key once the alias names it and the key before is the previous one', async () => { + const kms = serverKmsOf(server); + const [first, next] = [await aKeyMadeOn(kms), await aKeyMadeOn(kms)]; + const aliasName = `alias/secrets-test-${crypto.randomUUID()}`; + const aliasArn = first.arn.replace(/key\/.+$/u, aliasName); + await kms.client().send(new CreateAliasCommand({ AliasName: aliasName, TargetKeyId: first.id })); + const entry = inMemoryEntry(); + const database = await entry.aDatabase(); + const sealing = await kms.custodianOf(aliasArn); + const id = await kept(await storeWith(entry, database, { current: sealing }), 'pasted-key'); + + await kms.client().send(new UpdateAliasCommand({ AliasName: aliasName, TargetKeyId: next.id })); + const current = await kms.custodianOf(aliasArn); + const previous = await kms.custodianOf(first.arn); + const rewrapped = await Effect.runPromise((await storeWith(entry, database, { current, previous })).rotated()); + + expect([sealing.kid, current.kid, rewrapped]).toEqual([first.id, next.id, 1]); + expect(await readBack(await storeWith(entry, database, { current }), id)).toBe('pasted-key'); + }); + }, +); + +describe.skipIf(skipped)( + `The secret store over keys in a server that answers the KMS API${notice}`, + { timeout: 120_000 }, + () => { + const warned = vi.spyOn(console, 'warn'); + + storeBehaviour(inMemoryEntry(), onTheServer()); + + it('is refused under a key the server does not know, in the words of the server naming the kid and never the account', async () => { + const unknown = await serverKmsOf(server).custodianOf( + `arn:aws:kms:${region}:123456789012:key/${crypto.randomUUID()}`, + ); + + const refused = await Effect.runPromise(Effect.flip(unknown.wrap(aDataKey()))); + + expect(refused).toMatchObject({ _tag: 'custodian_refused', kid: unknown.kid }); + expect(refused.detail).toMatch( + new RegExp(`^The custodian of the key ${unknown.kid} refused to wrap a data key: NotFoundException: `, 'u'), + ); + expect(refused.detail).not.toMatch(/\d{12}:/u); + }); + + it('binds each data key to the context that names the purpose, which the server opens under no other context', async () => { + const kms = serverKmsOf(server); + const { arn } = await aKeyMadeOn(kms); + const dataKey = aDataKey(); + + const custodian = await kms.custodianOf(arn); + const wrapped = await Effect.runPromise(custodian.wrap(dataKey)); + const unwrapped = await Effect.runPromise(custodian.unwrap(wrapped)); + const elsewhere = new DecryptCommand({ KeyId: arn, CiphertextBlob: Buffer.from(wrapped, 'base64') }); + + expect(Buffer.from(Redacted.value(unwrapped)).equals(Buffer.from(Redacted.value(dataKey)))).toBe(true); + await expect(kms.client().send(elsewhere)).rejects.toMatchObject({ name: 'InvalidCiphertextException' }); + }); + + it('writes nothing to console.warn from the SDK while it wraps and unwraps', () => { + expect(warned).not.toHaveBeenCalled(); + }); + }, +); diff --git a/packages/secrets/src/aws/kms-store.test.ts b/packages/secrets/src/aws/kms-store.test.ts new file mode 100644 index 000000000..87f322144 --- /dev/null +++ b/packages/secrets/src/aws/kms-store.test.ts @@ -0,0 +1,36 @@ +import { randomUUID } from 'node:crypto'; + +import { Effect } from 'effect'; +import { afterAll, describe } from 'vitest'; + +import { serveFakeKms } from '../testing/fake-kms.ts'; +import { inMemoryEntry, type TestCustodians } from '../testing/secrets-entry.ts'; +import { storeBehaviour } from '../testing/store-behaviour.ts'; +import { kmsCustodian } from './kms-custodian.ts'; + +const keys = Array.from({ length: 3 }, () => `arn:aws:kms:eu-west-1:111122223333:key/${randomUUID()}`); + +const fake = await serveFakeKms({ keys }); + +afterAll(fake.close); + +const made = { custodians: 0 }; + +const onTheFake: TestCustodians = { + aCustodian: () => { + made.custodians += 1; + return Effect.runPromise( + kmsCustodian({ + keyId: String(keys[made.custodians % keys.length]), + region: 'eu-west-1', + proxy: { enabled: false, environment: {} }, + credentials: { accessKeyId: 'an-access-key', secretAccessKey: 'a-secret-access-key' }, + endpoint: fake.url, + }), + ); + }, +}; + +describe('the secret store over keys in KMS, each data key wrapped by Encrypt and unwrapped by Decrypt', () => { + storeBehaviour(inMemoryEntry(), onTheFake); +}); diff --git a/packages/secrets/src/custodians/custodian-probe.test.ts b/packages/secrets/src/custodians/custodian-probe.test.ts new file mode 100644 index 000000000..7fe621832 --- /dev/null +++ b/packages/secrets/src/custodians/custodian-probe.test.ts @@ -0,0 +1,47 @@ +import { Effect, Fiber } from 'effect'; +import { TestClock } from 'effect/testing'; +import { describe, expect, it } from 'vitest'; + +import { custodianRefusal, recordingCustodian } from '../testing/recording-custodian.ts'; +import { custodianProbed } from './custodian-probe.ts'; + +describe('the probe of a custodian', () => { + it('has one data key wrapped and unwrapped, and fills both copies with zeros once done', async () => { + const custodian = recordingCustodian(); + + await Effect.runPromise(custodianProbed(custodian)); + + expect([custodian.wraps(), custodian.unwraps()]).toEqual([1, 1]); + expect(custodian.dataKeys().map((dataKey) => [dataKey.length, dataKey.every((byte) => byte === 0)])).toEqual([ + [32, true], + [32, true], + ]); + }); + + it('fails in the words of a custodian that refuses, and as a keeper that does not answer for one silent for 10 seconds', async () => { + const refusing = recordingCustodian(); + const stalling = recordingCustodian(); + refusing.refuse(); + + const refused = await Effect.runPromise(Effect.flip(custodianProbed(refusing))); + const stalled = await Effect.runPromise( + Effect.gen(function* () { + const asked = stalling.stall(); + const probing = yield* Effect.forkChild(Effect.flip(custodianProbed(stalling))); + yield* Effect.promise(() => asked); + yield* TestClock.adjust('10 seconds'); + return yield* Fiber.join(probing); + }).pipe(Effect.provide(TestClock.layer())), + ); + + expect(refused).toMatchObject({ + _tag: 'custodian_refused', + detail: `The custodian of the key ${refusing.kid} refused to wrap a data key: ${custodianRefusal}`, + }); + expect(stalled).toMatchObject({ + _tag: 'keeper_unavailable', + detail: 'The keeper of the secret, or the custodian of its key, did not answer within 10 seconds', + }); + expect(refusing.dataKeys().map((dataKey) => dataKey.every((byte) => byte === 0))).toEqual([true]); + }); +}); diff --git a/packages/secrets/src/custodians/custodian-probe.ts b/packages/secrets/src/custodians/custodian-probe.ts new file mode 100644 index 000000000..bf6479f06 --- /dev/null +++ b/packages/secrets/src/custodians/custodian-probe.ts @@ -0,0 +1,16 @@ +import { Effect } from 'effect'; + +import { aDataKey, zeroedAfter } from '../keepers/data-key.ts'; +import type { KeepingFailure } from '../keepers/keeper.ts'; +import { answeredWithinItsBound } from '../store/answer-bound.ts'; +import type { Custodian } from './custodian.ts'; + +export function custodianProbed(custodian: Custodian): Effect.Effect { + return Effect.suspend(() => { + const dataKey = aDataKey(); + const wrappedAndUnwrapped = Effect.flatMap(custodian.wrap(dataKey), (wrapped) => + Effect.flatMap(custodian.unwrap(wrapped), (unwrapped) => zeroedAfter(unwrapped, Effect.void)), + ); + return zeroedAfter(dataKey, answeredWithinItsBound(wrappedAndUnwrapped)); + }); +} diff --git a/packages/secrets/src/index.ts b/packages/secrets/src/index.ts index c7836bd39..b7b42bf56 100644 --- a/packages/secrets/src/index.ts +++ b/packages/secrets/src/index.ts @@ -1,3 +1,4 @@ +export { custodianProbed } from './custodians/custodian-probe.ts'; export { CustodianRefused } from './custodians/custodian-refused.ts'; export type { Custodian, Custodians } from './custodians/custodian.ts'; export { localCustodian, secretKeyBytes } from './custodians/local-custodian.ts'; diff --git a/packages/secrets/src/keepers/keeper-unavailable.ts b/packages/secrets/src/keepers/keeper-unavailable.ts index f1d894528..2733cd5f1 100644 --- a/packages/secrets/src/keepers/keeper-unavailable.ts +++ b/packages/secrets/src/keepers/keeper-unavailable.ts @@ -1,6 +1,9 @@ import { Data } from 'effect'; +import type { CustodianAct } from '../custodians/custodian-refused.ts'; + export class KeeperUnavailable extends Data.TaggedError('keeper_unavailable')<{ + readonly kid?: string | undefined; readonly detail: string; }> {} @@ -9,3 +12,10 @@ export function keeperUnavailable(seconds: number): KeeperUnavailable { detail: `The keeper of the secret, or the custodian of its key, did not answer within ${seconds} seconds`, }); } + +export function custodianUnanswered(kid: string, act: CustodianAct, words: string): KeeperUnavailable { + return new KeeperUnavailable({ + kid, + detail: `The custodian of the key ${kid} did not answer a request to ${act} a data key: ${words}`, + }); +} diff --git a/packages/secrets/src/store/answer-bound.ts b/packages/secrets/src/store/answer-bound.ts index fdff4a1bb..a977c6b10 100644 --- a/packages/secrets/src/store/answer-bound.ts +++ b/packages/secrets/src/store/answer-bound.ts @@ -2,7 +2,9 @@ import { Duration, Effect } from 'effect'; import { keeperUnavailable, type KeeperUnavailable } from '../keepers/keeper-unavailable.ts'; -const answerBoundSeconds = 30; +const answerBoundSeconds = 10; + +export const answerBoundMs = answerBoundSeconds * 1000; const answerBound = Duration.seconds(answerBoundSeconds); diff --git a/packages/secrets/src/testing/failure-behaviour.ts b/packages/secrets/src/testing/failure-behaviour.ts index f4ff32bf4..432831131 100644 --- a/packages/secrets/src/testing/failure-behaviour.ts +++ b/packages/secrets/src/testing/failure-behaviour.ts @@ -3,22 +3,22 @@ import { TestClock } from 'effect/testing'; import { describe, expect, it } from 'vitest'; import type { SecretStore } from '../store/secret-store.ts'; -import { aSecretsKey, custodianRefusal, recordingCustodian, type RecordingCustodian } from './recording-custodian.ts'; -import { kept, kidOf, readBack, rowsIn, storeOn, storeWith, type SecretsEntry } from './secrets-entry.ts'; +import { custodianRefusal, recordingCustodian, type RecordingCustodian } from './recording-custodian.ts'; +import { kept, readBack, rowsIn, storeWith, type SecretsEntry, type TestCustodians } from './secrets-entry.ts'; -function unknownKeyRefused(entry: SecretsEntry): void { +function unknownKeyRefused(entry: SecretsEntry, custodians: TestCustodians): void { it('cannot read a value sealed under another key, and names that key and never the value', async () => { const database = await entry.aDatabase(); - const sealing = aSecretsKey(); - const id = await kept(await storeOn(entry, database, { current: sealing }), 'pasted-key'); - const other = await storeOn(entry, database, { current: aSecretsKey() }); + const sealing = await custodians.aCustodian(); + const id = await kept(await storeWith(entry, database, { current: sealing }), 'pasted-key'); + const other = await storeWith(entry, database, { current: await custodians.aCustodian() }); const refused = await Effect.runPromise(Effect.flip(other.read(id))); expect(refused).toMatchObject({ _tag: 'secret_key_unknown', - kid: kidOf(sealing), - detail: `The secret is sealed under the key ${kidOf(sealing)}, which is neither the current key nor the previous one`, + kid: sealing.kid, + detail: `The secret is sealed under the key ${sealing.kid}, which is neither the current key nor the previous one`, }); expect(JSON.stringify(refused)).not.toContain('pasted-key'); }); @@ -32,10 +32,10 @@ function refusedTo(custodian: RecordingCustodian, act: string) { }; } -function custodianRefusing(entry: SecretsEntry): void { +function custodianRefusing(entry: SecretsEntry, custodians: TestCustodians): void { it('keeps nothing, as a value or as a replacement, whose data key the custodian refuses to wrap, failing in its words', async () => { const database = await entry.aDatabase(); - const custodian = recordingCustodian(); + const custodian = recordingCustodian(await custodians.aCustodian()); const store = await storeWith(entry, database, { current: custodian }); const id = await kept(store, 'pasted-key'); const before = await rowsIn(entry, database); @@ -60,7 +60,7 @@ function readStalled(store: SecretStore, id: string, custodian: RecordingCustodi const asked = custodian.stall(); const reading = yield* Effect.forkChild(Effect.flip(store.read(id))); yield* Effect.promise(() => asked); - yield* TestClock.adjust('29999 millis'); + yield* TestClock.adjust('9999 millis'); const before = reading.pollUnsafe(); yield* TestClock.adjust('1 millis'); return { before, failure: yield* Fiber.join(reading) }; @@ -68,10 +68,10 @@ function readStalled(store: SecretStore, id: string, custodian: RecordingCustodi ); } -function unreadableNow(entry: SecretsEntry): void { - it('cannot read now, in words that name no value, a value whose custodian refuses or does not answer within 30 seconds, and reads it once the custodian answers again', async () => { +function unreadableNow(entry: SecretsEntry, custodians: TestCustodians): void { + it('cannot read now, in words that name no value, a value whose custodian refuses or does not answer within 10 seconds, and reads it once the custodian answers again', async () => { const database = await entry.aDatabase(); - const custodian = recordingCustodian(); + const custodian = recordingCustodian(await custodians.aCustodian()); const store = await storeWith(entry, database, { current: custodian }); const id = await kept(store, 'pasted-key'); const before = await rowsIn(entry, database); @@ -86,7 +86,7 @@ function unreadableNow(entry: SecretsEntry): void { expect(stalled.before).toBeUndefined(); expect(stalled.failure).toMatchObject({ _tag: 'keeper_unavailable', - detail: 'The keeper of the secret, or the custodian of its key, did not answer within 30 seconds', + detail: 'The keeper of the secret, or the custodian of its key, did not answer within 10 seconds', }); expect(answered).toBe('pasted-key'); expect(JSON.stringify([refused, stalled])).not.toContain('pasted-key'); @@ -94,10 +94,10 @@ function unreadableNow(entry: SecretsEntry): void { }); } -export function storeFailureBehaviour(entry: SecretsEntry): void { +export function storeFailureBehaviour(entry: SecretsEntry, custodians: TestCustodians): void { describe('the failures of the secret store', () => { - unknownKeyRefused(entry); - custodianRefusing(entry); - unreadableNow(entry); + unknownKeyRefused(entry, custodians); + custodianRefusing(entry, custodians); + unreadableNow(entry, custodians); }); } diff --git a/packages/secrets/src/testing/fake-kms.test.ts b/packages/secrets/src/testing/fake-kms.test.ts new file mode 100644 index 000000000..d03005364 --- /dev/null +++ b/packages/secrets/src/testing/fake-kms.test.ts @@ -0,0 +1,130 @@ +import { Schema } from 'effect'; +import { afterAll, describe, expect, it } from 'vitest'; + +import { serveFakeKms } from './fake-kms.ts'; + +const key = 'arn:aws:kms:eu-west-1:111122223333:key/0f0e0d0c-0b0a-4908-8706-050403020100'; + +const alias = 'arn:aws:kms:eu-west-1:111122223333:alias/auto-runtime'; + +const fake = await serveFakeKms({ keys: [key], aliases: { [alias]: key } }); + +afterAll(fake.close); + +async function asked(operation: string, body: Readonly>) { + const answer = await fetch(fake.url, { + method: 'POST', + headers: { 'x-amz-target': `TrentService.${operation}`, 'content-type': 'application/x-amz-json-1.1' }, + body: JSON.stringify(body), + }); + return { status: answer.status, body: await answer.json() }; +} + +const context = { 'on.auto/runtime': 'secret data key' }; + +const blobOf = (body: unknown) => + Schema.decodeUnknownSync(Schema.Struct({ CiphertextBlob: Schema.String }))(body).CiphertextBlob; + +describe('the fake KMS', () => { + it('answers Encrypt with a blob that opens only under the key and the context it was made with, recording each call', async () => { + const plaintext = Buffer.alloc(32, 7).toString('base64'); + const before = fake.calls().length; + + const encrypted = await asked('Encrypt', { KeyId: key, Plaintext: plaintext, EncryptionContext: context }); + const blob = blobOf(encrypted.body); + const decrypted = await asked('Decrypt', { KeyId: key, CiphertextBlob: blob, EncryptionContext: context }); + const otherwise = await asked('Decrypt', { KeyId: key, CiphertextBlob: blob, EncryptionContext: {} }); + + expect(encrypted).toMatchObject({ status: 200, body: { KeyId: key, EncryptionAlgorithm: 'SYMMETRIC_DEFAULT' } }); + expect(Buffer.from(blob, 'base64').includes(Buffer.alloc(32, 7))).toBe(false); + expect(blob.includes(plaintext)).toBe(false); + expect(decrypted).toEqual({ + status: 200, + body: { Plaintext: plaintext, KeyId: key, EncryptionAlgorithm: 'SYMMETRIC_DEFAULT' }, + }); + expect(otherwise).toEqual({ + status: 400, + body: { + __type: 'InvalidCiphertextException', + message: 'The encryption context is not the one the ciphertext was made with', + }, + }); + expect(fake.calls().slice(before)).toEqual([ + { operation: 'Encrypt', keyId: key, context }, + { operation: 'Decrypt', keyId: key, context }, + { operation: 'Decrypt', keyId: key, context: {} }, + ]); + }); +}); + +describe('the context of a blob of the fake KMS', () => { + it('opens the blob under its entries in any order', async () => { + const written = { purpose: 'secret data key', runtime: 'on.auto' }; + const encrypted = await asked('Encrypt', { KeyId: key, Plaintext: 'AAAA', EncryptionContext: written }); + + const decrypted = await asked('Decrypt', { + KeyId: key, + CiphertextBlob: blobOf(encrypted.body), + EncryptionContext: { runtime: 'on.auto', purpose: 'secret data key' }, + }); + + expect(decrypted).toMatchObject({ status: 200, body: { Plaintext: 'AAAA' } }); + }); +}); + +const described = { status: 200, body: { KeyMetadata: { KeyId: '0f0e0d0c-0b0a-4908-8706-050403020100', Arn: key } } }; + +describe('the keys the fake KMS describes', () => { + it('describes a key by its ARN or by an alias that names it, enabled, disabled or pending deletion, and records each call', async () => { + const before = fake.calls().length; + + const answers = [await asked('DescribeKey', { KeyId: alias })]; + fake.tell(key, 'disabled'); + answers.push(await asked('DescribeKey', { KeyId: key })); + fake.tell(key, 'pending_deletion'); + answers.push(await asked('DescribeKey', { KeyId: key })); + fake.tell(key, 'enabled'); + + expect(answers).toEqual([described, described, described]); + expect(fake.calls().slice(before)).toEqual([ + { operation: 'DescribeKey', keyId: alias }, + { operation: 'DescribeKey', keyId: key }, + { operation: 'DescribeKey', keyId: key }, + ]); + }); + + it('refuses to describe an alias or a key it does not know, a body it cannot read, and a key it is told to fail', async () => { + const unknownAlias = 'arn:aws:kms:eu-west-1:111122223333:alias/unknown'; + const unknownKey = 'arn:aws:kms:eu-west-1:111122223333:key/0a0b0c0d-0e0f-4000-8000-000000000000'; + fake.failWith(key, { type: 'ThrottlingException', status: 400 }); + + const failed = await asked('DescribeKey', { KeyId: alias }); + fake.tell(key, 'enabled'); + + expect([ + await asked('DescribeKey', { KeyId: unknownAlias }), + await asked('DescribeKey', { KeyId: unknownKey }), + await asked('DescribeKey', {}), + failed, + ]).toEqual([ + { status: 400, body: { __type: 'NotFoundException', message: `Alias ${unknownAlias} is not found.` } }, + { status: 400, body: { __type: 'NotFoundException', message: `Key '${unknownKey}' does not exist` } }, + { status: 400, body: { __type: 'SerializationException', message: 'The request is not one the fake KMS reads' } }, + { status: 400, body: { __type: 'ThrottlingException', message: 'Rate exceeded' } }, + ]); + }); +}); + +describe('a request the fake KMS does not take', () => { + it('is an operation it does not answer, or one it cannot read, refused as KMS refuses them', async () => { + expect([ + await asked('CreateKey', {}), + await asked('Encrypt', { KeyId: key }), + await asked('Decrypt', { KeyId: key }), + ]).toEqual([ + { status: 400, body: { __type: 'UnknownOperationException', message: 'The fake KMS does not answer CreateKey' } }, + { status: 400, body: { __type: 'SerializationException', message: 'The request is not one the fake KMS reads' } }, + { status: 400, body: { __type: 'SerializationException', message: 'The request is not one the fake KMS reads' } }, + ]); + }); +}); diff --git a/packages/secrets/src/testing/fake-kms.ts b/packages/secrets/src/testing/fake-kms.ts new file mode 100644 index 000000000..182ad655f --- /dev/null +++ b/packages/secrets/src/testing/fake-kms.ts @@ -0,0 +1,296 @@ +import { createCipheriv, createDecipheriv, randomBytes } from 'node:crypto'; +import { once } from 'node:events'; +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; + +import { Option, Schema } from 'effect'; + +export type FakeKeyState = 'enabled' | 'disabled' | 'pending_deletion' | 'denied' | 'stalling'; + +export type FakeKmsOperation = 'Encrypt' | 'Decrypt' | 'DescribeKey'; + +export interface FakeKmsCall { + readonly operation: FakeKmsOperation; + readonly keyId: string; + readonly context?: Readonly>; +} + +export interface FakeKmsError { + readonly type: string; + readonly status: number; +} + +export interface FakeKms { + readonly url: string; + readonly tell: (keyId: string, state: FakeKeyState) => void; + readonly failWith: (keyId: string, error: FakeKmsError) => void; + readonly point: (aliasArn: string, keyArn: string) => void; + readonly calls: () => readonly FakeKmsCall[]; + readonly close: () => Promise; +} + +export interface FakeKmsOptions { + readonly keys: readonly string[]; + readonly aliases?: Readonly>; +} + +interface Answer { + readonly status: number; + readonly body: Readonly>; +} + +type Answering = Answer | 'stalled'; + +const ContextSchema = Schema.Record(Schema.String, Schema.String); + +const EncryptSchema = Schema.Struct({ + KeyId: Schema.String, + Plaintext: Schema.String, + EncryptionContext: ContextSchema, +}); + +const DecryptSchema = Schema.Struct({ + KeyId: Schema.String, + CiphertextBlob: Schema.String, + EncryptionContext: ContextSchema, +}); + +const DescribeSchema = Schema.Struct({ KeyId: Schema.String }); + +const SealedSchema = Schema.fromJsonString( + Schema.Struct({ keyId: Schema.String, context: ContextSchema, plaintext: Schema.String }), +); + +type Sealed = typeof SealedSchema.Type; + +type Context = Readonly>; + +const decodeEncrypt = Schema.decodeUnknownOption(Schema.fromJsonString(EncryptSchema)); + +const decodeDecrypt = Schema.decodeUnknownOption(Schema.fromJsonString(DecryptSchema)); + +const decodeDescribe = Schema.decodeUnknownOption(Schema.fromJsonString(DescribeSchema)); + +const decodeSealed = Schema.decodeUnknownOption(SealedSchema); + +const encodeSealed = Schema.encodeSync(SealedSchema); + +const decodeAddress = Schema.decodeUnknownSync(Schema.Struct({ port: Schema.Number })); + +const targetPrefix = 'TrentService.'; + +const ivBytes = 12; + +const tagBytes = 16; + +const unread: Answer = refusal('SerializationException', 'The request is not one the fake KMS reads'); + +const aliasResource = ':alias/'; + +function refusal(type: string, message: string): Answer { + return { status: 400, body: { __type: type, message } }; +} + +function stateRefusal(keyId: string, state: FakeKeyState, operation: FakeKmsOperation): Answering | undefined { + const account = String(keyId.split(':')[4]); + const refusals: Readonly> = { + enabled: undefined, + stalling: 'stalled', + disabled: refusal('DisabledException', `${keyId} is disabled.`), + pending_deletion: refusal('KMSInvalidStateException', `${keyId} is pending deletion.`), + denied: refusal( + 'AccessDeniedException', + `User: arn:aws:iam::${account}:user/runtime is not authorized to perform: kms:${operation} on resource: ${keyId} because no identity-based policy allows the kms:${operation} action`, + ), + }; + return refusals[state]; +} + +function writtenOf(context: Context): string { + const names = Object.keys(context).toSorted((one, other) => one.localeCompare(other)); + return JSON.stringify(names.map((name) => [name, context[name]])); +} + +function sealedUnder(seal: Readonly, content: Sealed): string { + const iv = randomBytes(ivBytes); + const cipher = createCipheriv('aes-256-gcm', seal, iv); + const text = Buffer.concat([cipher.update(encodeSealed(content)), cipher.final()]); + return Buffer.concat([iv, cipher.getAuthTag(), text]).toString('base64'); +} + +function openedUnder(seal: Readonly, blob: string): Option.Option { + const bytes = Buffer.from(blob, 'base64'); + try { + const decipher = createDecipheriv('aes-256-gcm', seal, bytes.subarray(0, ivBytes)); + decipher.setAuthTag(bytes.subarray(ivBytes, ivBytes + tagBytes)); + const text = Buffer.concat([decipher.update(bytes.subarray(ivBytes + tagBytes)), decipher.final()]); + return decodeSealed(text.toString()); + } catch { + return Option.none(); + } +} + +function openedFor(keyId: string, context: Context, { keyId: sealedBy, context: sealedWith, plaintext }: Sealed) { + if (sealedBy !== keyId) { + return refusal('IncorrectKeyException', 'The key in the request is not the key that encrypted the ciphertext'); + } + return writtenOf(context) === writtenOf(sealedWith) + ? { status: 200, body: { Plaintext: plaintext, KeyId: keyId, EncryptionAlgorithm: 'SYMMETRIC_DEFAULT' } } + : refusal('InvalidCiphertextException', 'The encryption context is not the one the ciphertext was made with'); +} + +interface Kept { + readonly seal: Readonly; + readonly states: ReadonlyMap; + readonly errors: ReadonlyMap; + readonly aliases: ReadonlyMap; + readonly noted: (call: FakeKmsCall) => void; +} + +function errorOf({ type, status }: FakeKmsError): Answer { + return { status, body: { __type: type, message: 'Rate exceeded' } }; +} + +function failedAs(kept: Kept, keyId: string): Answer | undefined { + return Option.getOrUndefined(Option.map(Option.fromNullishOr(kept.errors.get(keyId)), errorOf)); +} + +function heard(kept: Kept, call: FakeKmsCall, answer: () => Answer): Answering { + kept.noted(call); + const state = kept.states.get(call.keyId); + if (state === undefined) { + return refusal('NotFoundException', `Key '${call.keyId}' does not exist`); + } + return failedAs(kept, call.keyId) ?? stateRefusal(call.keyId, state, call.operation) ?? answer(); +} + +function describingRefusal(keyArn: string, state: FakeKeyState): Answering | undefined { + return state === 'disabled' || state === 'pending_deletion' ? undefined : stateRefusal(keyArn, state, 'DescribeKey'); +} + +function unknownToDescribe(keyId: string): Answer { + return keyId.includes(aliasResource) + ? refusal('NotFoundException', `Alias ${keyId} is not found.`) + : refusal('NotFoundException', `Key '${keyId}' does not exist`); +} + +function describedAs(keyId: string, kept: Kept): Answering { + kept.noted({ operation: 'DescribeKey', keyId }); + const keyArn = kept.aliases.get(keyId) ?? keyId; + const state = kept.states.get(keyArn); + if (state === undefined) { + return unknownToDescribe(keyId); + } + const keyMetadata = { KeyId: keyArn.slice(keyArn.lastIndexOf('/') + 1), Arn: keyArn }; + return ( + failedAs(kept, keyArn) ?? describingRefusal(keyArn, state) ?? { status: 200, body: { KeyMetadata: keyMetadata } } + ); +} + +function encrypted(body: string, kept: Kept): Answering { + return Option.match(decodeEncrypt(body), { + onNone: () => unread, + onSome: ({ KeyId, Plaintext, EncryptionContext }) => + heard(kept, { operation: 'Encrypt', keyId: KeyId, context: EncryptionContext }, () => ({ + status: 200, + body: { + CiphertextBlob: sealedUnder(kept.seal, { keyId: KeyId, context: EncryptionContext, plaintext: Plaintext }), + KeyId, + EncryptionAlgorithm: 'SYMMETRIC_DEFAULT', + }, + })), + }); +} + +function decrypted(body: string, kept: Kept): Answering { + return Option.match(decodeDecrypt(body), { + onNone: () => unread, + onSome: ({ KeyId, CiphertextBlob, EncryptionContext }) => + heard(kept, { operation: 'Decrypt', keyId: KeyId, context: EncryptionContext }, () => + Option.match(openedUnder(kept.seal, CiphertextBlob), { + onNone: () => refusal('InvalidCiphertextException', 'The ciphertext is not one this KMS made'), + onSome: (sealed) => openedFor(KeyId, EncryptionContext, sealed), + }), + ), + }); +} + +function described(body: string, kept: Kept): Answering { + return Option.match(decodeDescribe(body), { + onNone: () => unread, + onSome: ({ KeyId }) => describedAs(KeyId, kept), + }); +} + +function answerTo(operation: string, body: string, kept: Kept): Answering { + switch (operation) { + case 'Encrypt': + return encrypted(body, kept); + case 'Decrypt': + return decrypted(body, kept); + case 'DescribeKey': + return described(body, kept); + default: + return refusal('UnknownOperationException', `The fake KMS does not answer ${operation}`); + } +} + +async function bodyOf(request: IncomingMessage): Promise { + request.setEncoding('utf8'); + let text = ''; + for await (const chunk of request) { + text += String(chunk); + } + return text; +} + +async function respond(request: IncomingMessage, response: ServerResponse, kept: Kept): Promise { + const operation = String(request.headers['x-amz-target']).replace(targetPrefix, ''); + const answer = answerTo(operation, await bodyOf(request), kept); + if (answer !== 'stalled') { + response + .writeHead(answer.status, { 'content-type': 'application/x-amz-json-1.1' }) + .end(JSON.stringify(answer.body)); + } +} + +export async function serveFakeKms({ keys, aliases = {} }: FakeKmsOptions): Promise { + const states = new Map(keys.map((key) => [key, 'enabled'])); + const errors = new Map(); + const pointed = new Map(Object.entries(aliases)); + const calls: FakeKmsCall[] = []; + const kept: Kept = { + seal: randomBytes(32), + states, + errors, + aliases: pointed, + noted: (call) => { + calls.push(call); + }, + }; + const server = createServer(); + server.on('request', (request: IncomingMessage, response: ServerResponse) => { + void respond(request, response, kept); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + return { + url: `http://127.0.0.1:${String(decodeAddress(server.address()).port)}`, + tell: (keyId, state) => { + states.set(keyId, state); + errors.delete(keyId); + }, + failWith: (keyId, error) => { + errors.set(keyId, error); + }, + point: (aliasArn, keyArn) => { + pointed.set(aliasArn, keyArn); + }, + calls: () => [...calls], + close: async () => { + const closed = once(server, 'close'); + server.close(); + server.closeAllConnections(); + await closed; + }, + }; +} diff --git a/packages/secrets/src/testing/index.ts b/packages/secrets/src/testing/index.ts index 4fe988dc8..229a15d9f 100644 --- a/packages/secrets/src/testing/index.ts +++ b/packages/secrets/src/testing/index.ts @@ -1,3 +1,12 @@ +export { + serveFakeKms, + type FakeKeyState, + type FakeKms, + type FakeKmsCall, + type FakeKmsError, + type FakeKmsOperation, + type FakeKmsOptions, +} from './fake-kms.ts'; export { memoryRows, type KeptColumns, type MemoryRows, type MemoryRowsOptions } from './memory-rows.ts'; export { custodianRefusal, recordingCustodian, type RecordingCustodian } from './recording-custodian.ts'; export { rowsBehaviour } from './rows-behaviour.ts'; diff --git a/packages/secrets/src/testing/lease-behaviour.ts b/packages/secrets/src/testing/lease-behaviour.ts index 422b46f6c..e84a50329 100644 --- a/packages/secrets/src/testing/lease-behaviour.ts +++ b/packages/secrets/src/testing/lease-behaviour.ts @@ -1,26 +1,25 @@ import { Effect, Redacted } from 'effect'; import { describe, expect, it } from 'vitest'; -import { aSecretsKey } from './recording-custodian.ts'; import { aLease, at, firstMoment, isoOf, kept, - kidOf, kidsIn, rowsIn, stateAt, - storeOn, + storeWith, type SecretsEntry, + type TestCustodians, } from './secrets-entry.ts'; -function replacedAtTheNextVersion(entry: SecretsEntry): void { +function replacedAtTheNextVersion(entry: SecretsEntry, custodians: TestCustodians): void { it('replaces the value of a row whose version is the one read, at the next version, under a new data key, kept when it was replaced', async () => { const database = await entry.aDatabase(); - const key = aSecretsKey(); - const store = await storeOn(entry, database, { current: key }); + const custodian = await custodians.aCustodian(); + const store = await storeWith(entry, database, { current: custodian }); const id = await at(firstMoment, store.keep(Redacted.make('first-tokens-1234'), '1234')); const [before] = await rowsIn(entry, database); @@ -33,17 +32,17 @@ function replacedAtTheNextVersion(entry: SecretsEntry): void { version: 2, claimedUntil: undefined, }); - expect(row).toMatchObject({ id, kept_by: 'ledger', kid: kidOf(key), kept_at: isoOf(firstMoment + 1000) }); + expect(row).toMatchObject({ id, kept_by: 'ledger', kid: custodian.kid, kept_at: isoOf(firstMoment + 1000) }); expect(row?.held.includes('second-tokens')).toBe(false); expect(row?.held).not.toBe(before?.held); expect(row?.wrapped_key).not.toBe(before?.wrapped_key); }); } -function replacementsRefused(entry: SecretsEntry): void { +function replacementsRefused(entry: SecretsEntry, custodians: TestCustodians): void { it('refuses an empty value and one of more than 8 KiB as it refuses to keep them, and keeps the value it holds', async () => { const database = await entry.aDatabase(); - const store = await storeOn(entry, database, { current: aSecretsKey() }); + const store = await storeWith(entry, database, { current: await custodians.aCustodian() }); const id = await kept(store, 'first-tokens'); const refusals = await Effect.runPromise( @@ -62,20 +61,20 @@ function replacementsRefused(entry: SecretsEntry): void { }); } -function leaseKeptByTheRotation(entry: SecretsEntry): void { +function leaseKeptByTheRotation(entry: SecretsEntry, custodians: TestCustodians): void { it('keeps the version and the claim of each row the rotation of its key wraps again', async () => { const database = await entry.aDatabase(); - const [previous, current] = [aSecretsKey(), aSecretsKey()]; - const sealing = await storeOn(entry, database, { current: previous }); + const [previous, current] = [await custodians.aCustodian(), await custodians.aCustodian()]; + const sealing = await storeWith(entry, database, { current: previous }); const id = await kept(sealing, 'first-tokens'); await Effect.runPromise(sealing.replace(id, 1, Redacted.make('second-tokens'))); await at(firstMoment, sealing.claim(id, 2, aLease)); - const rotating = await storeOn(entry, database, { current, previous }); + const rotating = await storeWith(entry, database, { current, previous }); const rewrapped = await Effect.runPromise(rotating.rotated()); expect(rewrapped).toBe(1); - expect(await kidsIn(entry, database)).toEqual({ [kidOf(current)]: 1 }); + expect(await kidsIn(entry, database)).toEqual({ [current.kid]: 1 }); expect(await stateAt(rotating, id, firstMoment)).toEqual({ value: 'second-tokens', version: 2, @@ -84,10 +83,10 @@ function leaseKeptByTheRotation(entry: SecretsEntry): void { }); } -export function storeLeaseBehaviour(entry: SecretsEntry): void { +export function storeLeaseBehaviour(entry: SecretsEntry, custodians: TestCustodians): void { describe('a value replaced under a lease', () => { - replacedAtTheNextVersion(entry); - replacementsRefused(entry); - leaseKeptByTheRotation(entry); + replacedAtTheNextVersion(entry, custodians); + replacementsRefused(entry, custodians); + leaseKeptByTheRotation(entry, custodians); }); } diff --git a/packages/secrets/src/testing/memory-rows.test.ts b/packages/secrets/src/testing/memory-rows.test.ts index 6370fd9d1..4e6db4b36 100644 --- a/packages/secrets/src/testing/memory-rows.test.ts +++ b/packages/secrets/src/testing/memory-rows.test.ts @@ -1,26 +1,10 @@ -import { DateTime, Effect, Option } from 'effect'; import { describe } from 'vitest'; -import { memoryRows, type MemoryRows } from './memory-rows.ts'; import { rowsBehaviour } from './rows-behaviour.ts'; -import type { SecretsEntry } from './secrets-entry.ts'; +import { inMemoryEntry } from './secrets-entry.ts'; import { storeBehaviour } from './store-behaviour.ts'; -const databases = new Map(); - -function rowsNamed(database: string): MemoryRows { - return Option.getOrThrow(Option.fromNullishOr(databases.get(database))); -} - -const inMemory: SecretsEntry = { - aDatabase: () => { - const database = `memory-${databases.size + 1}`; - databases.set(database, memoryRows({ now: Effect.map(DateTime.now, DateTime.toEpochMillis) })); - return Promise.resolve(database); - }, - rowsOn: (database) => Effect.sync(() => rowsNamed(database)), - rowsIn: (database) => Promise.resolve(rowsNamed(database).kept()), -}; +const inMemory = inMemoryEntry(); describe('the rows kept in memory', () => { rowsBehaviour(inMemory); diff --git a/packages/secrets/src/testing/recording-custodian.ts b/packages/secrets/src/testing/recording-custodian.ts index 6f1508770..df8d7ad32 100644 --- a/packages/secrets/src/testing/recording-custodian.ts +++ b/packages/secrets/src/testing/recording-custodian.ts @@ -66,8 +66,7 @@ function answersOf(kid: string): Answers { }; } -export function recordingCustodian(key: Redacted.Redacted = aSecretsKey()): RecordingCustodian { - const custodian = localCustodian(key); +export function recordingCustodian(custodian: Custodian = localCustodian(aSecretsKey())): RecordingCustodian { const { answered, refuse, stall, answer } = answersOf(custodian.kid); const asked = { wraps: 0, unwraps: 0 }; const dataKeys: Readonly[] = []; diff --git a/packages/secrets/src/testing/secrets-entry.ts b/packages/secrets/src/testing/secrets-entry.ts index b7df315d1..d8680a97e 100644 --- a/packages/secrets/src/testing/secrets-entry.ts +++ b/packages/secrets/src/testing/secrets-entry.ts @@ -1,13 +1,14 @@ -import { sha256Of } from '@on.auto/operations'; -import { Effect, Option, Redacted, Schema } from 'effect'; +import { DateTime, Effect, Option, Redacted, Schema } from 'effect'; import { TestClock } from 'effect/testing'; import { onTestFinished } from 'vitest'; -import type { Custodians } from '../custodians/custodian.ts'; +import type { Custodian, Custodians } from '../custodians/custodian.ts'; import { localCustodian } from '../custodians/local-custodian.ts'; import { sealedKeeper } from '../keepers/sealed-keeper.ts'; import type { SecretRows } from '../rows/secret-rows.ts'; import { secretStoreOn, type SecretStore } from '../store/secret-store.ts'; +import { memoryRows, type MemoryRows } from './memory-rows.ts'; +import { aSecretsKey } from './recording-custodian.ts'; export interface SecretsEntry { readonly aDatabase: () => Promise; @@ -15,13 +16,14 @@ export interface SecretsEntry { readonly rowsIn: (database: string) => Promise; } -type Key = Redacted.Redacted; - -export interface Keys { - readonly current: Key; - readonly previous?: Key; +export interface TestCustodians { + readonly aCustodian: () => Promise; } +export const localCustodians: TestCustodians = { + aCustodian: () => Promise.resolve(localCustodian(aSecretsKey())), +}; + const KeptRows = Schema.Array( Schema.Struct({ id: Schema.String, @@ -48,10 +50,6 @@ export function at(moment: number, effect: Effect.Effect): Promise { const rows = await Effect.runPromise(entry.rowsOn(database)); onTestFinished(() => rows.close()); @@ -62,13 +60,6 @@ export async function storeWith(entry: SecretsEntry, database: string, custodian return secretStoreOn(await rowsOpenedOn(entry, database), sealedKeeper(custodians), custodians); } -export function storeOn(entry: SecretsEntry, database: string, { current, previous }: Keys): Promise { - return storeWith(entry, database, { - current: localCustodian(current), - previous: previous === undefined ? undefined : localCustodian(previous), - }); -} - export function kept(store: SecretStore, value: string): Promise { return Effect.runPromise(store.keep(Redacted.make(value))); } @@ -103,3 +94,17 @@ export async function kidsIn(entry: SecretsEntry, database: string): Promise(); + const rowsNamed = (database: string) => Option.getOrThrow(Option.fromNullishOr(databases.get(database))); + return { + aDatabase: () => { + const database = `memory-${databases.size + 1}`; + databases.set(database, memoryRows({ now: Effect.map(DateTime.now, DateTime.toEpochMillis) })); + return Promise.resolve(database); + }, + rowsOn: (database) => Effect.sync(() => rowsNamed(database)), + rowsIn: (database) => Promise.resolve(rowsNamed(database).kept()), + }; +} diff --git a/packages/secrets/src/testing/store-behaviour.ts b/packages/secrets/src/testing/store-behaviour.ts index 8626736e8..1b3ec4624 100644 --- a/packages/secrets/src/testing/store-behaviour.ts +++ b/packages/secrets/src/testing/store-behaviour.ts @@ -3,49 +3,52 @@ import { Effect, Redacted } from 'effect'; import { compactDecrypt, decodeProtectedHeader } from 'jose'; import { describe, expect, it } from 'vitest'; -import { localCustodian } from '../custodians/local-custodian.ts'; +import type { Custodian } from '../custodians/custodian.ts'; import { storeFailureBehaviour } from './failure-behaviour.ts'; import { storeLeaseBehaviour } from './lease-behaviour.ts'; -import { aSecretsKey, recordingCustodian } from './recording-custodian.ts'; +import { recordingCustodian } from './recording-custodian.ts'; import { at, firstMoment, isoOf, kept, keptEach, - kidOf, kidsIn, + localCustodians, readBack, rowsIn, - storeOn, storeWith, type SecretsEntry, + type TestCustodians, } from './secrets-entry.ts'; const uuidV7 = /^[\da-f]{8}-[\da-f]{4}-7[\da-f]{3}-[89ab][\da-f]{3}-[\da-f]{12}$/u; -type Key = Redacted.Redacted; - -function dataKeyOf(key: Key, wrappedKey: string | null): Promise { - return Effect.runPromise(localCustodian(key).unwrap(String(wrappedKey))); +interface Wrapped { + readonly held: string; + readonly wrapped_key: string | null; } -async function hexOfTheDataKey(key: Key, wrappedKey: string | null): Promise { - return Buffer.from(Redacted.value(await dataKeyOf(key, wrappedKey))).toString('hex'); +async function hexOfTheDataKey(custodian: Custodian, { wrapped_key }: Wrapped): Promise { + return Buffer.from(Redacted.value(await Effect.runPromise(custodian.unwrap(String(wrapped_key))))).toString('hex'); } -async function openedWith( - key: Key, - { held, wrapped_key }: { readonly held: string; readonly wrapped_key: string | null }, -): Promise { - const { plaintext } = await compactDecrypt(held, Redacted.value(await dataKeyOf(key, wrapped_key))); +async function openedWith(custodian: Custodian, row: Wrapped): Promise { + const { plaintext } = await compactDecrypt(row.held, Buffer.from(await hexOfTheDataKey(custodian, row), 'hex')); return new TextDecoder().decode(plaintext); } -function keptUnderAnIdOfItsOwn(entry: SecretsEntry): void { +function holdsInTheClear({ wrapped_key }: Wrapped, hex: string): boolean { + const dataKey = Buffer.from(hex, 'hex'); + const wrapping = String(wrapped_key); + const written = [dataKey.toString('base64'), dataKey.toString('base64url'), dataKey.toString('hex')]; + return written.some((form) => wrapping.includes(form)) || Buffer.from(wrapping, 'base64').includes(dataKey); +} + +function keptUnderAnIdOfItsOwn(entry: SecretsEntry, custodians: TestCustodians): void { it('keeps a value under an id of its own, a UUIDv7, at its first version with the ending it is given, and reads it back redacted', async () => { const database = await entry.aDatabase(); - const store = await storeOn(entry, database, { current: aSecretsKey() }); + const store = await storeWith(entry, database, { current: await custodians.aCustodian() }); const ids = await at( firstMoment, @@ -74,34 +77,33 @@ function keptUnderAnIdOfItsOwn(entry: SecretsEntry): void { }); } -function sealedUnderADataKeyOfItsOwn(entry: SecretsEntry): void { - it('seals each value as one JWE under a data key of its own, which the current key wraps beside it under its kid', async () => { +function sealedUnderADataKeyOfItsOwn(entry: SecretsEntry, custodians: TestCustodians): void { + it('seals each value as one JWE under a data key of its own, which the current custodian wraps beside it under its kid and never holds in the clear', async () => { const database = await entry.aDatabase(); - const key = aSecretsKey(); - const store = await storeOn(entry, database, { current: key }); + const custodian = await custodians.aCustodian(); + const store = await storeWith(entry, database, { current: custodian }); const ids = await keptEach(store, ['first-key', 'second-key']); const rows = await rowsIn(entry, database); - const dataKeys = await Promise.all(rows.map(({ wrapped_key }) => hexOfTheDataKey(key, wrapped_key))); + const dataKeys = await Promise.all(rows.map((row) => hexOfTheDataKey(custodian, row))); + const inTheClear = await Promise.all( + rows.map(async (row) => holdsInTheClear(row, await hexOfTheDataKey(custodian, row))), + ); const opened = await Promise.all( - rows.map(async (row): Promise => [row.id, await openedWith(key, row)]), + rows.map(async (row): Promise => [row.id, await openedWith(custodian, row)]), ); - expect(kidOf(key)).toMatch(/^[\da-f]{16}$/u); expect(rows.map(({ kept_by, kid }) => [kept_by, kid])).toEqual([ - ['ledger', kidOf(key)], - ['ledger', kidOf(key)], + ['ledger', custodian.kid], + ['ledger', custodian.kid], ]); expect(rows.map(({ held }) => [held.split('.').length, decodeProtectedHeader(held)])).toEqual([ [5, { alg: 'dir', enc: 'A256GCM' }], [5, { alg: 'dir', enc: 'A256GCM' }], ]); - expect(rows.map(({ wrapped_key }) => decodeProtectedHeader(String(wrapped_key)))).toEqual([ - { alg: 'A256KW', enc: 'A256GCM' }, - { alg: 'A256KW', enc: 'A256GCM' }, - ]); expect(dataKeys.map((hex) => hex.length)).toEqual([64, 64]); - expect(dataKeys[0]).not.toBe(dataKeys[1]); + expect(new Set(dataKeys).size).toBe(2); + expect(inTheClear).toEqual([false, false]); expect(new Map(opened)).toEqual( new Map([ [ids[0], 'first-key'], @@ -111,10 +113,10 @@ function sealedUnderADataKeyOfItsOwn(entry: SecretsEntry): void { }); } -function unwrappedAtEachRead(entry: SecretsEntry): void { +function unwrappedAtEachRead(entry: SecretsEntry, custodians: TestCustodians): void { it('asks the custodian to wrap once for each value it keeps and to unwrap once for each read, keeping no data key and filling each with zeros once used', async () => { const database = await entry.aDatabase(); - const custodian = recordingCustodian(); + const custodian = recordingCustodian(await custodians.aCustodian()); const store = await storeWith(entry, database, { current: custodian }); const id = await kept(store, 'pasted-key'); @@ -126,10 +128,10 @@ function unwrappedAtEachRead(entry: SecretsEntry): void { }); } -function forgottenTwiceAsOnce(entry: SecretsEntry): void { +function forgottenTwiceAsOnce(entry: SecretsEntry, custodians: TestCustodians): void { it('answers nothing for an id it does not keep, and forgets a value, twice as once', async () => { const database = await entry.aDatabase(); - const store = await storeOn(entry, database, { current: aSecretsKey() }); + const store = await storeWith(entry, database, { current: await custodians.aCustodian() }); const id = await kept(store, 'pasted-key'); await Effect.runPromise(Effect.andThen(store.forget(id), store.forget(id))); @@ -139,10 +141,10 @@ function forgottenTwiceAsOnce(entry: SecretsEntry): void { }); } -function valuesRefused(entry: SecretsEntry): void { +function valuesRefused(entry: SecretsEntry, custodians: TestCustodians): void { it('refuses an empty value and one of more than 8 KiB of UTF-8 before anything is sealed, and keeps neither', async () => { const database = await entry.aDatabase(); - const custodian = recordingCustodian(); + const custodian = recordingCustodian(await custodians.aCustodian()); const store = await storeWith(entry, database, { current: custodian }); const refusals = await Effect.runPromise( @@ -164,38 +166,42 @@ function valuesRefused(entry: SecretsEntry): void { }); } -function rewrappedInBatches(entry: SecretsEntry): void { - it('reads with its previous key, and wraps again in batches under the current one the data keys the previous wrapped, touching no value', async () => { +async function threeOf(custodians: TestCustodians): Promise { + return [await custodians.aCustodian(), await custodians.aCustodian(), await custodians.aCustodian()]; +} + +function rewrappedInBatches(entry: SecretsEntry, custodians: TestCustodians): void { + it('reads with its previous custodian, and wraps again in batches under the current one the data keys the previous wrapped, touching no value', async () => { const database = await entry.aDatabase(); - const [previous, current, stray] = [aSecretsKey(), aSecretsKey(), aSecretsKey()]; + const [previous, current, stray] = await threeOf(custodians); const values = Array.from({ length: 101 }, (_, index) => `pasted-key-${index}`); - const ids = await keptEach(await storeOn(entry, database, { current: previous }), values); - const strayId = await kept(await storeOn(entry, database, { current: stray }), 'stray-key'); - const rotating = await storeOn(entry, database, { current, previous }); + const ids = await keptEach(await storeWith(entry, database, { current: previous }), values); + const strayId = await kept(await storeWith(entry, database, { current: stray }), 'stray-key'); + const rotating = await storeWith(entry, database, { current, previous }); const before = (await rowsIn(entry, database)).map(({ held, kept_at, version }) => ({ held, kept_at, version })); const readBefore = await readBack(rotating, String(ids[0])); const rewrapped = await Effect.runPromise(rotating.rotated()); - const afterwards = await storeOn(entry, database, { current }); + const afterwards = await storeWith(entry, database, { current }); expect({ readBefore, rewrapped }).toEqual({ readBefore: 'pasted-key-0', rewrapped: 101 }); - expect(await kidsIn(entry, database)).toEqual({ [kidOf(current)]: 101, [kidOf(stray)]: 1 }); + expect(await kidsIn(entry, database)).toEqual({ [current.kid]: 101, [stray.kid]: 1 }); expect((await rowsIn(entry, database)).map(({ held, kept_at, version }) => ({ held, kept_at, version }))).toEqual( before, ); expect(await Promise.all(ids.map((id) => readBack(afterwards, id)))).toEqual(values); expect(await Effect.runPromise(rotating.rotated())).toBe(0); - expect(await Effect.runPromise(Effect.flip(rotating.read(strayId)))).toMatchObject({ kid: kidOf(stray) }); + expect(await Effect.runPromise(Effect.flip(rotating.read(strayId)))).toMatchObject({ kid: stray.kid }); }); } -function rewrappedNothing(entry: SecretsEntry): void { - it('wraps nothing again without a previous key, or with the current key given as the previous one', async () => { +function rewrappedNothing(entry: SecretsEntry, custodians: TestCustodians): void { + it('wraps nothing again without a previous custodian, or with the current one given as the previous one', async () => { const database = await entry.aDatabase(); - const key = aSecretsKey(); - const alone = await storeOn(entry, database, { current: key }); + const custodian = await custodians.aCustodian(); + const alone = await storeWith(entry, database, { current: custodian }); const id = await kept(alone, 'pasted-key'); - const twice = await storeOn(entry, database, { current: key, previous: key }); + const twice = await storeWith(entry, database, { current: custodian, previous: custodian }); const rewrapped = [await Effect.runPromise(alone.rotated()), await Effect.runPromise(twice.rotated())]; @@ -204,16 +210,16 @@ function rewrappedNothing(entry: SecretsEntry): void { }); } -export function storeBehaviour(entry: SecretsEntry): void { +export function storeBehaviour(entry: SecretsEntry, custodians: TestCustodians = localCustodians): void { describe('the secret store over the rows', () => { - keptUnderAnIdOfItsOwn(entry); - sealedUnderADataKeyOfItsOwn(entry); - unwrappedAtEachRead(entry); - forgottenTwiceAsOnce(entry); - valuesRefused(entry); - rewrappedInBatches(entry); - rewrappedNothing(entry); - storeFailureBehaviour(entry); - storeLeaseBehaviour(entry); + keptUnderAnIdOfItsOwn(entry, custodians); + sealedUnderADataKeyOfItsOwn(entry, custodians); + unwrappedAtEachRead(entry, custodians); + forgottenTwiceAsOnce(entry, custodians); + valuesRefused(entry, custodians); + rewrappedInBatches(entry, custodians); + rewrappedNothing(entry, custodians); + storeFailureBehaviour(entry, custodians); + storeLeaseBehaviour(entry, custodians); }); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 28e047ced..b631d6938 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -825,9 +825,18 @@ importers: packages/secrets: dependencies: + '@aws-sdk/client-kms': + specifier: 3.1149.0 + version: 3.1149.0 '@on.auto/operations': specifier: workspace:* version: link:../operations + '@on.auto/outbound': + specifier: workspace:* + version: link:../outbound + '@smithy/core': + specifier: 3.35.2 + version: 3.35.2 effect: specifier: 'catalog:' version: 4.0.0 @@ -1272,6 +1281,10 @@ packages: resolution: {integrity: sha512-gKRj5IXZ/O0ehk2rwfb55YygdNKNhK5N1l/Ztg/w06tk21AJtCPuyErETk2RS/6XOmpBkcZY6CN3GlQ2fWTh1g==} engines: {node: '>=20.0.0'} + '@aws-sdk/client-kms@3.1149.0': + resolution: {integrity: sha512-+nkx9W8OA4MAZAmDAvDsTSZ3lwidQyOF/lha5FwHUXxOcb5suwfkst2uSM3PsnB/dtYDvfCzGzRXouh+EWzQOg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/client-s3@3.1149.0': resolution: {integrity: sha512-wEccG+3/ap7iIBO4DbWwqNHh2u5Qm5OCQJzJaAvtVFIoZbp0qia8HIXv4rpnNmJJEX8205mlq67F9QJ0nKlcxg==} engines: {node: '>=20.0.0'} @@ -5705,6 +5718,17 @@ snapshots: '@smithy/types': 4.19.0 tslib: 2.8.1 + '@aws-sdk/client-kms@3.1149.0': + dependencies: + '@aws-sdk/core': 3.978.3 + '@aws-sdk/credential-provider-node': 3.972.86 + '@aws-sdk/types': 3.974.6 + '@smithy/core': 3.35.2 + '@smithy/fetch-http-handler': 5.8.0 + '@smithy/node-http-handler': 4.12.1 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + '@aws-sdk/client-s3@3.1149.0': dependencies: '@aws-sdk/checksums': 3.1001.2 From 0abd4896ff686071b30e69256bf94f1c09f87aed Mon Sep 17 00:00:00 2001 From: Rami Date: Tue, 20 Oct 2026 20:24:17 +0100 Subject: [PATCH 3/4] feat(server): the secrets key held in AWS KMS by SECRETS_KEY_KMS, probed at start and rotated to A runtime can name a key in AWS KMS in place of a key in hand: SECRETS_KEY_KMS and SECRETS_KEY_KMS_PREVIOUS join the settings of the secrets key, the server makes the KMS custodian where one names it, probes it before it reads anything, and rotates to it and from it as it rotates between keys in hand. - The value is the ARN of a symmetric key or of an alias, arn:aws:kms:::key/, a multi-Region mrk- id among them, or arn:aws:kms:::alias/, refused in words otherwise; the region is the ARN's. One of SECRETS_KEY, SECRETS_KEY_FILE and SECRETS_KEY_KMS at most, the same for the previous key, which stands only beside a current one; a ledger in PostgreSQL takes any of the three. secretsVariables lists all six, so connections.variables refuses them. - database/secret-store.ts makes a localCustodian of a key in hand, and database/kms-custodians.ts loads kmsCustodian through import('@on.auto/secrets/aws') where a setting names a key in KMS, with the region, the proxy settings and awsCredentialChain(proxy), so a runtime without the setting loads no part of @aws-sdk/client-kms. ledgerStoresOpenedBy takes the loader, which the tests replace. An alias is resolved to its key there, and a resolution that fails stops the start, "The secrets settings are invalid. SECRETS_KEY_KMS: The KMS alias alias/ it names could not be resolved to its key at start: ". - Before the rows open, a current key in KMS is probed with custodianProbed: a data key wrapped, unwrapped and zeroed. A probe that fails stops the start, "The secrets settings are invalid. SECRETS_KEY_KMS: The KMS key it names could not wrap and unwrap a data key at start: ", never a credential or the account. - The start's line gains its third form, "... sealed under the key that the KMS key holds, and sealed under the previous key were sealed again under it", annotated secrets_key_from: kms, and the words of an own key that cannot be read name SECRETS_KEY_KMS_PREVIOUS. secretsKeysOf gives way to keyInHandOf, of one key in hand. - Tests: the six variables, the ARNs of a key, a multi-Region key and an alias, eight values that are no such ARN, the kinds of key refused side by side; no module of @aws-sdk/client-kms loaded by a server whose key is in hand, from the environment or from the file, and the client loaded once a setting names a key in KMS; the loader called for a current or a previous key in KMS with its region, the credential chain and the proxy settings; the probe and its context, and the start stopped for a key KMS does not know, disabled, pending deletion, denied or silent, against the fake; an alias resolved before the probe, the start stopped for an alias KMS does not know, and the rows of an alias pointed at another key wrapped again under it with the key before as the previous one; the rotation from a key in hand to a key in KMS and back, two each way, with nothing of a key, a data key or a credential in the log; a connection of a server whose key is disabled meanwhile failing its call as connection_unavailable in the words of KMS and called again once the key is enabled, through the SDK's own endpoint and credential variables; a spawned server logging the KMS line at two starts and stopping for a disabled key; and the page's settings, policy and quoted refusal. - docs/self-host/production.md gains "Hold the secrets key in a KMS", with kms:DescribeKey for an alias, how to move to another key, and what a first read and a renewal answer under a disabled key; configuration.md has its rows, security.md's "Secrets at rest" its sentence and the wrapping KMS makes, postgresql.md, docker.md, connections.md and the npm README the third setting where they named two, and auto-runtime.example.yaml and .env.example SECRETS_KEY_KMS beside the others. The server README sets out the secrets key, and the outbound README the custodian among its AWS clients. Co-Authored-By: Claude Fable 5.1 --- .env.example | 3 +- auto-runtime.example.yaml | 5 +- docs/self-host/configuration.md | 23 +- docs/self-host/connections.md | 2 +- docs/self-host/docker.md | 4 +- docs/self-host/postgresql.md | 2 +- docs/self-host/production.md | 42 ++- docs/self-host/security.md | 2 +- packages/outbound/README.md | 4 +- packages/server/README.md | 6 +- packages/server/npm/README.md | 2 +- .../connections/connections-over-mcp.test.ts | 2 +- .../connections/connections-under-kms.test.ts | 71 +++++ .../database/kms-custodian-loading.test.ts | 56 ++++ .../server/src/database/kms-custodians.ts | 43 +++ packages/server/src/database/kms-page.test.ts | 126 ++++++++ .../src/database/ledger-stores-kms.test.ts | 282 ++++++++++++++++++ packages/server/src/database/ledger-stores.ts | 34 ++- packages/server/src/database/own-keys.test.ts | 2 +- packages/server/src/database/own-keys.ts | 2 +- .../server/src/database/secret-store.test.ts | 107 +++++++ packages/server/src/database/secret-store.ts | 18 +- .../server/src/database/secrets-keys.test.ts | 37 ++- packages/server/src/database/secrets-keys.ts | 18 +- .../server/src/lifecycle/kms-startup.test.ts | 103 +++++++ .../src/lifecycle/secrets-startup.test.ts | 10 +- packages/server/src/logging/logging.ts | 3 + .../src/settings/secrets-settings.test.ts | 96 +++++- .../server/src/settings/secrets-settings.ts | 56 +++- .../entries/serve-with-waiting-tool.ts | 4 +- .../src/testing/records/kept-credentials.ts | 11 +- 31 files changed, 1069 insertions(+), 107 deletions(-) create mode 100644 packages/server/src/connections/connections-under-kms.test.ts create mode 100644 packages/server/src/database/kms-custodian-loading.test.ts create mode 100644 packages/server/src/database/kms-custodians.ts create mode 100644 packages/server/src/database/kms-page.test.ts create mode 100644 packages/server/src/database/ledger-stores-kms.test.ts create mode 100644 packages/server/src/database/secret-store.test.ts create mode 100644 packages/server/src/lifecycle/kms-startup.test.ts diff --git a/.env.example b/.env.example index 8fc57eda1..59e2d4836 100644 --- a/.env.example +++ b/.env.example @@ -19,7 +19,8 @@ # Keep the ledger in PostgreSQL instead of packages/server/.data. Empty LEDGER_FILE # too, since dev.env names a ledger file and the server refuses both. A ledger in # PostgreSQL needs SECRETS_KEY, the key its secrets are sealed under: put in its -# place the line openssl rand -base64 32 prints. +# place the line openssl rand -base64 32 prints, or name a file that holds it in +# SECRETS_KEY_FILE, or a key in AWS KMS by its ARN in SECRETS_KEY_KMS, instead. # DATABASE_URL=postgresql://postgres:your-password@127.0.0.1:5432/postgres # LEDGER_FILE= # SECRETS_KEY=cmVwbGFjZS13aXRoLW9wZW5zc2wtcmFuZC1iYXNlNjQ= diff --git a/auto-runtime.example.yaml b/auto-runtime.example.yaml index adfa76435..69b585be4 100644 --- a/auto-runtime.example.yaml +++ b/auto-runtime.example.yaml @@ -10,8 +10,9 @@ # write ${NAME} where one belongs and set NAME in .env or in the container. # DATABASE_URL, the PostgreSQL database of the ledger, holds a password, and # SECRETS_KEY is the key the secrets kept in the ledger are sealed under, so both -# are read from the environment only and have no key here. SECRETS_KEY_FILE is an -# environment variable too, which names a file that holds that key instead. +# are read from the environment only and have no key here. SECRETS_KEY_FILE and +# SECRETS_KEY_KMS are environment variables too, which name a file that holds that +# key and a key in AWS KMS that wraps the key of each secret instead. # OpenAI-compatible gateways; a reasoning function then names gateway/. model_gateways: diff --git a/docs/self-host/configuration.md b/docs/self-host/configuration.md index 46ee7d5c2..f6c2b3b3b 100644 --- a/docs/self-host/configuration.md +++ b/docs/self-host/configuration.md @@ -68,6 +68,7 @@ Inside an argument, only the value is judged, never the name the argument gives | An outbound proxy or a private certificate authority | `NODE_USE_ENV_PROXY=1`, `HTTPS_PROXY` and `NODE_EXTRA_CA_CERTS` | [Proxy and CA](models.md#behind-an-outbound-proxy-with-a-private-certificate-authority) | | The ledger in Amazon RDS without a database password | `DATABASE_URL` without a password and `DATABASE_CREDENTIALS=aws-iam` | [Without a password](postgresql.md#without-a-password-on-amazon-rds) | | Secrets sealed under a key of your own | `SECRETS_KEY` or `SECRETS_KEY_FILE`, and `SECRETS_KEY_PREVIOUS` or `SECRETS_KEY_FILE_PREVIOUS` while it replaces another | [Secrets at rest](security.md#secrets-at-rest) | +| Secrets sealed under a key in AWS KMS | `SECRETS_KEY_KMS`, the ARN of the key or of its alias, and `SECRETS_KEY_KMS_PREVIOUS` while it replaces another | [Hold the secrets key in a KMS](production.md#hold-the-secrets-key-in-a-kms) | | Content kept in an object store instead of the ledger | `CONTENT_STORE_URL`, an `s3://bucket/prefix` URL | [Object storage](object-storage.md) | | Tools that functions may call | `tool_servers` in the file, each entry's `allowed` to allow only some of its tools | [Tool servers](#tool-servers) | | People who sign in through the Auto identity service | `PUBLIC_URL`, which turns sign-in on, and `IDENTITY_ISSUER` for a development instance of the service | [Let people sign in](security.md#let-people-sign-in) | @@ -115,16 +116,18 @@ Every setting is an environment variable. The tables below group them by area. ### Storage -| Variable | Default | Purpose | -| --------------------------- | ----------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `LEDGER_FILE` | `data/ledger.db`, which in the image is on the `/data` volume; `~/.auto-runtime/ledger.db` with `npx` | The SQLite file of the ledger, absolute or relative to the working directory. Its folder is created when missing | -| `DATABASE_URL` | none | A PostgreSQL URL. When set, the ledger is kept in that database instead, and setting `LEDGER_FILE` too stops the server at start. Environment only, since it holds a password. See [Use PostgreSQL](postgresql.md) | -| `DATABASE_CREDENTIALS` | `url` | How the server signs in to that database: `url`, with the password the URL holds, or `aws-iam`, with a token signed for each connection. See [Without a password, on Amazon RDS](postgresql.md#without-a-password-on-amazon-rds) | -| `SECRETS_KEY` | none | The key the secrets kept in the ledger's database are sealed under: the base64 of 32 random bytes, as `openssl rand -base64 32` prints it. Set it or `SECRETS_KEY_FILE`, not both; a ledger in PostgreSQL needs one of them. The keys the runtime keeps for itself are among those secrets, such as the one that signs the state an MCP client sends back when it retries a call, so a server whose key cannot read them does not start. Environment only. See [Secrets at rest](security.md#secrets-at-rest) | -| `SECRETS_KEY_FILE` | `secrets.key` beside a SQLite ledger file, made at the first start | A file that holds the same key, such as a secret your platform mounts into the container. A file named here is read as the platform mounts it, and the file the server makes is its owner's alone. A file named here is never made; one that is missing or that the server cannot read, one that holds anything but the key, and a default file that others than its owner may read stop the start. Without either setting, a SQLite ledger keeps its key in the default file. See [Keep the secrets key](production.md#keep-the-secrets-key) | -| `SECRETS_KEY_PREVIOUS` | none | The key that `SECRETS_KEY` or `SECRETS_KEY_FILE` replaces, in the same form and only beside one of them. Set it or `SECRETS_KEY_FILE_PREVIOUS`, not both. At start the server unwraps with it the key of each secret it wrapped and wraps that again under the current key, after which it can go. See [Rotate the secrets key](production.md#rotate-the-secrets-key) | -| `SECRETS_KEY_FILE_PREVIOUS` | none | A file that holds the key being replaced, read as a file `SECRETS_KEY_FILE` names is, and otherwise as `SECRETS_KEY_PREVIOUS` | -| `CONTENT_STORE_URL` | none | An `s3://bucket/prefix` URL. When set, content is kept in that bucket instead of the ledger's database. Content kept before a change is not moved; see [Keep content on object storage](object-storage.md#switch-a-deployment-that-already-has-content) | +| Variable | Default | Purpose | +| --------------------------- | ----------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `LEDGER_FILE` | `data/ledger.db`, which in the image is on the `/data` volume; `~/.auto-runtime/ledger.db` with `npx` | The SQLite file of the ledger, absolute or relative to the working directory. Its folder is created when missing | +| `DATABASE_URL` | none | A PostgreSQL URL. When set, the ledger is kept in that database instead, and setting `LEDGER_FILE` too stops the server at start. Environment only, since it holds a password. See [Use PostgreSQL](postgresql.md) | +| `DATABASE_CREDENTIALS` | `url` | How the server signs in to that database: `url`, with the password the URL holds, or `aws-iam`, with a token signed for each connection. See [Without a password, on Amazon RDS](postgresql.md#without-a-password-on-amazon-rds) | +| `SECRETS_KEY` | none | The key the secrets kept in the ledger's database are sealed under: the base64 of 32 random bytes, as `openssl rand -base64 32` prints it. Set one of it, `SECRETS_KEY_FILE` and `SECRETS_KEY_KMS`; a ledger in PostgreSQL needs one of them. The keys the runtime keeps for itself are among those secrets, such as the one that signs the state an MCP client sends back when it retries a call, so a server whose key cannot read them does not start. Environment only. See [Secrets at rest](security.md#secrets-at-rest) | +| `SECRETS_KEY_FILE` | `secrets.key` beside a SQLite ledger file, made at the first start | A file that holds the same key, such as a secret your platform mounts into the container. A file named here is read as the platform mounts it, and the file the server makes is its owner's alone. A file named here is never made; one that is missing or that the server cannot read, one that holds anything but the key, and a default file that others than its owner may read stop the start. Without it, `SECRETS_KEY` or `SECRETS_KEY_KMS`, a SQLite ledger keeps its key in the default file. See [Keep the secrets key](production.md#keep-the-secrets-key) | +| `SECRETS_KEY_KMS` | none | The ARN of a symmetric AWS KMS key, `arn:aws:kms:::key/`, or of an alias of one, `arn:aws:kms:::alias/`, which wraps the key of each secret in place of a key in hand; the region is the ARN's, and the credentials the AWS default credential chain's. The identity needs `kms:Encrypt` and `kms:Decrypt` on the key, and `kms:DescribeKey` where the setting names an alias, which the server resolves to its key at start; a key that cannot wrap and unwrap at start stops the start. See [Hold the secrets key in a KMS](production.md#hold-the-secrets-key-in-a-kms) | +| `SECRETS_KEY_PREVIOUS` | none | The key that `SECRETS_KEY`, `SECRETS_KEY_FILE` or `SECRETS_KEY_KMS` replaces, in the same form as `SECRETS_KEY` and only beside one of them. Set one of it, `SECRETS_KEY_FILE_PREVIOUS` and `SECRETS_KEY_KMS_PREVIOUS`. At start the server unwraps with it the key of each secret it wrapped and wraps that again under the current key, after which it can go. See [Rotate the secrets key](production.md#rotate-the-secrets-key) | +| `SECRETS_KEY_FILE_PREVIOUS` | none | A file that holds the key being replaced, read as a file `SECRETS_KEY_FILE` names is, and otherwise as `SECRETS_KEY_PREVIOUS` | +| `SECRETS_KEY_KMS_PREVIOUS` | none | The ARN of the key in AWS KMS being replaced, in the form of `SECRETS_KEY_KMS`, and otherwise as `SECRETS_KEY_PREVIOUS`; the identity needs `kms:Decrypt` on it. See [Hold the secrets key in a KMS](production.md#hold-the-secrets-key-in-a-kms) | +| `CONTENT_STORE_URL` | none | An `s3://bucket/prefix` URL. When set, content is kept in that bucket instead of the ledger's database. Content kept before a change is not moved; see [Keep content on object storage](object-storage.md#switch-a-deployment-that-already-has-content) | ### Models diff --git a/docs/self-host/connections.md b/docs/self-host/connections.md index 1b770680e..ba6eb3151 100644 --- a/docs/self-host/connections.md +++ b/docs/self-host/connections.md @@ -159,7 +159,7 @@ Auto checks an entry as it checks an entry of the settings file, and refuses one A `secret` or a `reference` sent in a header may name its `scheme`, such as `Bearer`: Auto writes the scheme and a space before the value, so the person pastes the token alone, the variable you set for a reference holds the token alone, and the server receives `Authorization: Bearer `. Auto refuses a token pasted with the scheme before it, saying to paste the token alone. -Auto seals a secret under its secret key, `SECRETS_KEY`, or the file `SECRETS_KEY_FILE` names, by default `secrets.key` beside a SQLite ledger, as [the configuration reference](configuration.md#storage) sets out. No record of the org, no answer and no history holds the value, and the link's token is kept only as its SHA-256. Of a secret or a client secret of twelve characters or more, Auto also keeps its last four characters beside the sealed value in the secret store, never in the org's record, and `list_connections` and `get_connection` answer them as `credential_ends_with`, so the person can tell which key a connection holds; a private key, or a shorter value, has none. Removing a connection forgets its secret, and the ending with it, and first revokes at the provider the tokens of a connection connected by consent. +Auto seals a secret under its secret key, `SECRETS_KEY`, the file `SECRETS_KEY_FILE` names, by default `secrets.key` beside a SQLite ledger, or a key in AWS KMS that `SECRETS_KEY_KMS` names, as [the configuration reference](configuration.md#storage) sets out. No record of the org, no answer and no history holds the value, and the link's token is kept only as its SHA-256. Of a secret or a client secret of twelve characters or more, Auto also keeps its last four characters beside the sealed value in the secret store, never in the org's record, and `list_connections` and `get_connection` answer them as `credential_ends_with`, so the person can tell which key a connection holds; a private key, or a shorter value, has none. Removing a connection forgets its secret, and the ending with it, and first revokes at the provider the tokens of a connection connected by consent. Every value of a credential is a secret Auto scrubs from what it records and answers, in each form it may take, as it scrubs the value of a reference of the settings file. A value in a header that is a scheme and a credential, such as `Bearer `, is scrubbed whole and its credential alone; where the credential names its `scheme`, the value is the token, scrubbed whole. diff --git a/docs/self-host/docker.md b/docs/self-host/docker.md index 018341179..edeef7af9 100644 --- a/docs/self-host/docker.md +++ b/docs/self-host/docker.md @@ -82,7 +82,7 @@ What the script does: - `--volume auto-runtime-data:/data` keeps the ledger on the named volume when the container is replaced. Without it, Docker gives each container an empty anonymous volume, and its playbooks last only as long as that container. - `--env-file auto-runtime.env` passes the settings and secrets. Docker reads the file when it creates the container, so `docker restart auto-runtime` does not pick up a change to it. Run `./start-runtime.sh` instead. -`SECRETS_KEY` stays out of the volume, so a copy of the volume holds no secret anyone can read without it. Without `SECRETS_KEY` or `SECRETS_KEY_FILE`, Auto would make a key itself, in `secrets.key` on the volume beside the ledger, where a copy of the volume would carry it. Keep a copy of the key with your other secrets: a secret sealed under a lost key cannot be read again. Where there is no `openssl`, `. ./.env && docker run --rm "$IMAGE" node --print "require('node:crypto').randomBytes(32).toString('base64')"` makes one too. +`SECRETS_KEY` stays out of the volume, so a copy of the volume holds no secret anyone can read without it. Without `SECRETS_KEY`, `SECRETS_KEY_FILE` or `SECRETS_KEY_KMS`, Auto would make a key itself, in `secrets.key` on the volume beside the ledger, where a copy of the volume would carry it. Keep a copy of the key with your other secrets: a secret sealed under a lost key cannot be read again. Where there is no `openssl`, `. ./.env && docker run --rm "$IMAGE" node --print "require('node:crypto').randomBytes(32).toString('base64')"` makes one too. ### On your own machine, without sign-in {#on-your-own-machine} @@ -172,7 +172,7 @@ In this variant the database holds the ledger, and Auto's container keeps nothin echo "SECRETS_KEY=$(openssl rand -base64 32)" >> auto-runtime.env ``` - Auto takes `SECRETS_KEY` from `auto-runtime.env`, as in [step 1](#start-the-server). A ledger in PostgreSQL needs it, or `SECRETS_KEY_FILE` naming a file that holds it, and Auto stops at start without either. + Auto takes `SECRETS_KEY` from `auto-runtime.env`, as in [step 1](#start-the-server). A ledger in PostgreSQL needs it, `SECRETS_KEY_FILE` naming a file that holds it, or `SECRETS_KEY_KMS` naming a key in AWS KMS, as [Hold the secrets key in a KMS](production.md#hold-the-secrets-key-in-a-kms) sets out, and Auto stops at start without one. 3. Save this as `compose.yaml` beside `auto-runtime.env`: diff --git a/docs/self-host/postgresql.md b/docs/self-host/postgresql.md index 35ef95318..93e2bc434 100644 --- a/docs/self-host/postgresql.md +++ b/docs/self-host/postgresql.md @@ -52,7 +52,7 @@ With `DATABASE_URL` the container needs no volume. At start: - The server logs that the ledger is kept in PostgreSQL, naming the database and its host, never the URL. - A database it cannot reach stops it, naming the host and port. - Setting `LEDGER_FILE` as well stops it, since the ledger can be in only one place. -- Without `SECRETS_KEY` or `SECRETS_KEY_FILE` it stops, naming the setting. +- Without `SECRETS_KEY`, `SECRETS_KEY_FILE` or `SECRETS_KEY_KMS` it stops, naming the setting. `SECRETS_KEY` is the key the secrets kept in the database are sealed under. Every server that shares the database takes the same key, as [Keep the secrets key](production.md#keep-the-secrets-key) describes. diff --git a/docs/self-host/production.md b/docs/self-host/production.md index 01bfb6434..866feffa8 100644 --- a/docs/self-host/production.md +++ b/docs/self-host/production.md @@ -56,7 +56,7 @@ Restrict access to the database, and to its backups, accordingly. ## Keep the secrets key -The runtime seals the secrets it keeps in the ledger's database under the secrets key, as [Secrets at rest](security.md#secrets-at-rest) describes. A container takes it from your platform's secret store, as it does the other secrets: in `SECRETS_KEY`, or in a file the platform mounts into the container, whose path `SECRETS_KEY_FILE` names. A file a setting names is read as the platform mounts it, and the file the server makes beside a SQLite ledger is its owner's alone. Either way the key is on no disk the ledger is on and in no backup of it; a ledger in PostgreSQL cannot start without one of them, and every server that shares the database takes the same key. Make one with either command, and keep a copy where your other secrets are kept. A secret sealed under a lost key cannot be read again, and the runtime keeps a key of its own under it, so a server started with another key does not start on that ledger: +The runtime seals the secrets it keeps in the ledger's database under the secrets key, as [Secrets at rest](security.md#secrets-at-rest) describes. A container takes it from your platform's secret store, as it does the other secrets: in `SECRETS_KEY`, or in a file the platform mounts into the container, whose path `SECRETS_KEY_FILE` names; or the key stays in AWS KMS, as [Hold the secrets key in a KMS](#hold-the-secrets-key-in-a-kms) sets out. A file a setting names is read as the platform mounts it, and the file the server makes beside a SQLite ledger is its owner's alone. Either way the key is on no disk the ledger is on and in no backup of it; a ledger in PostgreSQL cannot start without one of them, and every server that shares the database takes the same key. Make one with either command, and keep a copy where your other secrets are kept. A secret sealed under a lost key cannot be read again, and the runtime keeps a key of its own under it, so a server started with another key does not start on that ledger: ```bash openssl rand -base64 32 @@ -67,6 +67,46 @@ docker run --rm docker.on.auto/onauto/runtime:0.9.1 node --print "require('node: To replace the key, give the old key to `SECRETS_KEY_PREVIOUS`, or the path of its file to `SECRETS_KEY_FILE_PREVIOUS`, the new key to `SECRETS_KEY` or the path of its file to `SECRETS_KEY_FILE`, and restart the server. At start it unwraps with the old key the key of each secret the old key wrapped, wraps it again under the new one, leaving the secrets themselves as they are, and logs `Secrets are kept in the ledger's database, sealed under the key that SECRETS_KEY holds, and sealed under the previous key were sealed again under it`, or `that the file holds` for a key in a file, where the id is the new key's, the first 16 hex characters of its SHA-256. Then remove the previous key's setting and restart it again: nothing is sealed under the old key any more. With several servers, stop them all and start them with both keys, as for an [upgrade](#upgrades), since a server that still runs with the old key alone cannot read what the new key seals. +### Hold the secrets key in a KMS + +The secrets key can be a key in AWS KMS instead, which wraps the key of each secret itself, so no process of yours ever holds it and every use of it is an entry in your account's trail. Set `SECRETS_KEY_KMS`, in place of `SECRETS_KEY` and `SECRETS_KEY_FILE`, to the ARN of a symmetric KMS key, or of an alias of one: + +```bash +SECRETS_KEY_KMS=arn:aws:kms:eu-west-1:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab +``` + +```bash +SECRETS_KEY_KMS=arn:aws:kms:eu-west-1:111122223333:alias/auto-runtime +``` + +The runtime asks KMS in the region the ARN names, with the credentials of the AWS default credential chain, the role of the task, the pod or the instance it runs as, as it does for [object storage](object-storage.md), and through your proxy where `NODE_USE_ENV_PROXY=1` is set. Where the setting names an alias, the server asks `DescribeKey` once at start for the key the alias names, and holds to that key until it starts again. It wraps with `Encrypt` and unwraps with `Decrypt`, each under that key and the encryption context `on.auto/runtime: secret data key`, which your trail records beside each request. The identity the container runs as needs those two actions on the key, the key itself where the setting names an alias, and `kms:DescribeKey` on it too where it names an alias: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["kms:Encrypt", "kms:Decrypt"], + "Resource": "arn:aws:kms:eu-west-1:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab" + } + ] +} +``` + +At start, before it reads any secret, the server wraps and unwraps one key of its own with KMS and forgets it, so a key that does not exist, one that is disabled or pending deletion, and one the identity may not use stop the start, naming the setting, the key's id and the answer of KMS and never a credential, as an alias KMS does not know or will not describe does, naming the alias, such as `The secrets settings are invalid. SECRETS_KEY_KMS: The KMS key 1234abcd-12ab-34cd-56ef-1234567890ab it names could not wrap and unwrap a data key at start: The custodian of the key 1234abcd-12ab-34cd-56ef-1234567890ab refused to wrap a data key: DisabledException: key/1234abcd-12ab-34cd-56ef-1234567890ab is disabled.`; a KMS that does not answer within 10 seconds stops it too. Once started, it logs `Secrets are kept in the ledger's database, sealed under the key that the KMS key holds, and 0 sealed under the previous key were sealed again under it`. + +The runtime keeps no key unwrapped: it asks KMS for the key of a secret at each read, when a connection is first used in a server's process and when its tokens are renewed, and waits 10 seconds for the answer. So disabling the key, scheduling its deletion or taking the permission away ends the runtime's reading at the next read KMS refuses: a connection's first read in a process fails the call as `connection_unavailable` in KMS's words, a renewal of its tokens fails the call in the same words, both leave the connection connected, and once the key is enabled again the next call reads it. A process that already uses a connection keeps the credential it read until it stops, and a server does not start while the key is disabled. + +To move secrets from a key in the environment or in a file to a key in KMS, set `SECRETS_KEY_KMS`, give the key it replaces to `SECRETS_KEY_PREVIOUS` or the path of its file to `SECRETS_KEY_FILE_PREVIOUS`, and restart once: at start the server unwraps with the old key the key of each secret, wraps it again with KMS, leaving the secrets themselves as they are, and logs how many, after which the previous setting can go: + +```bash +SECRETS_KEY_KMS=arn:aws:kms:eu-west-1:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab +SECRETS_KEY_FILE_PREVIOUS=/run/secrets/secrets-key +``` + +Moving back is the same with `SECRETS_KEY` or `SECRETS_KEY_FILE` and the ARN in `SECRETS_KEY_KMS_PREVIOUS`, for which the identity needs `kms:Decrypt` alone. To move to another key in KMS, restart once with the new key's ARN, or the alias once you have pointed it at the new key, in `SECRETS_KEY_KMS`, and the old key's ARN in `SECRETS_KEY_KMS_PREVIOUS`. The rotation of a key's material that KMS makes itself, on its schedule or on demand, needs nothing of the runtime, since KMS decrypts with the material that encrypted. + ## Run it as an always-on service These all run inside the server: workflows, their timers and retries, the deliveries of interaction functions, and the folding of recall views. diff --git a/docs/self-host/security.md b/docs/self-host/security.md index f50e0a59f..8ac587250 100644 --- a/docs/self-host/security.md +++ b/docs/self-host/security.md @@ -191,7 +191,7 @@ A web page may call the API from the Studio host, `https://studio.on.auto` unles ## Secrets at rest -The runtime has a store for the secrets it must hold, such as the credentials of a connection, in the ledger's database, apart from the ledger's records and never in the clear. Each secret is a row of its own: its value is sealed as a JWE under a key made for that value alone, which encrypts it with AES-GCM (`A256GCM`), and beside it the row keeps that key wrapped by the secrets key with AES key wrap (`A256KW`), and the id of the secrets key that wrapped it. So neither the database nor a backup of it holds a secret anyone can read without the secrets key, a new secrets key wraps the key of each value again and leaves the values as they are, and the store hands a secret back only as a redacted value, which prints as ``, so none reaches a log by accident. The secrets key is the base64 of 32 random bytes, in `SECRETS_KEY` or in the file `SECRETS_KEY_FILE` names, such as a secret your platform mounts into the container, and never in both. With a ledger in SQLite and neither set, the server makes a key at its first start, in `secrets.key` beside the ledger file with the mode `0600`, and reads it at every later start; a file `SECRETS_KEY_FILE` names is read and never made. A file a setting names is read as the platform mounts it, and the file the server makes is its owner's alone. A key file a setting names that is missing, a key file the server cannot read or that holds anything but such a key, and a file the server made that others than its owner may read, stop the start, naming the setting and the file. The file beside the ledger keeps a copy of the database unreadable, but not a copy of the folder that holds both, so a container [takes the key](production.md#keep-the-secrets-key) from its platform's secret store. A ledger in PostgreSQL, which several runtimes may share, needs `SECRETS_KEY` or `SECRETS_KEY_FILE`, and the server stops at start without either. At start the server logs where the key is and its id, the first 16 hex characters of the key's SHA-256, and never the key. The store also keeps the keys the runtime keeps for itself, each named in the ledger by a record of the runtime's own: the first is the key that signs the state an MCP client sends back when it retries a call, 32 random bytes the first server of a ledger makes, so every server that shares the ledger answers a retry another asked for. A server whose secrets key cannot read that key does not start: it names the key it was given and why, and says to start it with the key the ledger's secrets were sealed under, or to give that key as `SECRETS_KEY_PREVIOUS` or `SECRETS_KEY_FILE_PREVIOUS` beside a new one. A secret sealed under a lost key cannot be read again by anyone, the runtime included, and a ledger whose key is lost for good can no longer be started, so keep a copy of the key with your other secrets, apart from the backups of the ledger; [Rotate the secrets key](production.md#rotate-the-secrets-key) replaces it, through `SECRETS_KEY_PREVIOUS` or `SECRETS_KEY_FILE_PREVIOUS`. +The runtime has a store for the secrets it must hold, such as the credentials of a connection, in the ledger's database, apart from the ledger's records and never in the clear. Each secret is a row of its own: its value is sealed as a JWE under a key made for that value alone, which encrypts it with AES-GCM (`A256GCM`), and beside it the row keeps that key wrapped by the secrets key with AES key wrap (`A256KW`), or by KMS itself for a key in KMS, and the id of the secrets key that wrapped it. So neither the database nor a backup of it holds a secret anyone can read without the secrets key, a new secrets key wraps the key of each value again and leaves the values as they are, and the store hands a secret back only as a redacted value, which prints as ``, so none reaches a log by accident. The secrets key is the base64 of 32 random bytes, in `SECRETS_KEY` or in the file `SECRETS_KEY_FILE` names, such as a secret your platform mounts into the container, and never in both; or it is a key in AWS KMS that `SECRETS_KEY_KMS` names, which wraps the key of each value itself at each keep and unwraps it at each read, so no process of yours holds it and disabling the key ends the runtime's reading at its next read, as [Hold the secrets key in a KMS](production.md#hold-the-secrets-key-in-a-kms) sets out. With a ledger in SQLite and none of the three set, the server makes a key at its first start, in `secrets.key` beside the ledger file with the mode `0600`, and reads it at every later start; a file `SECRETS_KEY_FILE` names is read and never made. A file a setting names is read as the platform mounts it, and the file the server makes is its owner's alone. A key file a setting names that is missing, a key file the server cannot read or that holds anything but such a key, and a file the server made that others than its owner may read, stop the start, naming the setting and the file. The file beside the ledger keeps a copy of the database unreadable, but not a copy of the folder that holds both, so a container [takes the key](production.md#keep-the-secrets-key) from its platform's secret store. A ledger in PostgreSQL, which several runtimes may share, needs `SECRETS_KEY`, `SECRETS_KEY_FILE` or `SECRETS_KEY_KMS`, and the server stops at start without one. At start the server logs where the key is and its id, the first 16 hex characters of the key's SHA-256 or the id of the key in KMS, and never the key. The store also keeps the keys the runtime keeps for itself, each named in the ledger by a record of the runtime's own: the first is the key that signs the state an MCP client sends back when it retries a call, 32 random bytes the first server of a ledger makes, so every server that shares the ledger answers a retry another asked for. A server whose secrets key cannot read that key does not start: it names the key it was given and why, and says to start it with the key the ledger's secrets were sealed under, or to give that key as `SECRETS_KEY_PREVIOUS`, `SECRETS_KEY_FILE_PREVIOUS` or `SECRETS_KEY_KMS_PREVIOUS` beside a new one. A secret sealed under a lost key cannot be read again by anyone, the runtime included, and a ledger whose key is lost for good can no longer be started, so keep a copy of the key with your other secrets, apart from the backups of the ledger; [Rotate the secrets key](production.md#rotate-the-secrets-key) replaces it, through `SECRETS_KEY_PREVIOUS` or `SECRETS_KEY_FILE_PREVIOUS`. ## The page at the root diff --git a/packages/outbound/README.md b/packages/outbound/README.md index c940c400a..9fb846e00 100644 --- a/packages/outbound/README.md +++ b/packages/outbound/README.md @@ -1,6 +1,6 @@ # @on.auto/outbound -How the server calls systems on the internet. It walks the causes of a failed call, and tells a certificate the server does not trust from any other failure of a call: the reasoning function adapter and the tool-servers package take the classification from here, so a model provider and a tool server whose certificate the server does not trust are named alike; every caller keeps its own client and bounds. It names the runtime in the `User-Agent` of each request a `fetch` it wraps sends, which the server gives its tool servers and the Studio host it asks. And it makes the settings of a client of the AWS SDK, which the Bedrock provider of the reasoning capability and the object store of [`@on.auto/object-storage`](../object-storage/README.md) share, and signs the tokens the server signs in to the ledger's database with when `DATABASE_CREDENTIALS=aws-iam`. +How the server calls systems on the internet. It walks the causes of a failed call, and tells a certificate the server does not trust from any other failure of a call: the reasoning function adapter and the tool-servers package take the classification from here, so a model provider and a tool server whose certificate the server does not trust are named alike; every caller keeps its own client and bounds. It names the runtime in the `User-Agent` of each request a `fetch` it wraps sends, which the server gives its tool servers and the Studio host it asks. And it makes the settings of a client of the AWS SDK, which the Bedrock provider of the reasoning capability, the object store of [`@on.auto/object-storage`](../object-storage/README.md) and the custodian of a key in KMS of [`@on.auto/secrets/aws`](../secrets/README.md#the-custodian) share, and signs the tokens the server signs in to the ledger's database with when `DATABASE_CREDENTIALS=aws-iam`. ## The causes of a failure @@ -20,7 +20,7 @@ How the server calls systems on the internet. It walks the causes of a failed ca ## AWS clients -`proxySettingsOf(environment)`, in the main entry, reads `ProxySettings`, `{ enabled, environment }`: the switch, `NODE_USE_ENV_PROXY=1`, and the proxy's variables that are set, `HTTPS_PROXY`, `HTTP_PROXY` and `NO_PROXY` in either case. The server reads them once and hands them to every client it makes, the model providers' and the object store's. +`proxySettingsOf(environment)`, in the main entry, reads `ProxySettings`, `{ enabled, environment }`: the switch, `NODE_USE_ENV_PROXY=1`, and the proxy's variables that are set, `HTTPS_PROXY`, `HTTP_PROXY` and `NO_PROXY` in either case. The server reads them once and hands them to every client it makes, the model providers', the object store's and the KMS custodian's. `@on.auto/outbound/aws` (`src/aws/aws-clients.ts`) is a subpath of its own, so that the packages that take the classification of certificates alone, the tool-servers package among them, load no part of the AWS SDK. `awsCredentialChain(proxy)` is the default credential chain of the AWS SDK, `fromNodeProviderChain` of `@aws-sdk/credential-providers`, which resolves credentials from the environment, the shared files, single sign-on, a process, a web identity, the container or the instance, looking nothing up until a call needs them. `awsClientSettingsOf(proxy)` is the part of a client's settings that sends its requests through the environment proxy, `HTTPS_PROXY`, `HTTP_PROXY` and `NO_PROXY`, when Node is told to with `NODE_USE_ENV_PROXY=1`, by the `proxyEnv` of the agents of its request handler, since a client of the AWS SDK makes agents of its own that the switch alone does not reach; without the switch it is empty. The chain's own clients take the same settings, by HTTP as well as by HTTPS: the agent of each carries the proxy's variables, so a credential endpoint reached by HTTP, such as a container's, goes through the proxy too unless `NO_PROXY` names it, where the reasoning capability's chain once set the HTTPS agent alone. diff --git a/packages/server/README.md b/packages/server/README.md index 6da5716be..f26568c57 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -2,11 +2,15 @@ The Node.js server of Auto Runtime: `src/main.ts` starts it on the composition root of `src/composition/composition-root.ts`, `dev.ts` restarts it on every save, and `auto-runtime.ts` is the command of the npm package `@on.auto/runtime`, which `npm/pack.ts` packs with the `README.md` in `npm/`; the `Dockerfile` builds its container image. +## The secrets key + +`src/settings/secrets-settings.ts` reads where the key that wraps the data key of each secret is held: in `SECRETS_KEY`, in the file `SECRETS_KEY_FILE` names, by default `secrets.key` beside a SQLite ledger, or in AWS KMS, under the ARN of a key or of an alias that `SECRETS_KEY_KMS` names, whose region is the ARN's; one of the three at most, and at most one key being replaced beside it, `SECRETS_KEY_PREVIOUS`, `SECRETS_KEY_FILE_PREVIOUS` or `SECRETS_KEY_KMS_PREVIOUS`. `src/database/secret-store.ts` makes a custodian of each, `localCustodian` of [`@on.auto/secrets`](../secrets/README.md#the-custodian) for a key in hand and, through `src/database/kms-custodians.ts`, its `kmsCustodian` for a key in KMS, with the proxy settings and `awsCredentialChain(proxy)` of `@on.auto/outbound/aws`, loaded through `import('@on.auto/secrets/aws')` only where a setting names a key in KMS, so a server without one loads no part of `@aws-sdk/client-kms`, as `src/database/kms-custodian-loading.test.ts` holds by the modules its process has loaded. The custodian of an alias resolves it to the key it names with one `DescribeKey` and is pinned to that key until the next start, and a resolution that fails stops the start, `The secrets settings are invalid. SECRETS_KEY_KMS: The KMS alias alias/ it names could not be resolved to its key at start: `. `src/database/ledger-stores.ts` probes a current key in KMS before it opens the rows, one data key wrapped, unwrapped and filled with zeros, and a probe that fails stops the start, `The secrets settings are invalid. SECRETS_KEY_KMS: The KMS key it names could not wrap and unwrap a data key at start: `; it then rotates and logs where the key is held, `… sealed under the key that the KMS key holds, and sealed under the previous key were sealed again under it`. + ## The runtime's own keys The runtime keeps keys of its own in its secret store, beside the secrets it keeps for others, and names each by a fact in a stream of the runtime's and of no org, `runtime/own-keys`, as `@on.auto/connections` names the clients it registers dynamically in `runtime/client-registrations` (`src/database/own-keys.ts`, beside `ledger-stores.ts`, which opens, rotates and logs the store the key is read from right after). The one fact, `own_key_kept { purpose, secret_id }`, by the runtime, `by: runtime`, is folded by purpose, the latest last, and the decider, `ownKeysDecider`, refuses a second for a purpose that has one, `conflict` of the kind `taken`, `The runtime keeps a key for retry_state already`. The one purpose is `retry_state`, the key that signs the state an MCP client sends back when it retries a call, as [`@on.auto/api`](../api/README.md) says. -At start, once the secret store is open and rotated, `retryStateKeysOf` folds the stream. Where no key is recorded for the purpose it keeps 32 random bytes in the store, as their base64, and records the fact, which the ledger appends under the version of the stream it read; where the record is refused because another server recorded one first, it forgets the row it kept and folds again, taking that one. It then reads the row and gives the API `retryStateKeys` with that key as `current` and no `previous`, so every server that shares the ledger signs under the same key, and a rotation of the secrets key wraps it again and leaves it as it is. A row it cannot read stops the start as an `OwnKeyUnavailable`, named as the server's other start failures are, in words that name the purpose, the key in force as the start's log names it, the store's words and the two ways out: `The key the runtime keeps for retry_state cannot be read under the key that SECRETS_KEY holds: . Start the server with the key the ledger's secrets were sealed under, or give that key as SECRETS_KEY_PREVIOUS or SECRETS_KEY_FILE_PREVIOUS beside a new key`. A row that is gone, `… is no longer in the secret store, so the server does not start`, one that holds something other than 32 bytes, and a key the store cannot keep at the first start, `… cannot be kept under the key that SECRETS_KEY holds, so the server does not start: `, stop it too; the store is closed before the start fails. A deliberate reset of the runtime's own keys is not offered yet. Nothing of the key is logged, and no key is derived from another. +At start, once the secret store is open and rotated, `retryStateKeysOf` folds the stream. Where no key is recorded for the purpose it keeps 32 random bytes in the store, as their base64, and records the fact, which the ledger appends under the version of the stream it read; where the record is refused because another server recorded one first, it forgets the row it kept and folds again, taking that one. It then reads the row and gives the API `retryStateKeys` with that key as `current` and no `previous`, so every server that shares the ledger signs under the same key, and a rotation of the secrets key wraps it again and leaves it as it is. A row it cannot read stops the start as an `OwnKeyUnavailable`, named as the server's other start failures are, in words that name the purpose, the key in force as the start's log names it, the store's words and the two ways out: `The key the runtime keeps for retry_state cannot be read under the key that SECRETS_KEY holds: . Start the server with the key the ledger's secrets were sealed under, or give that key as SECRETS_KEY_PREVIOUS, SECRETS_KEY_FILE_PREVIOUS or SECRETS_KEY_KMS_PREVIOUS beside a new key`. A row that is gone, `… is no longer in the secret store, so the server does not start`, one that holds something other than 32 bytes, and a key the store cannot keep at the first start, `… cannot be kept under the key that SECRETS_KEY holds, so the server does not start: `, stop it too; the store is closed before the start fails. A deliberate reset of the runtime's own keys is not offered yet. Nothing of the key is logged, and no key is derived from another. ## Testing diff --git a/packages/server/npm/README.md b/packages/server/npm/README.md index 534af32eb..f34d98cab 100644 --- a/packages/server/npm/README.md +++ b/packages/server/npm/README.md @@ -43,7 +43,7 @@ The command shows only the server's errors; set `LOG_LEVEL=warn` to see its warn Each time it starts, Auto Runtime sends one anonymous usage ping: its version, the operating system, the processor, the version of Node.js, how it was installed and a random id for this copy, kept in `~/.auto-runtime/install-id`, never anything of your work or your keys. `DO_NOT_TRACK=1` or `USAGE_PING=false` turns it off. [Usage ping](https://on.auto/docs/self-host/configuration#usage-ping) says exactly what it sends. -The ledger, where playbooks and their runs are kept, is the SQLite file `~/.auto-runtime/ledger.db` unless `LEDGER_FILE` names another file or `DATABASE_URL` names a PostgreSQL database. The secrets it keeps are sealed under the key in `secrets.key` beside the ledger file, which the command makes at its first start, unless `SECRETS_KEY` gives one or `SECRETS_KEY_FILE` names another file, as a ledger in PostgreSQL needs. `PORT` changes the port. +The ledger, where playbooks and their runs are kept, is the SQLite file `~/.auto-runtime/ledger.db` unless `LEDGER_FILE` names another file or `DATABASE_URL` names a PostgreSQL database. The secrets it keeps are sealed under the key in `secrets.key` beside the ledger file, which the command makes at its first start, unless `SECRETS_KEY` gives one, `SECRETS_KEY_FILE` names another file or `SECRETS_KEY_KMS` names a key in AWS KMS, as a ledger in PostgreSQL needs. `PORT` changes the port. ## Commands diff --git a/packages/server/src/connections/connections-over-mcp.test.ts b/packages/server/src/connections/connections-over-mcp.test.ts index db96717ca..b0bbd3902 100644 --- a/packages/server/src/connections/connections-over-mcp.test.ts +++ b/packages/server/src/connections/connections-over-mcp.test.ts @@ -181,7 +181,7 @@ describe('a server whose secrets key cannot read the key the runtime keeps for i ); await expect(started).rejects.toThrow( - `The key the runtime keeps for retry_state cannot be read under the key that SECRETS_KEY holds: The secret is sealed under the key ${sha256Of(secretsKey).slice(0, 16)}, which is neither the current key nor the previous one. Start the server with the key the ledger's secrets were sealed under, or give that key as SECRETS_KEY_PREVIOUS or SECRETS_KEY_FILE_PREVIOUS beside a new key`, + `The key the runtime keeps for retry_state cannot be read under the key that SECRETS_KEY holds: The secret is sealed under the key ${sha256Of(secretsKey).slice(0, 16)}, which is neither the current key nor the previous one. Start the server with the key the ledger's secrets were sealed under, or give that key as SECRETS_KEY_PREVIOUS, SECRETS_KEY_FILE_PREVIOUS or SECRETS_KEY_KMS_PREVIOUS beside a new key`, ); }); }); diff --git a/packages/server/src/connections/connections-under-kms.test.ts b/packages/server/src/connections/connections-under-kms.test.ts new file mode 100644 index 000000000..8d8f6f0fd --- /dev/null +++ b/packages/server/src/connections/connections-under-kms.test.ts @@ -0,0 +1,71 @@ +import { randomUUID } from 'node:crypto'; + +import { serveFakeKms } from '@on.auto/secrets/testing'; +import { serveFakeMcp } from '@on.auto/tool-servers/testing'; +import { afterAll, describe, expect, it, onTestFinished, vi } from 'vitest'; + +import { aSecretHeader, connectedWith } from '../testing/servers/connected-servers.ts'; +import { alpha, servingReasoning, type ReasoningServer } from '../testing/servers/reasoning-server.ts'; + +const keyId = randomUUID(); + +const keyArn = `arn:aws:kms:eu-west-1:111122223333:key/${keyId}`; + +const fake = await serveFakeKms({ keys: [keyArn] }); + +afterAll(fake.close); + +const crmKey = 'crm-api-key-5d1c7e20'; + +function kmsOfTheEnvironmentAtTheFake(): void { + vi.stubEnv('AWS_ENDPOINT_URL_KMS', fake.url); + vi.stubEnv('AWS_ACCESS_KEY_ID', 'an-access-key-of-the-tests'); + vi.stubEnv('AWS_SECRET_ACCESS_KEY', 'a-secret-access-key-of-the-tests'); + onTestFinished(() => { + vi.unstubAllEnvs(); + }); +} + +async function servedUnderTheKeyInKms(): Promise { + kmsOfTheEnvironmentAtTheFake(); + const server = await servingReasoning([], { SECRETS_KEY_KMS: keyArn }); + onTestFinished(server.stop); + await server.call('POST', '/v1/orgs/acme/playbooks', { body: { playbook: 'alpha', name: 'Alpha' } }); + return server; +} + +function executed(server: ReasoningServer) { + return server.call('POST', `${alpha}/tool-servers/crm/tools/echo/execute`, { body: { arguments: {} } }); +} + +describe('a connection whose credential is sealed under a key in KMS that is disabled meanwhile', () => { + it('fails its call as connection_unavailable in the words of KMS, stays connected, and is called once the key is enabled again', async () => { + const crm = await serveFakeMcp({ bearer: crmKey }); + onTestFinished(crm.close); + const server = await servedUnderTheKeyInKms(); + const id = await connectedWith(server, aSecretHeader('crm', crm.url), `Bearer ${crmKey}`); + + fake.tell(keyArn, 'disabled'); + onTestFinished(() => { + fake.tell(keyArn, 'enabled'); + }); + const refused = await executed(server); + const read = await server.call('GET', `/v1/orgs/acme/connections/${id}`); + fake.tell(keyArn, 'enabled'); + const called = await executed(server); + + expect(refused).toMatchObject({ + status: 503, + body: { + kind: 'tool_not_offered', + because: 'connection_unavailable', + detail: `The credential of the connection crm cannot be read now: The custodian of the key ${keyId} refused to unwrap a data key: DisabledException: key/${keyId} is disabled.`, + }, + }); + expect(read.body).toMatchObject({ status: 'connected' }); + expect(called.status).toBe(200); + expect(new Set(fake.calls().map(({ keyId: asked, context }) => `${asked} ${JSON.stringify(context)}`))).toEqual( + new Set([`${keyArn} {"on.auto/runtime":"secret data key"}`]), + ); + }); +}); diff --git a/packages/server/src/database/kms-custodian-loading.test.ts b/packages/server/src/database/kms-custodian-loading.test.ts new file mode 100644 index 000000000..83a0569c2 --- /dev/null +++ b/packages/server/src/database/kms-custodian-loading.test.ts @@ -0,0 +1,56 @@ +import { createRequire } from 'node:module'; + +import { serveFakeKms } from '@on.auto/secrets/testing'; +import { Effect, Logger } from 'effect'; +import { afterAll, describe, expect, it, onTestFinished, vi } from 'vitest'; + +import { readSettings } from '../settings/settings.ts'; +import { temporaryLedger } from '../testing/records/temporary-ledger.ts'; +import { ledgerStoresOpenedBy } from './ledger-stores.ts'; + +const modulesLoaded = createRequire(import.meta.url).cache; + +function modulesOfTheKmsClientLoaded(): readonly string[] { + return Object.keys(modulesLoaded).filter((path) => path.includes('/@aws-sdk/client-kms/')); +} + +const keyArn = `arn:aws:kms:eu-west-1:111122223333:key/${crypto.randomUUID()}`; + +const fake = await serveFakeKms({ keys: [keyArn] }); + +afterAll(fake.close); + +async function openedAndClosed(environment: Readonly>): Promise { + const { fileName, remove } = temporaryLedger(); + onTestFinished(remove); + const context = await Effect.runPromise(Effect.context().pipe(Effect.provide(Logger.layer([])))); + const stores = await ledgerStoresOpenedBy(context, readSettings({ LEDGER_FILE: fileName, ...environment }), { + store: 'sqlite', + file: fileName, + }); + await stores.secrets.close(); +} + +describe('a server that opens its secret store under a key in hand', () => { + it('loads no module of the client of KMS, from the environment or from the file beside the ledger', async () => { + await openedAndClosed({ SECRETS_KEY: Buffer.alloc(32, 5).toString('base64') }); + await openedAndClosed({}); + + expect(modulesOfTheKmsClientLoaded()).toEqual([]); + }); +}); + +describe('a server that opens its secret store under a key in KMS', () => { + it('loads the client of KMS with the custodian of the aws subpath of the secrets package', async () => { + vi.stubEnv('AWS_ENDPOINT_URL_KMS', fake.url); + vi.stubEnv('AWS_ACCESS_KEY_ID', 'an-access-key-of-the-tests'); + vi.stubEnv('AWS_SECRET_ACCESS_KEY', 'a-secret-access-key-of-the-tests'); + onTestFinished(() => { + vi.unstubAllEnvs(); + }); + + await openedAndClosed({ SECRETS_KEY_KMS: keyArn }); + + expect(modulesOfTheKmsClientLoaded()).toContainEqual(expect.stringContaining('/@aws-sdk/client-kms/dist-cjs/')); + }); +}); diff --git a/packages/server/src/database/kms-custodians.ts b/packages/server/src/database/kms-custodians.ts new file mode 100644 index 000000000..6d8829f6c --- /dev/null +++ b/packages/server/src/database/kms-custodians.ts @@ -0,0 +1,43 @@ +import type { ProxySettings } from '@on.auto/outbound'; +import { awsCredentialChain } from '@on.auto/outbound/aws'; +import type { Custodian, KeepingFailure } from '@on.auto/secrets'; +import type { KmsCustodianOptions } from '@on.auto/secrets/aws'; +import { Effect } from 'effect'; + +import { InvalidSettingsError } from '../settings/invalid-settings-error.ts'; +import type { SecretsKeySource } from '../settings/secrets-settings.ts'; + +export interface KmsCustodians { + readonly kmsCustodian: (options: KmsCustodianOptions) => Effect.Effect; +} + +export type KmsCustodiansLoader = () => Promise; + +export const loadKmsCustodians: KmsCustodiansLoader = () => import('@on.auto/secrets/aws'); + +export interface CustodiansMaking { + readonly proxy: ProxySettings; + readonly loadKms: KmsCustodiansLoader; +} + +export type KeyInKms = Extract; + +function unresolved({ setting, arn }: KeyInKms, detail: string): InvalidSettingsError { + const alias = arn.slice(arn.lastIndexOf(':') + 1); + return new InvalidSettingsError({ + message: `The secrets settings are invalid. ${setting}: The KMS alias ${alias} it names could not be resolved to its key at start: ${detail}`, + }); +} + +export async function kmsCustodianOf(source: KeyInKms, { proxy, loadKms }: CustodiansMaking): Promise { + const { kmsCustodian } = await loadKms(); + const made = kmsCustodian({ + keyId: source.arn, + region: source.region, + proxy, + credentials: awsCredentialChain(proxy), + }); + return Effect.runPromise( + Effect.mapError(made, ({ detail }: { readonly detail: string }) => unresolved(source, detail)), + ); +} diff --git a/packages/server/src/database/kms-page.test.ts b/packages/server/src/database/kms-page.test.ts new file mode 100644 index 000000000..1bd4e9a26 --- /dev/null +++ b/packages/server/src/database/kms-page.test.ts @@ -0,0 +1,126 @@ +import { readFileSync } from 'node:fs'; + +import { kmsCustodian, type KmsCustodianOptions } from '@on.auto/secrets/aws'; +import { serveFakeKms } from '@on.auto/secrets/testing'; +import { Effect, Schema } from 'effect'; +import { afterAll, describe, expect, it, onTestFinished } from 'vitest'; + +import { readSettings } from '../settings/settings.ts'; +import { temporaryLedger } from '../testing/records/temporary-ledger.ts'; +import { ledgerStoresOpenedBy } from './ledger-stores.ts'; + +const page = readFileSync(new URL('../../../../docs/self-host/production.md', import.meta.url), 'utf8'); + +const section = page.slice( + page.indexOf('### Hold the secrets key in a KMS'), + page.indexOf('## Run it as an always-on service'), +); + +interface Block { + readonly language: string; + readonly body: string; +} + +const blocks = [...section.matchAll(/```(\w+)\n([\s\S]*?)```/gu)].map( + ([, language = '', body = '']: readonly string[]): Block => ({ language, body }), +); + +function environmentOf(body: string): Readonly> { + return Object.fromEntries( + body + .trim() + .split('\n') + .map((line) => [line.slice(0, line.indexOf('=')), line.slice(line.indexOf('=') + 1)]), + ); +} + +const shellBlocks = blocks + .filter(({ language }: Block) => language === 'bash') + .map(({ body }: Block) => environmentOf(body)); + +const PolicySchema = Schema.fromJsonString( + Schema.Struct({ + Version: Schema.String, + Statement: Schema.Array( + Schema.Struct({ Effect: Schema.String, Action: Schema.Array(Schema.String), Resource: Schema.String }), + ), + }), +); + +const policies = blocks + .filter(({ language }: Block) => language === 'json') + .map(({ body }: Block) => Schema.decodeUnknownSync(PolicySchema)(body)); + +const keyArn = 'arn:aws:kms:eu-west-1:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab'; + +const quotedRefusal = String(/`(The secrets settings are invalid\. SECRETS_KEY_KMS: [^`]+)`/u.exec(section)?.[1]); + +describe('the page that holds the secrets key in a KMS', () => { + it('sets SECRETS_KEY_KMS to the ARN of a key and of an alias, and rotates from a key in a file in one restart, as the settings read them', () => { + expect(shellBlocks.map((environment) => readSettings(environment).secrets)).toEqual([ + { current: { from: 'kms', setting: 'SECRETS_KEY_KMS', arn: keyArn, region: 'eu-west-1' }, previous: undefined }, + { + current: { + from: 'kms', + setting: 'SECRETS_KEY_KMS', + arn: 'arn:aws:kms:eu-west-1:111122223333:alias/auto-runtime', + region: 'eu-west-1', + }, + previous: undefined, + }, + { + current: { from: 'kms', setting: 'SECRETS_KEY_KMS', arn: keyArn, region: 'eu-west-1' }, + previous: { + from: 'file', + setting: 'SECRETS_KEY_FILE_PREVIOUS', + file: '/run/secrets/secrets-key', + madeByTheServer: false, + }, + }, + ]); + }); + + it('grants the identity kms:Encrypt and kms:Decrypt on the key it names', () => { + expect(policies).toEqual([ + { + Version: '2012-10-17', + Statement: [{ Effect: 'Allow', Action: ['kms:Encrypt', 'kms:Decrypt'], Resource: keyArn }], + }, + ]); + }); +}); + +const fake = await serveFakeKms({ keys: [keyArn] }); + +afterAll(fake.close); + +const onTheFake = { + kmsCustodian: (options: KmsCustodianOptions) => + kmsCustodian({ + ...options, + credentials: { accessKeyId: 'an-access-key', secretAccessKey: 'a-secret-access-key' }, + endpoint: fake.url, + }), +}; + +async function startRefusedFor(environment: Readonly>): Promise { + const { fileName, remove } = temporaryLedger(); + onTestFinished(remove); + const context = await Effect.runPromise(Effect.context()); + return ledgerStoresOpenedBy( + context, + readSettings({ LEDGER_FILE: fileName, ...environment }), + { store: 'sqlite', file: fileName }, + () => Promise.resolve(onTheFake), + ).then(() => 'started', String); +} + +describe('the refusal the page quotes', () => { + it('is the words a start under a disabled key stops in', async () => { + fake.tell(keyArn, 'disabled'); + + const refused = await startRefusedFor({ SECRETS_KEY_KMS: keyArn }); + + expect(refused).toBe(`InvalidSettingsError: ${quotedRefusal}`); + }); +}); diff --git a/packages/server/src/database/ledger-stores-kms.test.ts b/packages/server/src/database/ledger-stores-kms.test.ts new file mode 100644 index 000000000..c4bcafbaf --- /dev/null +++ b/packages/server/src/database/ledger-stores-kms.test.ts @@ -0,0 +1,282 @@ +import { sqliteSecretRows } from '@on.auto/ledger/sqlite3'; +import { sha256Of } from '@on.auto/operations'; +import { kmsCustodian, type KmsCustodianOptions } from '@on.auto/secrets/aws'; +import { serveFakeKms, type FakeKeyState } from '@on.auto/secrets/testing'; +import { Effect, Logger, Redacted, type Context } from 'effect'; +import { afterAll, describe, expect, it, onTestFinished } from 'vitest'; + +import { readSettings } from '../settings/settings.ts'; +import { secretStoreInFile } from '../testing/records/kept-credentials.ts'; +import { temporaryLedger } from '../testing/records/temporary-ledger.ts'; +import { ledgerStoresOpenedBy } from './ledger-stores.ts'; + +const account = '111122223333'; + +const keyId = 'a2e5c1f4-6b0d-4c3e-9f21-7d8e6b5a4c3f'; + +const keyArn = `arn:aws:kms:eu-west-1:${account}:key/${keyId}`; + +const nextKeyId = '5b7f3d1e-8c2a-4e6b-a0d9-1f3e5c7a9b2d'; + +const nextKeyArn = `arn:aws:kms:eu-west-1:${account}:key/${nextKeyId}`; + +const aliasArn = `arn:aws:kms:eu-west-1:${account}:alias/auto-runtime`; + +const movingAliasArn = `arn:aws:kms:eu-west-1:${account}:alias/team/auto-runtime`; + +const fake = await serveFakeKms({ + keys: [keyArn, nextKeyArn], + aliases: { [aliasArn]: keyArn, [movingAliasArn]: keyArn }, +}); + +afterAll(fake.close); + +const credentials = { accessKeyId: 'an-access-key-of-the-tests', secretAccessKey: 'a-secret-access-key-of-the-tests' }; + +const onTheFake = { + kmsCustodian: (options: KmsCustodianOptions) => + kmsCustodian({ ...options, credentials, endpoint: fake.url, waits: { connectionMs: 100, requestMs: 100 } }), +}; + +const keyInHand = Buffer.alloc(32, 4); + +async function aContextLogging(): Promise<{ readonly context: Context.Context; readonly lines: string[] }> { + const lines: string[] = []; + const capture = Logger.map(Logger.formatJson, (line: string) => { + lines.push(line); + }); + const context = await Effect.runPromise(Effect.context().pipe(Effect.provide(Logger.layer([capture])))); + return { context, lines }; +} + +function aLedgerFile(): string { + const { fileName, remove } = temporaryLedger(); + onTestFinished(remove); + return fileName; +} + +async function openedWith(fileName: string, environment: Readonly>) { + const { context, lines } = await aContextLogging(); + const stores = await ledgerStoresOpenedBy( + context, + readSettings({ LEDGER_FILE: fileName, ...environment }), + { store: 'sqlite', file: fileName }, + () => Promise.resolve(onTheFake), + ); + await stores.secrets.close(); + return lines; +} + +async function storesOf(fileName: string, environment: Readonly>) { + const { context } = await aContextLogging(); + return ledgerStoresOpenedBy( + context, + readSettings({ LEDGER_FILE: fileName, ...environment }), + { store: 'sqlite', file: fileName }, + () => Promise.resolve(onTheFake), + ); +} + +async function keptUnder(fileName: string, environment: Readonly>) { + const { secrets } = await storesOf(fileName, environment); + const ids = await Effect.runPromise( + Effect.forEach(['pasted-key-1', 'pasted-key-2'], (value) => secrets.keep(Redacted.make(value))), + ); + await secrets.close(); + return ids; +} + +async function readUnder(fileName: string, environment: Readonly>, ids: readonly string[]) { + const { secrets } = await storesOf(fileName, environment); + const read = await Effect.runPromise(Effect.forEach(ids, (id) => secrets.read(id))); + await secrets.close(); + return read.map((stored) => (stored === undefined ? '' : Redacted.value(stored.value))); +} + +function secretsLineOf(lines: readonly string[]): string { + return lines.filter((line) => line.includes('Secrets are kept')).join('\n'); +} + +describe('the stores of a ledger whose secrets a key in KMS holds', () => { + it('probe the key at start with one data key under the context of the purpose, before anything is read, and log the KMS key', async () => { + const before = fake.calls().length; + + const lines = await openedWith(aLedgerFile(), { SECRETS_KEY_KMS: keyArn }); + + expect(fake.calls().slice(before)).toEqual([ + { operation: 'Encrypt', keyId: keyArn, context: { 'on.auto/runtime': 'secret data key' } }, + { operation: 'Decrypt', keyId: keyArn, context: { 'on.auto/runtime': 'secret data key' } }, + ]); + expect(secretsLineOf(lines)).toContain( + `"message":"Secrets are kept in the ledger's database, sealed under the key ${keyId} that the KMS key ${keyArn} holds, and 0 sealed under the previous key were sealed again under it"`, + ); + expect(secretsLineOf(lines)).toContain( + `"annotations":{"secrets_kid":"${keyId}","secrets_resealed":0,"secrets_key_from":"kms"}`, + ); + }); +}); + +const refusals: readonly (readonly [FakeKeyState, string])[] = [ + ['disabled', `DisabledException: key/${keyId} is disabled.`], + ['pending_deletion', `KMSInvalidStateException: key/${keyId} is pending deletion.`], + [ + 'denied', + `AccessDeniedException: User: user/runtime is not authorized to perform: kms:Encrypt on resource: key/${keyId} because no identity-based policy allows the kms:Encrypt action`, + ], +]; + +const unprobed = `The secrets settings are invalid. SECRETS_KEY_KMS: The KMS key ${keyId} it names could not wrap and unwrap a data key at start`; + +describe('the stores of a ledger whose key in KMS cannot be used', () => { + it.each(refusals)( + 'stop the start for a key %s, naming the setting, the key and the words of KMS, and never a credential or the account', + async (state, words) => { + fake.tell(keyArn, state); + onTestFinished(() => { + fake.tell(keyArn, 'enabled'); + }); + + const opened = openedWith(aLedgerFile(), { SECRETS_KEY_KMS: keyArn }); + + await expect(opened).rejects.toThrow( + `${unprobed}: The custodian of the key ${keyId} refused to wrap a data key: ${words}`, + ); + await expect(opened).rejects.not.toThrow(new RegExp(`${account}|${credentials.secretAccessKey}`, 'u')); + }, + ); + + it('stop the start for a key KMS does not know, naming the setting, the key and the words of KMS', async () => { + const unknownId = 'c0ffee00-1234-4abc-9def-0123456789ab'; + const unknownArn = `arn:aws:kms:eu-west-1:${account}:key/${unknownId}`; + + await expect(openedWith(aLedgerFile(), { SECRETS_KEY_KMS: unknownArn })).rejects.toThrow( + `The secrets settings are invalid. SECRETS_KEY_KMS: The KMS key ${unknownId} it names could not wrap and unwrap a data key at start: The custodian of the key ${unknownId} refused to wrap a data key: NotFoundException: Key 'key/${unknownId}' does not exist`, + ); + }); + + it('stop the start for a key in a KMS that does not answer, naming the setting and the key', async () => { + fake.tell(keyArn, 'stalling'); + onTestFinished(() => { + fake.tell(keyArn, 'enabled'); + }); + + await expect(openedWith(aLedgerFile(), { SECRETS_KEY_KMS: keyArn })).rejects.toThrow( + `${unprobed}: The custodian of the key ${keyId} did not answer a request to wrap a data key: TimeoutError`, + ); + }); +}); + +async function keptUnderTheKeyInHand(fileName: string): Promise { + const store = await secretStoreInFile(fileName, keyInHand); + await Effect.runPromise( + Effect.forEach(['pasted-key-1', 'pasted-key-2'], (value) => store.keep(Redacted.make(value))), + ); + await store.close(); +} + +async function valuesReadUnderTheKeyInHand(fileName: string, ids: readonly string[]): Promise { + const store = await secretStoreInFile(fileName, keyInHand); + const read = await Effect.runPromise(Effect.forEach(ids, (id) => store.read(id))); + await store.close(); + return read.map((stored) => (stored === undefined ? '' : Redacted.value(stored.value))); +} + +async function keptRowsIn(fileName: string) { + const rows = await Effect.runPromise(sqliteSecretRows({ fileName })); + const held = await Effect.runPromise(rows.heldBy({ keptBy: 'ledger' }, 10)); + await rows.close(); + return held; +} + +async function dataKeysIn(fileName: string): Promise { + const custodian = await Effect.runPromise( + onTheFake.kmsCustodian({ + keyId: keyArn, + region: 'eu-west-1', + proxy: { enabled: false, environment: {} }, + credentials, + }), + ); + const held = await keptRowsIn(fileName); + const dataKeys = await Effect.runPromise( + Effect.forEach(held, ({ holding }) => custodian.unwrap(String(holding.wrappedKey))), + ); + return dataKeys.flatMap((dataKey) => { + const bytes = Buffer.from(Redacted.value(dataKey)); + return [bytes.toString('base64'), bytes.toString('hex')]; + }); +} + +describe('the rotation of the secrets between a key in hand and a key in KMS', () => { + it('wraps again under the key in KMS what the key in hand wrapped, and back, counting each, and logs nothing of a key, a data key or a credential', async () => { + const fileName = aLedgerFile(); + await keptUnderTheKeyInHand(fileName); + const inHand = keyInHand.toString('base64'); + + const intoKms = await openedWith(fileName, { SECRETS_KEY_KMS: keyArn, SECRETS_KEY_PREVIOUS: inHand }); + const kidsInKms = (await keptRowsIn(fileName)).map(({ holding }) => holding.kid); + const dataKeys = await dataKeysIn(fileName); + const back = await openedWith(fileName, { SECRETS_KEY: inHand, SECRETS_KEY_KMS_PREVIOUS: keyArn }); + const readBack = await valuesReadUnderTheKeyInHand( + fileName, + (await keptRowsIn(fileName)).map(({ id }) => id), + ); + + expect(secretsLineOf(intoKms)).toContain(`that the KMS key ${keyArn} holds, and 2 sealed under the previous key`); + expect(kidsInKms).toEqual([keyId, keyId]); + expect(secretsLineOf(back)).toContain( + `sealed under the key ${sha256Of(keyInHand).slice(0, 16)} that SECRETS_KEY holds, and 2 sealed under the previous key`, + ); + expect(readBack).toEqual(['pasted-key-1', 'pasted-key-2']); + const logged = [...intoKms, ...back].join('\n'); + expect( + [inHand, credentials.secretAccessKey, credentials.accessKeyId, ...dataKeys].filter((secret) => + logged.includes(secret), + ), + ).toEqual([]); + }); +}); + +describe('the stores of a ledger whose secrets the key behind an alias holds', () => { + it('resolve the alias with DescribeKey before the probe, which then pins the key, and log the id of that key', async () => { + const before = fake.calls().length; + + const lines = await openedWith(aLedgerFile(), { SECRETS_KEY_KMS: aliasArn }); + + expect(fake.calls().slice(before)).toEqual([ + { operation: 'DescribeKey', keyId: aliasArn }, + { operation: 'Encrypt', keyId: keyArn, context: { 'on.auto/runtime': 'secret data key' } }, + { operation: 'Decrypt', keyId: keyArn, context: { 'on.auto/runtime': 'secret data key' } }, + ]); + expect(secretsLineOf(lines)).toContain(`sealed under the key ${keyId} that the KMS key ${aliasArn} holds`); + }); + + it('stop the start for an alias KMS does not know, naming the setting, the alias and the words of KMS, never the account', async () => { + const unknownAlias = `arn:aws:kms:eu-west-1:${account}:alias/unknown`; + + const opened = openedWith(aLedgerFile(), { SECRETS_KEY_KMS: unknownAlias }); + + await expect(opened).rejects.toThrow( + 'The secrets settings are invalid. SECRETS_KEY_KMS: The KMS alias alias/unknown it names could not be resolved to its key at start: The custodian of the alias alias/unknown refused to describe the key it names: NotFoundException: Alias alias/unknown is not found.', + ); + await expect(opened).rejects.not.toThrow(new RegExp(account, 'u')); + }); +}); + +describe('the stores of a ledger whose alias is pointed at another key', () => { + it('wrap again under the key the alias names now what the key before wrapped, given as the previous one, and read every secret back', async () => { + const fileName = aLedgerFile(); + const ids = await keptUnder(fileName, { SECRETS_KEY_KMS: movingAliasArn }); + fake.point(movingAliasArn, nextKeyArn); + + const moved = await openedWith(fileName, { SECRETS_KEY_KMS: movingAliasArn, SECRETS_KEY_KMS_PREVIOUS: keyArn }); + const kids = (await keptRowsIn(fileName)).map(({ holding }) => holding.kid); + const values = await readUnder(fileName, { SECRETS_KEY_KMS: movingAliasArn }, ids); + + expect(secretsLineOf(moved)).toContain( + `sealed under the key ${nextKeyId} that the KMS key ${movingAliasArn} holds, and 2 sealed under the previous key`, + ); + expect(kids).toEqual([nextKeyId, nextKeyId]); + expect(values).toEqual(['pasted-key-1', 'pasted-key-2']); + }); +}); diff --git a/packages/server/src/database/ledger-stores.ts b/packages/server/src/database/ledger-stores.ts index dfa458747..8f2509ceb 100644 --- a/packages/server/src/database/ledger-stores.ts +++ b/packages/server/src/database/ledger-stores.ts @@ -1,4 +1,4 @@ -import type { SecretStore } from '@on.auto/secrets'; +import { custodianProbed, type Custodian, type Custodians, type SecretStore } from '@on.auto/secrets'; import type { DatabaseSettings } from '@on.auto/workflow-host'; import { Effect, type Context } from 'effect'; @@ -6,8 +6,8 @@ import type { Served } from '../lifecycle/lifecycle.ts'; import { logContentStore, logLedger, logSecretStore } from '../logging/logging.ts'; import { InvalidSettingsError } from '../settings/invalid-settings-error.ts'; import type { Settings } from '../settings/settings.ts'; +import { loadKmsCustodians, type KmsCustodiansLoader } from './kms-custodians.ts'; import { custodiansOf, secretRowsOn, secretStoreOver } from './secret-store.ts'; -import { secretsKeysOf, type SecretKeys } from './secrets-keys.ts'; export interface LedgerStores { readonly secrets: SecretStore; @@ -18,10 +18,23 @@ type SecretsSettings = Settings['secrets']; type PreviousKeyGiven = NonNullable; +function invalid(setting: string, detail: string): InvalidSettingsError { + return new InvalidSettingsError({ message: `The secrets settings are invalid. ${setting}: ${detail}` }); +} + function sealedAgainRefused({ setting }: PreviousKeyGiven, detail: string): InvalidSettingsError { - return new InvalidSettingsError({ - message: `The secrets settings are invalid. ${setting}: The secrets its key sealed could not be sealed again under the current key: ${detail}`, - }); + return invalid(setting, `The secrets its key sealed could not be sealed again under the current key: ${detail}`); +} + +function probedAtStart(custodian: Custodian, { current }: SecretsSettings): Effect.Effect { + return current.from === 'kms' + ? Effect.mapError(custodianProbed(custodian), ({ detail }: { readonly detail: string }) => + invalid( + current.setting, + `The KMS key ${custodian.kid} it names could not wrap and unwrap a data key at start: ${detail}`, + ), + ) + : Effect.void; } function rotatedUnder( @@ -37,11 +50,11 @@ function rotatedUnder( function openedAndRotated( database: DatabaseSettings, - keys: SecretKeys, + custodians: Custodians, settings: SecretsSettings, ): Effect.Effect { return Effect.gen(function* () { - const custodians = custodiansOf(keys); + yield* probedAtStart(custodians.current, settings); const secrets = secretStoreOver(yield* secretRowsOn(database), custodians); const resealed = yield* rotatedUnder(secrets, settings); yield* logSecretStore({ kid: custodians.current.kid, resealed, keyFrom: settings.current }); @@ -51,13 +64,14 @@ function openedAndRotated( export async function ledgerStoresOpenedBy( context: Context.Context, - { ledger, content, secrets }: Pick, + { ledger, content, secrets, proxy }: Pick, database: DatabaseSettings, + loadKms: KmsCustodiansLoader = loadKmsCustodians, ): Promise { const run = Effect.runPromiseWith(context); await run(Effect.andThen(logLedger(ledger), logContentStore(content))); - const keys = secretsKeysOf(secrets); - const store = await run(openedAndRotated(database, keys, secrets)); + const custodians = await custodiansOf(secrets, { proxy, loadKms }); + const store = await run(openedAndRotated(database, custodians, secrets)); return { secrets: store, withSecretsClosed: ({ routes, stopWork }) => ({ diff --git a/packages/server/src/database/own-keys.test.ts b/packages/server/src/database/own-keys.test.ts index 700525aac..5852b1ef4 100644 --- a/packages/server/src/database/own-keys.test.ts +++ b/packages/server/src/database/own-keys.test.ts @@ -36,7 +36,7 @@ function started(services: Started[0]) { } const waysOut = - "Start the server with the key the ledger's secrets were sealed under, or give that key as SECRETS_KEY_PREVIOUS or SECRETS_KEY_FILE_PREVIOUS beside a new key"; + "Start the server with the key the ledger's secrets were sealed under, or give that key as SECRETS_KEY_PREVIOUS, SECRETS_KEY_FILE_PREVIOUS or SECRETS_KEY_KMS_PREVIOUS beside a new key"; function storeOver(rows: SecretRows, custodian: RecordingCustodian): SecretStore { return secretStoreOn(rows, sealedKeeper({ current: custodian }), { current: custodian }); diff --git a/packages/server/src/database/own-keys.ts b/packages/server/src/database/own-keys.ts index 62870d390..338320ca0 100644 --- a/packages/server/src/database/own-keys.ts +++ b/packages/server/src/database/own-keys.ts @@ -73,7 +73,7 @@ interface Asked { function unreadableUnder({ purpose, keyFrom }: Asked, words: string): OwnKeyUnavailable { return new OwnKeyUnavailable({ - message: `The key the runtime keeps for ${purpose} cannot be read under the key ${keyHeldBy(keyFrom)}: ${words}. Start the server with the key the ledger's secrets were sealed under, or give that key as SECRETS_KEY_PREVIOUS or SECRETS_KEY_FILE_PREVIOUS beside a new key`, + message: `The key the runtime keeps for ${purpose} cannot be read under the key ${keyHeldBy(keyFrom)}: ${words}. Start the server with the key the ledger's secrets were sealed under, or give that key as SECRETS_KEY_PREVIOUS, SECRETS_KEY_FILE_PREVIOUS or SECRETS_KEY_KMS_PREVIOUS beside a new key`, }); } diff --git a/packages/server/src/database/secret-store.test.ts b/packages/server/src/database/secret-store.test.ts new file mode 100644 index 000000000..af2c524a7 --- /dev/null +++ b/packages/server/src/database/secret-store.test.ts @@ -0,0 +1,107 @@ +import { sha256Of } from '@on.auto/operations'; +import { Effect } from 'effect'; +import { describe, expect, it } from 'vitest'; + +import { readSettings } from '../settings/settings.ts'; +import { loadKmsCustodians, type KmsCustodians } from './kms-custodians.ts'; +import { custodiansOf } from './secret-store.ts'; + +const keyArn = 'arn:aws:kms:eu-central-1:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab'; + +const [current, previous] = [Buffer.alloc(32, 1), Buffer.alloc(32, 2)]; + +interface Asked { + readonly keyId: string; + readonly region: string; + readonly credentials: string; + readonly proxied: boolean; +} + +function recordingLoader() { + const asked: Asked[] = []; + const loads = { made: 0 }; + const loaded: KmsCustodians = { + kmsCustodian: ({ keyId, region, credentials, proxy }) => { + asked.push({ keyId, region, credentials: typeof credentials, proxied: proxy.enabled }); + return Effect.succeed({ kid: `kms ${keyId}`, wrap: () => Effect.never, unwrap: () => Effect.never }); + }, + }; + return { + loadKms: () => { + loads.made += 1; + return Promise.resolve(loaded); + }, + asked: () => [...asked], + loads: () => loads.made, + }; +} + +function kidOf(key: Readonly): string { + return sha256Of(key).slice(0, 16); +} + +describe('the custodians of the secrets settings', () => { + it('are custodians of keys in hand, and load nothing of KMS, where no setting names a key in KMS', async () => { + const loader = recordingLoader(); + const settings = readSettings({ + SECRETS_KEY: current.toString('base64'), + SECRETS_KEY_PREVIOUS: previous.toString('base64'), + }); + + const custodians = await custodiansOf(settings.secrets, { proxy: settings.proxy, loadKms: loader.loadKms }); + const alone = await custodiansOf(readSettings({ LEDGER_FILE: ':memory:' }).secrets, { + proxy: settings.proxy, + loadKms: loader.loadKms, + }); + + expect([custodians.current.kid, custodians.previous?.kid, alone.previous]).toEqual([ + kidOf(current), + kidOf(previous), + undefined, + ]); + expect(alone.current.kid).toMatch(/^[\da-f]{16}$/u); + expect(loader.loads()).toBe(0); + }); +}); + +describe('the custodians of the secrets settings that name a key in KMS', () => { + it('load the custodian of KMS where a setting names a key there, for the key the ARN names in its region, with the default credential chain and the proxy settings', async () => { + const loader = recordingLoader(); + const inKms = readSettings({ + SECRETS_KEY_KMS: keyArn, + SECRETS_KEY_PREVIOUS: previous.toString('base64'), + NODE_USE_ENV_PROXY: '1', + }); + const outOfKms = readSettings({ SECRETS_KEY: current.toString('base64'), SECRETS_KEY_KMS_PREVIOUS: keyArn }); + + const intoKms = await custodiansOf(inKms.secrets, { proxy: inKms.proxy, loadKms: loader.loadKms }); + const back = await custodiansOf(outOfKms.secrets, { proxy: outOfKms.proxy, loadKms: loader.loadKms }); + + expect([intoKms.current.kid, intoKms.previous?.kid, back.current.kid, back.previous?.kid]).toEqual([ + `kms ${keyArn}`, + kidOf(previous), + kidOf(current), + `kms ${keyArn}`, + ]); + expect(loader.asked()).toEqual([ + { keyId: keyArn, region: 'eu-central-1', credentials: 'function', proxied: true }, + { keyId: keyArn, region: 'eu-central-1', credentials: 'function', proxied: false }, + ]); + expect(loader.loads()).toBe(2); + }); + + it('load the custodian of KMS from its own subpath of the secrets package', async () => { + const { kmsCustodian } = await loadKmsCustodians(); + + const made = await Effect.runPromise( + kmsCustodian({ + keyId: keyArn, + region: 'eu-central-1', + proxy: { enabled: false, environment: {} }, + credentials: { accessKeyId: 'a', secretAccessKey: 'b' }, + }), + ); + + expect(made.kid).toBe('1234abcd-12ab-34cd-56ef-1234567890ab'); + }); +}); diff --git a/packages/server/src/database/secret-store.ts b/packages/server/src/database/secret-store.ts index 79337c90e..55c71ff7e 100644 --- a/packages/server/src/database/secret-store.ts +++ b/packages/server/src/database/secret-store.ts @@ -4,6 +4,7 @@ import { localCustodian, sealedKeeper, secretStoreOn, + type Custodian, type Custodians, type SecretRows, type SecretStore, @@ -11,12 +12,21 @@ import { import type { DatabaseSettings } from '@on.auto/workflow-host'; import type { Effect } from 'effect'; -import type { SecretKeys } from './secrets-keys.ts'; +import type { SecretsKeySource, SecretsSettings } from '../settings/secrets-settings.ts'; +import { kmsCustodianOf, type CustodiansMaking } from './kms-custodians.ts'; +import { keyInHandOf } from './secrets-keys.ts'; -export function custodiansOf({ current, previous }: SecretKeys): Custodians { +function custodianOf(source: SecretsKeySource, making: CustodiansMaking): Custodian | Promise { + return source.from === 'kms' ? kmsCustodianOf(source, making) : localCustodian(keyInHandOf(source)); +} + +export async function custodiansOf( + { current, previous }: SecretsSettings, + making: CustodiansMaking, +): Promise { return { - current: localCustodian(current), - previous: previous === undefined ? undefined : localCustodian(previous), + current: await custodianOf(current, making), + previous: previous === undefined ? undefined : await custodianOf(previous, making), }; } diff --git a/packages/server/src/database/secrets-keys.test.ts b/packages/server/src/database/secrets-keys.test.ts index ede3b4ec8..9b4283c27 100644 --- a/packages/server/src/database/secrets-keys.test.ts +++ b/packages/server/src/database/secrets-keys.test.ts @@ -5,8 +5,7 @@ import { join } from 'node:path'; import { Redacted } from 'effect'; import { describe, expect, it, onTestFinished } from 'vitest'; -import type { SecretsKeyGiven, SecretsKeySource } from '../settings/secrets-settings.ts'; -import { secretsKeysOf } from './secrets-keys.ts'; +import { keyInHandOf, type KeyInHand } from './secrets-keys.ts'; function aFolder(): string { const folder = mkdtempSync(join(tmpdir(), 'auto-runtime-secrets-')); @@ -16,21 +15,21 @@ function aFolder(): string { return folder; } -function besideTheLedger(file: string): SecretsKeyGiven { +function besideTheLedger(file: string): KeyInHand { return { from: 'file', setting: 'SECRETS_KEY_FILE', file, madeByTheServer: true }; } -function named(file: string, setting = 'SECRETS_KEY_FILE'): SecretsKeyGiven { +function named(file: string, setting = 'SECRETS_KEY_FILE'): KeyInHand { return { from: 'file', setting, file, madeByTheServer: false }; } -function bytesOfTheKeyIn(source: SecretsKeySource): readonly number[] { - return [...Redacted.value(secretsKeysOf({ current: source, previous: undefined }).current)]; +function bytesOfTheKeyIn(source: KeyInHand): readonly number[] { + return [...Redacted.value(keyInHandOf(source))]; } -function refusalOf(source: SecretsKeySource): string { +function refusalOf(source: KeyInHand): string { try { - secretsKeysOf({ current: source, previous: undefined }); + keyInHandOf(source); } catch (error) { return String(error); } @@ -76,9 +75,9 @@ describe('the secrets key in a file a setting names', () => { writeFileSync(current, Buffer.alloc(32, 3).toString('base64'), { mode: 0o644 }); writeFileSync(previous, `${Buffer.alloc(32, 4).toString('base64')}\n`, { mode: 0o444 }); - const keys = secretsKeysOf({ current: named(current), previous: named(previous, 'SECRETS_KEY_FILE_PREVIOUS') }); + const keys = [keyInHandOf(named(current)), keyInHandOf(named(previous, 'SECRETS_KEY_FILE_PREVIOUS'))]; - expect([bytesOf(keys.current), bytesOf(keys.previous)]).toEqual([ + expect(keys.map((key) => bytesOf(key))).toEqual([ Array.from({ length: 32 }, () => 3), Array.from({ length: 32 }, () => 4), ]); @@ -123,20 +122,18 @@ describe('the secrets keys of the other settings', () => { it('are the keys SECRETS_KEY and SECRETS_KEY_PREVIOUS give, or a key made at each start', () => { const [current, previous] = [Redacted.make(new Uint8Array(32)), Redacted.make(new Uint8Array(32).fill(1))]; - const given = secretsKeysOf({ - current: { from: 'environment', setting: 'SECRETS_KEY', key: current }, - previous: { from: 'environment', setting: 'SECRETS_KEY_PREVIOUS', key: previous }, - }); - const [first, second] = [1, 2].map(() => - Array.from(Redacted.value(secretsKeysOf({ current: { from: 'start' }, previous: undefined }).current)), - ); + const given = [ + keyInHandOf({ from: 'environment', setting: 'SECRETS_KEY', key: current }), + keyInHandOf({ from: 'environment', setting: 'SECRETS_KEY_PREVIOUS', key: previous }), + ]; + const [first, second] = [1, 2].map(() => Array.from(Redacted.value(keyInHandOf({ from: 'start' })))); - expect(given).toEqual({ current, previous }); + expect(given).toEqual([current, previous]); expect([first?.length, second?.length]).toEqual([32, 32]); expect(first).not.toEqual(second); }); }); -function bytesOf(key: Redacted.Redacted | undefined): readonly number[] { - return key === undefined ? [] : Array.from(Redacted.value(key)); +function bytesOf(key: Redacted.Redacted): readonly number[] { + return Array.from(Redacted.value(key)); } diff --git a/packages/server/src/database/secrets-keys.ts b/packages/server/src/database/secrets-keys.ts index eda8f369a..a4b357d15 100644 --- a/packages/server/src/database/secrets-keys.ts +++ b/packages/server/src/database/secrets-keys.ts @@ -6,17 +6,9 @@ import { secretKeyBytes } from '@on.auto/secrets'; import { Redacted } from 'effect'; import { InvalidSettingsError } from '../settings/invalid-settings-error.ts'; -import { - aSecretsKeyInWords, - secretsKeyOfBase64, - type SecretsKeySource, - type SecretsSettings, -} from '../settings/secrets-settings.ts'; +import { aSecretsKeyInWords, secretsKeyOfBase64, type SecretsKeySource } from '../settings/secrets-settings.ts'; -export interface SecretKeys { - readonly current: Redacted.Redacted; - readonly previous?: Redacted.Redacted | undefined; -} +export type KeyInHand = Exclude; type KeyFile = Extract; @@ -80,13 +72,9 @@ function keyInFile(keyFile: KeyFile): Redacted.Redacted { return keyReadFrom(keyFile); } -function keyOf(source: SecretsKeySource): Redacted.Redacted { +export function keyInHandOf(source: KeyInHand): Redacted.Redacted { if (source.from === 'environment') { return source.key; } return source.from === 'file' ? keyInFile(source) : Redacted.make(Uint8Array.from(randomBytes(secretKeyBytes))); } - -export function secretsKeysOf({ current, previous }: SecretsSettings): SecretKeys { - return { current: keyOf(current), previous: previous === undefined ? undefined : keyOf(previous) }; -} diff --git a/packages/server/src/lifecycle/kms-startup.test.ts b/packages/server/src/lifecycle/kms-startup.test.ts new file mode 100644 index 000000000..e77fc2100 --- /dev/null +++ b/packages/server/src/lifecycle/kms-startup.test.ts @@ -0,0 +1,103 @@ +import { randomUUID } from 'node:crypto'; +import { fileURLToPath } from 'node:url'; + +import { sqliteSecretRows } from '@on.auto/ledger/sqlite3'; +import { kmsCustodian } from '@on.auto/secrets/aws'; +import { serveFakeKms } from '@on.auto/secrets/testing'; +import { Effect, Redacted } from 'effect'; +import { afterAll, describe, expect, it, onTestFinished } from 'vitest'; + +import { spawnServer } from '../testing/processes/spawned-server.ts'; +import { temporaryLedger } from '../testing/records/temporary-ledger.ts'; + +const mainModule = fileURLToPath(new URL('../main.ts', import.meta.url)); + +const keyId = randomUUID(); + +const keyArn = `arn:aws:kms:eu-west-1:111122223333:key/${keyId}`; + +const fake = await serveFakeKms({ keys: [keyArn] }); + +afterAll(fake.close); + +const credentials = { accessKeyId: 'an-access-key-of-the-tests', secretAccessKey: 'a-secret-access-key-of-the-tests' }; + +function aLedgerFile(): string { + const { fileName, remove } = temporaryLedger(); + onTestFinished(remove); + return fileName; +} + +function underTheKeyInKms(fileName: string): Readonly> { + return { + HOST: '127.0.0.1', + PORT: '0', + LEDGER_FILE: fileName, + SECRETS_KEY_KMS: keyArn, + AWS_ENDPOINT_URL_KMS: fake.url, + AWS_ACCESS_KEY_ID: credentials.accessKeyId, + AWS_SECRET_ACCESS_KEY: credentials.secretAccessKey, + }; +} + +async function startedOnce(fileName: string): Promise { + const child = spawnServer(mainModule, underTheKeyInKms(fileName)); + await child.port; + child.signal('SIGTERM'); + await child.exited; + return child.output().stderr; +} + +async function dataKeysIn(fileName: string): Promise { + const rows = await Effect.runPromise(sqliteSecretRows({ fileName })); + const held = await Effect.runPromise(rows.heldBy({ keptBy: 'ledger' }, 10)); + await rows.close(); + const proxy = { enabled: false, environment: {} }; + const custodian = await Effect.runPromise( + kmsCustodian({ keyId: keyArn, region: 'eu-west-1', proxy, credentials, endpoint: fake.url }), + ); + const dataKeys = await Effect.runPromise( + Effect.forEach(held, ({ holding }) => custodian.unwrap(String(holding.wrappedKey))), + ); + return dataKeys.flatMap((dataKey) => { + const bytes = Buffer.from(Redacted.value(dataKey)); + return [bytes.toString('base64'), bytes.toString('hex')]; + }); +} + +describe('a server whose secrets a key in KMS holds, as it starts', () => { + it('logs the KMS key its secrets are sealed under, at each start, and nothing of a data key or a credential', async () => { + const fileName = aLedgerFile(); + const first = await startedOnce(fileName); + const next = await startedOnce(fileName); + const dataKeys = await dataKeysIn(fileName); + const stderr = `${first}\n${next}`; + + expect(stderr.split('\n').filter((line) => line.includes('Secrets are kept'))).toEqual([ + expect.stringContaining( + `"message":"Secrets are kept in the ledger's database, sealed under the key ${keyId} that the KMS key ${keyArn} holds, and 0 sealed under the previous key were sealed again under it"`, + ), + expect.stringContaining(`that the KMS key ${keyArn} holds, and 0 sealed`), + ]); + expect(dataKeys.length).toBeGreaterThan(0); + expect( + [credentials.accessKeyId, credentials.secretAccessKey, ...dataKeys].filter((secret) => stderr.includes(secret)), + ).toEqual([]); + }); + + it('stops the process at start for a key that is disabled, naming the setting, the key and the words of KMS', async () => { + fake.tell(keyArn, 'disabled'); + onTestFinished(() => { + fake.tell(keyArn, 'enabled'); + }); + const child = spawnServer(mainModule, underTheKeyInKms(aLedgerFile())); + + const exitCode = await child.exited; + + expect(exitCode).toBe(1); + expect(child.output().stderr.split('\n').slice(-2)).toEqual([ + `Auto Runtime could not start: InvalidSettingsError: The secrets settings are invalid. SECRETS_KEY_KMS: The KMS key ${keyId} it names could not wrap and unwrap a data key at start: The custodian of the key ${keyId} refused to wrap a data key: DisabledException: key/${keyId} is disabled.`, + '', + ]); + }); +}); diff --git a/packages/server/src/lifecycle/secrets-startup.test.ts b/packages/server/src/lifecycle/secrets-startup.test.ts index 8c315c8fd..ed5d3065d 100644 --- a/packages/server/src/lifecycle/secrets-startup.test.ts +++ b/packages/server/src/lifecycle/secrets-startup.test.ts @@ -203,7 +203,7 @@ describe('the key the runtime keeps for itself as the server starts', () => { const stderr = await refusedStart({ LEDGER_FILE: fileName, SECRETS_KEY: previousKey.toString('base64') }); expect(lastLinesOf(stderr)).toEqual([ - `Auto Runtime could not start: OwnKeyUnavailable: The key the runtime keeps for retry_state cannot be read under the key that SECRETS_KEY holds: The secret is sealed under the key ${kidOf(currentKey)}, which is neither the current key nor the previous one. Start the server with the key the ledger's secrets were sealed under, or give that key as SECRETS_KEY_PREVIOUS or SECRETS_KEY_FILE_PREVIOUS beside a new key`, + `Auto Runtime could not start: OwnKeyUnavailable: The key the runtime keeps for retry_state cannot be read under the key that SECRETS_KEY holds: The secret is sealed under the key ${kidOf(currentKey)}, which is neither the current key nor the previous one. Start the server with the key the ledger's secrets were sealed under, or give that key as SECRETS_KEY_PREVIOUS, SECRETS_KEY_FILE_PREVIOUS or SECRETS_KEY_KMS_PREVIOUS beside a new key`, '', ]); expect(secretsIn(stderr, [ownKey])).toEqual([]); @@ -211,7 +211,7 @@ describe('the key the runtime keeps for itself as the server starts', () => { }); describe('a server given its secrets keys in more places than one', () => { - it('stops the process at start for two previous keys, naming both settings and never a key', async () => { + it('stops the process at start for two previous keys, naming the settings and never a key', async () => { const fileName = aLedgerFile(); const stderr = await refusedStart({ @@ -222,19 +222,19 @@ describe('a server given its secrets keys in more places than one', () => { }); expect(lastLinesOf(stderr)).toEqual([ - `${refusedAs} SECRETS_KEY_PREVIOUS: Set SECRETS_KEY_PREVIOUS or SECRETS_KEY_FILE_PREVIOUS, not both`, + `${refusedAs} SECRETS_KEY_PREVIOUS: Set one of SECRETS_KEY_PREVIOUS, SECRETS_KEY_FILE_PREVIOUS and SECRETS_KEY_KMS_PREVIOUS, not more`, '', ]); expect(secretsIn(stderr)).toEqual([]); }); - it('stops the process at start for a ledger in PostgreSQL without SECRETS_KEY or SECRETS_KEY_FILE, naming both and no URL', async () => { + it('stops the process at start for a ledger in PostgreSQL without SECRETS_KEY, SECRETS_KEY_FILE or SECRETS_KEY_KMS, naming them and no URL', async () => { const stderr = await refusedStart({ DATABASE_URL: 'postgresql://playbooks:a-secret-password@db.example.com/playbooks', }); expect(stderr).toBe( - `${refusedAs} SECRETS_KEY: Set it, or SECRETS_KEY_FILE, when the ledger is kept in PostgreSQL, since a database that runtimes share needs a key kept on none of their disks\n`, + `${refusedAs} SECRETS_KEY: Set it, SECRETS_KEY_FILE or SECRETS_KEY_KMS when the ledger is kept in PostgreSQL, since a database that runtimes share needs a key kept on none of their disks\n`, ); }); }); diff --git a/packages/server/src/logging/logging.ts b/packages/server/src/logging/logging.ts index d698e98fd..9dbc1216f 100644 --- a/packages/server/src/logging/logging.ts +++ b/packages/server/src/logging/logging.ts @@ -153,6 +153,9 @@ export function keyHeldBy(source: SecretsKeySource): string { if (source.from === 'environment') { return `that ${source.setting} holds`; } + if (source.from === 'kms') { + return `that the KMS key ${source.arn} holds`; + } return source.from === 'file' ? `that the file ${resolve(source.file)} holds` : 'that the server made at this start'; } diff --git a/packages/server/src/settings/secrets-settings.test.ts b/packages/server/src/settings/secrets-settings.test.ts index cd78c82b4..f765e1e38 100644 --- a/packages/server/src/settings/secrets-settings.test.ts +++ b/packages/server/src/settings/secrets-settings.test.ts @@ -38,12 +38,14 @@ const refusedAs = 'InvalidSettingsError: The secrets settings are invalid.'; const aKeyInWords = 'the base64 of 32 bytes, as openssl rand -base64 32 prints it'; describe('the secrets settings', () => { - it('are the four variables the runtime reads for itself, a key and a file each for the current key and the previous one', () => { + it('are the six variables the runtime reads for itself, a key, a file and a key in KMS each for the current key and the previous one', () => { expect(secretsVariables).toEqual([ 'SECRETS_KEY', 'SECRETS_KEY_FILE', + 'SECRETS_KEY_KMS', 'SECRETS_KEY_PREVIOUS', 'SECRETS_KEY_FILE_PREVIOUS', + 'SECRETS_KEY_KMS_PREVIOUS', ]); }); @@ -132,13 +134,13 @@ describe('the secrets settings that cannot be read', () => { ); }); - it('stop the server at start for a key in the environment and in a file both, of the current key or of the previous one, naming both', () => { + it('stop the server at start for a key in the environment and in a file both, of the current key or of the previous one, naming the settings', () => { expect([ errorFrom({ SECRETS_KEY: current, SECRETS_KEY_FILE: 'secrets.key' }), errorFrom({ SECRETS_KEY: current, SECRETS_KEY_PREVIOUS: previous, SECRETS_KEY_FILE_PREVIOUS: 'previous.key' }), ]).toEqual([ - `${refusedAs} SECRETS_KEY: Set SECRETS_KEY or SECRETS_KEY_FILE, not both`, - `${refusedAs} SECRETS_KEY_PREVIOUS: Set SECRETS_KEY_PREVIOUS or SECRETS_KEY_FILE_PREVIOUS, not both`, + `${refusedAs} SECRETS_KEY: Set one of SECRETS_KEY, SECRETS_KEY_FILE and SECRETS_KEY_KMS, not more`, + `${refusedAs} SECRETS_KEY_PREVIOUS: Set one of SECRETS_KEY_PREVIOUS, SECRETS_KEY_FILE_PREVIOUS and SECRETS_KEY_KMS_PREVIOUS, not more`, ]); }); @@ -147,17 +149,95 @@ describe('the secrets settings that cannot be read', () => { errorFrom({ SECRETS_KEY_PREVIOUS: previous }), errorFrom({ SECRETS_KEY_FILE_PREVIOUS: 'previous.key' }), ]).toEqual([ - `${refusedAs} SECRETS_KEY_PREVIOUS: Set it only beside SECRETS_KEY or SECRETS_KEY_FILE, to the key that one replaces`, - `${refusedAs} SECRETS_KEY_FILE_PREVIOUS: Set it only beside SECRETS_KEY or SECRETS_KEY_FILE, to the key that one replaces`, + `${refusedAs} SECRETS_KEY_PREVIOUS: Set it only beside SECRETS_KEY, SECRETS_KEY_FILE or SECRETS_KEY_KMS, to the key that one replaces`, + `${refusedAs} SECRETS_KEY_FILE_PREVIOUS: Set it only beside SECRETS_KEY, SECRETS_KEY_FILE or SECRETS_KEY_KMS, to the key that one replaces`, ]); }); - it('stop the server at start for a ledger in PostgreSQL without SECRETS_KEY or SECRETS_KEY_FILE, naming the setting and no URL', () => { + it('stop the server at start for a ledger in PostgreSQL without SECRETS_KEY, SECRETS_KEY_FILE or SECRETS_KEY_KMS, naming the setting and no URL', () => { const error = errorFrom({ DATABASE_URL: databaseUrl }); expect(error).toBe( - `${refusedAs} SECRETS_KEY: Set it, or SECRETS_KEY_FILE, when the ledger is kept in PostgreSQL, since a database that runtimes share needs a key kept on none of their disks`, + `${refusedAs} SECRETS_KEY: Set it, SECRETS_KEY_FILE or SECRETS_KEY_KMS when the ledger is kept in PostgreSQL, since a database that runtimes share needs a key kept on none of their disks`, ); expect(error).not.toMatch(/a-secret-password|db\.example\.com/u); }); }); + +const keyArn = 'arn:aws:kms:eu-central-1:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab'; + +const aliasArn = 'arn:aws:kms:us-west-2:111122223333:alias/team/auto-runtime'; + +const multiRegionArn = 'arn:aws:kms:ap-southeast-3:111122223333:key/mrk-1234abcd12ab34cd56ef1234567890ab'; + +describe('the secrets settings that name a key in AWS KMS', () => { + it('read SECRETS_KEY_KMS as the ARN of a key, a multi-Region key or an alias, the region the ARN names, beside a ledger in PostgreSQL too', () => { + expect([ + secretsOf({ SECRETS_KEY_KMS: ` ${keyArn}\n`, DATABASE_URL: databaseUrl }), + secretsOf({ SECRETS_KEY_KMS: aliasArn }), + secretsOf({ SECRETS_KEY_KMS: multiRegionArn }), + ]).toEqual([ + { + current: { from: 'kms', setting: 'SECRETS_KEY_KMS', arn: keyArn, region: 'eu-central-1' }, + previous: undefined, + }, + { + current: { from: 'kms', setting: 'SECRETS_KEY_KMS', arn: aliasArn, region: 'us-west-2' }, + previous: undefined, + }, + { + current: { from: 'kms', setting: 'SECRETS_KEY_KMS', arn: multiRegionArn, region: 'ap-southeast-3' }, + previous: undefined, + }, + ]); + }); +}); + +describe('the secrets settings that rotate to or from a key in AWS KMS', () => { + it('take a key in KMS as the key that replaces a key in hand, and a key in hand as the key that replaces one in KMS', () => { + expect([ + secretsOf({ SECRETS_KEY_KMS: keyArn, SECRETS_KEY_PREVIOUS: previous }), + secretsOf({ SECRETS_KEY: current, SECRETS_KEY_KMS_PREVIOUS: aliasArn }), + ]).toEqual([ + { + current: { from: 'kms', setting: 'SECRETS_KEY_KMS', arn: keyArn, region: 'eu-central-1' }, + previous: { from: 'environment', setting: 'SECRETS_KEY_PREVIOUS', key: twos }, + }, + { + current: { from: 'environment', setting: 'SECRETS_KEY', key: ones }, + previous: { from: 'kms', setting: 'SECRETS_KEY_KMS_PREVIOUS', arn: aliasArn, region: 'us-west-2' }, + }, + ]); + }); +}); + +describe('the secrets settings that name a key in AWS KMS and cannot be read', () => { + it.each([ + ['the id of a key alone', '1234abcd-12ab-34cd-56ef-1234567890ab'], + ['an alias name alone', 'alias/auto-runtime'], + ['the ARN of a key of another service', 'arn:aws:secretsmanager:eu-central-1:111122223333:secret:runtime'], + ['an ARN without its account', 'arn:aws:kms:eu-central-1::key/1234abcd-12ab-34cd-56ef-1234567890ab'], + ['an ARN without a region', 'arn:aws:kms::111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab'], + ['an ARN whose key is no key id', 'arn:aws:kms:eu-central-1:111122223333:key/runtime'], + ['an ARN of a resource other than a key or an alias', 'arn:aws:kms:eu-central-1:111122223333:grant/1234'], + ['an ARN of another partition', 'arn:aws-cn:kms:cn-north-1:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab'], + ])('stop the server at start for a SECRETS_KEY_KMS that is %s, naming the setting', (_, value) => { + expect(errorFrom({ SECRETS_KEY_KMS: value })).toBe( + `${refusedAs} SECRETS_KEY_KMS: Expected the ARN of a symmetric AWS KMS key, arn:aws:kms:::key/, or of an alias of one, arn:aws:kms:::alias/`, + ); + }); + + it('stop the server at start for a key in KMS beside a key in hand, and for a previous key in KMS without a current one', () => { + expect([ + errorFrom({ SECRETS_KEY_FILE: 'secrets.key', SECRETS_KEY_KMS: keyArn }), + errorFrom({ SECRETS_KEY: current, SECRETS_KEY_KMS_PREVIOUS: keyArn, SECRETS_KEY_PREVIOUS: previous }), + errorFrom({ SECRETS_KEY_KMS_PREVIOUS: keyArn }), + errorFrom({ SECRETS_KEY_KMS: keyArn, SECRETS_KEY_KMS_PREVIOUS: 'alias/auto-runtime' }), + ]).toEqual([ + `${refusedAs} SECRETS_KEY: Set one of SECRETS_KEY, SECRETS_KEY_FILE and SECRETS_KEY_KMS, not more`, + `${refusedAs} SECRETS_KEY_PREVIOUS: Set one of SECRETS_KEY_PREVIOUS, SECRETS_KEY_FILE_PREVIOUS and SECRETS_KEY_KMS_PREVIOUS, not more`, + `${refusedAs} SECRETS_KEY_KMS_PREVIOUS: Set it only beside SECRETS_KEY, SECRETS_KEY_FILE or SECRETS_KEY_KMS, to the key that one replaces`, + expect.stringContaining(`${refusedAs} SECRETS_KEY_KMS_PREVIOUS: Expected the ARN of a symmetric AWS KMS key`), + ]); + }); +}); diff --git a/packages/server/src/settings/secrets-settings.ts b/packages/server/src/settings/secrets-settings.ts index 746062c39..01d1ae36a 100644 --- a/packages/server/src/settings/secrets-settings.ts +++ b/packages/server/src/settings/secrets-settings.ts @@ -5,9 +5,10 @@ import { Config, ConfigProvider, Effect, Option, Redacted } from 'effect'; import { InvalidSettingsError } from './invalid-settings-error.ts'; import { besideTheLedgerFile, ledgerVariables, readLedgerSettings, type LedgerSettings } from './ledger-settings.ts'; -export type SecretsKeyGiven = +type SecretsKeyGiven = | { readonly from: 'environment'; readonly setting: string; readonly key: Redacted.Redacted } - | { readonly from: 'file'; readonly setting: string; readonly file: string; readonly madeByTheServer: boolean }; + | { readonly from: 'file'; readonly setting: string; readonly file: string; readonly madeByTheServer: boolean } + | { readonly from: 'kms'; readonly setting: string; readonly arn: string; readonly region: string }; export type SecretsKeySource = SecretsKeyGiven | { readonly from: 'start' }; @@ -24,20 +25,32 @@ export interface LedgerAndSecrets { interface KeySettings { readonly key: string; readonly file: string; + readonly kms: string; } -const currentKey: KeySettings = { key: 'SECRETS_KEY', file: 'SECRETS_KEY_FILE' }; +const currentKey: KeySettings = { key: 'SECRETS_KEY', file: 'SECRETS_KEY_FILE', kms: 'SECRETS_KEY_KMS' }; -const previousKey: KeySettings = { key: 'SECRETS_KEY_PREVIOUS', file: 'SECRETS_KEY_FILE_PREVIOUS' }; +const previousKey: KeySettings = { + key: 'SECRETS_KEY_PREVIOUS', + file: 'SECRETS_KEY_FILE_PREVIOUS', + kms: 'SECRETS_KEY_KMS_PREVIOUS', +}; -export const secretsVariables: readonly string[] = [currentKey, previousKey].flatMap(({ key, file }) => [key, file]); +function settingsOf({ key, file, kms }: KeySettings): readonly string[] { + return [key, file, kms]; +} + +export const secretsVariables: readonly string[] = [currentKey, previousKey].flatMap((settings) => + settingsOf(settings), +); export const ledgerAndSecretsVariables: readonly string[] = [...ledgerVariables, ...secretsVariables]; -function sourcesOf({ key, file }: KeySettings) { +function sourcesOf({ key, file, kms }: KeySettings) { return Config.all({ key: Config.map(Config.option(Config.Redacted(key)), Option.getOrUndefined), file: Config.map(Config.option(Config.String(file)), Option.getOrUndefined), + kms: Config.map(Config.option(Config.String(kms)), Option.getOrUndefined), }); } @@ -46,12 +59,19 @@ const sources = Config.all({ current: sourcesOf(currentKey), previous: sourcesOf interface Given { readonly key: Redacted.Redacted | undefined; readonly file: string | undefined; + readonly kms: string | undefined; } const keyFileName = 'secrets.key'; +const kmsKeyArn = + /^arn:aws:kms:([a-z]{2}(?:-[a-z]+)+-\d+):\d{12}:(?:key\/(?:[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}|mrk-[\da-f]{32})|alias\/[\w/-]+)$/u; + export const aSecretsKeyInWords = `the base64 of ${secretKeyBytes} bytes, as openssl rand -base64 ${secretKeyBytes} prints it`; +const aKmsKeyInWords = + 'the ARN of a symmetric AWS KMS key, arn:aws:kms:::key/, or of an alias of one, arn:aws:kms:::alias/'; + function invalid(detail: string): InvalidSettingsError { return new InvalidSettingsError({ message: `The secrets settings are invalid. ${detail}` }); } @@ -71,16 +91,30 @@ function keyIn(setting: string, given: Redacted.Redacted): Effect.Effect { + const arn = given.trim(); + const [, region] = kmsKeyArn.exec(arn) ?? []; + return region === undefined + ? Effect.fail(invalid(`${setting}: Expected ${aKmsKeyInWords}`)) + : Effect.succeed({ from: 'kms', setting, arn, region }); +} + function sourceGiven( settings: KeySettings, - { key, file }: Given, + { key, file, kms }: Given, ): Effect.Effect { - if (key !== undefined && file !== undefined) { - return Effect.fail(invalid(`${settings.key}: Set ${settings.key} or ${settings.file}, not both`)); + const set = [key, file, kms].filter((value) => value !== undefined).length; + if (set > 1) { + return Effect.fail( + invalid(`${settings.key}: Set one of ${settings.key}, ${settings.file} and ${settings.kms}, not more`), + ); } if (key !== undefined) { return keyIn(settings.key, key); } + if (kms !== undefined) { + return kmsKeyIn(settings.kms, kms); + } return Effect.succeed( file === undefined ? undefined : { from: 'file', setting: settings.file, file, madeByTheServer: false }, ); @@ -90,7 +124,7 @@ function withoutAKey(ledger: LedgerSettings): Effect.Effect( servedBy( routesServing([...playbookOperations, waitForeverAnnouncedIn(process.env)], { identityIssuer: settings.identityIssuer, - retryStateKeys: secretsKeysOf({ current: { from: 'start' }, previous: undefined }), + retryStateKeys: { current: keyInHandOf({ from: 'start' }) }, })(runtime), ), }), diff --git a/packages/server/src/testing/records/kept-credentials.ts b/packages/server/src/testing/records/kept-credentials.ts index 01070960b..47599731b 100644 --- a/packages/server/src/testing/records/kept-credentials.ts +++ b/packages/server/src/testing/records/kept-credentials.ts @@ -1,19 +1,18 @@ import { connectionsFold, connectionsStreamOfOrg } from '@on.auto/connections/testing'; import { ledgerLayer, sqliteSecretRows } from '@on.auto/ledger/sqlite3'; import { Ledger } from '@on.auto/operations'; -import type { SecretStore } from '@on.auto/secrets'; +import { localCustodian, type SecretStore } from '@on.auto/secrets'; import { Effect, ManagedRuntime, Redacted } from 'effect'; import { ownKeysDecider, ownKeysStream, type OwnKeyEvent } from '../../database/own-keys.ts'; -import { custodiansOf, secretStoreOver } from '../../database/secret-store.ts'; +import { secretStoreOver } from '../../database/secret-store.ts'; export const secretsKey = Buffer.alloc(32, 7); export async function secretStoreInFile(fileName: string, key: Readonly): Promise { - return secretStoreOver( - await Effect.runPromise(sqliteSecretRows({ fileName })), - custodiansOf({ current: Redacted.make(Uint8Array.from(key)) }), - ); + return secretStoreOver(await Effect.runPromise(sqliteSecretRows({ fileName })), { + current: localCustodian(Redacted.make(Uint8Array.from(key))), + }); } export async function secretIdOf(fileName: string, org: string, connection: string): Promise { From 5b132fe710314bebf7a502e338e2ca18f10862b3 Mon Sep 17 00:00:00 2001 From: Rami Date: Tue, 20 Oct 2026 20:24:17 +0100 Subject: [PATCH 4/4] ci(global): a job runs the secrets suite against moto and the runtime under a key it makes there The custodian of a key in KMS is tried against a server of the KMS API on every pull request, and so is a runtime whose secrets that key holds. - The job kms starts moto's server mode, motoserver/moto 5.2.4 pinned by its digest, on the port 19760, and makes a key there through the KMS API, CreateKey, before anything else runs. - It runs the tests of @on.auto/secrets with SECRETS_TEST_KMS_URL, and checks that the suite against a server of the KMS API ran, that vitest's Test Files and Tests summary lines name nothing skipped, and that the credential is in no line of their output. - It starts the runtime twice under SECRETS_KEY_KMS, the ARN of that key, on the port 19761, and checks, as the PostgreSQL smoke checks its own, that each start logs "Secrets are kept in the ledger's database, sealed under the key that the KMS key holds, and 0 sealed under the previous key", and that neither the secret access key nor the data key of the row the runtime keeps for itself, unwrapped by moto under the purpose's context and masked in both its forms before it is used, reaches its log. A step that fails ends the runtime it started and shows the log of each start. - ci-ok needs the job, and turbo passes SECRETS_TEST_KMS_URL to the tests, as it passes the variables of PostgreSQL and the object store. - scripts/test-groups.test.ts judges the groups that start PostgreSQL and the object store by the variables the test job sets, and checks that the job against moto runs the tests of exactly the packages that read the variables it sets, and that one of the two jobs sets each variable turbo passes to the tests. - The secrets README says what CI runs. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 115 +++++++++++++++++++++++++++++++++++- packages/secrets/README.md | 2 +- scripts/test-groups.test.ts | 61 +++++++++++++++++-- turbo.json | 2 +- 4 files changed, 172 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21cfd1791..199d1b7d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -186,6 +186,119 @@ jobs: docker logs postgresql docker logs object-store + kms: + name: Secrets under a key in KMS + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + with: + egress-policy: audit + - name: Start moto's server mode, which answers the KMS API, once the runner is hardened, as the test job starts PostgreSQL and the object store + run: | + docker run --detach --name kms --publish 19760:5000 \ + motoserver/moto:5.2.4@sha256:120b1589c6df7e35d2092935f6ff724486bb2c4189d0e36ce795baba1400ffd2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: .nvmrc + cache: pnpm + - run: pnpm install --frozen-lockfile + - name: Wait until moto answers on its port, and make through the KMS API the key the runtime starts under + run: | + for attempt in $(seq 1 30); do + curl --fail --silent --output /dev/null http://127.0.0.1:19760/moto-api/ && break + sleep 1 + done + curl --fail --silent --output "$RUNNER_TEMP/created-key.json" http://127.0.0.1:19760/ \ + --header 'x-amz-target: TrentService.CreateKey' --header 'content-type: application/x-amz-json-1.1' \ + --header "authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/20260101/us-east-1/kms/aws4_request, SignedHeaders=host, Signature=0" \ + --data '{"Description":"The key the runtime of this job keeps its secrets under"}' + jq --raw-output .KeyMetadata.Arn "$RUNNER_TEMP/created-key.json" > "$RUNNER_TEMP/kms-key-arn" + grep --quiet --extended-regexp '^arn:aws:kms:us-east-1:[0-9]{12}:key/[0-9a-f-]{36}$' "$RUNNER_TEMP/kms-key-arn" + env: + AWS_ACCESS_KEY_ID: kms-ci + - name: Run the tests of the secrets package with SECRETS_TEST_KMS_URL, so its suite against a server of the KMS API runs on moto and is skipped nowhere, and no key reaches their output + run: | + pnpm --filter @on.auto/secrets test --reporter=verbose > "$RUNNER_TEMP/kms-tests.log" 2>&1 || { cat "$RUNNER_TEMP/kms-tests.log"; exit 1; } + cat "$RUNNER_TEMP/kms-tests.log" + test "$(grep --count --fixed-strings ' > The secret store over keys in a server that answers the KMS API > ' "$RUNNER_TEMP/kms-tests.log")" -gt 0 + test "$(grep --count --extended-regexp '^ +(Test Files|Tests) ' "$RUNNER_TEMP/kms-tests.log")" = 2 + test "$(grep --extended-regexp '^ +(Test Files|Tests) ' "$RUNNER_TEMP/kms-tests.log" | grep --count skipped)" = 0 + test "$(grep --count --fixed-strings kms-ci-secret "$RUNNER_TEMP/kms-tests.log")" = 0 + env: + NO_COLOR: '1' + SECRETS_TEST_KMS_URL: http://kms-ci:kms-ci-secret@127.0.0.1:19760 + - name: Keep the runtime's own key under the key in KMS across a restart, logging that key and nothing of a credential or of a data key + timeout-minutes: 3 + run: | + arn="$(cat "$RUNNER_TEMP/kms-key-arn")" + secrets_line="\"message\":\"Secrets are kept in the ledger's database, sealed under the key ${arn##*/} that the KMS key $arn holds, and 0 sealed under the previous key" + export SECRETS_KEY_KMS="$arn" LEDGER_FILE="$RUNNER_TEMP/kms/ledger.db" HOST=127.0.0.1 PORT=19761 DO_NOT_TRACK=1 + started='' + shown_on_failure() { + status=$? + if [ "$status" -ne 0 ]; then + if [ -n "$started" ]; then + kill -TERM "$started" 2> /dev/null || true + fi + for start in first second; do + echo "::group::The log of the runtime's $start start" + cat "$RUNNER_TEMP/runtime-$start.log" 2> /dev/null || true + echo '::endgroup::' + done + fi + exit "$status" + } + trap shown_on_failure EXIT + started_and_stopped() { + node packages/server/src/main.ts > "$RUNNER_TEMP/runtime-$1.log" 2>&1 & + started=$! + for attempt in $(seq 1 60); do + curl --silent --fail --max-time 5 --output /dev/null http://127.0.0.1:19761/health && break + sleep 1 + done + test "$(curl --silent --fail --max-time 5 http://127.0.0.1:19761/health)" = '{"status":"ok"}' + kill -TERM "$started" + wait "$started" + started='' + } + started_and_stopped first + started_and_stopped second + log="$RUNNER_TEMP/runtime.log" + cat "$RUNNER_TEMP/runtime-first.log" "$RUNNER_TEMP/runtime-second.log" > "$log" + test "$(grep --count --fixed-strings "$secrets_line" "$log")" = 2 + test "$(grep --count --fixed-strings "$AWS_SECRET_ACCESS_KEY" "$log")" = 0 + node --input-type=module --eval " + import { DatabaseSync } from 'node:sqlite'; + const kept = new DatabaseSync(process.env.LEDGER_FILE, { readOnly: true }); + for (const { wrapped_key } of kept.prepare('SELECT wrapped_key FROM kept_secrets').all()) console.log(wrapped_key); + " > "$RUNNER_TEMP/wrapped-keys" + test "$(wc --lines < "$RUNNER_TEMP/wrapped-keys")" -gt 0 + while read -r wrapped; do + curl --fail --silent --output "$RUNNER_TEMP/unwrapped.json" http://127.0.0.1:19760/ \ + --header 'x-amz-target: TrentService.Decrypt' --header 'content-type: application/x-amz-json-1.1' \ + --header "authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/20260101/us-east-1/kms/aws4_request, SignedHeaders=host, Signature=0" \ + --data "$(jq --null-input --compact-output --arg blob "$wrapped" '{CiphertextBlob: $blob, EncryptionContext: {"on.auto/runtime": "secret data key"}}')" + data_key="$(jq --raw-output .Plaintext "$RUNNER_TEMP/unwrapped.json")" + echo "::add-mask::$data_key" + data_key_hex="$(printf '%s' "$data_key" | base64 --decode | od --address-radix=n --format=x1 --output-duplicates | tr --delete ' \n')" + echo "::add-mask::$data_key_hex" + test "${#data_key_hex}" = 64 + test "$(grep --count --fixed-strings --regexp="$data_key" --regexp="$data_key_hex" "$log")" = 0 + done < "$RUNNER_TEMP/wrapped-keys" + env: + AWS_ENDPOINT_URL_KMS: http://127.0.0.1:19760 + AWS_ACCESS_KEY_ID: kms-ci + AWS_SECRET_ACCESS_KEY: kms-ci-secret + - name: Show the log of moto when a step failed + if: failure() + run: docker logs kms + coverage: name: Coverage (server) needs: test @@ -989,7 +1102,7 @@ jobs: ci-ok: name: ci-ok if: always() - needs: [verify, test, coverage, container, package, pull-request-title, workflows] + needs: [verify, test, kms, coverage, container, package, pull-request-title, workflows] runs-on: ubuntu-latest permissions: {} steps: diff --git a/packages/secrets/README.md b/packages/secrets/README.md index b03506c32..3967ae85a 100644 --- a/packages/secrets/README.md +++ b/packages/secrets/README.md @@ -104,7 +104,7 @@ The same suite runs against a real server of the KMS API when `SECRETS_TEST_KMS_ SECRETS_TEST_KMS_URL=http://kms-test:kms-test-secret@127.0.0.1:19760 pnpm --filter @on.auto/secrets test ``` -It passes against moto's server mode, `motoserver/moto`, which answers `Encrypt` and `Decrypt` with their context but not the state of a key, so a key disabled, pending deletion or denied is the fake's to show. +It passes against moto's server mode, `motoserver/moto`, which answers `Encrypt` and `Decrypt` with their context but not the state of a key, so a key disabled, pending deletion or denied is the fake's to show. CI runs the package's tests with the variable in a job of their own, `kms`, against moto on the port 19760, and in the same job starts the runtime twice under a key it makes through the KMS API, checking the line the start logs and that neither the credential nor the data key of the secret the runtime keeps reaches its log. `src/store/secret-store.test.ts` adds what only a keeper or rows of a test's own show: the id the store hands its keeper and the holding it releases, a replacement made between a rotation's read of a row and its write, which the rotation leaves as it is, the data key a rotation unwrapped, filled with zeros once wrapped again, and a rotation whose previous custodian refuses, which fails with that refusal and holds nothing again. diff --git a/scripts/test-groups.test.ts b/scripts/test-groups.test.ts index cfa6b7073..214bd91cd 100644 --- a/scripts/test-groups.test.ts +++ b/scripts/test-groups.test.ts @@ -54,6 +54,22 @@ const testGroupsOf = Schema.decodeUnknownSync( }), ); +const stepsOf = Schema.decodeUnknownSync( + Schema.Struct({ + jobs: Schema.Record( + Schema.String, + Schema.Struct({ + steps: Schema.Array( + Schema.Struct({ + run: Schema.optionalKey(Schema.String), + env: Schema.optionalKey(Schema.Record(Schema.String, Schema.Unknown)), + }), + ), + }), + ), + }), +); + function isFolderOrEveryFolderIn(pattern: string): boolean { const folder = pattern.endsWith(everyFolderIn) ? pattern.slice(0, -everyFolderIn.length) : pattern; return folder !== '' && !folder.includes('*') && !folder.startsWith('!'); @@ -100,22 +116,46 @@ const testedPackages: readonly TestedPackage[] = workspacePatterns .filter(({ manifest }) => manifest.scripts?.['test'] !== undefined) .map(({ folder, manifest }): TestedPackage => ({ name: manifest.name, folder })); -const groups = testGroupsOf(parse(textOf('.github/workflows/ci.yml'))).jobs.test.strategy.matrix.include.map( +const workflow: unknown = parse(textOf('.github/workflows/ci.yml')); + +const groups = testGroupsOf(workflow).jobs.test.strategy.matrix.include.map( ({ packages, shard, services }): TestGroup => ({ names: packages.map((name) => `${scope}${name}`), shard, services }), ); -const variablesOfTheServices = testEnvironmentOf(textOf('turbo.json')).tasks.test.env; +const variablesOfTheTests = testEnvironmentOf(textOf('turbo.json')).tasks.test.env; + +const jobs = stepsOf(workflow).jobs; + +const kmsJob = 'kms'; + +function variablesSetBy(job: string): readonly string[] { + return (jobs[job]?.steps ?? []) + .flatMap(({ env }) => Object.keys(env ?? {})) + .filter((name) => variablesOfTheTests.includes(name)); +} + +const variablesOfTheServices = variablesSetBy('test'); -function readsTheServices({ folder }: TestedPackage): boolean { +const variablesOfKms = variablesSetBy(kmsJob); + +const variablesSetByAJob = new Set([...variablesOfTheServices, ...variablesOfKms]); + +function readsAnyOf({ folder }: TestedPackage, variables: readonly string[]): boolean { const sources = join(root, folder, 'src'); return readdirSync(sources, { recursive: true, encoding: 'utf8' }) .filter((file) => file.endsWith('.ts')) .some((file) => { const source = readFileSync(join(sources, file), 'utf8'); - return variablesOfTheServices.some((variable) => source.includes(variable)); + return variables.some((variable) => source.includes(variable)); }); } +function packagesRunBy(job: string): readonly string[] { + return (jobs[job]?.steps ?? []).flatMap(({ run = '' }) => + [...run.matchAll(/--filter (@on\.auto\/[\w-]+) test/gu)].map(([, name = '']: readonly string[]) => name), + ); +} + await test('the workspace names its packages by folder, or as every folder in a folder, so the groups can be checked against it', () => { assert.deepEqual( workspacePatterns.filter((pattern) => !isFolderOrEveryFolderIn(pattern)), @@ -153,7 +193,7 @@ await test('every package with tests runs in exactly one group of the test job, }); await test('a group starts PostgreSQL and the object store exactly when the tests of one of its packages read the variables that reach them', () => { - const readers = testedPackages.filter((tested) => readsTheServices(tested)).map(({ name }) => name); + const readers = testedPackages.filter((tested) => readsAnyOf(tested, variablesOfTheServices)).map(({ name }) => name); assert.ok(variablesOfTheServices.length > 0); assert.ok(readers.length > 0); assert.deepEqual( @@ -163,3 +203,14 @@ await test('a group starts PostgreSQL and the object store exactly when the test [], ); }); + +await test('the job against moto runs the tests of exactly the packages that read the variables it sets, and every variable that reaches the tests is set by it or by the test job', () => { + const readers = testedPackages.filter((tested) => readsAnyOf(tested, variablesOfKms)).map(({ name }) => name); + assert.ok(variablesOfKms.length > 0); + assert.ok(readers.length > 0); + assert.deepEqual(packagesRunBy(kmsJob), readers); + assert.deepEqual( + variablesOfTheTests.filter((name) => !variablesSetByAJob.has(name)), + [], + ); +}); diff --git a/turbo.json b/turbo.json index f01144117..d5152fc2c 100644 --- a/turbo.json +++ b/turbo.json @@ -29,7 +29,7 @@ "!$TURBO_ROOT$/docs/.vitepress/cache/**", "!$TURBO_ROOT$/docs/.vitepress/dist/**" ], - "env": ["LEDGER_TEST_POSTGRESQL_URL", "CONTENT_TEST_OBJECT_STORE_URL"], + "env": ["LEDGER_TEST_POSTGRESQL_URL", "CONTENT_TEST_OBJECT_STORE_URL", "SECRETS_TEST_KMS_URL"], "outputs": ["coverage/**"] }, "test:shard": {