feat(secrets): the secrets key in AWS KMS; a store that cannot answer now fails the call

.env.example

@@ -19,7 +19,8 @@
 # Keep the ledger in PostgreSQL instead of packages/server/.data. Empty LEDGER_FILE
 # too, since dev.env names a ledger file and the server refuses both. A ledger in
 # PostgreSQL needs SECRETS_KEY, the key its secrets are sealed under: put in its
-# place the line openssl rand -base64 32 prints.
+# place the line openssl rand -base64 32 prints, or name a file that holds it in
+# SECRETS_KEY_FILE, or a key in AWS KMS by its ARN in SECRETS_KEY_KMS, instead.
 # DATABASE_URL=postgresql://postgres:your-password@127.0.0.1:5432/postgres
 # LEDGER_FILE=
 # SECRETS_KEY=cmVwbGFjZS13aXRoLW9wZW5zc2wtcmFuZC1iYXNlNjQ=

.github/workflows/ci.yml

@@ -186,6 +186,119 @@ jobs:
           docker logs postgresql
           docker logs object-store
 
+  kms:
+    name: Secrets under a key in KMS
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1
+        with:
+          egress-policy: audit
+      - name: Start moto's server mode, which answers the KMS API, once the runner is hardened, as the test job starts PostgreSQL and the object store
+        run: |
+          docker run --detach --name kms --publish 19760:5000 \
+            motoserver/moto:5.2.4@sha256:120b1589c6df7e35d2092935f6ff724486bb2c4189d0e36ce795baba1400ffd2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
+      - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+        with:
+          node-version-file: .nvmrc
+          cache: pnpm
+      - run: pnpm install --frozen-lockfile
+      - name: Wait until moto answers on its port, and make through the KMS API the key the runtime starts under
+        run: |
+          for attempt in $(seq 1 30); do
+            curl --fail --silent --output /dev/null http://127.0.0.1:19760/moto-api/ && break
+            sleep 1
+          done
+          curl --fail --silent --output "$RUNNER_TEMP/created-key.json" http://127.0.0.1:19760/ \
+            --header 'x-amz-target: TrentService.CreateKey' --header 'content-type: application/x-amz-json-1.1' \
+            --header "authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/20260101/us-east-1/kms/aws4_request, SignedHeaders=host, Signature=0" \
+            --data '{"Description":"The key the runtime of this job keeps its secrets under"}'
+          jq --raw-output .KeyMetadata.Arn "$RUNNER_TEMP/created-key.json" > "$RUNNER_TEMP/kms-key-arn"
+          grep --quiet --extended-regexp '^arn:aws:kms:us-east-1:[0-9]{12}:key/[0-9a-f-]{36}$' "$RUNNER_TEMP/kms-key-arn"
+        env:
+          AWS_ACCESS_KEY_ID: kms-ci
+      - name: Run the tests of the secrets package with SECRETS_TEST_KMS_URL, so its suite against a server of the KMS API runs on moto and is skipped nowhere, and no key reaches their output
+        run: |
+          pnpm --filter @on.auto/secrets test --reporter=verbose > "$RUNNER_TEMP/kms-tests.log" 2>&1 || { cat "$RUNNER_TEMP/kms-tests.log"; exit 1; }
+          cat "$RUNNER_TEMP/kms-tests.log"
+          test "$(grep --count --fixed-strings ' > The secret store over keys in a server that answers the KMS API > ' "$RUNNER_TEMP/kms-tests.log")" -gt 0
+          test "$(grep --count --extended-regexp '^ +(Test Files|Tests) ' "$RUNNER_TEMP/kms-tests.log")" = 2
+          test "$(grep --extended-regexp '^ +(Test Files|Tests) ' "$RUNNER_TEMP/kms-tests.log" | grep --count skipped)" = 0
+          test "$(grep --count --fixed-strings kms-ci-secret "$RUNNER_TEMP/kms-tests.log")" = 0
+        env:
+          NO_COLOR: '1'
+          SECRETS_TEST_KMS_URL: http://kms-ci:kms-ci-secret@127.0.0.1:19760
+      - name: Keep the runtime's own key under the key in KMS across a restart, logging that key and nothing of a credential or of a data key
+        timeout-minutes: 3
+        run: |
+          arn="$(cat "$RUNNER_TEMP/kms-key-arn")"
+          secrets_line="\"message\":\"Secrets are kept in the ledger's database, sealed under the key ${arn##*/} that the KMS key $arn holds, and 0 sealed under the previous key"
+          export SECRETS_KEY_KMS="$arn" LEDGER_FILE="$RUNNER_TEMP/kms/ledger.db" HOST=127.0.0.1 PORT=19761 DO_NOT_TRACK=1
+          started=''
+          shown_on_failure() {
+            status=$?
+            if [ "$status" -ne 0 ]; then
+              if [ -n "$started" ]; then
+                kill -TERM "$started" 2> /dev/null || true
+              fi
+              for start in first second; do
+                echo "::group::The log of the runtime's $start start"
+                cat "$RUNNER_TEMP/runtime-$start.log" 2> /dev/null || true
+                echo '::endgroup::'
+              done
+            fi
+            exit "$status"
+          }
+          trap shown_on_failure EXIT
+          started_and_stopped() {
+            node packages/server/src/main.ts > "$RUNNER_TEMP/runtime-$1.log" 2>&1 &
+            started=$!
+            for attempt in $(seq 1 60); do
+              curl --silent --fail --max-time 5 --output /dev/null http://127.0.0.1:19761/health && break
+              sleep 1
+            done
+            test "$(curl --silent --fail --max-time 5 http://127.0.0.1:19761/health)" = '{"status":"ok"}'
+            kill -TERM "$started"
+            wait "$started"
+            started=''
+          }
+          started_and_stopped first
+          started_and_stopped second
+          log="$RUNNER_TEMP/runtime.log"
+          cat "$RUNNER_TEMP/runtime-first.log" "$RUNNER_TEMP/runtime-second.log" > "$log"
+          test "$(grep --count --fixed-strings "$secrets_line" "$log")" = 2
+          test "$(grep --count --fixed-strings "$AWS_SECRET_ACCESS_KEY" "$log")" = 0
+          node --input-type=module --eval "
+            import { DatabaseSync } from 'node:sqlite';
+            const kept = new DatabaseSync(process.env.LEDGER_FILE, { readOnly: true });
+            for (const { wrapped_key } of kept.prepare('SELECT wrapped_key FROM kept_secrets').all()) console.log(wrapped_key);
+          " > "$RUNNER_TEMP/wrapped-keys"
+          test "$(wc --lines < "$RUNNER_TEMP/wrapped-keys")" -gt 0
+          while read -r wrapped; do
+            curl --fail --silent --output "$RUNNER_TEMP/unwrapped.json" http://127.0.0.1:19760/ \
+              --header 'x-amz-target: TrentService.Decrypt' --header 'content-type: application/x-amz-json-1.1' \
+              --header "authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/20260101/us-east-1/kms/aws4_request, SignedHeaders=host, Signature=0" \
+              --data "$(jq --null-input --compact-output --arg blob "$wrapped" '{CiphertextBlob: $blob, EncryptionContext: {"on.auto/runtime": "secret data key"}}')"
+            data_key="$(jq --raw-output .Plaintext "$RUNNER_TEMP/unwrapped.json")"
+            echo "::add-mask::$data_key"
+            data_key_hex="$(printf '%s' "$data_key" | base64 --decode | od --address-radix=n --format=x1 --output-duplicates | tr --delete ' \n')"
+            echo "::add-mask::$data_key_hex"
+            test "${#data_key_hex}" = 64
+            test "$(grep --count --fixed-strings --regexp="$data_key" --regexp="$data_key_hex" "$log")" = 0
+          done < "$RUNNER_TEMP/wrapped-keys"
+        env:
+          AWS_ENDPOINT_URL_KMS: http://127.0.0.1:19760
+          AWS_ACCESS_KEY_ID: kms-ci
+          AWS_SECRET_ACCESS_KEY: kms-ci-secret
+      - name: Show the log of moto when a step failed
+        if: failure()
+        run: docker logs kms
+
   coverage:
     name: Coverage (server)
     needs: test
@@ -989,7 +1102,7 @@ jobs:
   ci-ok:
     name: ci-ok
     if: always()
-    needs: [verify, test, coverage, container, package, pull-request-title, workflows]
+    needs: [verify, test, kms, coverage, container, package, pull-request-title, workflows]
     runs-on: ubuntu-latest
     permissions: {}
     steps:

auto-runtime.example.yaml

@@ -10,8 +10,9 @@
 # write ${NAME} where one belongs and set NAME in .env or in the container.
 # DATABASE_URL, the PostgreSQL database of the ledger, holds a password, and
 # SECRETS_KEY is the key the secrets kept in the ledger are sealed under, so both
-# are read from the environment only and have no key here. SECRETS_KEY_FILE is an
-# environment variable too, which names a file that holds that key instead.
+# are read from the environment only and have no key here. SECRETS_KEY_FILE and
+# SECRETS_KEY_KMS are environment variables too, which name a file that holds that
+# key and a key in AWS KMS that wraps the key of each secret instead.
 
 # OpenAI-compatible gateways; a reasoning function then names gateway/<model id>.
 model_gateways:

docs/assets/openapi.json

@@ -3733,7 +3733,7 @@
       "post": {
         "operationId": "give_credential",
         "summary": "Give credential",
-        "description": "Gives the credential of a connection through its open link, which approves the connection and uses the link up, and answers it connected. The runtime keeps the value sealed in its secret store, and no answer, record or history of the org ever holds it. On a runtime that signs people in, only the person who added the connection may give it.",
+        "description": "Gives the credential of a connection through its open link, which approves the connection and uses the link up, and answers it connected. The runtime keeps the value sealed in its secret store, and no answer, record or history of the org ever holds it. Where the secret store cannot keep it just now, it answers unavailable in the words of the store, keeps nothing and leaves the link open, so the same credential is given again. On a runtime that signs people in, only the person who added the connection may give it.",
         "tags": [
           "connections"
         ],
@@ -4311,7 +4311,7 @@
       "post": {
         "operationId": "finish",
         "summary": "Finish consent",
-        "description": "Finishes the consent of a connection with the code and the state its authorization server sent the person back with, which connects it, and answers it connected. A consent begun through Auto's consent service is finished with the handle and the state the service sent the person back with instead. The runtime keeps the tokens sealed in its secret store, and no answer, record or history of the org ever holds them. On a runtime that signs people in, only the person who added the connection may finish it.",
+        "description": "Finishes the consent of a connection with the code and the state its authorization server sent the person back with, which connects it, and answers it connected. A consent begun through Auto's consent service is finished with the handle and the state the service sent the person back with instead. The runtime keeps the tokens sealed in its secret store, and no answer, record or history of the org ever holds them. Where the secret store cannot read the verifier just now, it answers unavailable and the same code may be tried again; where it cannot keep the tokens once the code is redeemed, it answers unavailable, revokes them, and the person starts again, since the code is spent. On a runtime that signs people in, only the person who added the connection may finish it.",
         "tags": [
           "connections"
         ],

docs/reference/http.md

@@ -1088,7 +1088,7 @@ Every operation answers the connection: `id`, a UUIDv7; `name`; `type`, `mcp` or
 - `approve` connects a connection whose credential is `none` or `reference` and uses its link up. `give_credential` keeps the value sealed in the runtime's secret store, connects the connection and uses its link up: giving the credential is its approval.
 - A connection is added by a person: on a runtime that signs people in, `add_connection` refuses a key, and a runtime of API keys alone, which signs nobody in, refuses every connection, since no person could approve it.
 - On a runtime that signs people in, `approve`, `give_credential`, `start` and `finish` need the session of the person who added the connection. In local mode the token alone binds.
-- `remove_connection` removes a connection for good, and the runtime forgets its credential, and what a consent started for it keeps, having first revoked at the provider the tokens of a connection connected by consent. Removing a removed connection succeeds and records nothing. Its name may then be given to a new connection.
+- `remove_connection` removes a connection for good, and the runtime forgets its credential, and what a consent started for it keeps, having first revoked at the provider the tokens of a connection connected by consent; tokens it cannot read or revoke now are logged and left to end on their own, and the removal goes on. Removing a removed connection succeeds and records nothing. Its name may then be given to a new connection.
 
 | Case                                                                                                                                                                                                                                                                         | Response                                   |
 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------ |
@@ -1112,6 +1112,8 @@ Every operation answers the connection: `id`, a UUIDv7; `name`; `type`, `mcp` or
 | `finish` with no consent started, or one started more than ten minutes ago                                                                                                                                                                                                   | `not_found`                                |
 | `finish` with a state the consent was not started with, at `/state`, an issuer that is not the server's, at `/iss`, or a code the server refuses, at `/code`                                                                                                                 | `invalid_input`                            |
 | `start` or `finish` where the authorization server does not answer                                                                                                                                                                                                           | `unavailable`                              |
+| `give_credential`, `start` or `finish` where the runtime's secret store cannot keep or read now what it needs, in the store's words; a `finish` whose code was redeemed then says the person starts again, its tokens revoked                                                | `unavailable`                              |
+| `start` whose client, registered by the runtime at the authorization server, is one its secret store refuses to keep, in the store's words                                                                                                                                   | `conflict`                                 |
 | `approve` of a connection whose credential is to be given, or `give_credential` of one that takes none                                                                                                                                                                       | `conflict`                                 |
 
 ### Providers