diff --git a/packages/ledger/README.md b/packages/ledger/README.md index a373215c9..2c3dad4f4 100644 --- a/packages/ledger/README.md +++ b/packages/ledger/README.md @@ -267,7 +267,7 @@ The ledger keeps content whole beside its messages rather than in them, since ev `content.stage(playbook, run, content)`, `content.staged(playbook, run, sha256)` and `content.unstage(playbook, run)` keep, read and remove content staged under a run, by its id, with the rules of a put and a get for the uploads, the heads and the digest, so a server reads what another staged on the same database. `unstage` records the run in `unstaged_runs` and then, in one transaction, removes the run's heads and then its chunks. The record stays, so `unstaged_runs` grows by one small row, the playbook key and the run id, for each run whose staging ended, as many as the runs that stage content; no capability of this repository stages content yet. A stage under a run already unstaged is refused with `StagingEnded` and leaves its stream unread, and a stage looks again once its head is written; if the run was unstaged meanwhile it deletes its own head and chunks and is refused the same way. Since the unstage records the run before it deletes and the stage writes its head before it looks, either the stage sees the record or the unstage's deletes see the head, so a stage that races an unstage leaves nothing behind. A read of a staged content whose run is unstaged while it is read fails at the first chunk it finds gone, as a defect, in words that say so: `The content staged under has no chunk , as when its run is unstaged while it is read`. -A database whose content tables have another shape than these, a chunk of text or a head without a media type, is refused when the ledger opens, with words that name the columns found and the columns kept and say to delete the local database, since nothing stored before launch is carried over. +A database whose content tables have another shape than these, a chunk of text or a head without a media type, is refused when the ledger opens, as a `LedgerShapeRefused` of `@on.auto/ledger` that names the table, what it holds, the columns found and the columns kept, in words that say to delete the local database, since nothing stored before launch is carried over. The server, which knows where the ledger is kept, answers it with words of its own: where the ledger is, what to do, and where to ask. `measure/recorded-content.ts` runs `measuredContent` of `measure/content-measurement.ts` on each store. It puts the answers the tests keep, `answerOfAbout` of `@on.auto/operations/testing`, of 100 bytes to 20 MB of JSON, three times each, the second time as it is and the third with its digest expected, and reads each back and parses it; it puts the 64 MiB of bytes the tests generate, `generatedStream`, as a stream the same three ways and streams them back, their digest checked. The content is made before each put is timed, and SQLite is opened as the server opens it, with a page cache of 8 MiB and no memory map. It then puts and gets the 64 MiB once more for another playbook, made as the stream is pulled, and samples two figures of the process every 5 ms: the heap in use and the array buffers after a full collection, `heapUsed + arrayBuffers` of `process.memoryUsage()` after `gc()`, and the resident set, `rss`, each reported as the most it rose above its figure before the put or the get. The routine takes any `RecordedContent`, and is the subpath `@on.auto/ledger/content-measurement`, so the object store of `@on.auto/object-storage` is measured by the same steps, as its README records: @@ -312,7 +312,7 @@ The ledger keeps the rows of the secret store of [`@on.auto/secrets`](../secrets | `wrapped_key` | the value's data key as the custodian wrapped it, or `NULL` for a keeper that wraps none | | `kid` | the kid of the custodian's key that wrapped it, or `NULL` for a keeper that wraps none | -A database whose table of secrets has another shape than this is refused when the ledger opens, in the words that refuse an older shape of the content tables, `requireTheShapeKept` of `src/content/content-tables.ts`, naming the columns found and the columns kept and saying to delete the local database. +A database whose table of secrets has another shape than this is refused when the ledger opens, as an older shape of the content tables is, by `requireTheShapeKept` of `src/content/content-tables.ts`, with the same `LedgerShapeRefused`. Each operation of the port is one statement, every value bound: `keep` an `INSERT`; `read` a `SELECT` that answers `claimed_until` only while the claim is open; `replace` and `claim` one conditional `UPDATE` each with `RETURNING`, under the version read, the claim's also under no claim being open; `endingsOf` one `SELECT` of the ids named, and nothing asked for no ids; `forget` a `DELETE` that returns the holding it deleted; `heldBy` a `SELECT` of a keeper's rows, under a kid where one is named and after an id where one is named, `ORDER BY id` with the `LIMIT` asked; and `reheld` a conditional `UPDATE` of the four columns of the holding, only while the row's version, `kept_by` and `held` are the ones read and its `wrapped_key` and `kid` too, compared through `COALESCE(…, '')` so that a holding that has none compares alike on both stores. Every time the rows write or compare, `kept_at`, the claim's end and the moment a claim is judged open, is read from Effect's `Clock`, the clock the times of the facts are read from, so a test drives it with `TestClock`. A failure of the database is a defect, as it is for the ledger. diff --git a/packages/ledger/src/content/content-tables.ts b/packages/ledger/src/content/content-tables.ts index 016800bd6..9dc2facaa 100644 --- a/packages/ledger/src/content/content-tables.ts +++ b/packages/ledger/src/content/content-tables.ts @@ -5,6 +5,7 @@ import type { StatementExecutor } from '../event-store.ts'; import { inTurn } from '../projections/inline-projection.ts'; import { createMissingIndexes, type PlaybookIndex } from '../recorded/missing-indexes.ts'; import type { ContentDialect } from './content-dialects.ts'; +import { ledgerShapeRefused } from './ledger-shape-refused.ts'; export interface Shelf { readonly chunks: string; @@ -115,10 +116,6 @@ export async function namesFound(execute: StatementExecutor, sql: SQL): Promise< return Schema.decodeUnknownSync(NameRows)(rows).map(({ name }) => name); } -function listed(columns: readonly string[]): string { - return columns.toSorted().join(', '); -} - export interface KeptTable { readonly name: string; readonly columns: readonly string[]; @@ -130,11 +127,10 @@ export async function requireTheShapeKept( dialect: ContentDialect, { name, columns, holds }: KeptTable, ): Promise { - const found = await namesFound(execute, dialect.columnsOf(name)); - if (listed(found) !== listed(columns)) { - throw new Error( - `The ledger's database keeps ${holds} in a shape this server no longer reads: ${name} has the columns ${listed(found)}, where this server keeps ${listed(columns)}. Nothing stored before launch is carried over, so delete the local database and start the server again`, - ); + const found = (await namesFound(execute, dialect.columnsOf(name))).toSorted(); + const kept = columns.toSorted(); + if (found.join() !== kept.join()) { + throw ledgerShapeRefused({ table: name, holds, found, kept }); } } diff --git a/packages/ledger/src/content/ledger-shape-refused.ts b/packages/ledger/src/content/ledger-shape-refused.ts new file mode 100644 index 000000000..987d6a1df --- /dev/null +++ b/packages/ledger/src/content/ledger-shape-refused.ts @@ -0,0 +1,22 @@ +import { Data } from 'effect'; + +export interface RefusedShape { + readonly table: string; + readonly holds: string; + readonly found: readonly string[]; + readonly kept: readonly string[]; +} + +export class LedgerShapeRefused extends Data.TaggedError('ledger_shape_refused')< + RefusedShape & { readonly message: string } +> {} + +export function ledgerShapeRefused({ table, holds, found, kept }: RefusedShape): LedgerShapeRefused { + return new LedgerShapeRefused({ + table, + holds, + found, + kept, + message: `The ledger's database keeps ${holds} in a shape this server no longer reads: ${table} has the columns ${found.join(', ')}, where this server keeps ${kept.join(', ')}. Nothing stored before launch is carried over, so delete the local database and start the server again`, + }); +} diff --git a/packages/ledger/src/index.ts b/packages/ledger/src/index.ts index 276a487d7..9975ba15a 100644 --- a/packages/ledger/src/index.ts +++ b/packages/ledger/src/index.ts @@ -20,6 +20,7 @@ export type { StoredPlace, StoredRecord, } from './event-store.ts'; +export { LedgerShapeRefused, type RefusedShape } from './content/ledger-shape-refused.ts'; export { cursorOf as recordedCursorOf } from './recorded/cursor.ts'; export { decisionLoop, diff --git a/packages/ledger/src/secrets/kept-secrets-behaviour.ts b/packages/ledger/src/secrets/kept-secrets-behaviour.ts index 9e2043cdd..14179b592 100644 --- a/packages/ledger/src/secrets/kept-secrets-behaviour.ts +++ b/packages/ledger/src/secrets/kept-secrets-behaviour.ts @@ -3,6 +3,7 @@ import { rowsBehaviour, storeBehaviour, type SecretsEntry } from '@on.auto/secre import type { Effect } from 'effect'; import { describe, expect, it } from 'vitest'; +import { LedgerShapeRefused } from '../content/ledger-shape-refused.ts'; import { aLedger, type LedgerEntry } from '../testing/ledger-entry.ts'; export interface KeptSecretsEntry extends LedgerEntry { @@ -25,16 +26,24 @@ function secretsEntryOf(entry: KeptSecretsEntry): SecretsEntry { } function olderShapeBehaviour(entry: KeptSecretsEntry): void { - it('refuses a database whose table of secrets has an older shape when the ledger opens, with words that say to delete it', async () => { + it('refuses a database whose table of secrets has an older shape when the ledger opens, naming the columns found and kept, with words that say to delete it', async () => { const database = await entry.aDatabase(); await entry.queried( database, 'CREATE TABLE kept_secrets (id TEXT PRIMARY KEY, version INTEGER NOT NULL, kept_at TEXT NOT NULL, kept_by TEXT NOT NULL, held TEXT NOT NULL)', ); - await expect(aLedger(entry, database)).rejects.toThrow( - "The ledger's database keeps secrets in a shape this server no longer reads: kept_secrets has the columns held, id, kept_at, kept_by, version, where this server keeps claimed_until, ends_with, held, id, kept_at, kept_by, kid, version, wrapped_key. Nothing stored before launch is carried over, so delete the local database and start the server again", - ); + const refused = await aLedger(entry, database).catch((error: unknown) => error); + + expect(refused).toBeInstanceOf(LedgerShapeRefused); + expect(refused).toMatchObject({ + table: 'kept_secrets', + holds: 'secrets', + found: ['held', 'id', 'kept_at', 'kept_by', 'version'], + kept: ['claimed_until', 'ends_with', 'held', 'id', 'kept_at', 'kept_by', 'kid', 'version', 'wrapped_key'], + message: + "The ledger's database keeps secrets in a shape this server no longer reads: kept_secrets has the columns held, id, kept_at, kept_by, version, where this server keeps claimed_until, ends_with, held, id, kept_at, kept_by, kid, version, wrapped_key. Nothing stored before launch is carried over, so delete the local database and start the server again", + }); }); } diff --git a/packages/server/src/command-line/failure-notice.test.ts b/packages/server/src/command-line/failure-notice.test.ts new file mode 100644 index 000000000..3da5da6c6 --- /dev/null +++ b/packages/server/src/command-line/failure-notice.test.ts @@ -0,0 +1,119 @@ +import { stripVTControlCharacters } from 'node:util'; + +import { Redacted } from 'effect'; +import { describe, expect, it } from 'vitest'; + +import { ledgerOfAnotherVersion } from '../database/ledger-of-another-version.ts'; +import type { ColorStream } from '../logging/terminal.ts'; +import { startupFailureNoticeFor } from './failure-notice.ts'; + +const secretsOfAnOlderShape = { + table: 'kept_secrets', + holds: 'secrets', + found: ['held', 'id', 'kept_at', 'kept_by', 'version'], + kept: ['claimed_until', 'ends_with', 'held', 'id', 'kept_at', 'kept_by', 'kid', 'version', 'wrapped_key'], +}; + +const ledgerFileOfAnotherVersion = ledgerOfAnotherVersion(secretsOfAnOlderShape, { + ledger: { store: 'sqlite', file: '/Users/ada/.auto-runtime/ledger.db' }, + distribution: 'npx', +}); + +const servicesFailed = new Error("The server's services could not start", { cause: ledgerFileOfAnotherVersion }); + +const notATerminal: ColorStream = {}; + +const aTerminalWithColors: ColorStream = { isTTY: true, getColorDepth: () => 24 }; + +describe('the notice that a server could not start on a ledger file of another version', () => { + it('says where the ledger is and why it is refused, what to do with the command that does it, where to ask, and the columns last', () => { + expect(startupFailureNoticeFor(notATerminal)(servicesFailed)?.split('\n')).toEqual([ + ' ✖ Auto Runtime could not start', + '', + ' The ledger is kept in the file /Users/ada/.auto-runtime/ledger.db, and was', + ' made by another version of Auto Runtime. This version keeps secrets in a', + ' different shape and cannot read the secrets this ledger holds.', + '', + ' What you can do', + ' Delete the ledger and start Auto Runtime again. Everything in the ledger', + ' goes with it, every playbook, definition, run and connection among it.', + '', + ' rm -f /Users/ada/.auto-runtime/ledger.db /Users/ada/.auto-runtime/ledger.db-wal /Users/ada/.auto-runtime/ledger.db-shm', + '', + ' Need a hand? Ask us on Discord: https://discord.gg/B8BKcKMRm8', + '', + ' kept_secrets has the columns held, id, kept_at, kept_by, version, where', + ' this version keeps claimed_until, ends_with, held, id, kept_at, kept_by,', + ' kid, version, wrapped_key.', + ]); + }); + + it('marks the failure in red, sets the command apart in cyan, links Discord and dims the columns where stderr is a terminal with colors', () => { + const notice = String(startupFailureNoticeFor(aTerminalWithColors)(ledgerFileOfAnotherVersion)); + + expect(notice).toContain('\u001B[31m✖\u001B[39m'); + expect(notice).toContain( + '\u001B[36mrm -f /Users/ada/.auto-runtime/ledger.db /Users/ada/.auto-runtime/ledger.db-wal /Users/ada/.auto-runtime/ledger.db-shm\u001B[39m', + ); + expect(notice).toContain('\u001B]8;;https://discord.gg/B8BKcKMRm8\u001B\\'); + expect(notice).toContain('\u001B[2m kid, version, wrapped_key.\u001B[22m'); + expect(stripVTControlCharacters(notice)).toBe(startupFailureNoticeFor(notATerminal)(servicesFailed)); + }); + + it('leaves any other failure to the line that names it', () => { + expect([ + startupFailureNoticeFor(aTerminalWithColors)(new Error('listen EADDRINUSE: address already in use :::5555')), + startupFailureNoticeFor(aTerminalWithColors)('a failure that is no error'), + ]).toEqual([undefined, undefined]); + }); +}); + +describe('the notice that a server in the image could not start on a ledger file of another version', () => { + it('says which files to delete from the volume, or to remove the volume, with no command', () => { + const inTheImage = ledgerOfAnotherVersion(secretsOfAnOlderShape, { + ledger: { store: 'sqlite', file: '/app/data/ledger.db' }, + distribution: 'docker', + }); + + expect(startupFailureNoticeFor(notATerminal)(inTheImage)?.split('\n').slice(6, 13)).toEqual([ + ' What you can do', + ' Delete ledger.db, ledger.db-wal and ledger.db-shm from the volume', + " mounted at /data, the ledger's folder /app/data in the image, or remove", + ' the volume, and start Auto Runtime again. Everything in the ledger goes', + ' with it, every playbook, definition, run and connection among it.', + '', + ' Need a hand? Ask us on Discord: https://discord.gg/B8BKcKMRm8', + ]); + }); +}); + +describe('the notice that a server could not start on a ledger in PostgreSQL of another version', () => { + it('says to point DATABASE_URL at an empty database, with no command', () => { + const inPostgreSQL = ledgerOfAnotherVersion(secretsOfAnOlderShape, { + ledger: { + store: 'postgresql', + url: Redacted.make('postgresql://playbooks:a-secret-password@db.example.com/auto_runtime'), + host: 'db.example.com:5432', + database: 'auto_runtime', + credentials: { kind: 'url' }, + }, + distribution: 'npx', + }); + + expect(startupFailureNoticeFor(notATerminal)(inPostgreSQL)?.split('\n').slice(2, 15)).toEqual([ + ' The ledger is kept in PostgreSQL, in the database auto_runtime on', + ' db.example.com:5432, and was made by another version of Auto Runtime. This', + ' version keeps secrets in a different shape and cannot read the secrets', + ' this ledger holds.', + '', + ' What you can do', + ' Point DATABASE_URL at an empty database, or empty this one, and start', + ' Auto Runtime again. The ledger then starts with no playbooks,', + ' definitions or runs.', + '', + ' Need a hand? Ask us on Discord: https://discord.gg/B8BKcKMRm8', + '', + ' kept_secrets has the columns held, id, kept_at, kept_by, version, where', + ]); + }); +}); diff --git a/packages/server/src/command-line/failure-notice.ts b/packages/server/src/command-line/failure-notice.ts new file mode 100644 index 000000000..4cf504461 --- /dev/null +++ b/packages/server/src/command-line/failure-notice.ts @@ -0,0 +1,56 @@ +import { + columnsCompared, + helpOfferedAt, + ledgerOfAnotherVersionIn, + type LedgerOfAnotherVersion, +} from '../database/ledger-of-another-version.ts'; +import { colorDepthOf, linkerFor, painterFor, type ColorDepth, type ColorStream } from '../logging/terminal.ts'; +import { discordInvite } from './local-banner.ts'; + +const noticeWidth = 76; + +function wrapped(text: string, indent: string): readonly string[] { + const width = noticeWidth - indent.length; + const lines: string[] = []; + for (const word of text.split(' ')) { + const last = lines.pop(); + if (last === undefined) { + lines.push(word); + } else if (last.length + 1 + word.length <= width) { + lines.push(`${last} ${word}`); + } else { + lines.push(last, word); + } + } + return lines.map((line) => `${indent}${line}`); +} + +type Refusal = Pick; + +function ledgerNotice(refusal: Refusal, depth: ColorDepth): string { + const paint = painterFor(depth); + const link = linkerFor(depth); + const { says, command } = refusal.remedy; + const shown = command === undefined ? [] : ['', ` ${paint('cyan', command)}`]; + return [ + ` ${paint('red', '✖')} ${paint('bold', 'Auto Runtime could not start')}`, + '', + ...wrapped(refusal.happened, ' '), + '', + ` ${paint('bold', 'What you can do')}`, + ...wrapped(says, ' '), + ...shown, + '', + ` ${helpOfferedAt(paint('cyan', link(discordInvite, discordInvite)))}`, + '', + ...wrapped(columnsCompared(refusal), ' ').map((line) => paint('dim', line)), + ].join('\n'); +} + +export function startupFailureNoticeFor(stderr: ColorStream): (failure: unknown) => string | undefined { + const depth = colorDepthOf(stderr); + return (failure) => { + const refusal = ledgerOfAnotherVersionIn(failure); + return refusal === undefined ? undefined : ledgerNotice(refusal, depth); + }; +} diff --git a/packages/server/src/command-line/local-banner.ts b/packages/server/src/command-line/local-banner.ts index 0cdbc744b..a5bab9e98 100644 --- a/packages/server/src/command-line/local-banner.ts +++ b/packages/server/src/command-line/local-banner.ts @@ -20,6 +20,8 @@ const tagline = 'Runs the playbooks your agent builds'; const documentation = 'https://on.auto/docs/'; +export const discordInvite = 'https://discord.gg/B8BKcKMRm8'; + function agentCommandsFor({ version, port, mode }: Ready, mcp: string): readonly (readonly [string, string])[] { if (mode !== 'local') { return [ diff --git a/packages/server/src/composition/composition-root.ts b/packages/server/src/composition/composition-root.ts index 24a433b00..09282928e 100644 --- a/packages/server/src/composition/composition-root.ts +++ b/packages/server/src/composition/composition-root.ts @@ -1,7 +1,7 @@ import type { DispatcherServices } from '@on.auto/operations'; import { storesAndConnectionsOf } from '../connections/served-connections.ts'; -import { ledgerDatabaseOf } from '../database/ledger-database.ts'; +import { ledgerDatabaseOf, withARefusedShapeExplained } from '../database/ledger-database.ts'; import { defaultServerOptions, type ServerOptions } from '../lifecycle/lifecycle.ts'; import { serveWorkflows, type WorkflowParts } from '../workflows/workflows.ts'; import { applicationLayer } from './application-layer.ts'; @@ -22,7 +22,9 @@ export function compositionRootWith( composed: (settings, telemetry) => { const database = ledgerDatabaseOf(settings, telemetry.appended); return { - services: applicationLayer(ledgerLayerFor(settings, database, wiring.projections.appends)), + services: applicationLayer( + withARefusedShapeExplained(ledgerLayerFor(settings, database, wiring.projections.appends), settings), + ), serve: async (runtime, ownOrigin) => { const { ledger, workflows, identityIssuer } = settings; const { stores, retryStateKeys, consent, connections, source, offering, clientMetadata } = diff --git a/packages/server/src/database/ledger-database.test.ts b/packages/server/src/database/ledger-database.test.ts index 4361f0dc9..034b910ca 100644 --- a/packages/server/src/database/ledger-database.test.ts +++ b/packages/server/src/database/ledger-database.test.ts @@ -1,8 +1,12 @@ +import { LedgerShapeRefused } from '@on.auto/ledger'; +import { Ledger } from '@on.auto/operations'; +import { Cause, Effect, Exit, Layer } from 'effect'; import { describe, expect, it } from 'vitest'; import { readSettings } from '../settings/settings.ts'; import { secretsKey } from '../testing/databases/secrets-key.ts'; -import { ledgerDatabaseOf } from './ledger-database.ts'; +import { ledgerDatabaseOf, withARefusedShapeExplained } from './ledger-database.ts'; +import { LedgerOfAnotherVersion } from './ledger-of-another-version.ts'; const somePools: unknown = expect.any(Function); @@ -25,3 +29,64 @@ describe('the database of the ledger and the workflow host', () => { ).toEqual({ store: 'postgresql', pools: somePools }); }); }); + +const refusedShape = new LedgerShapeRefused({ + table: 'kept_secrets', + holds: 'secrets', + found: ['held', 'id', 'kept_at', 'kept_by', 'version'], + kept: ['claimed_until', 'ends_with', 'held', 'id', 'kept_at', 'kept_by', 'kid', 'version', 'wrapped_key'], + message: "The ledger's database keeps secrets in a shape this server no longer reads", +}); + +async function builtFrom(ledgerLayer: Layer.Layer): Promise { + const exit = await Effect.runPromiseExit(Effect.scoped(Layer.build(ledgerLayer))); + return Exit.isFailure(exit) ? Cause.squash(exit.cause) : 'built'; +} + +describe('a ledger that refuses the shape of its database as it opens', () => { + it('stops the start with words that say where the ledger is kept, from the settings', async () => { + const refused = await builtFrom( + withARefusedShapeExplained( + Layer.effect(Ledger, Effect.die(refusedShape)), + readSettings({ LEDGER_FILE: '/data/ledger.db' }), + ), + ); + + expect(refused).toBeInstanceOf(LedgerOfAnotherVersion); + expect(refused).toMatchObject({ + table: 'kept_secrets', + found: refusedShape.found, + kept: refusedShape.kept, + remedy: { command: 'rm -f /data/ledger.db /data/ledger.db-wal /data/ledger.db-shm' }, + }); + }); + + it('says in the image which files to delete from the volume, from the distribution the settings name', async () => { + const refused = await builtFrom( + withARefusedShapeExplained( + Layer.effect(Ledger, Effect.die(refusedShape)), + readSettings({ LEDGER_FILE: '/app/data/ledger.db', RUNTIME_DISTRIBUTION: 'docker', DO_NOT_TRACK: '1' }), + ), + ); + + expect(refused).toMatchObject({ + remedy: { + says: "Delete ledger.db, ledger.db-wal and ledger.db-shm from the volume mounted at /data, the ledger's folder /app/data in the image, or remove the volume, and start Auto Runtime again. Everything in the ledger goes with it, every playbook, definition, run and connection among it.", + command: undefined, + }, + }); + }); + + it('leaves any other failure of the ledger as it was', async () => { + const unreachable = new Error('connect ECONNREFUSED 127.0.0.1:5432'); + + expect( + await builtFrom( + withARefusedShapeExplained( + Layer.effect(Ledger, Effect.die(unreachable)), + readSettings({ LEDGER_FILE: '/data/ledger.db' }), + ), + ), + ).toBe(unreachable); + }); +}); diff --git a/packages/server/src/database/ledger-database.ts b/packages/server/src/database/ledger-database.ts index 81c24f98e..e9d3021ba 100644 --- a/packages/server/src/database/ledger-database.ts +++ b/packages/server/src/database/ledger-database.ts @@ -1,8 +1,11 @@ -import type { AppendObserver } from '@on.auto/ledger'; +import { LedgerShapeRefused, type AppendObserver } from '@on.auto/ledger'; +import { Ledger } from '@on.auto/operations'; import type { DatabaseSettings } from '@on.auto/workflow-host'; +import { Cause, Effect, Layer } from 'effect'; import type { Settings } from '../settings/settings.ts'; import { databasePoolsOf } from './database-pools.ts'; +import { ledgerOfAnotherVersion } from './ledger-of-another-version.ts'; export function ledgerDatabaseOf( { ledger, proxy }: Pick, @@ -12,3 +15,18 @@ export function ledgerDatabaseOf( ? { store: 'sqlite', file: ledger.file, observer } : { store: 'postgresql', pools: databasePoolsOf(ledger, proxy), observer }; } + +export function withARefusedShapeExplained( + ledgerLayer: Layer.Layer, + settings: Pick, +): Layer.Layer { + return Layer.catchCause(ledgerLayer, (cause) => { + const failure = Cause.squash(cause); + return Layer.effect( + Ledger, + failure instanceof LedgerShapeRefused + ? Effect.die(ledgerOfAnotherVersion(failure, settings)) + : Effect.failCause(cause), + ); + }); +} diff --git a/packages/server/src/database/ledger-of-another-version.test.ts b/packages/server/src/database/ledger-of-another-version.test.ts new file mode 100644 index 000000000..8b892187f --- /dev/null +++ b/packages/server/src/database/ledger-of-another-version.test.ts @@ -0,0 +1,111 @@ +import { execFileSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { Redacted } from 'effect'; +import { describe, expect, it, onTestFinished } from 'vitest'; + +import { ledgerOfAnotherVersion } from './ledger-of-another-version.ts'; + +const secretsOfAnOlderShape = { + table: 'kept_secrets', + holds: 'secrets', + found: ['held', 'id', 'kept_at', 'kept_by', 'version'], + kept: ['claimed_until', 'ends_with', 'held', 'id', 'kept_at', 'kept_by', 'kid', 'version', 'wrapped_key'], +}; + +const inPostgreSQL = { + store: 'postgresql', + url: Redacted.make('postgresql://playbooks:a-secret-password@db.example.com/auto_runtime'), + host: 'db.example.com:5432', + database: 'auto_runtime', + credentials: { kind: 'url' }, +} as const; + +function refusedFileAt(file: string, distribution: 'npx' | 'docker' = 'npx') { + return ledgerOfAnotherVersion(secretsOfAnOlderShape, { ledger: { store: 'sqlite', file }, distribution }); +} + +const awkwardFolder = "Application Support/ada's $HOME"; + +describe('a ledger made by another version of Auto Runtime', () => { + it('says in one message where the ledger is, why it is refused, what to do and where to ask, ending with the command that deletes it', () => { + const refused = refusedFileAt('/Users/ada/.auto-runtime/ledger.db'); + + expect(refused.message).toBe( + 'The ledger is kept in the file /Users/ada/.auto-runtime/ledger.db, and was made by another version of Auto Runtime. This version keeps secrets in a different shape and cannot read the secrets this ledger holds. kept_secrets has the columns held, id, kept_at, kept_by, version, where this version keeps claimed_until, ends_with, held, id, kept_at, kept_by, kid, version, wrapped_key. Delete the ledger and start Auto Runtime again. Everything in the ledger goes with it, every playbook, definition, run and connection among it. Need a hand? Ask us on Discord: https://discord.gg/B8BKcKMRm8 To delete the ledger, run: rm -f /Users/ada/.auto-runtime/ledger.db /Users/ada/.auto-runtime/ledger.db-wal /Users/ada/.auto-runtime/ledger.db-shm', + ); + expect(String(refused)).toMatch(/^LedgerOfAnotherVersion: The ledger is kept in the file /u); + }); + + it('places a ledger file given as a relative path in the directory the server started in', () => { + const file = join(process.cwd(), 'data', 'ledger.db'); + + const refused = refusedFileAt('data/ledger.db'); + + expect(refused.happened).toContain(`The ledger is kept in the file ${file}, and was made by another version`); + expect(refused.remedy.command).toBe(`rm -f ${file} ${file}-wal ${file}-shm`); + }); + + it('names the PostgreSQL database and its host, never the URL, and says to point DATABASE_URL at an empty one', () => { + const refused = ledgerOfAnotherVersion(secretsOfAnOlderShape, { ledger: inPostgreSQL, distribution: 'npx' }); + + expect(refused.message).toBe( + 'The ledger is kept in PostgreSQL, in the database auto_runtime on db.example.com:5432, and was made by another version of Auto Runtime. This version keeps secrets in a different shape and cannot read the secrets this ledger holds. kept_secrets has the columns held, id, kept_at, kept_by, version, where this version keeps claimed_until, ends_with, held, id, kept_at, kept_by, kid, version, wrapped_key. Point DATABASE_URL at an empty database, or empty this one, and start Auto Runtime again. The ledger then starts with no playbooks, definitions or runs. Need a hand? Ask us on Discord: https://discord.gg/B8BKcKMRm8', + ); + expect(refused.remedy.command).toBeUndefined(); + expect(refused.message).not.toContain('a-secret-password'); + }); + + it('says in the image which files to delete from the volume, with no command for a container that has stopped', () => { + const refused = refusedFileAt('/app/data/ledger.db', 'docker'); + + expect(refused.remedy).toEqual({ + says: "Delete ledger.db, ledger.db-wal and ledger.db-shm from the volume mounted at /data, the ledger's folder /app/data in the image, or remove the volume, and start Auto Runtime again. Everything in the ledger goes with it, every playbook, definition, run and connection among it.", + command: undefined, + }); + expect(refused.message).toBe( + "The ledger is kept in the file /app/data/ledger.db, and was made by another version of Auto Runtime. This version keeps secrets in a different shape and cannot read the secrets this ledger holds. kept_secrets has the columns held, id, kept_at, kept_by, version, where this version keeps claimed_until, ends_with, held, id, kept_at, kept_by, kid, version, wrapped_key. Delete ledger.db, ledger.db-wal and ledger.db-shm from the volume mounted at /data, the ledger's folder /app/data in the image, or remove the volume, and start Auto Runtime again. Everything in the ledger goes with it, every playbook, definition, run and connection among it. Need a hand? Ask us on Discord: https://discord.gg/B8BKcKMRm8", + ); + }); +}); + +describe('the words for a ledger file of another version in the image', () => { + it('name the volume the image mounts at /data when the ledger is kept in its folder /app/data', () => { + expect(refusedFileAt('/app/data/ledger.db', 'docker').remedy.says).toMatch( + /^Delete ledger\.db, ledger\.db-wal and ledger\.db-shm from the volume mounted at \/data, the ledger's folder \/app\/data in the image, or remove the volume, /u, + ); + }); + + it('name only the folder of a ledger kept anywhere else, from the volume that holds it', () => { + expect(refusedFileAt('/mnt/ledger/runtime.db', 'docker').remedy).toEqual({ + says: 'Delete runtime.db, runtime.db-wal and runtime.db-shm from the volume that holds /mnt/ledger, or remove the volume, and start Auto Runtime again. Everything in the ledger goes with it, every playbook, definition, run and connection among it.', + command: undefined, + }); + }); +}); + +describe('the command that deletes a ledger file of another version', () => { + it('names the file and the two beside it one by one, each quoted where a shell would split it or read a $ in it', () => { + expect(refusedFileAt(`/Users/ada/${awkwardFolder}/ledger.db`).remedy.command).toBe( + String.raw`rm -f '/Users/ada/Application Support/ada'\''s $HOME/ledger.db' '/Users/ada/Application Support/ada'\''s $HOME/ledger.db-wal' '/Users/ada/Application Support/ada'\''s $HOME/ledger.db-shm'`, + ); + }); + + it('deletes in sh exactly the ledger file and the two beside it, on a path with a space, a quote and a $', () => { + const top = mkdtempSync(join(tmpdir(), 'auto-runtime-ledger-')); + onTestFinished(() => { + rmSync(top, { recursive: true, force: true }); + }); + const folder = join(top, awkwardFolder); + mkdirSync(folder, { recursive: true }); + for (const name of ['ledger.db', 'ledger.db-wal', 'ledger.db-shm', 'ledger.db-journal', 'secrets.key']) { + writeFileSync(join(folder, name), name); + } + + execFileSync('/bin/sh', ['-c', String(refusedFileAt(join(folder, 'ledger.db')).remedy.command)]); + + expect(readdirSync(folder).toSorted()).toEqual(['ledger.db-journal', 'secrets.key']); + }); +}); diff --git a/packages/server/src/database/ledger-of-another-version.ts b/packages/server/src/database/ledger-of-another-version.ts new file mode 100644 index 000000000..4c5e63a75 --- /dev/null +++ b/packages/server/src/database/ledger-of-another-version.ts @@ -0,0 +1,96 @@ +import { basename, dirname } from 'node:path'; + +import type { RefusedShape } from '@on.auto/ledger'; +import { Data } from 'effect'; + +import { discordInvite } from '../command-line/local-banner.ts'; +import { resolvedLedgerFile, whereTheLedgerIsKept, type LedgerSettings } from '../settings/ledger-settings.ts'; +import type { Settings } from '../settings/settings.ts'; + +interface LedgerRemedy { + readonly says: string; + readonly command: string | undefined; +} + +export class LedgerOfAnotherVersion extends Data.TaggedError('LedgerOfAnotherVersion')< + RefusedShape & { readonly happened: string; readonly remedy: LedgerRemedy; readonly message: string } +> {} + +export function helpOfferedAt(invite: string): string { + return `Need a hand? Ask us on Discord: ${invite}`; +} + +const unquotedInAShell = /^[\w./@%+=:,-]+$/u; + +function shellWord(text: string): string { + return unquotedInAShell.test(text) ? text : `'${text.replaceAll("'", String.raw`'\''`)}'`; +} + +function whatHappened(ledger: LedgerSettings, holds: string): string { + return `${whereTheLedgerIsKept(ledger)}, and was made by another version of Auto Runtime. This version keeps ${holds} in a different shape and cannot read the ${holds} this ledger holds.`; +} + +export function columnsCompared({ table, found, kept }: RefusedShape): string { + return `${table} has the columns ${found.join(', ')}, where this version keeps ${kept.join(', ')}.`; +} + +const everythingInItGoes = + 'Everything in the ledger goes with it, every playbook, definition, run and connection among it.'; + +function deletingTheFiles(file: string): LedgerRemedy { + return { + says: `Delete the ledger and start Auto Runtime again. ${everythingInItGoes}`, + command: `rm -f ${[file, `${file}-wal`, `${file}-shm`].map((word) => shellWord(word)).join(' ')}`, + }; +} + +const ledgerFolderOfTheImage = '/app/data'; + +function volumeHolding(folder: string): string { + return folder === ledgerFolderOfTheImage + ? `the volume mounted at /data, the ledger's folder ${folder} in the image` + : `the volume that holds ${folder}`; +} + +function deletingTheFilesOnTheVolume(file: string): LedgerRemedy { + const name = basename(file); + return { + says: `Delete ${name}, ${name}-wal and ${name}-shm from ${volumeHolding(dirname(file))}, or remove the volume, and start Auto Runtime again. ${everythingInItGoes}`, + command: undefined, + }; +} + +const emptyingTheDatabase: LedgerRemedy = { + says: 'Point DATABASE_URL at an empty database, or empty this one, and start Auto Runtime again. The ledger then starts with no playbooks, definitions or runs.', + command: undefined, +}; + +function remedyFor({ ledger, distribution }: Pick): LedgerRemedy { + if (ledger.store === 'postgresql') { + return emptyingTheDatabase; + } + const file = resolvedLedgerFile(ledger.file); + return distribution === 'docker' ? deletingTheFilesOnTheVolume(file) : deletingTheFiles(file); +} + +export function ledgerOfAnotherVersion( + shape: RefusedShape, + settings: Pick, +): LedgerOfAnotherVersion { + const happened = whatHappened(settings.ledger, shape.holds); + const remedy = remedyFor(settings); + const deleting = remedy.command === undefined ? [] : [`To delete the ledger, run: ${remedy.command}`]; + return new LedgerOfAnotherVersion({ + ...shape, + happened, + remedy, + message: [happened, columnsCompared(shape), remedy.says, helpOfferedAt(discordInvite), ...deleting].join(' '), + }); +} + +export function ledgerOfAnotherVersionIn(failure: unknown): LedgerOfAnotherVersion | undefined { + if (failure instanceof LedgerOfAnotherVersion) { + return failure; + } + return failure instanceof Error ? ledgerOfAnotherVersionIn(failure.cause) : undefined; +} diff --git a/packages/server/src/lifecycle/ledger-startup.test.ts b/packages/server/src/lifecycle/ledger-startup.test.ts index d74d922ae..9de17dd27 100644 --- a/packages/server/src/lifecycle/ledger-startup.test.ts +++ b/packages/server/src/lifecycle/ledger-startup.test.ts @@ -1,11 +1,13 @@ import { fileURLToPath } from 'node:url'; import { unreachableDatabase } from '@on.auto/ledger/testing'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, onTestFinished } from 'vitest'; import { compositionRoot } from '../composition/composition-root.ts'; import { secretsKey } from '../testing/databases/secrets-key.ts'; import { spawnServer } from '../testing/processes/spawned-server.ts'; +import { aLedgerOfAnotherVersionAt } from '../testing/records/a-ledger-of-another-version.ts'; +import { temporaryLedger } from '../testing/records/temporary-ledger.ts'; import { startServer } from './lifecycle.ts'; const mainModule = fileURLToPath(new URL('../main.ts', import.meta.url)); @@ -61,3 +63,47 @@ describe('a server given both DATABASE_URL and LEDGER_FILE', () => { }); }); }); + +function refusedFor(file: string): string { + return `Auto Runtime could not start: The ledger is kept in the file ${file}, and was made by another version of Auto Runtime. This version keeps secrets in a different shape and cannot read the secrets this ledger holds. kept_secrets has the columns held, id, kept_at, kept_by, version, where this version keeps claimed_until, ends_with, held, id, kept_at, kept_by, kid, version, wrapped_key.`; +} + +const goesWithIt = 'Everything in the ledger goes with it, every playbook, definition, run and connection among it.'; + +const askedAt = 'Need a hand? Ask us on Discord: https://discord.gg/B8BKcKMRm8'; + +describe('a ledger file made by another version of Auto Runtime', () => { + it('stops the process with one line of stderr, its words alone, that says where the ledger is, what to do and where to ask, ending with the command that deletes it', async () => { + const ledger = temporaryLedger(); + onTestFinished(ledger.remove); + aLedgerOfAnotherVersionAt(ledger.fileName); + const file = ledger.fileName; + + const child = spawnServer(mainModule, { HOST: '127.0.0.1', PORT: '0', LEDGER_FILE: file }); + + expect(await child.exited).toBe(1); + expect(child.output()).toEqual({ + stdout: '', + stderr: `${refusedFor(file)} Delete the ledger and start Auto Runtime again. ${goesWithIt} ${askedAt} To delete the ledger, run: rm -f ${file} ${file}-wal ${file}-shm\n`, + }); + }); + + it('says in the image which files to delete from the volume, with no command', async () => { + const ledger = temporaryLedger(); + onTestFinished(ledger.remove); + aLedgerOfAnotherVersionAt(ledger.fileName); + + const child = spawnServer(mainModule, { + HOST: '127.0.0.1', + PORT: '0', + LEDGER_FILE: ledger.fileName, + RUNTIME_DISTRIBUTION: 'docker', + }); + + expect(await child.exited).toBe(1); + expect(child.output()).toEqual({ + stdout: '', + stderr: `${refusedFor(ledger.fileName)} Delete ledger.db, ledger.db-wal and ledger.db-shm from the volume that holds ${ledger.directory}, or remove the volume, and start Auto Runtime again. ${goesWithIt} ${askedAt}\n`, + }); + }); +}); diff --git a/packages/server/src/lifecycle/lifecycle.ts b/packages/server/src/lifecycle/lifecycle.ts index 82dc90472..c8d3ad78d 100644 --- a/packages/server/src/lifecycle/lifecycle.ts +++ b/packages/server/src/lifecycle/lifecycle.ts @@ -35,6 +35,7 @@ import { exitDeadlineMs, shutDown, shutdownTimeoutMs } from './shutdown.ts'; class StartupError extends Data.TaggedError('StartupError')<{ readonly message: string; + readonly cause: unknown; }> {} export interface Install { @@ -119,7 +120,7 @@ async function startRuntime( try { return await makeAppRuntime(services.pipe(Layer.provideMerge(telemetry), Layer.provideMerge(serverLogs(logs)))); } catch (failure) { - throw new StartupError({ message: `The server's services could not start: ${String(failure)}` }); + throw new StartupError({ message: `The server's services could not start: ${String(failure)}`, cause: failure }); } } diff --git a/packages/server/src/lifecycle/run-server.test.ts b/packages/server/src/lifecycle/run-server.test.ts index f792247a2..e5ed3e826 100644 --- a/packages/server/src/lifecycle/run-server.test.ts +++ b/packages/server/src/lifecycle/run-server.test.ts @@ -2,6 +2,7 @@ import type { Environment } from '@on.auto/config'; import { Effect, Layer } from 'effect'; import { describe, expect, it } from 'vitest'; +import { ledgerOfAnotherVersion } from '../database/ledger-of-another-version.ts'; import { optionsComposing, optionsServing } from '../testing/entries/serving-options.ts'; import { isAcceptingConnections } from '../testing/processes/accepting-connections.ts'; import { defaultServerOptions, servedBy } from './lifecycle.ts'; @@ -195,6 +196,32 @@ describe('a server that cannot start', () => { ); }); +describe('a server that cannot start on a ledger of another version', () => { + const refused = ledgerOfAnotherVersion( + { + table: 'kept_secrets', + holds: 'secrets', + found: ['held', 'id', 'kept_at', 'kept_by', 'version'], + kept: ['claimed_until', 'ends_with', 'held', 'id', 'kept_at', 'kept_by', 'kid', 'version', 'wrapped_key'], + }, + { ledger: { store: 'sqlite', file: '/data/ledger.db' }, distribution: 'source' }, + ); + + it('says the refusal in its own words on one line of stderr, with no name of an error before them', async () => { + const { serverProcess, written, errors, exited } = fakeProcess(); + const servicesFailed = new Error("The server's services could not start", { cause: refused }); + + exitOnStartupFailure(serverProcess)(servicesFailed); + + expect(await exited).toBe(1); + expect({ written, errors }).toEqual({ + written: [], + errors: [`Auto Runtime could not start: ${refused.message}\n`], + }); + expect(errors[0]).toMatch(/^Auto Runtime could not start: The ledger is kept in the file \/data\/ledger\.db, /u); + }); +}); + describe('runServer told to stop twice', () => { it('exits 1 at the second signal, saying so, without waiting for its shutdown to finish', async () => { const { serverProcess, stopRequested, stopInsisted, send, exited, errors } = fakeProcess(); diff --git a/packages/server/src/lifecycle/run-server.ts b/packages/server/src/lifecycle/run-server.ts index e2bd4728b..795b16e69 100644 --- a/packages/server/src/lifecycle/run-server.ts +++ b/packages/server/src/lifecycle/run-server.ts @@ -2,6 +2,7 @@ import { setTimeout } from 'node:timers/promises'; import type { Environment } from '@on.auto/config'; +import { ledgerOfAnotherVersionIn } from '../database/ledger-of-another-version.ts'; import { firstStartNotice } from '../usage-ping/usage-ping.ts'; import { startServer, type RunningServer, type ServerOptions } from './lifecycle.ts'; @@ -53,8 +54,20 @@ export async function runServer( return server; } -export function exitOnStartupFailure(serverProcess: ServerProcess): (failure: unknown) => void { +function startupFailureLineOf(failure: unknown): string { + return `Auto Runtime could not start: ${String(failure).replaceAll(/\s*\n\s*/gu, ' ')}`; +} + +function ledgerRefusalLineOf(failure: unknown): string | undefined { + const refusal = ledgerOfAnotherVersionIn(failure); + return refusal === undefined ? undefined : startupFailureLineOf(refusal.message); +} + +export function exitOnStartupFailure( + serverProcess: ServerProcess, + noticeOf: (failure: unknown) => string | undefined = ledgerRefusalLineOf, +): (failure: unknown) => void { return (failure) => { - sayThenExit(serverProcess, `Auto Runtime could not start: ${String(failure).replaceAll(/\s*\n\s*/gu, ' ')}`, 1); + sayThenExit(serverProcess, noticeOf(failure) ?? startupFailureLineOf(failure), 1); }; } diff --git a/packages/server/src/local-runtime/local-start.ts b/packages/server/src/local-runtime/local-start.ts index a2daa0074..e303d0f86 100644 --- a/packages/server/src/local-runtime/local-start.ts +++ b/packages/server/src/local-runtime/local-start.ts @@ -2,6 +2,7 @@ import type { RuntimeAction, StartedBy } from '@on.auto/api'; import { Function } from 'effect'; import { coalescedSignals } from '../command-line/coalesced-signals.ts'; +import { startupFailureNoticeFor } from '../command-line/failure-notice.ts'; import { servedLocally, type LocalServerHost } from '../command-line/local-server.ts'; import { release } from '../composition/release.ts'; import { stopInsistedBy, stopRequestedBy } from '../lifecycle/stop-request.ts'; @@ -64,7 +65,7 @@ export async function startedLocally( stopRequested, }).catch(async (failure: unknown) => { const { exitOnStartupFailure } = await import('../lifecycle/run-server.ts'); - exitOnStartupFailure(serverProcess)(failure); + exitOnStartupFailure(serverProcess, startupFailureNoticeFor(serverProcess.stderr))(failure); }); return asked; } diff --git a/packages/server/src/local-runtime/terminal-control.test.ts b/packages/server/src/local-runtime/terminal-control.test.ts index 07243f7cf..2f53d7ace 100644 --- a/packages/server/src/local-runtime/terminal-control.test.ts +++ b/packages/server/src/local-runtime/terminal-control.test.ts @@ -1,4 +1,4 @@ -import { existsSync } from 'node:fs'; +import { existsSync, mkdirSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; import { createApiKey } from '@on.auto/identity'; @@ -7,6 +7,7 @@ import { describe, expect, it } from 'vitest'; import { aPortNothingListensOn } from '../testing/processes/free-port.ts'; import { homeOfTheTest, spawnRuntimeCommand } from '../testing/processes/runtime-command.ts'; +import { aLedgerOfAnotherVersionAt } from '../testing/records/a-ledger-of-another-version.ts'; import { until } from '../testing/waiting/until.ts'; const homePrefix = 'auto-runtime-terminal-'; @@ -83,3 +84,38 @@ describe('a runtime started in a terminal on a taken port', () => { expect(second.stderr()).toContain('Auto Runtime could not start'); }); }); + +function aHomeWithALedgerOfAnotherVersion(): { readonly home: string; readonly ledger: string } { + const home = homeOfTheTest(homePrefix); + const ledger = join(home, '.auto-runtime', 'ledger.db'); + mkdirSync(join(home, '.auto-runtime')); + aLedgerOfAnotherVersionAt(ledger); + return { home, ledger }; +} + +describe('a runtime started in a terminal on a ledger made by another version', () => { + it('says on stderr, in a notice of its own, where the ledger is, what to do and where to ask, and exits', async () => { + const { home, ledger } = aHomeWithALedgerOfAnotherVersion(); + + const runtime = spawnRuntimeCommand([], { home, port: await aPortNothingListensOn() }); + + expect(await runtime.exited).toBe(1); + expect(runtime.stderr()).toContain(' ✖ Auto Runtime could not start\n\n'); + expect(runtime.stderr()).toContain(`\n rm -f ${ledger} ${ledger}-wal ${ledger}-shm\n`); + expect(runtime.stderr()).toContain('\n Need a hand? Ask us on Discord: https://discord.gg/B8BKcKMRm8\n'); + }); + + it('writes the notice into a file that stderr is sent to with no color or link code', async () => { + const { home, ledger } = aHomeWithALedgerOfAnotherVersion(); + const written = join(home, 'stderr.txt'); + + const runtime = spawnRuntimeCommand([], { home, port: await aPortNothingListensOn(), stderrInto: written }); + const exitCode = await runtime.exited; + const notice = readFileSync(written, 'utf8'); + + expect(exitCode).toBe(1); + expect(notice).toContain(' ✖ Auto Runtime could not start\n\n'); + expect(notice).toContain(`\n rm -f ${ledger} ${ledger}-wal ${ledger}-shm\n`); + expect(notice).not.toContain('\u001B'); + }); +}); diff --git a/packages/server/src/logging/logging.ts b/packages/server/src/logging/logging.ts index 1ef2dba41..6d20fe38d 100644 --- a/packages/server/src/logging/logging.ts +++ b/packages/server/src/logging/logging.ts @@ -1,12 +1,14 @@ -import { resolve } from 'node:path'; - import type { AccessMode, SignInSettings, StudioSettings } from '@on.auto/identity'; import type { Incident } from '@on.auto/operations'; import { Cause, Effect, Layer } from 'effect'; import type { ContentSettings } from '../settings/content-settings.ts'; -import { privateMemory, type LedgerSettings, type PostgreSQLLedgerSettings } from '../settings/ledger-settings.ts'; -import type { SecretsKeySource } from '../settings/secrets-settings.ts'; +import { + resolvedLedgerFile, + whereTheLedgerIsKept, + type LedgerSettings, + type PostgreSQLLedgerSettings, +} from '../settings/ledger-settings.ts'; import type { SignalExport, TelemetrySettings } from '../telemetry/telemetry-settings.ts'; import { logsFrom, logsToStderr, type LogFormat, type LogLevelSetting } from './log-formats.ts'; @@ -100,12 +102,11 @@ export function logConfigFile(report?: ConfigFileReport): Effect.Effect { ); } -function ledgerFileLine(file: string): Effect.Effect { - return Effect.logInfo(`The ledger is kept in the file ${file}`).pipe(Effect.annotateLogs({ ledger_file: file })); +function ledgerFileLine(kept: string, file: string): Effect.Effect { + return Effect.logInfo(kept).pipe(Effect.annotateLogs({ ledger_file: file })); } -function postgresqlLine({ database, host, credentials }: PostgreSQLLedgerSettings): Effect.Effect { - const kept = `The ledger is kept in PostgreSQL, in the database ${database} on ${host}`; +function postgresqlLine(kept: string, { database, host, credentials }: PostgreSQLLedgerSettings): Effect.Effect { const annotations = { database, database_host: host, database_credentials: credentials.kind }; if (credentials.kind === 'url') { return Effect.logInfo(kept).pipe(Effect.annotateLogs(annotations)); @@ -117,9 +118,10 @@ function postgresqlLine({ database, host, credentials }: PostgreSQLLedgerSetting } export function logLedger(ledger: LedgerSettings): Effect.Effect { + const kept = whereTheLedgerIsKept(ledger); return ledger.store === 'postgresql' - ? postgresqlLine(ledger) - : ledgerFileLine(ledger.file === privateMemory ? ledger.file : resolve(ledger.file)); + ? postgresqlLine(kept, ledger) + : ledgerFileLine(kept, resolvedLedgerFile(ledger.file)); } function objectStorageLine(bucket: string, prefix: string, endpoint: string | undefined): Effect.Effect { @@ -143,59 +145,6 @@ export function logContentStore(content: ContentSettings): Effect.Effect { : objectStorageLine(content.location.bucket, content.location.prefix, content.endpoint); } -export interface SecretStoreReport { - readonly kid: string; - readonly resealed: number; - readonly keyFrom: SecretsKeySource; - readonly moved?: number | undefined; -} - -export interface ParameterStoreReport { - readonly prefix: string; - readonly region: string; - readonly moved: number; -} - -export function keyHeldBy(source: SecretsKeySource): string { - if (source.from === 'environment') { - return `that ${source.setting} holds`; - } - if (source.from === 'kms') { - return `that the KMS key ${source.arn} holds`; - } - return source.from === 'file' ? `that the file ${resolve(source.file)} holds` : 'that the server made at this start'; -} - -function movedThere(moved: number | undefined): string { - return moved === undefined ? '' : `, and ${moved} kept by the previous keeper were moved there`; -} - -export function logSecretStore({ kid, resealed, keyFrom, moved }: SecretStoreReport): Effect.Effect { - return Effect.logInfo( - `Secrets are kept in the ledger's database, sealed under the key ${kid} ${keyHeldBy(keyFrom)}, and ${resealed} sealed under the previous key were sealed again under it${movedThere(moved)}`, - ).pipe( - Effect.annotateLogs({ - secrets_kid: kid, - secrets_resealed: resealed, - secrets_key_from: keyFrom.from, - ...(moved === undefined ? {} : { secrets_moved: moved }), - }), - ); -} - -export function logSecretsInParameterStore({ prefix, region, moved }: ParameterStoreReport): Effect.Effect { - return Effect.logInfo( - `Secrets are kept in AWS Parameter Store under ${prefix} in ${region}${movedThere(moved)}`, - ).pipe( - Effect.annotateLogs({ - secrets_keeper: 'aws-parameter-store', - secrets_prefix: prefix, - secrets_region: region, - secrets_moved: moved, - }), - ); -} - const causeNotFormattable = Effect.logError('Unexpected error whose cause could not be formatted'); function withoutRequestContent(error: Readonly): string { diff --git a/packages/server/src/secrets/own-keys.ts b/packages/server/src/secrets/own-keys.ts index 0f3d92c9f..1cb3389eb 100644 --- a/packages/server/src/secrets/own-keys.ts +++ b/packages/server/src/secrets/own-keys.ts @@ -17,9 +17,9 @@ import { secretKeyBytes, type SecretStore } from '@on.auto/secrets'; import { Data, DateTime, Effect, Option, Redacted, Result, Schema } from 'effect'; import { runtimeLedger } from '../composition/runtime-ledger.ts'; -import { keyHeldBy } from '../logging/logging.ts'; import { secretsKeyOfBase64 } from '../settings/secrets-settings.ts'; import type { Settings } from '../settings/settings.ts'; +import { keyHeldBy } from './secrets-logging.ts'; export const ownKeysStream = 'runtime/own-keys'; diff --git a/packages/server/src/secrets/secret-store-start.ts b/packages/server/src/secrets/secret-store-start.ts index afadac19e..332be6899 100644 --- a/packages/server/src/secrets/secret-store-start.ts +++ b/packages/server/src/secrets/secret-store-start.ts @@ -10,11 +10,11 @@ import { import type { DatabaseSettings } from '@on.auto/workflow-host'; import { Effect, Option } from 'effect'; -import { logSecretsInParameterStore, logSecretStore } from '../logging/logging.ts'; import { InvalidSettingsError } from '../settings/invalid-settings-error.ts'; import { currentKeeper, type KeeperGiven, type KeeperSettings } from '../settings/keeper-settings.ts'; import type { SecretsSettings } from '../settings/secrets-settings.ts'; import { secretRowsOn } from './secret-store.ts'; +import { logSecretsInParameterStore, logSecretStore } from './secrets-logging.ts'; export interface SecretsMade { readonly custodians: Custodians | undefined; diff --git a/packages/server/src/secrets/secrets-logging.ts b/packages/server/src/secrets/secrets-logging.ts new file mode 100644 index 000000000..562836471 --- /dev/null +++ b/packages/server/src/secrets/secrets-logging.ts @@ -0,0 +1,58 @@ +import { resolve } from 'node:path'; + +import { Effect } from 'effect'; + +import type { SecretsKeySource } from '../settings/secrets-settings.ts'; + +export interface SecretStoreReport { + readonly kid: string; + readonly resealed: number; + readonly keyFrom: SecretsKeySource; + readonly moved?: number | undefined; +} + +export interface ParameterStoreReport { + readonly prefix: string; + readonly region: string; + readonly moved: number; +} + +export function keyHeldBy(source: SecretsKeySource): string { + if (source.from === 'environment') { + return `that ${source.setting} holds`; + } + if (source.from === 'kms') { + return `that the KMS key ${source.arn} holds`; + } + return source.from === 'file' ? `that the file ${resolve(source.file)} holds` : 'that the server made at this start'; +} + +function movedThere(moved: number | undefined): string { + return moved === undefined ? '' : `, and ${moved} kept by the previous keeper were moved there`; +} + +export function logSecretStore({ kid, resealed, keyFrom, moved }: SecretStoreReport): Effect.Effect { + return Effect.logInfo( + `Secrets are kept in the ledger's database, sealed under the key ${kid} ${keyHeldBy(keyFrom)}, and ${resealed} sealed under the previous key were sealed again under it${movedThere(moved)}`, + ).pipe( + Effect.annotateLogs({ + secrets_kid: kid, + secrets_resealed: resealed, + secrets_key_from: keyFrom.from, + ...(moved === undefined ? {} : { secrets_moved: moved }), + }), + ); +} + +export function logSecretsInParameterStore({ prefix, region, moved }: ParameterStoreReport): Effect.Effect { + return Effect.logInfo( + `Secrets are kept in AWS Parameter Store under ${prefix} in ${region}${movedThere(moved)}`, + ).pipe( + Effect.annotateLogs({ + secrets_keeper: 'aws-parameter-store', + secrets_prefix: prefix, + secrets_region: region, + secrets_moved: moved, + }), + ); +} diff --git a/packages/server/src/settings/ledger-settings.ts b/packages/server/src/settings/ledger-settings.ts index cbe55c627..2d4cb4f24 100644 --- a/packages/server/src/settings/ledger-settings.ts +++ b/packages/server/src/settings/ledger-settings.ts @@ -1,4 +1,4 @@ -import { dirname, join } from 'node:path'; +import { dirname, join, resolve } from 'node:path'; import type { Environment } from '@on.auto/config'; import { Config, ConfigProvider, Effect, Option, Redacted } from 'effect'; @@ -23,12 +23,22 @@ export type LedgerSettings = { readonly store: 'sqlite'; readonly file: string } const defaultFile = 'data/ledger.db'; -export const privateMemory = ':memory:'; +const privateMemory = ':memory:'; export function besideTheLedgerFile(ledger: LedgerSettings, name: string): string | undefined { return ledger.store === 'sqlite' && ledger.file !== privateMemory ? join(dirname(ledger.file), name) : undefined; } +export function resolvedLedgerFile(file: string): string { + return file === privateMemory ? file : resolve(file); +} + +export function whereTheLedgerIsKept(ledger: LedgerSettings): string { + return ledger.store === 'postgresql' + ? `The ledger is kept in PostgreSQL, in the database ${ledger.database} on ${ledger.host}` + : `The ledger is kept in the file ${resolvedLedgerFile(ledger.file)}`; +} + const postgresqlSchemes: ReadonlySet = new Set(['postgres:', 'postgresql:']); export const ledgerVariables: readonly string[] = ['LEDGER_FILE', 'DATABASE_URL', 'DATABASE_CREDENTIALS', 'AWS_REGION']; diff --git a/packages/server/src/settings/settings.test.ts b/packages/server/src/settings/settings.test.ts index 53151ce74..86d7a63c3 100644 --- a/packages/server/src/settings/settings.test.ts +++ b/packages/server/src/settings/settings.test.ts @@ -79,6 +79,7 @@ describe('readSettings', () => { providerValues: new Map(), telemetry: undefined, usagePing: { distribution: 'source' }, + distribution: 'source', configFile: undefined, }); }); @@ -153,9 +154,19 @@ const everySettingRead = { providerValues: new Map(), telemetry: undefined, usagePing: { distribution: 'source' }, + distribution: 'source', configFile: undefined, }; +describe('readSettings given the distribution', () => { + it('reads where the runtime came from even when the usage ping is turned off', () => { + expect(readSettings({ RUNTIME_DISTRIBUTION: 'docker', DO_NOT_TRACK: '1' })).toMatchObject({ + usagePing: undefined, + distribution: 'docker', + }); + }); +}); + describe('readSettings given every setting', () => { it('reads every setting from the environment it is given', () => { const { models, toolServers, interaction, studio, accessMode, publicUrl, ...server } = readSettings(everySetting); diff --git a/packages/server/src/settings/settings.ts b/packages/server/src/settings/settings.ts index 4a499c108..3aa32925d 100644 --- a/packages/server/src/settings/settings.ts +++ b/packages/server/src/settings/settings.ts @@ -4,7 +4,12 @@ import { Effect, Result } from 'effect'; import { fileSettings } from '../config-file/file-settings.ts'; import { readTelemetrySettings, type TelemetrySettings } from '../telemetry/telemetry-settings.ts'; -import { readUsagePingSettings, type UsagePingSettings } from '../usage-ping/usage-ping-settings.ts'; +import { + distributionIn, + readUsagePingSettings, + type Distribution, + type UsagePingSettings, +} from '../usage-ping/usage-ping-settings.ts'; import { readAccessSettings, readCredentialSettings, @@ -28,6 +33,7 @@ export interface Settings extends ServedSettings, AccessSettings, CredentialSett readonly proxy: ProxySettings; readonly telemetry: TelemetrySettings | undefined; readonly usagePing: UsagePingSettings | undefined; + readonly distribution: Distribution; readonly configFile: ConfigFileSources | undefined; readonly providerSetup?: string | undefined; } @@ -61,6 +67,7 @@ export function readSettings(given: Environment): Settings { ...served, telemetry, usagePing, + distribution: distributionIn(environment), configFile: sourcesOf(file), }; } diff --git a/packages/server/src/testing/processes/runtime-command.ts b/packages/server/src/testing/processes/runtime-command.ts index a98bafe95..d817b1584 100644 --- a/packages/server/src/testing/processes/runtime-command.ts +++ b/packages/server/src/testing/processes/runtime-command.ts @@ -29,6 +29,7 @@ export interface RuntimeCommandPlace { readonly port: number; readonly settings?: Readonly>; readonly inAGroupOfItsOwn?: boolean; + readonly stderrInto?: string; } export interface RuntimeOnThePort { @@ -79,11 +80,17 @@ export function homeOfTheTest(prefix: string): string { return home; } +function commandLineOf(args: readonly string[], stderrInto: string | undefined): readonly [string, readonly string[]] { + return stderrInto === undefined + ? [process.execPath, [command, ...args]] + : ['/bin/sh', ['-c', 'exec "$@" 2>"$0"', stderrInto, process.execPath, command, ...args]]; +} + export function spawnRuntimeCommand( args: readonly string[], - { home, port, settings = {}, inAGroupOfItsOwn = false }: RuntimeCommandPlace, + { home, port, settings = {}, inAGroupOfItsOwn = false, stderrInto }: RuntimeCommandPlace, ): RuntimeCommandRun { - const child = spawnForTheTest(process.execPath, [command, ...args], { + const child = spawnForTheTest(...commandLineOf(args, stderrInto), { detached: inAGroupOfItsOwn, cwd: home, env: { diff --git a/packages/server/src/testing/records/a-ledger-of-another-version.ts b/packages/server/src/testing/records/a-ledger-of-another-version.ts new file mode 100644 index 000000000..cde6dcda5 --- /dev/null +++ b/packages/server/src/testing/records/a-ledger-of-another-version.ts @@ -0,0 +1,10 @@ +import { DatabaseSync } from 'node:sqlite'; + +const secretsOfAnOlderShape = + 'CREATE TABLE kept_secrets (id TEXT PRIMARY KEY, version INTEGER NOT NULL, kept_at TEXT NOT NULL, kept_by TEXT NOT NULL, held TEXT NOT NULL)'; + +export function aLedgerOfAnotherVersionAt(file: string): void { + const database = new DatabaseSync(file); + database.exec(secretsOfAnOlderShape); + database.close(); +} diff --git a/packages/server/src/usage-ping/usage-ping-settings.ts b/packages/server/src/usage-ping/usage-ping-settings.ts index e3482a0c6..eeab9efc0 100644 --- a/packages/server/src/usage-ping/usage-ping-settings.ts +++ b/packages/server/src/usage-ping/usage-ping-settings.ts @@ -3,7 +3,7 @@ import { Effect } from 'effect'; import { InvalidSettingsError } from '../settings/invalid-settings-error.ts'; -type Distribution = 'npx' | 'docker' | 'source'; +export type Distribution = 'npx' | 'docker' | 'source'; export interface UsagePingSettings { readonly distribution: Distribution; @@ -28,7 +28,7 @@ function valueIn(environment: Environment, name: string): string { return (environment[name] ?? '').trim().toLowerCase(); } -function distributionIn(environment: Environment): Distribution { +export function distributionIn(environment: Environment): Distribution { const given = valueIn(environment, distributionVariable); return given === 'npx' || given === 'docker' ? given : 'source'; }