diff --git a/packages/discern/README.md b/packages/discern/README.md index c62d222..a57829a 100644 --- a/packages/discern/README.md +++ b/packages/discern/README.md @@ -114,6 +114,8 @@ console.log(await Effect.runPromise(guarded('bump lodash').pipe(Effect.provide(D `policy.runWithTrace(input)` returns the value together with a trace of every case evaluated, how it resolved, and the answers it used. +A bound that cannot hold is refused when the pattern is built, not accepted silently. `above` and `atLeast` refuse a `missBelow` greater than the threshold, `below` and `atMost` refuse a `missAbove` less than it, and a `between` whose low end comes after its high end is refused too (ratings compare by level position). Equal bounds stay legal. A policy run, `Eval.run`, `Eval.sweep` and `Eval.calibrate` fail before the model is asked, with `InvalidThresholdError` for a bound and `InvalidRangeError` for a range, whichever case the input would have reached. `Discern.evaluate` on a refused pattern answers `Uncertain` with the refusal message. + ### Record, replay, cache, budget Interceptors decorate the model. `recording` stores every answer, and `Discern.Model.replayLayer` reruns any program from those answers with no model at all: @@ -146,6 +148,8 @@ await Effect.runPromise(review('drop Node 18').pipe(Effect.provide(metered))) console.log(await Effect.runPromise(Discern.Model.spent(spend))) // { decisions: 2, calls: 1 } ``` +`replaying` and `caching` hold a stored answer to the rules a live provider is held to. An answer whose probability, rating or label breaks them fails with an error that `Discern.Model.isInvalidObservation` recognises, before the model is asked. + The same stack is a builder: `Discern.Model.model(cautious).recording(store).budgeted(spend).layer` appends one interceptor at a time and compiles the same layer. Answers are keyed by the question's content and the input's structure. Rewording instructions or reordering criteria is a new question; reordering an input object's keys is not. @@ -156,6 +160,8 @@ Stores and budgets are handles: `Discern.Model.snapshot`, `load`, `get`, `set`, `Discern.Eval` runs a pattern over labelled examples and reports accuracy, precision, recall and F1. `Eval.calibrate` asks the model once per example and picks the best threshold from those answers; `Eval.sweep` skips examples a deterministic pattern already settles. +`Eval.calibrate` refuses an empty `values` list with `CalibrationValuesRefused` before it builds a pattern or asks the model. + ### Route between procedures `Discern.Procedure` routes a request to one of several named procedures with a single classification. The registry takes its members as a record, and each key is that member's id: diff --git a/packages/discern/etc/discern.api.md b/packages/discern/etc/discern.api.md index f7791c7..1187177 100644 --- a/packages/discern/etc/discern.api.md +++ b/packages/discern/etc/discern.api.md @@ -6,7 +6,7 @@ // @public (undocumented) export namespace Discern { - export { Answer, Answers, AnyDecision, BandOptions, CalibrateOptions, CaseInspection, CaseTrace, ClassificationMatcher, ClassifyDecision, ClassifyOptions, ClassifyThresholds, CompiledPlan, DecisionIdCollisionError, DecisionInspection, DecisionNode, DecisionScope, Eval, EvalExample, EvalMetrics, EvalRecord, EvalReport, ExhaustiveMatchError, FinishedMatcher, HandlerResult, InvalidThresholdError, LeafOptions, Matcher, MatcherFlavor, MeasureCommandRejected, MeasureError, MeasureInputRefused, mod_d_exports$2 as Model, NodeCore, Pattern, PatternAst, PatternEvaluator, PatternMatched, PatternMissed, PatternRefusal, PatternResult, PatternStatus, PatternUncertain, Policy, PolicyCommandRejected, PolicyRun, PolicyTraced, Preview, ProbabilityBand, ProbabilityDecision, ProbabilityOptions, mod_d_exports$1 as Procedure, RateDecision, RateOptions, SweepOptions, SweepResult, Trace, UncertainContext, UncertainMatchError, above, and, ask, atLeast, atMost, band, below, between, caseOf as case, classify, compile, decision, deterministic, evaluate, exhaustive, inspect, is, margin, match, matched, missed, not, on, onUncertain, oneOf, or, orElse, preview, probability, rate, reasonOf, runWithTrace, statusIs, statusOf, type, uncertain, value, when, where, whereResult }; + export { Answer, Answers, AnyDecision, BandOptions, CalibrateOptions, CalibrationValuesRefused, CaseInspection, CaseTrace, ClassificationMatcher, ClassifyDecision, ClassifyOptions, ClassifyThresholds, CompiledPlan, DecisionIdCollisionError, DecisionInspection, DecisionNode, DecisionScope, Eval, EvalExample, EvalMetrics, EvalRecord, EvalReport, ExhaustiveMatchError, FinishedMatcher, HandlerResult, InvalidRangeError, InvalidThresholdError, LeafOptions, Matcher, MatcherFlavor, MeasureCommandRejected, MeasureError, MeasureInputRefused, mod_d_exports$2 as Model, NodeCore, Pattern, PatternAst, PatternEvaluator, PatternMatched, PatternMissed, PatternRefusal, PatternResult, PatternStatus, PatternUncertain, Policy, PolicyCommandRejected, PolicyRun, PolicyTraced, Preview, ProbabilityBand, ProbabilityDecision, ProbabilityOptions, mod_d_exports$1 as Procedure, RangeRefusal, RateDecision, RateOptions, SweepOptions, SweepResult, ThresholdRefusal, Trace, UncertainContext, UncertainMatchError, above, and, ask, atLeast, atMost, band, below, between, caseOf as case, classify, compile, decision, deterministic, evaluate, exhaustive, inspect, is, margin, match, matched, missed, not, on, onUncertain, oneOf, or, orElse, preview, probability, rate, reasonOf, runWithTrace, statusIs, statusOf, type, uncertain, value, when, where, whereResult }; } // (No @packageDocumentation comment for this package) diff --git a/packages/discern/src/Discern/mod.ts b/packages/discern/src/Discern/mod.ts index 0ce7981..bb182ab 100644 --- a/packages/discern/src/Discern/mod.ts +++ b/packages/discern/src/Discern/mod.ts @@ -52,6 +52,8 @@ export type { PatternEvaluator, PatternRefusal, Preview, + RangeRefusal, + ThresholdRefusal, UncertainContext, } from '../pattern.blueprint.js' export { matched, missed, reasonOf, statusIs, statusOf, uncertain } from '../Verdict.schema.js' @@ -77,8 +79,10 @@ export { Eval } from '../eval.js' export type { CalibrateOptions, EvalExample, MeasureError, SweepOptions, SweepResult } from '../measure-pattern.cell.js' export { + CalibrationValuesRefused, DecisionIdCollisionError, ExhaustiveMatchError, + InvalidRangeError, InvalidThresholdError, MeasureCommandRejected, MeasureInputRefused, diff --git a/packages/discern/src/DiscernError.schema.ts b/packages/discern/src/DiscernError.schema.ts index f6aad64..de0005e 100644 --- a/packages/discern/src/DiscernError.schema.ts +++ b/packages/discern/src/DiscernError.schema.ts @@ -37,6 +37,29 @@ export class InvalidThresholdError extends Schema.TaggedError()( + 'InvalidRangeError', + { + range: Schema.String, + low: Schema.Union([Schema.Finite, Schema.String]), + high: Schema.Union([Schema.Finite, Schema.String]), + message: Schema.String, + }, +) {} + +export class CalibrationValuesRefused extends Schema.TaggedError()( + 'CalibrationValuesRefused', + {}, +) { + override get message(): string { + return 'Calibration needs at least one candidate value; the list was empty' + } +} + /** * The policy run's command was rejected before any case could be answered. * Carries the cases the read had evaluated, so the refusal names the run it diff --git a/packages/discern/src/Model/mod.ts b/packages/discern/src/Model/mod.ts index 1cd82e7..b21d1e9 100644 --- a/packages/discern/src/Model/mod.ts +++ b/packages/discern/src/Model/mod.ts @@ -7,6 +7,7 @@ export { fromProvider, intercept, isBudgetExceeded, + isInvalidObservation, isReplayMiss, layer, provider, diff --git a/packages/discern/src/cache-provider.cell.ts b/packages/discern/src/cache-provider.cell.ts index c1e4504..a9d0761 100644 --- a/packages/discern/src/cache-provider.cell.ts +++ b/packages/discern/src/cache-provider.cell.ts @@ -3,7 +3,7 @@ import * as AiError from 'effect/ai/AiError' import type * as Decision from 'effect/ai/Decision' import type * as DecisionModel from 'effect/ai/DecisionModel' import * as Effect from 'effect/Effect' -import type { ObservationSplit, Provider } from './decision-model.blueprint.js' +import type { ObservationSplit, Provider, StoredHitCheck } from './decision-model.blueprint.js' import type { ProviderAnswer } from './Observation.schema.js' import { CurrentRegion } from './region.service.js' import { SelectObservationSource, selectObservationSource } from './select-observation-source.workflow.js' @@ -12,6 +12,7 @@ export interface CacheRequest { readonly options: DecisionModel.ProviderOptions readonly inner: Provider readonly split: ObservationSplit + readonly validate: StoredHitCheck readonly record: ( answers: Readonly>, regionPath: ReadonlyArray, @@ -23,6 +24,7 @@ export type CacheRead = (typeof SelectObservationSource)['Encoded'] & { readonly missingDecisions: Readonly> readonly state: DecisionModel.ProviderOptions['state'] readonly inner: Provider + readonly validate: StoredHitCheck readonly record: ( answers: Readonly>, regionPath: ReadonlyArray, @@ -39,20 +41,25 @@ const readCache = (request: CacheRequest): Effect.Effect => missingDecisions: request.split.missingDecisions, state: request.options.state, inner: request.inner, + validate: request.validate, record: request.record, }) const askAndRecord = (read: CacheRead): Effect.Effect => Effect.flatMap( - CurrentRegion.useSync((path) => path), - (regionPath) => + read.validate(read.hits), + () => Effect.flatMap( - read.inner.decide({ state: read.state, decisions: read.missingDecisions }), - (response) => - Effect.map(read.record(response.answers, regionPath), () => ({ - answers: { ...read.hits, ...response.answers }, - usage: response.usage, - })), + CurrentRegion.useSync((path) => path), + (regionPath) => + Effect.flatMap( + read.inner.decide({ state: read.state, decisions: read.missingDecisions }), + (response) => + Effect.map(read.record(response.answers, regionPath), () => ({ + answers: { ...read.hits, ...response.answers }, + usage: response.usage, + })), + ), ), ) @@ -60,7 +67,10 @@ export const cacheObservations = Sandwich.named('discern.model.cache')(readCache .decide(selectObservationSource) .write({ AllRecorded: (_recorded, read) => - Effect.succeed({ answers: read.hits, usage: { inputTokens: undefined, outputTokens: undefined } }), + Effect.as(read.validate(read.hits), { + answers: read.hits, + usage: { inputTokens: undefined, outputTokens: undefined }, + }), AskForMissing: (_asked, read) => askAndRecord(read), RecordingMissing: (refusal, _read) => Effect.succeed(refusal), CommandRejected: (rejected, read) => diff --git a/packages/discern/src/decision-model.blueprint.ts b/packages/discern/src/decision-model.blueprint.ts index c8229ac..a80ab3b 100644 --- a/packages/discern/src/decision-model.blueprint.ts +++ b/packages/discern/src/decision-model.blueprint.ts @@ -28,6 +28,7 @@ export const DiscernMethod = { caching: 'caching', budgeted: 'budgeted', refusing: 'refusing', + observation: 'observation', } as const const DISCERN_MODULE = 'Discern' @@ -45,6 +46,10 @@ export const isBudgetExceeded = (error: unknown): error is AiError.AiError => export const isReplayMiss = (error: unknown): error is AiError.AiError => AiError.isAiError(error) && isDiscernMethodOf(error, DiscernMethod.replaying) +/** True for the failure raised when an answer read back from a store fails validation. */ +export const isInvalidObservation = (error: unknown): error is AiError.AiError => + AiError.isAiError(error) && isDiscernMethodOf(error, DiscernMethod.observation) + const replayMissFailure = (refusal: (typeof RecordingMissing)['Encoded']): AiError.AiError => discernFailure(DiscernMethod.replaying, new AiError.InvalidRequestError({ description: refusal.detail })) @@ -164,6 +169,49 @@ const recordAnswers = ( { discard: true }, ) +export type StoredHitCheck = (hits: Readonly>) => Effect.Effect + +const invalidObservationFailure = (error: AiError.AiError): AiError.AiError => + discernFailure( + DiscernMethod.observation, + new AiError.InvalidOutputError({ + description: Match.value(error.reason).pipe( + Match.tag('InvalidOutputError', (reason) => reason.description), + Match.orElse((reason) => reason.message), + ), + }), + ) + +const hitDecisionsOf = ( + decisions: Readonly>, + hits: Readonly>, +): Readonly> => + Object.fromEntries(Object.entries(decisions).filter(([id]) => Object.hasOwn(hits, id))) + +/** + * Holds stored answers to the rules a provider's answers are held to, by running + * them through a throwaway `DecisionModel` whose provider returns them. A gate + * only: the stored hits pass on unchanged, because the validator's output drops + * the `_tag` the stored form carries. + */ +const storedHitCheckOf = (request: DecisionModel.ProviderOptions): StoredHitCheck => (hits) => { + const decisions = hitDecisionsOf(request.decisions, hits) + return Object.keys(decisions).length === 0 + ? Effect.void + : Effect.flatMap( + DecisionModel.make({ + decide: () => Effect.succeed({ answers: hits, usage: { inputTokens: undefined, outputTokens: undefined } }), + }), + (checker) => + Effect.asVoid( + Effect.mapError( + checker.decide(Decision.make({ input: Schema.Json, decisions }), { input: request.state }), + invalidObservationFailure, + ), + ), + ) +} + /** * A `DecisionModel` decorator. Interceptors are listed outermost-first, so in * `[recording, caching, budgeted]` the recorder observes every answer, the cache @@ -230,7 +278,13 @@ export const replaying: { splitObservations(request.decisions, request.state, lookup), (split) => Effect.flatMap( - replayObservations.run({ options: request, inner, split, onMissing }), + replayObservations.run({ + options: request, + inner, + split, + onMissing, + validate: storedHitCheckOf(request), + }), (answer) => Match.value(answer).pipe( Match.tag('RecordingMissing', (refusal) => Effect.fail(replayMissFailure(refusal))), @@ -257,6 +311,7 @@ export const caching = (into: ObservationStore): Interceptor => (inner) => options: request, inner, split, + validate: storedHitCheckOf(request), record: (answers, regionPath) => recordAnswers(into, request.decisions, request.state, answers, regionPath), }), (answer) => diff --git a/packages/discern/src/decision.blueprint.ts b/packages/discern/src/decision.blueprint.ts index 54abcd9..9dda14c 100644 --- a/packages/discern/src/decision.blueprint.ts +++ b/packages/discern/src/decision.blueprint.ts @@ -9,6 +9,7 @@ * Judging an answer against a threshold, band, label, or range is pure * blueprint code (KD7) — never a workflow. */ +/// import { Blueprint } from '@systemfsoftware/effect-cell-types' import { Array as Arr, Match, Predicate, Schema } from 'effect' import * as AiError from 'effect/ai/AiError' @@ -358,6 +359,7 @@ const missAtOf = (limits: ClassifyThresholds): number => limits.miss ?? matchAtO const thresholdRefusalOf = (limits: ClassifyThresholds): Option.Option => missAtOf(limits) > matchAtOf(limits) ? Option.some({ + kind: 'threshold', threshold: 'miss', value: missAtOf(limits), limit: matchAtOf(limits), @@ -425,11 +427,7 @@ const classifyVerdictOf = ( ) } -const refusedLeaf = ( - node: DecisionNode, S>, - description: string, - refusal: PatternRefusal, -): Pattern => +const refusedLeaf = (node: NodeCore, description: string, refusal: PatternRefusal): Pattern => semanticLeaf({ node, id: `p_${hash({ node: node.id, description })}`, @@ -438,6 +436,14 @@ const refusedLeaf = ( + node: NodeCore, + description: string, + refusal: Option.Option, + accepted: () => Pattern, +): Pattern => + Option.match(refusal, { onNone: accepted, onSome: (refused) => refusedLeaf(node, description, refused) }) + const thresholdLeaf = ( node: DecisionNode, S>, labels: ReadonlyArray, @@ -449,7 +455,7 @@ const thresholdLeaf = classifyVerdictOf(answer, labels, label, limits), { description: `${node.id} is ${label} with confidence thresholds`, }), - onSome: (refusal) => refusedLeaf(node, `${node.id} is ${label} with confidence thresholds`, refusal), + onSome: (refusal) => refusedLeaf(node, `${node.id} is ${label} with confidence thresholds`, refusal), }) const classifyIs = ( @@ -546,27 +552,68 @@ const belowVerdictOf = (probability: number, threshold: number, inclusive: boole ) } +const bothFinite = (first: number, second: number): boolean => Number.isFinite(first) && Number.isFinite(second) + +const finitelyAfter = (later: number, earlier: number): boolean => bothFinite(later, earlier) && later > earlier + +const invertedMissRefusalOf = ( + bound: { + readonly subject: string + readonly name: 'missBelow' | 'missAbove' + readonly relation: '<=' | '>=' + readonly threshold: number + readonly miss: number + }, + inverted: boolean, +): Option.Option => + inverted + ? Option.some({ + kind: 'threshold', + threshold: bound.name, + value: bound.miss, + limit: bound.threshold, + message: + `Probability pattern \`${bound.subject}\`: \`${bound.name}\` (${bound.miss}) must be ${bound.relation} the threshold (${bound.threshold})`, + }) + : Option.none() + const aboveLeaf = ( node: DecisionNode, threshold: number, inclusive: boolean, missAt: number, -): Pattern => - node.whereResult( - (answer) => aboveVerdictOf(answer.probability, threshold, inclusive, missAt), - { description: `${node.id} ${inclusive ? '>=' : '>'} ${threshold}` }, +): Pattern => { + const description = `${node.id} ${inclusive ? '>=' : '>'} ${threshold}` + return refusedOr( + node, + `${description} with missBelow ${missAt}`, + invertedMissRefusalOf( + { subject: description, name: 'missBelow', relation: '<=', threshold, miss: missAt }, + finitelyAfter(missAt, threshold), + ), + () => + node.whereResult((answer) => aboveVerdictOf(answer.probability, threshold, inclusive, missAt), { description }), ) +} const belowLeaf = ( node: DecisionNode, threshold: number, inclusive: boolean, missAt: number, -): Pattern => - node.whereResult( - (answer) => belowVerdictOf(answer.probability, threshold, inclusive, missAt), - { description: `${node.id} ${inclusive ? '<=' : '<'} ${threshold}` }, +): Pattern => { + const description = `${node.id} ${inclusive ? '<=' : '<'} ${threshold}` + return refusedOr( + node, + `${description} with missAbove ${missAt}`, + invertedMissRefusalOf( + { subject: description, name: 'missAbove', relation: '>=', threshold, miss: missAt }, + finitelyAfter(threshold, missAt), + ), + () => + node.whereResult((answer) => belowVerdictOf(answer.probability, threshold, inclusive, missAt), { description }), ) +} const probabilityFor = ( options: ProbabilityOptions, @@ -583,6 +630,7 @@ const probabilityFor = ( const bandRefusalOf = (limits: ProbabilityBand): Option.Option => limits.miss > limits.match ? Option.some({ + kind: 'threshold', threshold: 'miss', value: limits.miss, limit: limits.match, @@ -1167,6 +1215,15 @@ export const between: { const probabilityBoundOf = (bound: number | string): number => (typeof bound === 'number' ? bound : Number.NaN) +const reversedRangeRefusalOf = ( + range: { readonly description: string; readonly low: number | string; readonly high: number | string }, + reversed: boolean, + message: string, +): Option.Option => + reversed + ? Option.some({ kind: 'range', range: range.description, low: range.low, high: range.high, message }) + : Option.none() + const betweenOfProbability = ( self: ProbabilityDecision, low: number | string, @@ -1174,19 +1231,50 @@ const betweenOfProbability = ( ): Pattern => { const lower = probabilityBoundOf(low) const upper = probabilityBoundOf(high) - return where(self, (answer) => answer.probability >= lower && answer.probability <= upper, { - description: `${self.id} between ${low} and ${high}`, - }) + const description = `${self.id} between ${low} and ${high}` + return refusedOr( + self, + description, + reversedRangeRefusalOf( + { description, low: lower, high: upper }, + finitelyAfter(lower, upper), + `Probability range \`${description}\`: \`low\` (${lower}) must be <= \`high\` (${upper})`, + ), + () => where(self, (answer) => answer.probability >= lower && answer.probability <= upper, { description }), + ) } +const declaredRatingPositionOf = (levels: ReadonlyArray, bound: number | string): Option.Option => + typeof bound === 'number' + ? Option.some(bound) + : Option.filter(Option.some(ratePositionOf(levels, bound)), (position) => position >= 0) + +const knownRatingPositionOf = (levels: ReadonlyArray, bound: number | string): Option.Option => + Option.filter(declaredRatingPositionOf(levels, bound), Number.isFinite) + +const rateRangeReversedOf = (levels: ReadonlyArray, low: number | string, high: number | string): boolean => + Option.getOrElse( + Option.zipWith(knownRatingPositionOf(levels, low), knownRatingPositionOf(levels, high), (from, to) => from > to), + () => false, + ) + const betweenOfRate = ( self: RateDecision, low: number | string, high: number | string, -): Pattern => - where(self, (answer) => betweenRatings(answer, self.levels, low, high), { - description: `${self.id} between ${low} and ${high}`, - }) +): Pattern => { + const description = `${self.id} between ${low} and ${high}` + return refusedOr( + self, + description, + reversedRangeRefusalOf( + { description, low, high }, + rateRangeReversedOf(self.levels, low, high), + `Rating range \`${description}\`: \`${low}\` must not come after \`${high}\``, + ), + () => where(self, (answer) => betweenRatings(answer, self.levels, low, high), { description }), + ) +} const ratingBoundOf = (levels: ReadonlyArray, bound: number | string): number => typeof bound === 'number' ? bound : ratePositionOf(levels, bound) @@ -1264,3 +1352,62 @@ const Rates = Blueprint.make()(TypeId).operations on(Schema.String).probability({ id: 'bounds-risk', instructions: 'Likely to regress' }) + +const isRefused = (pattern: Pattern): boolean => pattern.refusals.length > 0 + +const aboveRefused = (threshold: number, miss: number): boolean => + isRefused(boundsRisk().above(threshold, { missBelow: miss })) + +const atLeastRefused = (threshold: number, miss: number): boolean => + isRefused(boundsRisk().atLeast(threshold, { missBelow: miss })) + +const belowRefused = (threshold: number, miss: number): boolean => + isRefused(boundsRisk().below(threshold, { missAbove: miss })) + +const atMostRefused = (threshold: number, miss: number): boolean => + isRefused(boundsRisk().atMost(threshold, { missAbove: miss })) + +const betweenRefused = (low: number, high: number): boolean => isRefused(boundsRisk().between(low, high)) + +if (import.meta.vitest !== void 0) { + const { it } = await import('@systemfsoftware/vitest') + + const pairsOf = (first: number, second: number): ReadonlyArray => + Arr.flatMap([0, 1, first], (left) => Arr.map([0, 1, second, left], (right) => [left, right] as const)) + + it.prop( + '∀t_AboveMissBound_≡StrictlyAboveThreshold', + { of: [Schema.Finite, Schema.Finite], subject: aboveRefused }, + (subject, [threshold, miss]) => + Arr.every(pairsOf(threshold, miss), ([line, bound]) => subject(line, bound) === (bound > line)), + ) + + it.prop( + '∀t_AtLeastMissBound_≡StrictlyAboveThreshold', + { of: [Schema.Finite, Schema.Finite], subject: atLeastRefused }, + (subject, [threshold, miss]) => + Arr.every(pairsOf(threshold, miss), ([line, bound]) => subject(line, bound) === (bound > line)), + ) + + it.prop( + '∀t_BelowMissBound_≡StrictlyBelowThreshold', + { of: [Schema.Finite, Schema.Finite], subject: belowRefused }, + (subject, [threshold, miss]) => + Arr.every(pairsOf(threshold, miss), ([line, bound]) => subject(line, bound) === (bound < line)), + ) + + it.prop( + '∀t_AtMostMissBound_≡StrictlyBelowThreshold', + { of: [Schema.Finite, Schema.Finite], subject: atMostRefused }, + (subject, [threshold, miss]) => + Arr.every(pairsOf(threshold, miss), ([line, bound]) => subject(line, bound) === (bound < line)), + ) + + it.prop( + '∀r_BetweenRange_≡LowAfterHigh', + { of: [Schema.Finite, Schema.Finite], subject: betweenRefused }, + (subject, [low, high]) => Arr.every(pairsOf(low, high), ([start, end]) => subject(start, end) === (start > end)), + ) +} diff --git a/packages/discern/src/eval-measure.ts b/packages/discern/src/eval-measure.ts index f40f7ae..856eb10 100644 --- a/packages/discern/src/eval-measure.ts +++ b/packages/discern/src/eval-measure.ts @@ -4,8 +4,10 @@ import type * as AiError from 'effect/ai/AiError' import type * as DecisionModel from 'effect/ai/DecisionModel' import { observe } from './decision.blueprint.js' import { + CalibrationValuesRefused, DecisionIdCollisionError, - InvalidThresholdError, + type InvalidRangeError, + type InvalidThresholdError, MeasureCommandRejected, MeasureInputRefused, } from './DiscernError.schema.js' @@ -13,6 +15,7 @@ import { EvalRecord, EvalReport, metricsOf } from './EvalReport.schema.js' import type { CalibrateOptions, EvalExample, MeasureError, SweepOptions, SweepResult } from './measure-pattern.cell.js' import type { Answers, Pattern, PatternRefusal, Preview } from './pattern.blueprint.js' import { evaluate, preview } from './pattern.blueprint.js' +import { refusalErrorOf } from './refusal-error.js' import { ScoreEvalRecord, scoreEvalRecord } from './score-eval-record.workflow.js' import type { EvalScore } from './score-eval-record.workflow.js' import { statusOf } from './Verdict.schema.js' @@ -50,13 +53,21 @@ type MeasureRead = (typeof ScoreEvalRecord)['Encoded'] & { * the read with the refusal it names, and the first one written is the one * reported. */ -const refusalGate = (refusals: ReadonlyArray): Effect.Effect => - Arr.reduce>( +export const refusalGate = ( + refusals: ReadonlyArray, +): Effect.Effect => + Arr.reduce>( refusals, Effect.void, - (gate, refusal) => Effect.andThen(gate, Effect.fail(new InvalidThresholdError(refusal))), + (gate, refusal) => Effect.andThen(gate, Effect.fail(refusalErrorOf(refusal))), ) +/** Every candidate pattern's refusals gate the run before any example is observed, the first candidate refused being the one reported. */ +const candidatesGate = ( + candidates: ReadonlyArray<{ readonly value: V; readonly pattern: Pattern }>, +): Effect.Effect => + Effect.forEach(candidates, (candidate) => refusalGate(candidate.pattern.refusals), { discard: true }) + /** The example's encoded input, or the refusal naming what could not be encoded. */ const encodedInputOf = ( measure: MeasureCase, @@ -197,6 +208,7 @@ export const sweep = ( ): Effect.Effect>, MeasureError, DecisionModel.DecisionModel | S['EncodingServices']> => Effect.gen(function*() { const candidates = candidatesOf(options) + yield* candidatesGate(candidates) const answeredExamples = yield* answeredExamplesOf(options, candidates) return yield* Effect.forEach(candidates, scoreCandidate(answeredExamples)) }) @@ -219,6 +231,10 @@ const betterOf = ( next: SweepResult, ): SweepResult => Order.min(bestOrderOf(metric))(best, next) +/** Calibration chooses among candidate values, so an empty list is refused before any pattern is built. */ +const valuesGate = (values: ReadonlyArray): Effect.Effect => + values.length === 0 ? Effect.fail(new CalibrationValuesRefused()) : Effect.void + /** Select the best sweep value by a metric, preferring higher coverage on ties. */ export const calibrate = ( options: CalibrateOptions, @@ -228,7 +244,9 @@ export const calibrate = ( DecisionModel.DecisionModel | S['EncodingServices'] > => Effect.gen(function*() { + yield* valuesGate(options.values) const candidates = candidatesOf(options) + yield* candidatesGate(candidates) const answeredExamples = yield* answeredExamplesOf(options, candidates) const score = scoreCandidate(answeredExamples) const [firstValue, ...restValues] = options.values diff --git a/packages/discern/src/matcher.blueprint.ts b/packages/discern/src/matcher.blueprint.ts index b55a199..992dcc0 100644 --- a/packages/discern/src/matcher.blueprint.ts +++ b/packages/discern/src/matcher.blueprint.ts @@ -17,6 +17,7 @@ import { decisionDefinition } from './DecisionDefinition.schema.js' import { type DecisionIdCollisionError, ExhaustiveMatchError, + type InvalidRangeError, type InvalidThresholdError, type PolicyCommandRejected, type UncertainMatchError, @@ -139,6 +140,7 @@ export type FinishedMatcher { /** Everything that can refuse one measurement: the model, the batch build, the pattern, or the example itself. */ export type MeasureError = | AiError.AiError + | CalibrationValuesRefused | DecisionIdCollisionError + | InvalidRangeError | InvalidThresholdError | MeasureCommandRejected | MeasureInputRefused @@ -74,8 +78,11 @@ export const run: { pattern: Pattern, examples: ReadonlyArray>, ): Effect.Effect => - Effect.map( - Effect.forEach(examples, (example) => measureOf(measureCaseOf({ schema, pattern, example }))), - (records) => EvalReport.make({ metrics: metricsOf(records), records }), + Effect.andThen( + refusalGate(pattern.refusals), + Effect.map( + Effect.forEach(examples, (example) => measureOf(measureCaseOf({ schema, pattern, example }))), + (records) => EvalReport.make({ metrics: metricsOf(records), records }), + ), ), ) diff --git a/packages/discern/src/pattern.blueprint.ts b/packages/discern/src/pattern.blueprint.ts index ce7d98d..c243b7d 100644 --- a/packages/discern/src/pattern.blueprint.ts +++ b/packages/discern/src/pattern.blueprint.ts @@ -51,14 +51,24 @@ export interface Preview { readonly decisions: ReadonlyArray } -/** The bounds a caller crossed when naming thresholds, carried to the run that evaluates the leaf. */ -export interface PatternRefusal { +export interface ThresholdRefusal { + readonly kind: 'threshold' readonly threshold: string readonly value: number readonly limit: number readonly message: string } +export interface RangeRefusal { + readonly kind: 'range' + readonly range: string + readonly low: number | string + readonly high: number | string + readonly message: string +} + +export type PatternRefusal = ThresholdRefusal | RangeRefusal + /** The blueprint identity of every pattern. `Symbol.for` identity is preserved. */ export const TypeId: unique symbol = Symbol.for('@systemfsoftware/discern/Pattern') export type TypeId = typeof TypeId diff --git a/packages/discern/src/refusal-error.ts b/packages/discern/src/refusal-error.ts new file mode 100644 index 0000000..0d5213c --- /dev/null +++ b/packages/discern/src/refusal-error.ts @@ -0,0 +1,17 @@ +import { InvalidRangeError, InvalidThresholdError } from './DiscernError.schema.js' +import type { PatternRefusal } from './pattern.blueprint.js' + +export const refusalErrorOf = (refusal: PatternRefusal): InvalidThresholdError | InvalidRangeError => + refusal.kind === 'threshold' + ? new InvalidThresholdError({ + threshold: refusal.threshold, + value: refusal.value, + limit: refusal.limit, + message: refusal.message, + }) + : new InvalidRangeError({ + range: refusal.range, + low: refusal.low, + high: refusal.high, + message: refusal.message, + }) diff --git a/packages/discern/src/replay-provider.cell.ts b/packages/discern/src/replay-provider.cell.ts index 54b2223..ce40443 100644 --- a/packages/discern/src/replay-provider.cell.ts +++ b/packages/discern/src/replay-provider.cell.ts @@ -3,7 +3,7 @@ import * as AiError from 'effect/ai/AiError' import type * as Decision from 'effect/ai/Decision' import type * as DecisionModel from 'effect/ai/DecisionModel' import * as Effect from 'effect/Effect' -import type { ObservationSplit, Provider } from './decision-model.blueprint.js' +import type { ObservationSplit, Provider, StoredHitCheck } from './decision-model.blueprint.js' import type { ProviderAnswer } from './Observation.schema.js' import { SelectObservationSource, selectObservationSource } from './select-observation-source.workflow.js' @@ -12,6 +12,7 @@ export interface ReplayRequest { readonly inner: Provider readonly split: ObservationSplit readonly onMissing: 'fail' | 'ask' + readonly validate: StoredHitCheck } export type ReplayRead = (typeof SelectObservationSource)['Encoded'] & { @@ -19,6 +20,7 @@ export type ReplayRead = (typeof SelectObservationSource)['Encoded'] & { readonly missingDecisions: Readonly> readonly state: DecisionModel.ProviderOptions['state'] readonly inner: Provider + readonly validate: StoredHitCheck } const readReplay = (request: ReplayRequest): Effect.Effect => @@ -31,18 +33,26 @@ const readReplay = (request: ReplayRequest): Effect.Effect => missingDecisions: request.split.missingDecisions, state: request.options.state, inner: request.inner, + validate: request.validate, }) export const replayObservations = Sandwich.named('discern.model.replay')(readReplay) .decide(selectObservationSource) .write({ AllRecorded: (_recorded, read) => - Effect.succeed({ answers: read.hits, usage: { inputTokens: undefined, outputTokens: undefined } }), + Effect.as(read.validate(read.hits), { + answers: read.hits, + usage: { inputTokens: undefined, outputTokens: undefined }, + }), AskForMissing: (_asked, read) => - Effect.map(read.inner.decide({ state: read.state, decisions: read.missingDecisions }), (response) => ({ - answers: { ...read.hits, ...response.answers }, - usage: response.usage, - })), + Effect.flatMap( + read.validate(read.hits), + () => + Effect.map(read.inner.decide({ state: read.state, decisions: read.missingDecisions }), (response) => ({ + answers: { ...read.hits, ...response.answers }, + usage: response.usage, + })), + ), RecordingMissing: (refusal, _read) => Effect.succeed(refusal), CommandRejected: (rejected, read) => Effect.fail( diff --git a/packages/discern/src/run-policy.cell.ts b/packages/discern/src/run-policy.cell.ts index 9d740d1..2d7326c 100644 --- a/packages/discern/src/run-policy.cell.ts +++ b/packages/discern/src/run-policy.cell.ts @@ -15,7 +15,8 @@ import type * as DecisionModel from 'effect/ai/DecisionModel' import { observe } from './decision.blueprint.js' import { DecisionIdCollisionError, - InvalidThresholdError, + type InvalidRangeError, + type InvalidThresholdError, PolicyCommandRejected, UncertainMatchError, } from './DiscernError.schema.js' @@ -23,6 +24,7 @@ import { SelectedCase, SelectedFallback, SelectedUncertain, Trace } from './Insp import type { CaseTrace, CompiledPlan, TraceSelection } from './Inspection.schema.js' import type { Answers, NodeCore, Pattern, PatternRefusal, Preview, UncertainContext } from './pattern.blueprint.js' import { distinctNodes, evaluate, preview } from './pattern.blueprint.js' +import { refusalErrorOf } from './refusal-error.js' import { SelectCase, selectCase } from './select-case.workflow.js' import { CaseVerdict } from './select-case.workflow.js' import { PatternResult, statusIs, statusOf } from './Verdict.schema.js' @@ -74,7 +76,6 @@ const prefixCasesOf = ( { onSome: (index) => Arr.take(cases, index + 1), onNone: () => cases }, ) -/** The first crossed threshold a case in the prefix carries, if any. */ const refusalOfCases = ( cases: ReadonlyArray>, ): Option.Option => Arr.head(Arr.flatMap(cases, (item) => item.pattern.refusals)) @@ -192,11 +193,11 @@ const readOf = (spec: PolicyS input: Input, ): Effect.Effect< PolicyRead, - AiError.AiError | DecisionIdCollisionError | InvalidThresholdError, + AiError.AiError | DecisionIdCollisionError | InvalidRangeError | InvalidThresholdError, DecisionModel.DecisionModel | S['EncodingServices'] > => - Option.match(refusalOfCases(prefixCasesOf(spec.cases, input)), { - onSome: (refusal) => Effect.fail(new InvalidThresholdError(refusal)), + Option.match(refusalOfCases(spec.cases), { + onSome: (refusal) => Effect.fail(refusalErrorOf(refusal)), onNone: () => Effect.flatMap(observe(spec.schema, neededNodesOf(spec.cases, input), input), (answers) => { const evaluated = evaluatedPrefixOf(spec.cases, input, answers) @@ -273,6 +274,7 @@ export interface Policy< | Err | AiError.AiError | DecisionIdCollisionError + | InvalidRangeError | InvalidThresholdError | PolicyCommandRejected | UncertainMatchError, @@ -288,6 +290,7 @@ export type PolicyTraced = Effect.Ef | Err | AiError.AiError | DecisionIdCollisionError + | InvalidRangeError | InvalidThresholdError | PolicyCommandRejected | UncertainMatchError, diff --git a/packages/discern/test-types/discern.tst.ts b/packages/discern/test-types/discern.tst.ts index f8bb9d4..6a18d1d 100644 --- a/packages/discern/test-types/discern.tst.ts +++ b/packages/discern/test-types/discern.tst.ts @@ -237,6 +237,7 @@ describe('the immediate value match', () => { string, | AiError.AiError | Discern.DecisionIdCollisionError + | Discern.InvalidRangeError | Discern.InvalidThresholdError | Discern.PolicyCommandRejected | Discern.UncertainMatchError, @@ -328,6 +329,7 @@ describe('the DecisionModel middleware', () => { string | number | boolean, | AiError.AiError | Discern.DecisionIdCollisionError + | Discern.InvalidRangeError | Discern.InvalidThresholdError | Discern.PolicyCommandRejected | Discern.UncertainMatchError, diff --git a/packages/discern/tests/evaluation.integration.test.ts b/packages/discern/tests/evaluation.integration.test.ts index 31e34d7..eba41c7 100644 --- a/packages/discern/tests/evaluation.integration.test.ts +++ b/packages/discern/tests/evaluation.integration.test.ts @@ -1,6 +1,7 @@ import { Discern } from '@systemfsoftware/discern' import { Gherkin, Given, it, makeFeature, Then, When } from '@systemfsoftware/effect-gherkin-spec' -import { Array as Arr, Effect, Schema } from 'effect' +import { Array as Arr, Effect, Predicate, Schema } from 'effect' +import type * as DecisionModel from 'effect/ai/DecisionModel' import { type AnswerFor, answering, @@ -8,6 +9,7 @@ import { CountingModel, probabilityAnswer, probabilityEverywhere, + tally, withProvider, } from './__fixtures__/counting-model.fixture.js' @@ -35,6 +37,46 @@ const guardableExamples = [ { input: 'c.md', expected: false }, ] +// The untyped-caller boundary. Calibrate's type refuses an empty tuple — that compile-time +// refusal is the typed caller's contract, already pinned in test-types. This boundary reads +// the entry dynamically, so the runtime refusal an untyped caller must meet can be +// exercised. No check is silenced: the scenarios through this shim fail today and pass +// only when the runtime guard exists. +type CalibrateArgs = { + readonly schema: typeof Schema.String + readonly values: ReadonlyArray + readonly pattern: (threshold: number) => Discern.Pattern + readonly examples: ReadonlyArray<{ readonly input: string; readonly expected: boolean }> +} +type CalibrateOutcome = Effect.Effect< + { readonly best: Discern.SweepResult; readonly results: ReadonlyArray> }, + Discern.MeasureError, + DecisionModel.DecisionModel +> + +const isCalibrateEntry = ( + candidate: unknown, +): candidate is (this: undefined, args_0: CalibrateArgs) => CalibrateOutcome => Predicate.isFunction(candidate) + +const calibrateUntyped = ( + values: ReadonlyArray, + pattern: (threshold: number) => Discern.Pattern, + examples: ReadonlyArray<{ readonly input: string; readonly expected: boolean }>, +): CalibrateOutcome => { + const surface: object = Discern.Eval + if (!Predicate.hasProperty(surface, 'calibrate')) { + return Effect.die(new Error('Discern.Eval.calibrate is missing from the public surface')) + } + if (!isCalibrateEntry(surface.calibrate)) { + return Effect.die(new Error('Discern.Eval.calibrate is not the entry the boundary expects')) + } + return Reflect.apply( + surface.calibrate, + undefined, + [{ schema: Schema.String, values, pattern, examples }], + ) +} + const answersWithTheExampleProbability: AnswerFor = (request) => answersFor({ request, answerOf: () => probabilityAnswer(Number(request.state)) }) @@ -190,4 +232,91 @@ Feature('Measuring a question against labelled examples') ), ), ) + + scenario( + 'Calibrating with no candidate values is refused before any pattern is built', + Gherkin.Do.pipe( + Given('labelled examples and a pattern builder that counts every value it is given')( + 'setup', + () => Effect.succeed({ examples: labelledExamples, built: tally() }), + ), + When('an empty list of thresholds is calibrated')( + 'outcome', + (s) => + Effect.gen(function*() { + const model = yield* CountingModel + return yield* Effect.flip( + withProvider( + calibrateUntyped( + [], + (threshold) => { + s.setup.built.bump() + return sweepRisk.atLeast(threshold) + }, + s.setup.examples, + ), + model.model, + ), + ) + }), + ), + Then('the calibration is refused as having no values and no pattern was built')((s, expect) => + expect({ refusal: s.outcome._tag, patternsBuilt: s.setup.built.count() }).toEqual({ + refusal: 'CalibrationValuesRefused', + patternsBuilt: 0, + }) + ), + ), + ) + + scenario( + 'Calibrating with no candidate values never asks the model', + Gherkin.Do.pipe( + Given('labelled examples')('examples', () => Effect.succeed(labelledExamples)), + When('an empty list of thresholds is calibrated')( + 'asked', + (s) => + Effect.gen(function*() { + const model = yield* CountingModel + yield* Effect.exit( + withProvider( + calibrateUntyped([], (threshold) => sweepRisk.atLeast(threshold), s.examples), + model.model, + ), + ) + return model.calls() + }), + ), + Then('the model was asked nothing')(({ asked }, expect) => expect(asked).toBe(0)), + ), + ) + + scenario( + 'Calibrating with one candidate value returns it as the best', + Gherkin.Do.pipe( + Given('labelled examples')('examples', () => Effect.succeed(labelledExamples)), + When('a single threshold is calibrated')( + 'result', + (s) => + Effect.gen(function*() { + const model = yield* CountingModel + return yield* withProvider( + Discern.Eval.calibrate({ + schema: Schema.String, + values: [0.7], + pattern: (threshold) => sweepRisk.atLeast(threshold), + examples: s.examples, + }), + model.model, + ) + }), + ), + Then('that threshold is the best and the only result')(({ result }, expect) => + expect({ bestValue: result.best.value, resultCount: result.results.length }).toEqual({ + bestValue: 0.7, + resultCount: 1, + }) + ), + ), + ) }) diff --git a/packages/discern/tests/pattern-bounds.integration.test.ts b/packages/discern/tests/pattern-bounds.integration.test.ts new file mode 100644 index 0000000..22d9c99 --- /dev/null +++ b/packages/discern/tests/pattern-bounds.integration.test.ts @@ -0,0 +1,464 @@ +import { Discern } from '@systemfsoftware/discern' +import { Gherkin, Given, it, makeFeature, Then, When } from '@systemfsoftware/effect-gherkin-spec' +import { Array as Arr, Context, Effect, Layer, MutableRef, Schema } from 'effect' +import type * as AiError from 'effect/ai/AiError' +import type * as DecisionModel from 'effect/ai/DecisionModel' + +const Feature = makeFeature({ it }) + +class CountingModel extends Context.Service number +}>()('@systemfsoftware/discern/tests/PatternBoundsModel') {} + +const answeringEverythingAt = (probability: number): Layer.Layer => + Layer.unwrap( + Effect.sync(() => { + const calls = MutableRef.make(0) + const model = Discern.Model.provider( + (request): Effect.Effect => { + MutableRef.set(calls, MutableRef.get(calls) + 1) + return Effect.succeed({ + answers: Object.fromEntries( + Arr.map(Object.keys(request.decisions), (id) => [id, { _tag: 'Probability' as const, probability }]), + ), + usage: { inputTokens: undefined, outputTokens: undefined }, + }) + }, + ) + return Layer.mergeAll( + Discern.Model.fromProvider(model), + Layer.succeed(CountingModel, { model, calls: () => MutableRef.get(calls) }), + ) + }), + ) + +const withProvider = ( + effect: Effect.Effect, + model: Discern.Model.Provider, +): Effect.Effect => Effect.provide(effect, Discern.Model.layer(model, [])) + +const Change = Discern.on(Schema.String) + +const risk = Change.probability({ id: 'bounds-risk', instructions: 'Likely to regress' }) +const impact = Change.classify({ + id: 'bounds-impact', + instructions: 'Classify API impact', + criteria: { none: 'none', breaking: 'breaking' }, +}) +const severity = Change.rate({ + id: 'bounds-severity', + instructions: 'Rate severity', + criteria: ['trivial', 'minor', 'major', 'critical'], +}) + +const policyOf = (pattern: Discern.Pattern) => + Discern.type(Schema.String).pipe( + Discern.when(pattern, () => 'matched'), + Discern.orElse(() => 'fallback'), + ) + +const refusedAfterAsking = (effect: Effect.Effect) => + Effect.gen(function*() { + const model = yield* CountingModel + const outcome = yield* Effect.flip(withProvider(effect, model.model)) + return { outcome, asked: model.calls() } + }) + +const probabilityAnswers = (probability: number): Discern.Answers => ({ [risk.id]: { probability } }) + +const ratingAnswers = (rating: number, label: string): Discern.Answers => ({ + [severity.id]: { rating, label, probabilities: { trivial: 0, minor: 0, major: 0, critical: 0 } }, +}) + +const statusesOf = (pattern: Discern.Pattern, answers: ReadonlyArray) => + Arr.map(answers, (answered) => Discern.statusOf(Discern.evaluate(pattern, 'change', answered))) + +interface InvertedBound { + readonly name: string + readonly build: () => Discern.Pattern + readonly value: number + readonly limit: number +} + +const invertedBounds: ReadonlyArray = [ + { name: 'above', build: () => risk.above(0.5, { missBelow: 0.8 }), value: 0.8, limit: 0.5 }, + { name: 'atLeast', build: () => risk.atLeast(0.5, { missBelow: 0.8 }), value: 0.8, limit: 0.5 }, + { name: 'below', build: () => risk.below(0.5, { missAbove: 0.2 }), value: 0.2, limit: 0.5 }, + { name: 'atMost', build: () => risk.atMost(0.5, { missAbove: 0.2 }), value: 0.2, limit: 0.5 }, +] + +interface ReversedRange { + readonly name: string + readonly build: () => Discern.Pattern + readonly low: number | string + readonly high: number | string +} + +const reversedRanges: ReadonlyArray = [ + { name: 'probability span', build: () => risk.between(0.6, 0.4), low: 0.6, high: 0.4 }, + { + name: 'rating span from major to minor', + build: () => severity.between('major', 'minor'), + low: 'major', + high: 'minor', + }, + { + name: 'rating span from critical to trivial', + build: () => severity.between('critical', 'trivial'), + low: 'critical', + high: 'trivial', + }, +] + +interface RefusedPattern { + readonly name: string + readonly build: () => Discern.Pattern + readonly refusal: Readonly> +} + +const refusedPatterns: ReadonlyArray = [ + { + name: 'an inverted miss bound', + build: () => risk.above(0.5, { missBelow: 0.8 }), + refusal: { _tag: 'InvalidThresholdError', value: 0.8, limit: 0.5 }, + }, + { + name: 'a reversed probability span', + build: () => risk.between(0.6, 0.4), + refusal: { _tag: 'InvalidRangeError', low: 0.6, high: 0.4 }, + }, +] + +const onlySourceFiles = Discern.deterministic((path: string) => path.endsWith('.ts'), { id: 'bounds-source-file' }) + +const twoExamples: ReadonlyArray> = [ + { input: 'change-1', expected: true }, + { input: 'change-2', expected: false }, +] + +const crossedClassifyBounds = impact.is('breaking', { match: 0.5, miss: 0.8 }) +const crossedBand = risk.band({ match: 0.5, miss: 0.8 }) + +const crossedThresholds: ReadonlyArray<{ readonly name: string; readonly build: () => Discern.Pattern }> = [ + { name: 'a probability band', build: () => crossedBand }, + { name: 'a classification threshold', build: () => crossedClassifyBounds }, +] + +Feature('Refusing probability and rating bounds that cannot judge') + .withScenarioLayer(answeringEverythingAt(0.99)) + .body(({ scenario }) => { + Arr.forEach(invertedBounds, (bound) => { + scenario( + `A ${bound.name} pattern whose miss bound sits past its threshold carries a refusal naming the bound`, + Gherkin.Do.pipe( + Given('a probability pattern whose miss bound sits on the wrong side of its threshold')( + 'pattern', + () => Effect.succeed(bound.build()), + ), + When('the pattern is built and its refusals are read')('refusals', (s) => Effect.succeed(s.pattern.refusals)), + Then('exactly one refusal names the bound and the threshold it crossed')(({ refusals }, expect) => + expect(refusals).toMatchObject([{ value: bound.value, limit: bound.limit }]) + ), + ), + ) + + scenario( + `A policy using a ${bound.name} pattern with an inverted miss bound is refused before the model is asked`, + Gherkin.Do.pipe( + Given('a policy whose only case is that pattern')('policy', () => Effect.succeed(policyOf(bound.build()))), + When('a change is judged')('outcome', (s) => refusedAfterAsking(s.policy('change'))), + Then('the run is refused naming the bound, and the model is never asked')(({ outcome }, expect) => + expect(outcome).toMatchObject({ + outcome: { _tag: 'InvalidThresholdError', value: bound.value, limit: bound.limit }, + asked: 0, + }) + ), + ), + ) + }) + + Arr.forEach(reversedRanges, (range) => { + scenario( + `A policy using a reversed ${range.name} is refused before the model is asked`, + Gherkin.Do.pipe( + Given('a policy whose only case is a span that ends before it starts')( + 'policy', + () => Effect.succeed(policyOf(range.build())), + ), + When('a change is judged')('outcome', (s) => refusedAfterAsking(s.policy('change'))), + Then('the run is refused naming both ends of the span, and the model is never asked')( + ({ outcome }, expect) => + expect(outcome).toMatchObject({ + outcome: { _tag: 'InvalidRangeError', low: range.low, high: range.high }, + asked: 0, + }), + ), + ), + ) + }) + + scenario( + 'A miss bound equal to the threshold leaves the pattern two-valued', + Gherkin.Do.pipe( + Given('a probability pattern whose miss bound equals its threshold')( + 'pattern', + () => Effect.succeed(risk.above(0.5, { missBelow: 0.5 })), + ), + When('it judges probabilities on both sides of the line')('statuses', (s) => + Effect.succeed( + statusesOf(s.pattern, Arr.map([0, 0.49, 0.5, 0.51, 1], probabilityAnswers)), + )), + Then('it matches above the line and misses at or below it, never uncertain')(({ statuses }, expect) => + expect(statuses).toEqual(['Miss', 'Miss', 'Miss', 'Match', 'Match']) + ), + ), + ) + + scenario( + 'A probability span with equal ends matches exactly that probability', + Gherkin.Do.pipe( + Given('a probability span whose two ends are the same value')( + 'pattern', + () => Effect.succeed(risk.between(0.5, 0.5)), + ), + When('it judges 0.5 and 0.51')( + 'statuses', + (s) => Effect.succeed(statusesOf(s.pattern, Arr.map([0.5, 0.51], probabilityAnswers))), + ), + Then('it matches 0.5 and misses 0.51')(({ statuses }, expect) => expect(statuses).toEqual(['Match', 'Miss'])), + ), + ) + + scenario( + 'A rating span with equal ends matches exactly that level', + Gherkin.Do.pipe( + Given('a rating span from major to major')('pattern', () => Effect.succeed(severity.between('major', 'major'))), + When('it judges a rating of 2 and of 1.5')( + 'statuses', + (s) => Effect.succeed(statusesOf(s.pattern, [ratingAnswers(2, 'major'), ratingAnswers(1.5, 'minor')])), + ), + Then('it matches the level itself and misses the rating between levels')(({ statuses }, expect) => + expect(statuses).toEqual(['Match', 'Miss']) + ), + ), + ) + + Arr.forEach(crossedThresholds, (crossed) => { + scenario( + `A policy using ${crossed.name} whose miss bound is past its match bound keeps its refusal`, + Gherkin.Do.pipe( + Given('a policy whose only case treats a miss as stricter than a match')( + 'policy', + () => Effect.succeed(policyOf(crossed.build())), + ), + When('a change is judged')('outcome', (s) => refusedAfterAsking(s.policy('change'))), + Then('the run is refused naming the crossed bounds, and the model is never asked')( + ({ outcome }, expect) => + expect(outcome).toMatchObject({ + outcome: { _tag: 'InvalidThresholdError', threshold: 'miss', value: 0.8, limit: 0.5 }, + asked: 0, + }), + ), + ), + ) + }) + + Arr.forEach(refusedPatterns, (refused) => { + scenario( + `A deterministic guard that already matches does not hide ${refused.name} in a later case`, + Gherkin.Do.pipe( + Given('a policy with a matching source-file guard ahead of a refused case')('policy', () => + Effect.succeed( + Discern.type(Schema.String).pipe( + Discern.when(onlySourceFiles, () => 'guarded'), + Discern.when(refused.build(), () => 'refused-case'), + Discern.orElse(() => 'fallback'), + ), + )), + When('a source file the guard matches is judged')('outcome', (s) => refusedAfterAsking(s.policy('index.ts'))), + Then('the run is refused all the same, and the model is never asked')(({ outcome }, expect) => + expect(outcome).toMatchObject({ outcome: refused.refusal, asked: 0 }) + ), + ), + ) + }) + + scenario( + 'A refused leaf stays refused under and', + Gherkin.Do.pipe( + Given('a crossed band conjoined with a source-file guard')( + 'pattern', + () => Effect.succeed(Discern.and(crossedBand, onlySourceFiles)), + ), + When('the composed pattern is built and its refusals are read')( + 'refusals', + (s) => Effect.succeed(s.pattern.refusals), + ), + Then('the composed pattern carries the band refusal')(({ refusals }, expect) => + expect(refusals).toMatchObject([{ threshold: 'miss', value: 0.8, limit: 0.5 }]) + ), + ), + ) + + scenario( + 'A refused leaf stays refused under or', + Gherkin.Do.pipe( + Given('a crossed band disjoined with a source-file guard')( + 'pattern', + () => Effect.succeed(Discern.or(crossedBand, onlySourceFiles)), + ), + When('the composed pattern is built and its refusals are read')( + 'refusals', + (s) => Effect.succeed(s.pattern.refusals), + ), + Then('the composed pattern carries the band refusal')(({ refusals }, expect) => + expect(refusals).toMatchObject([{ threshold: 'miss', value: 0.8, limit: 0.5 }]) + ), + ), + ) + + scenario( + 'A refused leaf stays refused under not', + Gherkin.Do.pipe( + Given('a negated crossed band')('pattern', () => Effect.succeed(Discern.not(crossedBand))), + When('the negated pattern is built and its refusals are read')( + 'refusals', + (s) => Effect.succeed(s.pattern.refusals), + ), + Then('the negated pattern carries the band refusal')(({ refusals }, expect) => + expect(refusals).toMatchObject([{ threshold: 'miss', value: 0.8, limit: 0.5 }]) + ), + ), + ) + + Arr.forEach(refusedPatterns, (refused) => { + Arr.forEach([ + { label: 'labelled examples', examples: twoExamples }, + { label: 'no examples', examples: [] }, + ], (given) => { + scenario( + `Evaluating ${refused.name} over ${given.label} is refused before the model is asked`, + Gherkin.Do.pipe( + Given('a pattern that cannot judge')('pattern', () => Effect.succeed(refused.build())), + When('the pattern is evaluated over the examples')( + 'outcome', + (s) => refusedAfterAsking(Discern.Eval.run(Schema.String, s.pattern, given.examples)), + ), + Then('the evaluation is refused, and the model is never asked')(({ outcome }, expect) => + expect(outcome).toMatchObject({ outcome: refused.refusal, asked: 0 }) + ), + ), + ) + }) + }) + + scenario( + 'Sweeping a valid candidate and then an inverted one is refused before the model is asked', + Gherkin.Do.pipe( + Given('labelled examples and a sweep of miss bounds from 0.3 to 0.8 under a 0.5 threshold')( + 'examples', + () => Effect.succeed(twoExamples), + ), + When('the miss bounds are swept')('outcome', (s) => + refusedAfterAsking(Discern.Eval.sweep({ + schema: Schema.String, + values: [0.3, 0.8], + pattern: (missBelow: number) => risk.above(0.5, { missBelow }), + examples: s.examples, + }))), + Then('the sweep is refused naming the inverted bound, and the model is never asked')(({ outcome }, expect) => + expect(outcome).toMatchObject({ + outcome: { _tag: 'InvalidThresholdError', value: 0.8, limit: 0.5 }, + asked: 0, + }) + ), + ), + ) + + scenario( + 'Sweeping an inverted candidate over no examples is still refused', + Gherkin.Do.pipe( + Given('a sweep of miss bounds from 0.3 to 0.8 under a 0.5 threshold with no examples')( + 'examples', + () => Effect.succeed>>([]), + ), + When('the miss bounds are swept')('outcome', (s) => + refusedAfterAsking(Discern.Eval.sweep({ + schema: Schema.String, + values: [0.3, 0.8], + pattern: (missBelow: number) => risk.above(0.5, { missBelow }), + examples: s.examples, + }))), + Then('the sweep is refused naming the inverted bound, and the model is never asked')(({ outcome }, expect) => + expect(outcome).toMatchObject({ + outcome: { _tag: 'InvalidThresholdError', value: 0.8, limit: 0.5 }, + asked: 0, + }) + ), + ), + ) + + scenario( + 'Calibrating a plain candidate and then an inverted one is refused before the model is asked', + Gherkin.Do.pipe( + Given('labelled examples and two candidates, the second with a miss bound past its threshold')( + 'examples', + () => Effect.succeed(twoExamples), + ), + When('the candidates are calibrated')('outcome', (s) => + refusedAfterAsking(Discern.Eval.calibrate({ + schema: Schema.String, + values: [0.5, 0.7], + pattern: (value: number) => value === 0.5 ? risk.atLeast(value) : risk.above(0.5, { missBelow: 0.8 }), + examples: s.examples, + }))), + Then('the calibration is refused naming the inverted bound, and the model is never asked')( + ({ outcome }, expect) => + expect(outcome).toMatchObject({ + outcome: { _tag: 'InvalidThresholdError', value: 0.8, limit: 0.5 }, + asked: 0, + }), + ), + ), + ) + + scenario( + 'Evaluating a refused probability pattern directly answers uncertain with the refusal message', + Gherkin.Do.pipe( + Given('a pattern whose miss bound sits past its threshold')( + 'pattern', + () => Effect.succeed(risk.above(0.5, { missBelow: 0.8 })), + ), + When('it is evaluated against an answer that would clear the threshold')( + 'result', + (s) => Effect.succeed(Discern.evaluate(s.pattern, 'change', probabilityAnswers(0.99))), + ), + Then('the answer is uncertain and carries the refusal message')(({ pattern, result }, expect) => + expect({ status: Discern.statusOf(result), reasons: [Discern.reasonOf(result)] }).toEqual({ + status: 'Uncertain', + reasons: Arr.map(pattern.refusals, (refusal) => refusal.message), + }) + ), + ), + ) + + scenario( + 'A miss bound of infinity builds without a refusal and runs as it always has', + Gherkin.Do.pipe( + Given('a policy whose pattern has an unbounded miss bound')( + 'pattern', + () => Effect.succeed(risk.above(0.5, { missBelow: Number.POSITIVE_INFINITY })), + ), + When('a change is judged')('run', (s) => + Effect.gen(function*() { + const model = yield* CountingModel + const verdict = yield* withProvider(policyOf(s.pattern)('change'), model.model) + return { refusals: s.pattern.refusals, verdict, asked: model.calls() } + })), + Then('the pattern carries no refusal and the run decides after one model round trip')(({ run }, expect) => + expect(run).toEqual({ refusals: [], verdict: 'matched', asked: 1 }) + ), + ), + ) + }) diff --git a/packages/discern/tests/stored-answer-validation.integration.test.ts b/packages/discern/tests/stored-answer-validation.integration.test.ts new file mode 100644 index 0000000..b4ecd97 --- /dev/null +++ b/packages/discern/tests/stored-answer-validation.integration.test.ts @@ -0,0 +1,501 @@ +import { Discern } from '@systemfsoftware/discern' +import { Gherkin, Given, it, makeFeature, Then, When } from '@systemfsoftware/effect-gherkin-spec' +import { Context, Effect, Layer, Match, MutableRef, Predicate, Schema } from 'effect' +import type * as AiError from 'effect/ai/AiError' +import type * as DecisionModel from 'effect/ai/DecisionModel' + +const Feature = makeFeature({ it }) + +class CountingModel extends Context.Service number +}>()('@systemfsoftware/discern/tests/stored-answer-validation/CountingModel') {} + +const probabilityAnswer = (probability: number): DecisionModel.ProviderProbabilityAnswer => ({ + _tag: 'Probability', + probability, +}) + +const classifyAnswer = ( + label: string, + probabilities: Readonly>, +): DecisionModel.ProviderClassifyAnswer => ({ _tag: 'Classify', label, probabilities }) + +const rateAnswer = ( + rating: number, + probabilities: Readonly>, +): DecisionModel.ProviderRateAnswer => ({ _tag: 'Rate', rating, probabilities }) + +const noImpact = { none: 1, additive: 0, behavioral: 0, breaking: 0 } +const minorSeverity = { trivial: 0, minor: 1, major: 0, critical: 0 } + +const validAnswerOf = (decision: Discern.AnyDecision): DecisionModel.ProviderAnswer => + Match.value(decision).pipe( + Match.tag('Classify', () => classifyAnswer('none', noImpact)), + Match.tag('Probability', () => probabilityAnswer(0.95)), + Match.tag('Rate', () => rateAnswer(1, minorSeverity)), + Match.exhaustive, + ) + +const uncountedUsage = { inputTokens: undefined, outputTokens: undefined } as const + +const answeringValidly: Layer.Layer = Layer.unwrap( + Effect.sync(() => { + const calls = MutableRef.make(0) + const model = Discern.Model.provider( + (request): Effect.Effect => { + MutableRef.set(calls, MutableRef.get(calls) + 1) + const answers: Record = {} + for (const [id, decision] of Object.entries(request.decisions)) { + answers[id] = validAnswerOf(decision) + } + return Effect.succeed({ answers, usage: uncountedUsage }) + }, + ) + return Layer.mergeAll( + Discern.Model.fromProvider(model), + Layer.succeed(CountingModel, { model, calls: () => MutableRef.get(calls) }), + ) + }), +) + +const withProvider = ( + effect: Effect.Effect, + model: Discern.Model.Provider, + interceptors: ReadonlyArray, +): Effect.Effect => Effect.provide(effect, Discern.Model.layer(model, interceptors)) + +const Change = Discern.on(Schema.String) + +const risk = Change.probability({ id: 'risk', instructions: 'How risky is this change' }) + +const riskPolicy = Discern.type(Schema.String).pipe( + Discern.when(risk.above(0.8), () => 'block'), + Discern.orElse(() => 'ship'), +) + +const impact = Change.classify({ + id: 'impact', + instructions: 'Classify impact', + criteria: { none: 'none', additive: 'additive', behavioral: 'behavioral', breaking: 'breaking' }, +}) + +const impactPolicy = Discern.type(Schema.String).pipe( + Discern.when(impact.is('breaking'), () => 'migrate'), + Discern.orElse(() => 'ship'), +) + +const severity = Change.rate({ + id: 'severity', + instructions: 'Rate severity', + criteria: ['trivial', 'minor', 'major', 'critical'], +}) + +const severityPolicy = Discern.type(Schema.String).pipe( + Discern.when(severity.atLeast('major'), () => 'escalate'), + Discern.orElse(() => 'queue'), +) + +const firstQuestion = Change.probability({ id: 'a', instructions: 'First' }) +const secondQuestion = Change.probability({ id: 'b', instructions: 'Second' }) + +const firstPolicy = Discern.type(Schema.String).pipe( + Discern.when(firstQuestion.above(0.8), () => 'a'), + Discern.orElse(() => 'none'), +) + +const bothPolicy = Discern.type(Schema.String).pipe( + Discern.when(Discern.and(firstQuestion.above(0.8), secondQuestion.above(0.8)), () => 'both'), + Discern.orElse(() => 'none'), +) + +const recordingOf = (run: Effect.Effect) => + Effect.gen(function*() { + const model = yield* CountingModel + const into = Discern.Model.store() + yield* withProvider(run, model.model, [Discern.Model.recording(into)]) + return yield* Discern.Model.snapshot(into) + }) + +const tamperedSnapshot = (taken: Discern.Model.Observations, answer: DecisionModel.ProviderAnswer) => + Effect.flatMap( + Schema.encodeEffect(Discern.Model.Observations)(taken), + (plain) => + Schema.encodeUnknownEffect(Schema.Json)({ + version: plain.version, + entries: Object.fromEntries( + Object.entries(plain.entries).map(([address, entry]) => [address, { ...entry, answer }]), + ), + }), + ) + +const storeFrom = (json: Schema.Json) => + Effect.gen(function*() { + const into = Discern.Model.store() + yield* Discern.Model.load(into, json) + return into + }) + +const observationsFrom = (json: Schema.Json) => Schema.decodeUnknownEffect(Discern.Model.Observations)(json) + +const reasonOf = (error: E): string => + Predicate.hasProperty(error, 'message') && Predicate.isString(error.message) ? error.message : '' + +const invalidObservationOf = (error: E): boolean | undefined => { + const surface: object = Discern.Model + return Predicate.hasProperty(surface, 'isInvalidObservation') && + Predicate.isFunction(surface.isInvalidObservation) + ? Reflect.apply(surface.isInvalidObservation, undefined, [error]) === true + : undefined +} + +const failureOf = (error: E) => ({ + invalidObservation: invalidObservationOf(error), + replayMiss: Discern.Model.isReplayMiss(error), + reason: reasonOf(error), +}) + +const tamperedReplayOf = ( + run: Effect.Effect, + answer: DecisionModel.ProviderAnswer, +) => + Effect.gen(function*() { + const model = yield* CountingModel + const taken = yield* recordingOf(run) + const into = yield* storeFrom(yield* tamperedSnapshot(taken, answer)) + const refused = yield* Effect.flip(withProvider(run, model.model, [Discern.Model.replaying(into)])) + return failureOf(refused) + }) + +const firstQuestionRecordedTampered = Effect.gen(function*() { + const model = yield* CountingModel + const taken = yield* recordingOf(firstPolicy('x')) + return { + observations: yield* tamperedSnapshot(taken, probabilityAnswer(7)), + callsAfterRecording: model.calls(), + } +}) + +Feature('Stored answers are held to the rules a provider is held to') + .withScenarioLayer(answeringValidly) + .body(({ scenario }) => { + scenario( + 'A recorded probability tampered to 7 is refused as an invalid observation, not a missing one', + Gherkin.Do.pipe( + Given('a blocking policy on a probability question')('policy', () => Effect.succeed(riskPolicy)), + When('a valid run is recorded, every answer is tampered to 7, and the tampered store replays')( + 'failure', + (s) => tamperedReplayOf(s.policy('x'), probabilityAnswer(7)), + ), + Then('the replay fails naming the probability range')(({ failure }, expect) => + expect(failure).toMatchObject({ + invalidObservation: true, + replayMiss: false, + reason: expect.stringMatching(/probability outside \[0, 1\]/), + }) + ), + ), + ) + + scenario( + 'A recorded classification naming an unknown label is refused as an invalid observation', + Gherkin.Do.pipe( + Given('a policy on a classification question')('policy', () => Effect.succeed(impactPolicy)), + When('a valid run is recorded, its label is tampered to one the question lacks, and the store replays')( + 'failure', + (s) => tamperedReplayOf(s.policy('x'), classifyAnswer('sideways', noImpact)), + ), + Then('the replay fails naming the unknown label')(({ failure }, expect) => + expect(failure).toMatchObject({ + invalidObservation: true, + replayMiss: false, + reason: expect.stringMatching(/unknown label/), + }) + ), + ), + ) + + scenario( + 'A recorded classification whose probabilities sum to 0.5 is refused as an invalid observation', + Gherkin.Do.pipe( + Given('a policy on a classification question')('policy', () => Effect.succeed(impactPolicy)), + When('a valid run is recorded, its distribution is tampered to sum to 0.5, and the store replays')( + 'failure', + (s) => + tamperedReplayOf( + s.policy('x'), + classifyAnswer('none', { none: 0.5, additive: 0, behavioral: 0, breaking: 0 }), + ), + ), + Then('the replay fails naming the distribution sum')(({ failure }, expect) => + expect(failure).toMatchObject({ + invalidObservation: true, + replayMiss: false, + reason: expect.stringMatching(/do not sum to 1/), + }) + ), + ), + ) + + scenario( + 'A recorded rating of 99 on a four-level scale is refused as an invalid observation', + Gherkin.Do.pipe( + Given('a policy on a four-level rating question')('policy', () => Effect.succeed(severityPolicy)), + When('a valid run is recorded, its rating is tampered to 99, and the store replays')( + 'failure', + (s) => tamperedReplayOf(s.policy('x'), rateAnswer(99, minorSeverity)), + ), + Then('the replay fails naming the rating range')(({ failure }, expect) => + expect(failure).toMatchObject({ + invalidObservation: true, + replayMiss: false, + reason: expect.stringMatching(/rating outside \[0, 3\]/), + }) + ), + ), + ) + + scenario( + 'A recorded rating of -1 on a four-level scale is refused as an invalid observation', + Gherkin.Do.pipe( + Given('a policy on a four-level rating question')('policy', () => Effect.succeed(severityPolicy)), + When('a valid run is recorded, its rating is tampered to -1, and the store replays')( + 'failure', + (s) => tamperedReplayOf(s.policy('x'), rateAnswer(-1, minorSeverity)), + ), + Then('the replay fails naming the rating range')(({ failure }, expect) => + expect(failure).toMatchObject({ + invalidObservation: true, + replayMiss: false, + reason: expect.stringMatching(/rating outside \[0, 3\]/), + }) + ), + ), + ) + + scenario( + 'A recorded answer of another decision kind is refused as an invalid observation', + Gherkin.Do.pipe( + Given('a policy on a probability question')('policy', () => Effect.succeed(riskPolicy)), + When('a valid run is recorded, its answer is tampered into a classification, and the store replays')( + 'failure', + (s) => tamperedReplayOf(s.policy('x'), classifyAnswer('none', noImpact)), + ), + Then('the replay fails naming the kind the decision expected')(({ failure }, expect) => + expect(failure).toMatchObject({ + invalidObservation: true, + replayMiss: false, + reason: expect.stringMatching(/expected "Probability"/), + }) + ), + ), + ) + + scenario( + 'A cache holding one tampered answer and missing another refuses before asking the model', + Gherkin.Do.pipe( + Given('a policy on two questions')('policy', () => Effect.succeed(bothPolicy)), + Given('a recording of only the first question, tampered to 7')( + 'tampered', + () => firstQuestionRecordedTampered, + ), + When('both questions are judged through a cache over the tampered store')( + 'failure', + (s) => + Effect.gen(function*() { + const model = yield* CountingModel + const into = yield* storeFrom(s.tampered.observations) + const refused = yield* Effect.flip( + withProvider(s.policy('x'), model.model, [Discern.Model.caching(into)]), + ) + return { ...failureOf(refused), modelCallsSinceRecording: model.calls() - s.tampered.callsAfterRecording } + }), + ), + Then('the run is refused as an invalid observation and the model is never asked')( + ({ failure }, expect) => + expect(failure).toMatchObject({ + invalidObservation: true, + reason: expect.stringMatching(/probability outside \[0, 1\]/), + modelCallsSinceRecording: 0, + }), + ), + ), + ) + + scenario( + 'A replay set to ask, with one tampered answer and one missing, refuses before asking the model', + Gherkin.Do.pipe( + Given('a policy on two questions')('policy', () => Effect.succeed(bothPolicy)), + Given('a recording of only the first question, tampered to 7')( + 'tampered', + () => firstQuestionRecordedTampered, + ), + When('both questions are judged replaying the tampered store with onMissing ask')( + 'failure', + (s) => + Effect.gen(function*() { + const model = yield* CountingModel + const into = yield* storeFrom(s.tampered.observations) + const refused = yield* Effect.flip( + withProvider(s.policy('x'), model.model, [Discern.Model.replaying(into, { onMissing: 'ask' })]), + ) + return { ...failureOf(refused), modelCallsSinceRecording: model.calls() - s.tampered.callsAfterRecording } + }), + ), + Then('the run is refused as an invalid observation and the model is never asked')( + ({ failure }, expect) => + expect(failure).toMatchObject({ + invalidObservation: true, + replayMiss: false, + reason: expect.stringMatching(/probability outside \[0, 1\]/), + modelCallsSinceRecording: 0, + }), + ), + ), + ) + + scenario( + 'Tampered observations handed over as typed data are refused like loaded ones', + Gherkin.Do.pipe( + Given('a blocking policy on a probability question')('policy', () => Effect.succeed(riskPolicy)), + Given('a recording tampered to 7 and read back as typed observations')('observations', () => + Effect.flatMap( + recordingOf(riskPolicy('x')), + (taken) => Effect.flatMap(tamperedSnapshot(taken, probabilityAnswer(7)), observationsFrom), + )), + When('the typed observations are replayed directly, through a store built from them, and cached')( + 'failures', + (s) => + Effect.gen(function*() { + const model = yield* CountingModel + const replayedDirectly = yield* Effect.flip( + withProvider(s.policy('x'), model.model, [Discern.Model.replaying(s.observations)]), + ) + const replayedFromStore = yield* Effect.flip( + withProvider(s.policy('x'), model.model, [ + Discern.Model.replaying(Discern.Model.store(s.observations)), + ]), + ) + const cached = yield* Effect.flip( + withProvider(s.policy('x'), model.model, [Discern.Model.caching(Discern.Model.store(s.observations))]), + ) + return { + replayedDirectly: failureOf(replayedDirectly), + replayedFromStore: failureOf(replayedFromStore), + cached: failureOf(cached), + } + }), + ), + Then('every entry point refuses the answer as an invalid observation')(({ failures }, expect) => + expect(failures).toMatchObject({ + replayedDirectly: { + invalidObservation: true, + replayMiss: false, + reason: expect.stringMatching(/probability outside \[0, 1\]/), + }, + replayedFromStore: { + invalidObservation: true, + replayMiss: false, + reason: expect.stringMatching(/probability outside \[0, 1\]/), + }, + cached: { + invalidObservation: true, + replayMiss: false, + reason: expect.stringMatching(/probability outside \[0, 1\]/), + }, + }) + ), + ), + ) + + scenario( + 'A replay set to fail names the missing answer, not the tampered one beside it', + Gherkin.Do.pipe( + Given('a policy on two questions')('policy', () => Effect.succeed(bothPolicy)), + Given('a recording of only the first question, tampered to 7')( + 'tampered', + () => firstQuestionRecordedTampered, + ), + When('both questions are judged replaying the tampered store with onMissing fail')( + 'outcome', + (s) => + Effect.gen(function*() { + const model = yield* CountingModel + const into = yield* storeFrom(s.tampered.observations) + const refused = yield* Effect.flip( + withProvider(s.policy('x'), model.model, [Discern.Model.replaying(into, { onMissing: 'fail' })]), + ) + return { + refused, + replayMiss: Discern.Model.isReplayMiss(refused), + reason: reasonOf(refused), + modelCallsSinceRecording: model.calls() - s.tampered.callsAfterRecording, + } + }), + ), + Then('the run fails as a replay miss, never reading the tampered answer, and the model is never asked')( + ({ outcome }, expect) => + expect(outcome).toMatchObject({ + refused: { _tag: 'AiError', module: 'Discern', method: 'replaying' }, + replayMiss: true, + reason: expect.not.stringMatching(/outside \[0, 1\]/), + modelCallsSinceRecording: 0, + }), + ), + ), + ) + + scenario( + 'A valid recording replays and caches without the model', + Gherkin.Do.pipe( + Given('a policy on two questions')('policy', () => Effect.succeed(bothPolicy)), + Given('a valid recording of both questions')('recorded', () => + Effect.gen(function*() { + const model = yield* CountingModel + const taken = yield* recordingOf(bothPolicy('x')) + return { taken, callsAfterRecording: model.calls() } + })), + When('the recording is replayed, then judged through a cache over it')( + 'verdicts', + (s) => + Effect.gen(function*() { + const model = yield* CountingModel + const replayed = yield* withProvider(s.policy('x'), model.model, [ + Discern.Model.replaying(Discern.Model.store(s.recorded.taken)), + ]) + const cached = yield* withProvider(s.policy('x'), model.model, [ + Discern.Model.caching(Discern.Model.store(s.recorded.taken)), + ]) + return { replayed, cached, modelCallsSinceRecording: model.calls() - s.recorded.callsAfterRecording } + }), + ), + Then('both runs reach the recorded verdict and the model was never asked')(({ verdicts }, expect) => + expect(verdicts).toEqual({ replayed: 'both', cached: 'both', modelCallsSinceRecording: 0 }) + ), + ), + ) + + scenario( + 'An empty store sends every decision to the model, whether caching or replaying set to ask', + Gherkin.Do.pipe( + Given('a policy on two questions')('policy', () => Effect.succeed(bothPolicy)), + When('both questions are judged through a cache over an empty store, and replaying an empty store set to ask')( + 'verdicts', + (s) => + Effect.gen(function*() { + const model = yield* CountingModel + const cached = yield* withProvider(s.policy('x'), model.model, [ + Discern.Model.caching(Discern.Model.store()), + ]) + const replayed = yield* withProvider(s.policy('x'), model.model, [ + Discern.Model.replaying(Discern.Model.store(), { onMissing: 'ask' }), + ]) + return { cached, replayed, modelCalls: model.calls() } + }), + ), + Then('the model was asked once per run, about both questions in one batch')(({ verdicts }, expect) => + expect(verdicts).toEqual({ cached: 'both', replayed: 'both', modelCalls: 2 }) + ), + ), + ) + })