fix(discern)!: refuse what upstream 0.5.0 refuses: stored answers, inverted bounds, empty calibration

packages/discern/README.md

@@ -114,6 +114,8 @@ console.log(await Effect.runPromise(guarded('bump lodash').pipe(Effect.provide(D
 
 `policy.runWithTrace(input)` returns the value together with a trace of every case evaluated, how it resolved, and the answers it used.
 
+A bound that cannot hold is refused when the pattern is built, not accepted silently. `above` and `atLeast` refuse a `missBelow` greater than the threshold, `below` and `atMost` refuse a `missAbove` less than it, and a `between` whose low end comes after its high end is refused too (ratings compare by level position). Equal bounds stay legal. A policy run, `Eval.run`, `Eval.sweep` and `Eval.calibrate` fail before the model is asked, with `InvalidThresholdError` for a bound and `InvalidRangeError` for a range, whichever case the input would have reached. `Discern.evaluate` on a refused pattern answers `Uncertain` with the refusal message.
+
 ### Record, replay, cache, budget
 
 Interceptors decorate the model. `recording` stores every answer, and `Discern.Model.replayLayer` reruns any program from those answers with no model at all:
@@ -146,6 +148,8 @@ await Effect.runPromise(review('drop Node 18').pipe(Effect.provide(metered)))
 console.log(await Effect.runPromise(Discern.Model.spent(spend))) // { decisions: 2, calls: 1 }
 ```
 
+`replaying` and `caching` hold a stored answer to the rules a live provider is held to. An answer whose probability, rating or label breaks them fails with an error that `Discern.Model.isInvalidObservation` recognises, before the model is asked.
+
 The same stack is a builder: `Discern.Model.model(cautious).recording(store).budgeted(spend).layer` appends one interceptor at a time and compiles the same layer.
 
 Answers are keyed by the question's content and the input's structure. Rewording instructions or reordering criteria is a new question; reordering an input object's keys is not.
@@ -156,6 +160,8 @@ Stores and budgets are handles: `Discern.Model.snapshot`, `load`, `get`, `set`,
 
 `Discern.Eval` runs a pattern over labelled examples and reports accuracy, precision, recall and F1. `Eval.calibrate` asks the model once per example and picks the best threshold from those answers; `Eval.sweep` skips examples a deterministic pattern already settles.
 
+`Eval.calibrate` refuses an empty `values` list with `CalibrationValuesRefused` before it builds a pattern or asks the model.
+
 ### Route between procedures
 
 `Discern.Procedure` routes a request to one of several named procedures with a single classification. The registry takes its members as a record, and each key is that member's id:

packages/discern/etc/discern.api.md

@@ -6,7 +6,7 @@
 
 // @public (undocumented)
 export namespace Discern {
-    export { Answer, Answers, AnyDecision, BandOptions, CalibrateOptions, CaseInspection, CaseTrace, ClassificationMatcher, ClassifyDecision, ClassifyOptions, ClassifyThresholds, CompiledPlan, DecisionIdCollisionError, DecisionInspection, DecisionNode, DecisionScope, Eval, EvalExample, EvalMetrics, EvalRecord, EvalReport, ExhaustiveMatchError, FinishedMatcher, HandlerResult, InvalidThresholdError, LeafOptions, Matcher, MatcherFlavor, MeasureCommandRejected, MeasureError, MeasureInputRefused, mod_d_exports$2 as Model, NodeCore, Pattern, PatternAst, PatternEvaluator, PatternMatched, PatternMissed, PatternRefusal, PatternResult, PatternStatus, PatternUncertain, Policy, PolicyCommandRejected, PolicyRun, PolicyTraced, Preview, ProbabilityBand, ProbabilityDecision, ProbabilityOptions, mod_d_exports$1 as Procedure, RateDecision, RateOptions, SweepOptions, SweepResult, Trace, UncertainContext, UncertainMatchError, above, and, ask, atLeast, atMost, band, below, between, caseOf as case, classify, compile, decision, deterministic, evaluate, exhaustive, inspect, is, margin, match, matched, missed, not, on, onUncertain, oneOf, or, orElse, preview, probability, rate, reasonOf, runWithTrace, statusIs, statusOf, type, uncertain, value, when, where, whereResult };
+    export { Answer, Answers, AnyDecision, BandOptions, CalibrateOptions, CalibrationValuesRefused, CaseInspection, CaseTrace, ClassificationMatcher, ClassifyDecision, ClassifyOptions, ClassifyThresholds, CompiledPlan, DecisionIdCollisionError, DecisionInspection, DecisionNode, DecisionScope, Eval, EvalExample, EvalMetrics, EvalRecord, EvalReport, ExhaustiveMatchError, FinishedMatcher, HandlerResult, InvalidRangeError, InvalidThresholdError, LeafOptions, Matcher, MatcherFlavor, MeasureCommandRejected, MeasureError, MeasureInputRefused, mod_d_exports$2 as Model, NodeCore, Pattern, PatternAst, PatternEvaluator, PatternMatched, PatternMissed, PatternRefusal, PatternResult, PatternStatus, PatternUncertain, Policy, PolicyCommandRejected, PolicyRun, PolicyTraced, Preview, ProbabilityBand, ProbabilityDecision, ProbabilityOptions, mod_d_exports$1 as Procedure, RangeRefusal, RateDecision, RateOptions, SweepOptions, SweepResult, ThresholdRefusal, Trace, UncertainContext, UncertainMatchError, above, and, ask, atLeast, atMost, band, below, between, caseOf as case, classify, compile, decision, deterministic, evaluate, exhaustive, inspect, is, margin, match, matched, missed, not, on, onUncertain, oneOf, or, orElse, preview, probability, rate, reasonOf, runWithTrace, statusIs, statusOf, type, uncertain, value, when, where, whereResult };
 }
 
 // (No @packageDocumentation comment for this package)

packages/discern/src/Discern/mod.ts

@@ -52,6 +52,8 @@ export type {
   PatternEvaluator,
   PatternRefusal,
   Preview,
+  RangeRefusal,
+  ThresholdRefusal,
   UncertainContext,
 } from '../pattern.blueprint.js'
 export { matched, missed, reasonOf, statusIs, statusOf, uncertain } from '../Verdict.schema.js'
@@ -77,8 +79,10 @@ export { Eval } from '../eval.js'
 export type { CalibrateOptions, EvalExample, MeasureError, SweepOptions, SweepResult } from '../measure-pattern.cell.js'
 
 export {
+  CalibrationValuesRefused,
   DecisionIdCollisionError,
   ExhaustiveMatchError,
+  InvalidRangeError,
   InvalidThresholdError,
   MeasureCommandRejected,
   MeasureInputRefused,

packages/discern/src/DiscernError.schema.ts

@@ -37,6 +37,29 @@ export class InvalidThresholdError extends Schema.TaggedError<InvalidThresholdEr
   },
 ) {}
 
+/**
+ * A span names its ends in the wrong order. `low` and `high` carry the ends as
+ * numbers or level names, so the refusal names both ends the caller wrote.
+ */
+export class InvalidRangeError extends Schema.TaggedError<InvalidRangeError>()(
+  'InvalidRangeError',
+  {
+    range: Schema.String,
+    low: Schema.Union([Schema.Finite, Schema.String]),
+    high: Schema.Union([Schema.Finite, Schema.String]),
+    message: Schema.String,
+  },
+) {}
+
+export class CalibrationValuesRefused extends Schema.TaggedError<CalibrationValuesRefused>()(
+  'CalibrationValuesRefused',
+  {},
+) {
+  override get message(): string {
+    return 'Calibration needs at least one candidate value; the list was empty'
+  }
+}
+
 /**
  * The policy run's command was rejected before any case could be answered.
  * Carries the cases the read had evaluated, so the refusal names the run it

packages/discern/src/Model/mod.ts

@@ -7,6 +7,7 @@ export {
   fromProvider,
   intercept,
   isBudgetExceeded,
+  isInvalidObservation,
   isReplayMiss,
   layer,
   provider,

packages/discern/src/cache-provider.cell.ts

@@ -3,7 +3,7 @@ import * as AiError from 'effect/ai/AiError'
 import type * as Decision from 'effect/ai/Decision'
 import type * as DecisionModel from 'effect/ai/DecisionModel'
 import * as Effect from 'effect/Effect'
-import type { ObservationSplit, Provider } from './decision-model.blueprint.js'
+import type { ObservationSplit, Provider, StoredHitCheck } from './decision-model.blueprint.js'
 import type { ProviderAnswer } from './Observation.schema.js'
 import { CurrentRegion } from './region.service.js'
 import { SelectObservationSource, selectObservationSource } from './select-observation-source.workflow.js'
@@ -12,6 +12,7 @@ export interface CacheRequest {
   readonly options: DecisionModel.ProviderOptions
   readonly inner: Provider
   readonly split: ObservationSplit
+  readonly validate: StoredHitCheck
   readonly record: (
     answers: Readonly<Record<string, DecisionModel.ProviderAnswer>>,
     regionPath: ReadonlyArray<string>,
@@ -23,6 +24,7 @@ export type CacheRead = (typeof SelectObservationSource)['Encoded'] & {
   readonly missingDecisions: Readonly<Record<string, Decision.Any>>
   readonly state: DecisionModel.ProviderOptions['state']
   readonly inner: Provider
+  readonly validate: StoredHitCheck
   readonly record: (
     answers: Readonly<Record<string, DecisionModel.ProviderAnswer>>,
     regionPath: ReadonlyArray<string>,
@@ -39,28 +41,36 @@ const readCache = (request: CacheRequest): Effect.Effect<CacheRead> =>
     missingDecisions: request.split.missingDecisions,
     state: request.options.state,
     inner: request.inner,
+    validate: request.validate,
     record: request.record,
   })
 
 const askAndRecord = (read: CacheRead): Effect.Effect<DecisionModel.ProviderResponse, AiError.AiError> =>
   Effect.flatMap(
-    CurrentRegion.useSync((path) => path),
-    (regionPath) =>
+    read.validate(read.hits),
+    () =>
       Effect.flatMap(
-        read.inner.decide({ state: read.state, decisions: read.missingDecisions }),
-        (response) =>
-          Effect.map(read.record(response.answers, regionPath), () => ({
-            answers: { ...read.hits, ...response.answers },
-            usage: response.usage,
-          })),
+        CurrentRegion.useSync((path) => path),
+        (regionPath) =>
+          Effect.flatMap(
+            read.inner.decide({ state: read.state, decisions: read.missingDecisions }),
+            (response) =>
+              Effect.map(read.record(response.answers, regionPath), () => ({
+                answers: { ...read.hits, ...response.answers },
+                usage: response.usage,
+              })),
+          ),
       ),
   )
 
 export const cacheObservations = Sandwich.named('discern.model.cache')(readCache)
   .decide(selectObservationSource)
   .write({
     AllRecorded: (_recorded, read) =>
-      Effect.succeed({ answers: read.hits, usage: { inputTokens: undefined, outputTokens: undefined } }),
+      Effect.as(read.validate(read.hits), {
+        answers: read.hits,
+        usage: { inputTokens: undefined, outputTokens: undefined },
+      }),
     AskForMissing: (_asked, read) => askAndRecord(read),
     RecordingMissing: (refusal, _read) => Effect.succeed(refusal),
     CommandRejected: (rejected, read) =>

packages/discern/src/decision-model.blueprint.ts

@@ -28,6 +28,7 @@ export const DiscernMethod = {
   caching: 'caching',
   budgeted: 'budgeted',
   refusing: 'refusing',
+  observation: 'observation',
 } as const
 const DISCERN_MODULE = 'Discern'
 
@@ -45,6 +46,10 @@ export const isBudgetExceeded = (error: unknown): error is AiError.AiError =>
 export const isReplayMiss = (error: unknown): error is AiError.AiError =>
   AiError.isAiError(error) && isDiscernMethodOf(error, DiscernMethod.replaying)
 
+/** True for the failure raised when an answer read back from a store fails validation. */
+export const isInvalidObservation = (error: unknown): error is AiError.AiError =>
+  AiError.isAiError(error) && isDiscernMethodOf(error, DiscernMethod.observation)
+
 const replayMissFailure = (refusal: (typeof RecordingMissing)['Encoded']): AiError.AiError =>
   discernFailure(DiscernMethod.replaying, new AiError.InvalidRequestError({ description: refusal.detail }))
 
@@ -164,6 +169,49 @@ const recordAnswers = (
     { discard: true },
   )
 
+export type StoredHitCheck = (hits: Readonly<Record<string, ProviderAnswer>>) => Effect.Effect<void, AiError.AiError>
+
+const invalidObservationFailure = (error: AiError.AiError): AiError.AiError =>
+  discernFailure(
+    DiscernMethod.observation,
+    new AiError.InvalidOutputError({
+      description: Match.value(error.reason).pipe(
+        Match.tag('InvalidOutputError', (reason) => reason.description),
+        Match.orElse((reason) => reason.message),
+      ),
+    }),
+  )
+
+const hitDecisionsOf = (
+  decisions: Readonly<Record<string, Decision.Any>>,
+  hits: Readonly<Record<string, ProviderAnswer>>,
+): Readonly<Record<string, Decision.Any>> =>
+  Object.fromEntries(Object.entries(decisions).filter(([id]) => Object.hasOwn(hits, id)))
+
+/**
+ * Holds stored answers to the rules a provider's answers are held to, by running
+ * them through a throwaway `DecisionModel` whose provider returns them. A gate
+ * only: the stored hits pass on unchanged, because the validator's output drops
+ * the `_tag` the stored form carries.
+ */
+const storedHitCheckOf = (request: DecisionModel.ProviderOptions): StoredHitCheck => (hits) => {
+  const decisions = hitDecisionsOf(request.decisions, hits)
+  return Object.keys(decisions).length === 0
+    ? Effect.void
+    : Effect.flatMap(
+      DecisionModel.make({
+        decide: () => Effect.succeed({ answers: hits, usage: { inputTokens: undefined, outputTokens: undefined } }),
+      }),
+      (checker) =>
+        Effect.asVoid(
+          Effect.mapError(
+            checker.decide(Decision.make({ input: Schema.Json, decisions }), { input: request.state }),
+            invalidObservationFailure,
+          ),
+        ),
+    )
+}
+
 /**
  * A `DecisionModel` decorator. Interceptors are listed outermost-first, so in
  * `[recording, caching, budgeted]` the recorder observes every answer, the cache
@@ -230,7 +278,13 @@ export const replaying: {
           splitObservations(request.decisions, request.state, lookup),
           (split) =>
             Effect.flatMap(
-              replayObservations.run({ options: request, inner, split, onMissing }),
+              replayObservations.run({
+                options: request,
+                inner,
+                split,
+                onMissing,
+                validate: storedHitCheckOf(request),
+              }),
               (answer) =>
                 Match.value(answer).pipe(
                   Match.tag('RecordingMissing', (refusal) => Effect.fail(replayMissFailure(refusal))),
@@ -257,6 +311,7 @@ export const caching = (into: ObservationStore): Interceptor => (inner) =>
             options: request,
             inner,
             split,
+            validate: storedHitCheckOf(request),
             record: (answers, regionPath) => recordAnswers(into, request.decisions, request.state, answers, regionPath),
           }),
           (answer) =>