diff --git a/db/migrations/0018_share_ref_assumptions.sql b/db/migrations/0018_share_ref_assumptions.sql new file mode 100644 index 0000000..9791eb4 --- /dev/null +++ b/db/migrations/0018_share_ref_assumptions.sql @@ -0,0 +1 @@ +ALTER TABLE "share_ref" ADD COLUMN "assumptions" jsonb; \ No newline at end of file diff --git a/db/migrations/meta/0018_snapshot.json b/db/migrations/meta/0018_snapshot.json new file mode 100644 index 0000000..858ac24 --- /dev/null +++ b/db/migrations/meta/0018_snapshot.json @@ -0,0 +1,1073 @@ +{ + "id": "a86a48f3-f8c6-4194-9286-af7e196938b3", + "prevId": "ab576e2a-72ab-416b-98c1-9290023712db", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.conversation": { + "name": "conversation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_session_id": { + "name": "agent_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "turn_count": { + "name": "turn_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "conversation_user_updated_idx": { + "name": "conversation_user_updated_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "conversation_user_id_user_id_fk": { + "name": "conversation_user_id_user_id_fk", + "tableFrom": "conversation", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.event": { + "name": "event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "event_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "visitor_id": { + "name": "visitor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "page": { + "name": "page", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "props": { + "name": "props", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + } + }, + "indexes": { + "event_created_at_idx": { + "name": "event_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "event_name_created_at_idx": { + "name": "event_name_created_at_idx", + "columns": [ + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.feedback": { + "name": "feedback", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "feedback_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "page": { + "name": "page", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "feedback_created_at_idx": { + "name": "feedback_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "feedback_message_length": { + "name": "feedback_message_length", + "value": "char_length(\"feedback\".\"message\") BETWEEN 1 AND 5000" + } + }, + "isRLSEnabled": false + }, + "public.limit_counter": { + "name": "limit_counter", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "window_start": { + "name": "window_start", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "limit_counter_window_start_idx": { + "name": "limit_counter_window_start_idx", + "columns": [ + { + "expression": "window_start", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.roadmap_vote": { + "name": "roadmap_vote", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entry_id": { + "name": "entry_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "roadmap_vote_entry_idx": { + "name": "roadmap_vote_entry_idx", + "columns": [ + { + "expression": "entry_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "roadmap_vote_user_id_user_id_fk": { + "name": "roadmap_vote_user_id_user_id_fk", + "tableFrom": "roadmap_vote", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "roadmap_vote_user_id_entry_id_pk": { + "name": "roadmap_vote_user_id_entry_id_pk", + "columns": [ + "user_id", + "entry_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.share": { + "name": "share", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "card_id": { + "name": "card_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "share_user_idx": { + "name": "share_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "share_user_id_user_id_fk": { + "name": "share_user_id_user_id_fk", + "tableFrom": "share", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.share_card_result": { + "name": "share_card_result", + "schema": "", + "columns": { + "share_id": { + "name": "share_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "card_id": { + "name": "card_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "card": { + "name": "card", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "frame": { + "name": "frame", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "retryable": { + "name": "retryable", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "retry_after_s": { + "name": "retry_after_s", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "checked_at": { + "name": "checked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "fresh_until": { + "name": "fresh_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "share_card_result_share_id_share_id_fk": { + "name": "share_card_result_share_id_share_id_fk", + "tableFrom": "share_card_result", + "tableTo": "share", + "columnsFrom": [ + "share_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "share_card_result_share_id_card_id_pk": { + "name": "share_card_result_share_id_card_id_pk", + "columns": [ + "share_id", + "card_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.share_ref": { + "name": "share_ref", + "schema": "", + "columns": { + "share_id": { + "name": "share_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "part_index": { + "name": "part_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "spec_id": { + "name": "spec_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identity": { + "name": "identity", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "entity_groups": { + "name": "entity_groups", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "variable_text": { + "name": "variable_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_version": { + "name": "content_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "artifact_digest": { + "name": "artifact_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operations": { + "name": "operations", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "assumptions": { + "name": "assumptions", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "share_ref_share_id_share_id_fk": { + "name": "share_ref_share_id_share_id_fk", + "tableFrom": "share_ref", + "tableTo": "share", + "columnsFrom": [ + "share_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "share_ref_share_id_part_index_pk": { + "name": "share_ref_share_id_part_index_pk", + "columns": [ + "share_id", + "part_index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/db/migrations/meta/_journal.json b/db/migrations/meta/_journal.json index 4ff5540..021eb8e 100644 --- a/db/migrations/meta/_journal.json +++ b/db/migrations/meta/_journal.json @@ -120,6 +120,13 @@ "when": 1792396643977, "tag": "0017_limit_counter", "breakpoints": true + }, + { + "idx": 18, + "version": "7", + "when": 1792532418871, + "tag": "0018_share_ref_assumptions", + "breakpoints": true } ] } \ No newline at end of file diff --git a/db/schema.ts b/db/schema.ts index 27530c1..52b870e 100644 --- a/db/schema.ts +++ b/db/schema.ts @@ -105,6 +105,9 @@ export const shareRef = pgTable( // Null for a live share and for snapshots stored before the column. artifactDigest: text('artifact_digest'), operations: jsonb('operations').notNull().default([]), + // The part's assumptions as the response gave them, [] included, + // which the replay echoes. Null for a share stored before the column. + assumptions: jsonb('assumptions'), }, (table) => [primaryKey({ columns: [table.shareId, table.partIndex] })], ); diff --git a/db/test/share-ref-round-trip.test.ts b/db/test/share-ref-round-trip.test.ts index 5775dcf..daf454c 100644 --- a/db/test/share-ref-round-trip.test.ts +++ b/db/test/share-ref-round-trip.test.ts @@ -12,9 +12,11 @@ const MIGRATIONS_FOLDER = path.resolve(import.meta.dirname, '../migrations'); /** * A `share_ref` row written through db/schema.ts as the runtime role reads * back with its groups in `entity_groups`, beside an identity that holds - * none, and with null for a part that has no group. + * none, and with null for a part that has no group. Its `assumptions` read + * back as stored, [] included, and null for a row written without them, as + * a share stored before the column holds. */ -test("a share_ref row keeps a part's groups beside its identity", async (t) => { +test("a share_ref row keeps a part's groups beside its identity, and its assumptions", async (t) => { const database = await createTestDatabase(); t.after(() => database.teardown()); @@ -29,6 +31,14 @@ test("a share_ref row keeps a part's groups beside its identity", async (t) => { const groups = { origin: { ref: 'africa', label: 'Africa', entity_type: 'Continent' }, }; + const assumptions = [ + { + kind: 'declared_default', + binding_ref: 'how:flowType', + refs: ['ODA'], + text: 'official development assistance', + }, + ]; await db.insert(share).values({ id: 'srch0001', @@ -38,8 +48,23 @@ test("a share_ref row keeps a part's groups beside its identity", async (t) => { data: { query: 'aid from the EU' }, }); await db.insert(shareRef).values([ - { shareId: 'srch0001', partIndex: 0, specId: 'spec_a', identity, entityGroups: groups }, - { shareId: 'srch0001', partIndex: 1, specId: 'spec_b', identity, entityGroups: null }, + { + shareId: 'srch0001', + partIndex: 0, + specId: 'spec_a', + identity, + entityGroups: groups, + assumptions, + }, + { + shareId: 'srch0001', + partIndex: 1, + specId: 'spec_b', + identity, + entityGroups: null, + assumptions: [], + }, + { shareId: 'srch0001', partIndex: 2, specId: 'spec_c', identity, entityGroups: null }, ]); const refs = await db @@ -49,10 +74,11 @@ test("a share_ref row keeps a part's groups beside its identity", async (t) => { .orderBy(shareRef.partIndex); assert.deepEqual( - refs.map((ref) => [ref.identity, ref.entityGroups]), + refs.map((ref) => [ref.identity, ref.entityGroups, ref.assumptions]), [ - [identity, groups], - [identity, null], + [identity, groups, assumptions], + [identity, null, []], + [identity, null, null], ], ); }); diff --git a/src/components/organisms/DataSearch/DataSearch.module.scss b/src/components/organisms/DataSearch/DataSearch.module.scss index d01c7ad..9ebc703 100644 --- a/src/components/organisms/DataSearch/DataSearch.module.scss +++ b/src/components/organisms/DataSearch/DataSearch.module.scss @@ -273,6 +273,7 @@ $table-head-cover: rem(48); // The muted lead above a heading that shows the nearest measure the data // holds. +.coverageLead, .closestLead { @include font-size(14px, 1.3); @@ -283,10 +284,11 @@ $table-head-cover: rem(48); letter-spacing: -0.01em; } -// The byline (unit and source) and the departures under the heading, in -// small type. +// The byline (unit and source), the departures and the assumptions clause +// under the heading, in small type. .byline, -.departures { +.departures, +.assumptions { @include font-size(15px, 1.3); font-family: var(--font-outfit); @@ -323,6 +325,10 @@ $table-head-cover: rem(48); list-style: none; } +.assumptions { + margin-top: rem(12); +} + .departureAction { padding: 0; font: inherit; @@ -349,9 +355,11 @@ $table-head-cover: rem(48); } @media (prefers-reduced-motion: reduce) { + .coverageLead, .closestLead, .byline, - .departures { + .departures, + .assumptions { animation: none; } } diff --git a/src/components/organisms/DataSearch/SearchCards.tsx b/src/components/organisms/DataSearch/SearchCards.tsx index 14ed80b..8fec4ef 100644 --- a/src/components/organisms/DataSearch/SearchCards.tsx +++ b/src/components/organisms/DataSearch/SearchCards.tsx @@ -6,7 +6,11 @@ import classNames from 'classnames'; import styles from './DataSearch.module.scss'; import Button from '@/components/atoms/Button/Button'; -import { buildAmbiguousPickPart, offersAmbiguousPicker } from '@/lib/search/parsedEdit'; +import { + entityChoiceGroups, + entityChoicePart, + offersAmbiguousPicker, +} from '@/lib/search/parsedEdit'; import { cardText, coverageSummary, @@ -110,36 +114,30 @@ const SearchCards = ({ } if (part?.status === 'no_data') { - // The single-part escape hatch for entity_ambiguous picking (see - // module doc). A pick pins the candidate's dcid onto the matching - // restatement entity and posts a parsed entry, a no-op if no entity - // matches. + // The entity_ambiguous picker of a part with no slot token to hang + // it off. A pick posts the refusal's option for it unchanged as a + // parsed entry (entityChoicePart). if (offersAmbiguousPicker(part) && onEditParsed && partIndex !== undefined) { - const restatement = part.restatement; - const groups = part.no_data.ambiguous_entities ?? []; + const groups = entityChoiceGroups(part.no_data); return (

Choose a Match

- {groups.flatMap((group: any) => - group.candidates.map((candidate: any, index: number) => ( + {groups.flatMap((group) => + group.options.map((option, index) => ( )), )} diff --git a/src/components/organisms/DataSearch/SearchSentence.tsx b/src/components/organisms/DataSearch/SearchSentence.tsx index 63f6f62..865a27f 100644 --- a/src/components/organisms/DataSearch/SearchSentence.tsx +++ b/src/components/organisms/DataSearch/SearchSentence.tsx @@ -40,6 +40,7 @@ import Button from '@/components/atoms/Button/Button'; import { capitalise } from '@/lib/capitalise'; import { answerDepartures, + assumptionClauses, isClosestMatch, refsStatedBesides, sentenceBlocks, @@ -57,15 +58,14 @@ import { } from '@/lib/search/pickerState'; import { spliceSentence } from '@/lib/search/refine'; import { keyRef } from '@/lib/search/searchClient'; -import { planSentenceText, tokenPart } from '@/lib/search/sentencePlan'; +import { planCoverageText, planSentenceText, tokenPart } from '@/lib/search/sentencePlan'; import type { - AmbiguousEntityCandidate, - AmbiguousEntityGroup, EntityRoleKey, GraphRef, HydratedPart, IdentityFacetAxis, IdentityFacets, + NoDataOption, ParsedPart, MeasureAxisValue, RoleAxis, @@ -84,7 +84,8 @@ import { type UnitSwitch, } from '@/lib/search/measureMenu'; import { - buildAmbiguousPickPart, + entityChoiceGroups, + entityChoicePart, offersAmbiguousPicker, resolveTypedEdit, } from '@/lib/search/parsedEdit'; @@ -672,11 +673,14 @@ const lowerFirst = (text: string): string => /^[A-Z][a-z]/.test(text) ? text.charAt(0).toLowerCase() + text.slice(1) : text; // The answer block over every part of the resolve response, from the -// envelope's own composed `sentence_plan`: a muted "Closest match" lead when +// envelope's own composed `sentence_plan`: a muted coverage lead ("Answered +// 1 of 2 parts.") on a mixed answer, a muted "Closest match" lead when // a part shows the nearest measure, the heading tokens inside the page's one -//

, the byline (unit and source) in a

after it, and every part's -// departures as a list, each with the one action it offers. The plan's own -// departure tokens are skipped, since the list reads `parts[].departures`. A +//

, the byline (unit and source) in a

after it, every part's +// departures as a list, each with the one action it offers, and the +// assumptions clause ("Assumed: every provider.") in a closing

. The +// plan's own departure tokens are skipped, since the list reads +// `parts[].departures`. A // part-indexed what/how token routes its menu to that part's own spec; a // part-indexed where/when/source/role token routes to that same part's // slot, window, or provider menu; a shared where/when/source token (part_index @@ -1831,21 +1835,17 @@ export const SearchSentenceHeading = ({ ); } - // An entity_ambiguous pick pins the candidate's dcid onto the - // restatement entity whose text is the picked wording and posts - // a parsed entry, a no-op if no entity carries that text. - // Reachable only when the response carried ambiguous_entities - // and a restatement to edit from. A wording the plan carries as - // its own slot token opens a menu of the nodes it could name; a - // part whose plan carries none opens one menu over every - // wording on its what-token. + // An entity_ambiguous pick posts the refusal's option for it + // unchanged as a parsed entry (entityChoicePart). A wording the + // plan carries as its own slot token opens a menu of the options + // that settle it; a part whose plan carries none opens one menu + // over every wording on its what-token. if (wirePart.status === 'no_data' && offersAmbiguousPicker(wirePart)) { - const groups = wirePart.no_data.ambiguous_entities; - const restatement = wirePart.restatement; - const pick = (group: AmbiguousEntityGroup, candidate: AmbiguousEntityCandidate) => { - const edited = buildAmbiguousPickPart(restatement, group.text, candidate); + const groups = entityChoiceGroups(wirePart.no_data); + const pick = (option: NoDataOption) => { + const posted = entityChoicePart(option); - if (edited) onEditParsed(partIndex, edited); + if (posted) onEditParsed(partIndex, posted); }; if (wording) { @@ -1860,13 +1860,13 @@ export const SearchSentenceHeading = ({ )} marked > - {wording.candidates.map((candidate) => ( + {wording.options.map((option, optionIndex) => ( pick(wording, candidate)} + onSelect={() => pick(option)} > - {candidate.label} + {option.label} ))} , @@ -1883,16 +1883,16 @@ export const SearchSentenceHeading = ({ marked > {groups.flatMap((group) => - group.candidates.map((candidate) => ( + group.options.map((option, optionIndex) => ( pick(group, candidate)} + onSelect={() => pick(option)} > {/* Two ambiguous wordings share one menu, each row naming its own. */} 1 ? group.text : null}> - {candidate.label} + {option.label} )), @@ -1981,10 +1981,15 @@ export const SearchSentenceHeading = ({ // Before the response lands the block is the

alone. const answerParts = response ? parts : []; + const clauses = assumptionClauses( + rawPlan, + parts.map((part) => part.variableText ?? null), + ); // A candidates part's lead sits above its reading cards instead. const closestParts = answerParts.filter( (part) => part.part?.status !== 'candidates' && isClosestMatch(part.part), ); + const coverageLead = response ? planCoverageText(rawPlan) : ''; const closestLead = closestParts.length === 0 ? null @@ -2092,6 +2097,7 @@ export const SearchSentenceHeading = ({ return ( <> + {coverageLead &&

{coverageLead}

} {closestLead &&

{closestLead}

}

{bylineNodes.length > 0 &&

{bylineNodes}

} {departureItems.length > 0 && } + {clauses.length > 0 && ( +

+ {clauses.map(({ key, prefix, text }) => ( + + {key > 0 && ' '} + {prefix && `${prefix}: `} + {text} + + ))} +

+ )} ); }; @@ -2363,6 +2380,7 @@ export const SearchSentenceBody = ({ variableText: part.variableText, spec: part.hydratedSpec, warnings: part.part?.warnings, + assumptions: part.part?.assumptions, })); return ( diff --git a/src/components/organisms/DataSearch/WhyPanel.tsx b/src/components/organisms/DataSearch/WhyPanel.tsx index 5079306..5682dec 100644 --- a/src/components/organisms/DataSearch/WhyPanel.tsx +++ b/src/components/organisms/DataSearch/WhyPanel.tsx @@ -10,7 +10,7 @@ import Button from '@/components/atoms/Button/Button'; import Icon from '@/components/atoms/Icon/Icon'; import SourceLink from '@/components/atoms/SourceLink/SourceLink'; import { aboutAnswerNotes, type AboutAnswerNotes } from '@/lib/search/resultNotes'; -import type { Warning } from '@/lib/search/types'; +import type { Assumption, Warning } from '@/lib/search/types'; import { buildWhySummary, formatCount, @@ -151,6 +151,7 @@ export type AboutPart = { variableText: string | null; spec: Parameters[0]; warnings?: Warning[]; + assumptions?: Assumption[]; }; const AboutSection = ({ @@ -243,14 +244,14 @@ const AboutPartSection = ({ { warnings: part.warnings ?? [], variable_text: part.variableText }, members, )} - notes={aboutAnswerNotes(part.spec, part.warnings, optionsBySpec)} + notes={aboutAnswerNotes(part.spec, part.warnings, optionsBySpec, part.assumptions)} /> ); }; /** The "About this answer" panel: one section per definite part, each with * the coverage sentence, what the search matched and assumed, the - * variables, how each facet and defaulted slot was chosen, and the sources + * variables, how each facet was chosen, and the sources * with their licence. A lone part carries no title. `open` is whether the * panel shows, which starts the members reads. */ const WhyPanel = ({ diff --git a/src/components/organisms/DataSearch/useSearchRun.ts b/src/components/organisms/DataSearch/useSearchRun.ts index 4163ec4..59b865d 100644 --- a/src/components/organisms/DataSearch/useSearchRun.ts +++ b/src/components/organisms/DataSearch/useSearchRun.ts @@ -151,6 +151,7 @@ import { withEntityEdit, withFacetEdit, withResolvedSlotEdit } from '@/lib/searc import type { PermalinkEdit, PermalinkState } from '@/lib/search/permalink'; import { buildDatasetEntries, + candidatePickEntry, sharedRefEntry, specEntityGroups, type DatasetEntriesOverride, @@ -1174,14 +1175,19 @@ export const useSearchRun = () => { [facetOverride, runDatasetEdit], ); - // Candidate promote: post the candidate's own identity verbatim. + // Candidate promote: post the candidate's own identity verbatim, with + // its reading's own assumptions. const pickCandidate = useCallback( - (partIndex: number, candidate: any) => { - const entry = identityOverrideEntry(partIndex, candidate, candidate.identity); + (partIndex: number, candidate: Spec) => { + const part = parts[partIndex]; + + if (!part) return; + + const entry = candidatePickEntry(part, candidate); runDatasetEdit({ index: partIndex, entry }, [partIndex], 'pick'); }, - [identityOverrideEntry, runDatasetEdit], + [parts, runDatasetEdit], ); // One post for every part a pick switches: a shared unit token's diff --git a/src/lib/search/datasetEntries.ts b/src/lib/search/datasetEntries.ts index a6215cb..9da61be 100644 --- a/src/lib/search/datasetEntries.ts +++ b/src/lib/search/datasetEntries.ts @@ -12,10 +12,16 @@ * preserving posted candidate order. A no_data/unsupported part becomes a * `placeholder` entry, holding its position without asking the engine to * re-derive anything. Every entry, overrides included, carries its part's - * `operations`, so an edit keeps what the question asked of the data. + * `operations`, so an edit keeps what the question asked of the data (a + * `parsed` entry's part carries its own). Every identity entry carries its + * part's `assumptions` as the response gave them, so an edit that leaves a + * declared default's slot alone keeps it. A candidates part's own + * `assumptions` are only what its readings share, so a pick and each + * candidate echo their reading's `reading_assumptions` list. */ import type { + Assumption, BaseIdentity, DatasetCandidateIdentity, DatasetCandidatesEntry, @@ -65,10 +71,13 @@ export type SharedEntryRef = { identity: unknown; entityGroups: Record | null; operations: Operation[]; + assumptions: Assumption[] | null; }; /** The `identity` entry a stored share part replays: the identity as - * stored, its groups beside it, and the operations its question asked. */ + * stored, its groups beside it, the operations its question asked, and + * the assumptions its response gave, [] included. A share with null + * assumptions posts none. */ export function sharedRefEntry(ref: SharedEntryRef): DatasetIdentityEntry { return { entry_kind: 'identity', @@ -76,6 +85,7 @@ export function sharedRefEntry(ref: SharedEntryRef): DatasetIdentityEntry { identity: ref.identity as BaseIdentity, ...specEntityGroups({ entity_groups: ref.entityGroups }), ...(ref.operations.length ? { operations: ref.operations } : {}), + ...(ref.assumptions ? { assumptions: ref.assumptions } : {}), }; } @@ -95,6 +105,35 @@ function identityEntryFor(part: HydratedPart): DatasetIdentityEntry { }; } +/** A reading's own assumptions on a candidates part as an entry member: + * `reading_assumptions[spec_id]`, [] included. A response without the + * field or the key echoes nothing, so the engine computes them fresh. */ +function readingAssumptions( + part: HydratedPart, + spec: Pick, +): { assumptions?: Assumption[] } { + const assumptions = part.part?.reading_assumptions?.[spec.spec_id]; + + return assumptions ? { assumptions } : {}; +} + +/** The `identity` entry a pick of `spec` on a candidates part posts: the + * reading's identity, its groups beside it, and the reading's own + * assumptions (see `readingAssumptions`). */ +export function candidatePickEntry(part: HydratedPart, spec: Spec): DatasetIdentityEntry { + if (!spec.identity) { + throw new Error(`candidatePickEntry: part ${part.index} picked a spec with no identity`); + } + + return { + entry_kind: 'identity', + variable_text: part.variableText ?? '', + identity: spec.identity, + ...specEntityGroups(spec), + ...readingAssumptions(part, spec), + }; +} + function candidatesEntryFor(part: HydratedPart): DatasetCandidatesEntry { const candidateSet = part.part?.candidates; if (!candidateSet) { @@ -107,7 +146,11 @@ function candidatesEntryFor(part: HydratedPart): DatasetCandidatesEntry { `buildDatasetEntries: part ${part.index} has a candidate spec with no identity`, ); } - return { identity: spec.identity, ...specEntityGroups(spec) }; + return { + identity: spec.identity, + ...specEntityGroups(spec), + ...readingAssumptions(part, spec), + }; }); return { @@ -183,25 +226,49 @@ function applyOverride( }); } -/** `entry` with its part's operations attached: on the entry itself, or on - * a `parsed` entry's `part`. A part with none leaves the entry as it is. */ +/** `entry` with its part's operations attached. A `parsed` entry posts its + * `part` unchanged, since that part (a refusal option's request or an + * edited restatement) carries its own operations. A part with none leaves + * the entry as it is. */ function withOperations(entry: DatasetResubmitEntry, part: HydratedPart): DatasetResubmitEntry { const operations = part.part?.operations; - if (!operations?.length) return entry; - if (entry.entry_kind === 'parsed') { - return { ...entry, part: { ...entry.part, operations } }; - } + if (!operations?.length || entry.entry_kind === 'parsed') return entry; return { ...entry, operations }; } +/** An identity `entry` with its definite part's assumptions echoed, + * unchanged. The engine decides which still hold after the edit. An entry + * that already carries an echo keeps it: a permalink replay posts each + * stored edit with the echo it was posted with, which a later edit may have + * narrowed below the fresh part's list. Every other entry kind, and a part + * with none, leaves the entry as it is. So does any other part: a + * candidates part's `assumptions` hold only what its readings share, and an + * identity entry there is a pick, which carries its reading's own list; a + * no_data part assumed nothing because it answered nothing, so its edit + * reads fresh. */ +function withAssumptions(entry: DatasetResubmitEntry, part: HydratedPart): DatasetResubmitEntry { + const assumptions = part.part?.assumptions; + + if ( + entry.entry_kind !== 'identity' || + entry.assumptions !== undefined || + part.part?.status !== 'definite' || + assumptions === undefined + ) { + return entry; + } + return { ...entry, assumptions }; +} + /** * Build the `entries` list for a `dataset_resubmit` post from the run's * current `parts`, in part order. With no override, every entry reflects * each part's own current content untouched: the shape a fresh post between * edits would carry. See `DatasetEntriesOverride` for what an edit changes; - * pass an array to apply more than one, in order. Operations attach after - * the overrides, so an edited entry keeps its part's operations too. + * pass an array to apply more than one, in order. Operations and + * assumptions attach after the overrides, so an edited entry keeps its + * part's too. */ export function buildDatasetEntries( parts: HydratedPart[], @@ -213,5 +280,5 @@ export function buildDatasetEntries( parts.map(entryForPart), ); - return entries.map((entry, i) => withOperations(entry, parts[i])); + return entries.map((entry, i) => withAssumptions(withOperations(entry, parts[i]), parts[i])); } diff --git a/src/lib/search/departures.ts b/src/lib/search/departures.ts index 7fc5967..4d7e898 100644 --- a/src/lib/search/departures.ts +++ b/src/lib/search/departures.ts @@ -1,26 +1,26 @@ -/** Pure logic behind the answer block: the heading and byline blocks of a - * sentence plan and the departures under them with the action each offers. - * No React. */ +/** Pure logic behind the answer block: the heading, byline and assumptions + * blocks of a sentence plan and the departures under them with the action + * each offers. No React. */ import { offersStudioCallout } from './resultNotes.ts'; import { tokenPart } from './sentencePlan.ts'; -import type { Departure, Part, SentenceToken } from './types.ts'; +import type { Departure, Part, SentenceSegment, SentenceToken } from './types.ts'; export type SentenceBlock = { - segment: 'heading' | 'byline'; + segment: Exclude; tokens: SentenceToken[]; }; const isSpacing = (token: SentenceToken) => token.kind === 'text' && token.text.trim() === ''; -/** The plan's heading and byline as blocks, in plan order: each run of - * consecutive tokens in one segment is one block. Departure tokens are - * left out, since the departure list draws from `parts[].departures`, so - * heading runs a departure separated join into one block. A block's - * leading and trailing whitespace-only text tokens are the space between - * blocks and are dropped; a block left empty is dropped too. A plan of - * several separate sentences (one per part) reads heading, byline, - * heading, byline. */ +/** The plan's coverage opener, heading, byline and assumptions clause as + * blocks, in plan order: each run of consecutive tokens in one segment is + * one block. Departure tokens are left out, since the departure list draws from + * `parts[].departures`, so heading runs a departure separated join into + * one block. A block's leading and trailing whitespace-only text tokens are + * the space between blocks and are dropped; a block left empty is dropped + * too. A plan of several separate sentences (one per part) reads heading, + * byline, assumption, heading, byline, assumption. */ export function sentenceBlocks(plan: readonly SentenceToken[] | null | undefined): SentenceBlock[] { const blocks: SentenceBlock[] = []; @@ -43,6 +43,43 @@ export function sentenceBlocks(plan: readonly SentenceToken[] | null | undefined .filter((block) => block.tokens.length > 0); } +/** One assumptions clause the answer block draws after the byline and + * departures: "Assumed: official development assistance; every provider." */ +export type AssumptionClause = { + key: number; + /** the variable text of the part the clause belongs to, set only in a + * response of several parts */ + prefix: string | null; + text: string; +}; + +/** The plan's assumptions clauses as text, in plan order. In a response of + * several parts a clause carries as `prefix` the variable text + * (`variableTexts`, by part index) of the part that owns its lead token + * ("Assumed: "). dfo-apis' render/envelope.py `_clause` gives the lead to + * the one part every item of the clause belongs to, or to no part when the + * items' owners differ, so a clause naming several parts' items carries no + * prefix. Empty when the answer assumed nothing. */ +export function assumptionClauses( + plan: readonly SentenceToken[] | null | undefined, + variableTexts: readonly (string | null)[], +): AssumptionClause[] { + const partCount = variableTexts.length; + + return sentenceBlocks(plan) + .filter((block) => block.segment === 'assumption') + .map((block, key) => { + const lead = block.tokens[0]; + const owner = partCount > 1 && lead ? tokenPart(lead, partCount) : null; + + return { + key, + prefix: owner != null ? (variableTexts[owner] ?? null) : null, + text: block.tokens.map((token) => token.text).join(''), + }; + }); +} + /** What a departure's one action does: open the menu of the visible token * that states `bindingRef` on part `partIndex`, open Refine at the * binding no token states, or hand the part to Studio. */ @@ -74,7 +111,7 @@ const isVisibleSlotOf = ( return ( token.kind === 'slot' && - token.segment !== 'departure' && + (token.segment === 'heading' || token.segment === 'byline') && token.text.trim() !== '' && (owner === partIndex || owner === null) ); diff --git a/src/lib/search/parsedEdit.ts b/src/lib/search/parsedEdit.ts index 9c74241..de4e450 100644 --- a/src/lib/search/parsedEdit.ts +++ b/src/lib/search/parsedEdit.ts @@ -1,15 +1,12 @@ /** Pure logic behind the parsed dataset_resubmit entry. A typed phrase edit * rewrites one restatement field chosen from the token's axis/role (how has - * none, falling back to raw-text refineAt). An entity_ambiguous pick - * rewrites the restatement entity whose text matches the picked group. - * canPinEntityDcid gates a pinned dcid to only when the response itself - * carried ambiguous_entities, never invented from a candidate in hand some - * other way. */ + * none, falling back to raw-text refineAt). An entity_ambiguous pick posts + * one of the refusal's own options (`no_data.options`) unchanged + * (entityChoiceGroups, entityChoicePart). */ import type { - AmbiguousEntityCandidate, - AmbiguousEntityGroup, NoData, + NoDataOption, ParsedPart, ParsedPartEntity, RoleAxis, @@ -101,50 +98,40 @@ export function resolveTypedEdit( return applyRestatementEdit(restatement, target, text); } -export function canPinEntityDcid( - noData: NoData | null | undefined, -): noData is NoData & { ambiguous_entities: AmbiguousEntityGroup[] } { - return Boolean(noData?.ambiguous_entities?.length); -} +/** One ambiguous wording of an entity_ambiguous refusal and the options + * that settle it (`no_data.options` whose `mentions` hold the wording), in + * the order the refusal lists them. */ +export type EntityChoiceGroup = { text: string; options: NoDataOption[] }; -type AmbiguousPickerPart = { no_data?: NoData | null; restatement?: ParsedPart | null }; +/** The refusal's options grouped by the wording each settles, wordings in + * the order their first option comes. An option settling two wordings + * stands in both groups. Empty for every reason but entity_ambiguous. */ +export function entityChoiceGroups(noData: NoData | null | undefined): EntityChoiceGroup[] { + if (noData?.reason !== 'entity_ambiguous') return []; -const hasAmbiguousPickerShape = (part: unknown): part is AmbiguousPickerPart => - typeof part === 'object' && part !== null; + const groups = new Map(); -/** Whether a no_data part renders SearchCards' "Choose a Match" picker - * rather than a generic Studio hand-off panel: an entity_ambiguous reason, - * a response that itself carried ambiguous_entities (canPinEntityDcid), and - * a restatement to pin the pick onto. The one gate SearchCards and - * studioHandoffParts must agree on, so a picker-eligible part always keeps - * its own picker instead of collapsing into the generic studio callout. */ -export function offersAmbiguousPicker(part: unknown): part is AmbiguousPickerPart & { - no_data: NoData & { ambiguous_entities: AmbiguousEntityGroup[] }; - restatement: ParsedPart; -} { - if (!hasAmbiguousPickerShape(part)) return false; - - return ( - part.no_data?.reason === 'entity_ambiguous' && - canPinEntityDcid(part.no_data) && - Boolean(part.restatement) - ); + for (const option of noData.options ?? []) { + for (const text of option.mentions) { + groups.set(text, [...(groups.get(text) ?? []), option]); + } + } + + return [...groups].map(([text, options]) => ({ text, options })); } -/** Rewrites the restatement entity whose text equals groupText to the - * candidate's label, pinned with its dcid. Returns null when no entity - * matches (the invariant broken), which the caller must treat as a no-op - * rather than re-confirming the same ambiguity. */ -export function buildAmbiguousPickPart( - restatement: ParsedPart, - groupText: string, - candidate: AmbiguousEntityCandidate, -): ParsedPart | null { - const entities = restatement.entities ?? []; - const index = entities.findIndex((entity) => entity.text === groupText); - if (index === -1) return null; +/** The part an entity choice posts: its option's one-part request, + * unchanged. Null for an option carrying no part. */ +export const entityChoicePart = (option: NoDataOption): ParsedPart | null => + option.data_query.parts[0] ?? null; + +/** Whether a no_data part renders a "Choose a Match" picker rather than a + * generic Studio hand-off panel: an entity_ambiguous refusal with at least + * one option (entityChoiceGroups). The one gate SearchCards, SearchSentence + * and studioHandoffParts agree on, so a picker-eligible part always keeps + * its own picker instead of collapsing into the generic studio callout. */ +export function offersAmbiguousPicker(part: unknown): part is { no_data: NoData } { + if (typeof part !== 'object' || part === null) return false; - const updated = entities.slice(); - updated[index] = { ...updated[index], text: candidate.label, dcid: candidate.ref }; - return { ...restatement, entities: updated }; + return entityChoiceGroups((part as { no_data?: NoData | null }).no_data).length > 0; } diff --git a/src/lib/search/permalink.ts b/src/lib/search/permalink.ts index f7b0e93..ff69c7e 100644 --- a/src/lib/search/permalink.ts +++ b/src/lib/search/permalink.ts @@ -18,6 +18,7 @@ * run anyway. */ +import { isAssumptions } from './searchShareRequest.ts'; import type { DatasetIdentityEntry, DatasetParsedEntry } from './types'; const PREFIX_DEFLATE = 'd.'; @@ -123,11 +124,15 @@ const isReplayableEditEntryKind = (kind: unknown): kind is PermalinkEdit['entry' /** An identity entry as the door takes it today. A link stored before the * door derived membership from the identity also carries the base it was - * edited from and the StatVars it served, which the door refuses. */ + * edited from and the StatVars it served, which the door refuses. The + * entry's `assumptions` echo survives as stored, [] included, when Search + * would take it. A list it would refuse is dropped, and + * `buildDatasetEntries` then echoes the reloaded definite part's own list, + * as it does for any edit. */ const replayableEntry = (entry: PermalinkEdit['entry']): PermalinkEdit['entry'] => { if (entry.entry_kind !== 'identity') return entry; - const { variable_text, identity, entity_groups, operations } = entry; + const { variable_text, identity, entity_groups, operations, assumptions } = entry; return { entry_kind: 'identity', @@ -135,6 +140,7 @@ const replayableEntry = (entry: PermalinkEdit['entry']): PermalinkEdit['entry'] identity, ...(entity_groups ? { entity_groups } : {}), ...(operations ? { operations } : {}), + ...(isAssumptions(assumptions) ? { assumptions } : {}), }; }; diff --git a/src/lib/search/readings.ts b/src/lib/search/readings.ts index da9dd16..836c249 100644 --- a/src/lib/search/readings.ts +++ b/src/lib/search/readings.ts @@ -54,10 +54,11 @@ export function readingOf(spec: { sentence_plan?: SentenceToken[] | null }): Rea } /** A candidates part's summary plan read as one line, for screen readers: - * "2 readings." and each reading's heading and byline. */ + * "2 readings." and each reading's heading and byline. Its departures and + * assumptions clause are left out, since the answer block states them. */ export function readingsSummary(plan: readonly SentenceToken[] | null | undefined): string { return (plan ?? []) - .filter((token) => token.segment !== 'departure') + .filter((token) => token.segment === 'heading' || token.segment === 'byline') .map((token) => token.text) .join('') .trim(); @@ -68,8 +69,9 @@ export function readingsSummary(plan: readonly SentenceToken[] | null | undefine * token, at the place of that part's first token: a what-token whose text * is the part's variable text ("debt flows to Ghana"), or the summary's own * heading when the part has none. The heading draws it as a marked phrase - * and the reading cards below carry the choice. Returns `plan` itself when - * no part is a candidates part. + * and the reading cards below carry the choice. The part's assumption + * tokens stay as they are, since the clause states what every reading + * assumed alike. Returns `plan` itself when no part is a candidates part. */ export function foldReadings( plan: readonly SentenceToken[], @@ -86,7 +88,9 @@ export function foldReadings( return plan.flatMap((token): SentenceToken[] => { const partIndex = tokenPart(token, partCount); - if (partIndex == null || !isReadings(partIndex)) return [token]; + if (partIndex == null || !isReadings(partIndex) || token.segment === 'assumption') { + return [token]; + } if (folded.has(partIndex)) return []; folded.add(partIndex); diff --git a/src/lib/search/resultNotes.ts b/src/lib/search/resultNotes.ts index 597512c..6a8de66 100644 --- a/src/lib/search/resultNotes.ts +++ b/src/lib/search/resultNotes.ts @@ -16,7 +16,7 @@ import { seriesPreviewCaptions, type SlotOptionsEntry, } from './view.ts'; -import type { Warning } from './types.ts'; +import type { Assumption, Warning } from './types.ts'; const andList = (labels: string[]): string => new Intl.ListFormat('en-GB', { type: 'conjunction' }).format(labels); @@ -45,12 +45,11 @@ export const CAPTION_WARNING_CODES: ReadonlySet = new Set([ /** Codes the download panel explains: what a download will hold. */ export const DOWNLOAD_WARNING_CODES: ReadonlySet = new Set(['membership_unverified']); -/** Codes the "About this answer" panel explains: the scopes the question - * left open (read through view.ts's scopeAssumptions), and UN reads that - * failed. */ +/** Codes the "About this answer" panel explains: a year the question left + * open (read through view.ts's scopeAssumptions), and UN reads that failed. + * The place, role and slot defaults it states come from the part's + * `assumptions`. */ export const ABOUT_WARNING_CODES: ReadonlySet = new Set([ - 'place_scope_defaulted', - 'role_scope_defaulted', 'time_window_defaulted', 'un_system_unavailable', ]); @@ -162,8 +161,8 @@ export type AboutAnswerNotes = { /** the coverage sentence ("2024 is the latest year with data, with 152 * values…"), null without a headline */ coverage: string | null; - /** what the answer assumed, one sentence per scope the question left - * open */ + /** what the answer assumed, one sentence per item of the part's + * `assumptions`, then the year when the question named none */ assumptions: string[]; /** one line per varying fact the measure menu cannot offer */ variance: string[]; @@ -171,18 +170,19 @@ export type AboutAnswerNotes = { unavailable: string[]; }; -/** The notes "About this answer" shows for one definite part. It reads - * only the warnings `ABOUT_WARNING_CODES` names. */ +/** The notes "About this answer" shows for one definite part: its + * `assumptions` and the warnings `ABOUT_WARNING_CODES` names. */ export function aboutAnswerNotes( spec: MeasureMenuSpec, warnings: readonly Warning[] | null | undefined, optionsBySpec: Record>, + assumptions?: readonly Assumption[] | null, ): AboutAnswerNotes { const about = (warnings ?? []).filter((warning) => ABOUT_WARNING_CODES.has(warning.code)); return { coverage: coverageSentence(spec), - assumptions: scopeAssumptions(spec, about), + assumptions: scopeAssumptions(spec, assumptions, about), variance: measureVarianceNotes(spec, optionsBySpec), unavailable: unSystemFailedReads(about), }; diff --git a/src/lib/search/searchClient.ts b/src/lib/search/searchClient.ts index 0158b38..834e337 100644 --- a/src/lib/search/searchClient.ts +++ b/src/lib/search/searchClient.ts @@ -26,6 +26,7 @@ import { type VersionLinkOutcome, } from './versionLink.ts'; import type { + Assumption, DatasetResubmitEntry, LateDepartures, EntityGroup, @@ -1272,6 +1273,10 @@ export type OpenedSearchShare = { * replay. Null when it had none. */ entityGroups: Record | null; operations: Operation[]; + /** The part's assumptions as the shared response gave them, echoed + * on replay. Null for a share stored without them and for a + * version link. */ + assumptions: Assumption[] | null; contentVersion: string | null; }[]; /** Set when the view opened from a version link (lib/search/versionLink) diff --git a/src/lib/search/searchShare.ts b/src/lib/search/searchShare.ts index 2ddfe0d..c5475cf 100644 --- a/src/lib/search/searchShare.ts +++ b/src/lib/search/searchShare.ts @@ -3,7 +3,7 @@ import { asc, eq } from 'drizzle-orm'; import { searchShareMode } from '@/lib/chat/shareMode'; import { shortId } from '@/lib/chat/shareStore'; import { db } from '@/lib/db'; -import type { EntityGroup, Operation } from '@/lib/search/types'; +import type { Assumption, EntityGroup, Operation } from '@/lib/search/types'; import { searchApiBase } from '@/lib/searchApi'; import { share, shareRef } from '../../../db/schema'; import { registerDefinitions } from './registerDefinitions.ts'; @@ -34,6 +34,8 @@ export type SearchShareRef = { identity: unknown; entityGroups: Record | null; operations: Operation[]; + // Null for a share stored without assumptions. + assumptions: Assumption[] | null; contentVersion: string | null; // The file a snapshot captured and pinned. Null for a live share, and // for a snapshot stored before digests were kept. @@ -109,6 +111,7 @@ export const createSearchShare = async ( entityGroups: part.entityGroups ?? null, variableText: part.variableText, operations: part.operations, + assumptions: part.assumptions ?? null, contentVersion: captured?.[index]?.contentVersion ?? null, artifactDigest: captured?.[index]?.artifactDigest ?? null, })), @@ -143,6 +146,7 @@ export const getSearchShare = async (id: string): Promise => identity: ref.identity, entityGroups: ref.entityGroups as Record | null, operations: ref.operations as Operation[], + assumptions: ref.assumptions as Assumption[] | null, contentVersion: ref.contentVersion, artifactDigest: ref.artifactDigest, })), diff --git a/src/lib/search/searchShareRequest.ts b/src/lib/search/searchShareRequest.ts index 84883ff..22d0e45 100644 --- a/src/lib/search/searchShareRequest.ts +++ b/src/lib/search/searchShareRequest.ts @@ -1,5 +1,12 @@ /* eslint-disable @typescript-eslint/no-explicit-any -- request bodies are validated field by field below */ -import type { EntityGroup, EntityRoleKey, HydratedPart, Operation } from './types.ts'; +import type { + Assumption, + AssumptionKind, + EntityGroup, + EntityRoleKey, + HydratedPart, + Operation, +} from './types.ts'; /** * The shape and limits of a search share request, checked before any @@ -23,6 +30,11 @@ export type SearchSharePart = { // What the question asked of the part beyond its data. The replay // carries them back. operations: Operation[]; + // The values the answer filled that the question did not ask for, as + // the response gave them, [] included. The replay echoes them, so a + // shared answer states the same defaults. Null when the response gave + // none. + assumptions?: Assumption[] | null; // The canonical definition from the resolve response, which register // and mint take. Not stored, since the identity is enough to replay. definition: unknown; @@ -74,6 +86,46 @@ const isEntityGroups = (value: any): value is Record => ([role, group]) => ENTITY_ROLE_KEYS.includes(role) && isEntityGroup(group), ); +const ASSUMPTION_KINDS: string[] = [ + 'declared_default', + 'open_role', + 'place_scope', +] satisfies AssumptionKind[]; + +// Search's own bounds on an echoed `Assumption`: a text of 1 to 200 code +// points, at most one ref of at most 1024 characters, a binding ref of at +// most 1036 characters, and at most 67 items in a part's list. A share +// storing one past them cannot open. +const ASSUMPTION_TEXT_MAX_LENGTH = 200; +const ASSUMPTION_MAX_REFS = 1; +const ASSUMPTION_REF_MAX_LENGTH = 1024; +const ASSUMPTION_BINDING_REF_MAX_LENGTH = 1036; +const ASSUMPTIONS_MAX = 67; + +const isAssumption = (value: any): value is Assumption => + isPlainObject(value) && + Object.keys(value).every((member) => + ['kind', 'binding_ref', 'refs', 'text'].includes(member), + ) && + ASSUMPTION_KINDS.includes(value.kind) && + typeof value.binding_ref === 'string' && + value.binding_ref.length <= ASSUMPTION_BINDING_REF_MAX_LENGTH && + Array.isArray(value.refs) && + value.refs.length <= ASSUMPTION_MAX_REFS && + value.refs.every( + (ref: unknown) => typeof ref === 'string' && ref.length <= ASSUMPTION_REF_MAX_LENGTH, + ) && + typeof value.text === 'string' && + value.text.length > 0 && + // A text within the bound spans at most two UTF-16 units per code point. + value.text.length <= ASSUMPTION_TEXT_MAX_LENGTH * 2 && + [...value.text].length <= ASSUMPTION_TEXT_MAX_LENGTH; + +/** A part's echoed list as Search takes it: within the item cap, every item + * an `Assumption` within its bounds. */ +export const isAssumptions = (value: any): value is Assumption[] => + Array.isArray(value) && value.length <= ASSUMPTIONS_MAX && value.every(isAssumption); + export const isSearchSharePart = (value: any): value is SearchSharePart => !!value && typeof value === 'object' && @@ -87,11 +139,12 @@ export const isSearchSharePart = (value: any): value is SearchSharePart => !Object.hasOwn(value.identity, 'entity_groups') && (value.entityGroups == null || isEntityGroups(value.entityGroups)) && Array.isArray(value.operations) && + (value.assumptions == null || isAssumptions(value.assumptions)) && !!value.definition && typeof value.definition === 'object'; /** The share part a definite part on screen posts: its Spec's identity, with - * the Spec's groups beside it. */ + * the Spec's groups beside it, and the part's operations and assumptions. */ export const searchSharePart = (part: HydratedPart): Required => { const spec = part.hydratedSpec as any; @@ -102,6 +155,7 @@ export const searchSharePart = (part: HydratedPart): Required = identity: spec.identity, entityGroups: spec.entity_groups ?? null, operations: part.part?.operations ?? [], + assumptions: part.part?.assumptions ?? null, definition: spec.definition, }; }; diff --git a/src/lib/search/sentencePlan.ts b/src/lib/search/sentencePlan.ts index c96a486..af4ab5c 100644 --- a/src/lib/search/sentencePlan.ts +++ b/src/lib/search/sentencePlan.ts @@ -50,14 +50,27 @@ export function earlySentenceTokens( return tokens; } -/** The heading a plan reads as: its heading tokens joined verbatim, since - * the engine's tokens carry their own spacing, and trimmed. The byline - * (unit and source) and departure tokens are left out. Empty for a - * missing plan. */ -export function planSentenceText(plan: readonly SentenceToken[] | null | undefined): string { - return (plan ?? []) - .filter((token) => token.segment === 'heading') +const segmentText = ( + plan: readonly SentenceToken[] | null | undefined, + segment: SentenceToken['segment'], +): string => + (plan ?? []) + .filter((token) => token.segment === segment) .map((token) => token.text) .join('') .trim(); + +/** The heading a plan reads as: its heading tokens joined verbatim, since + * the engine's tokens carry their own spacing, and trimmed. The coverage + * line, byline (unit and source), departure and assumption tokens are left + * out. Empty for a missing plan. */ +export function planSentenceText(plan: readonly SentenceToken[] | null | undefined): string { + return segmentText(plan, 'heading'); +} + +/** The coverage line a mixed answer's plan opens with ("Answered 1 of 2 + * parts."), its coverage tokens joined and trimmed. Empty when the plan has + * none. */ +export function planCoverageText(plan: readonly SentenceToken[] | null | undefined): string { + return segmentText(plan, 'coverage'); } diff --git a/src/lib/search/test/datasetEntries.test.ts b/src/lib/search/test/datasetEntries.test.ts index 0d1bc02..205c783 100644 --- a/src/lib/search/test/datasetEntries.test.ts +++ b/src/lib/search/test/datasetEntries.test.ts @@ -1,11 +1,12 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { buildDatasetEntries, sharedRefEntry } from '../datasetEntries.ts'; -import { buildAmbiguousPickPart } from '../parsedEdit.ts'; +import { buildDatasetEntries, candidatePickEntry, sharedRefEntry } from '../datasetEntries.ts'; import { withEntityEdit } from '../promote.ts'; import { normalizeResponse, restorePlaceholderParts } from '../searchClient.ts'; import type { + Assumption, + DatasetCandidatesEntry, DatasetIdentityEntry, DatasetParsedEntry, HydratedPart, @@ -29,20 +30,18 @@ function hydratedParts(name: FixtureName): HydratedPart[] { })); } -/** The `parsed` entry an entity_ambiguous pick posts for `part`: its - * restatement with `dcid` pinned on the "Congo" wording. */ +/** A `parsed` entry for `part`: its restatement with `dcid` pinned on the + * "Congo" wording, as an entity_ambiguous option posts it. */ function congoPick(part: HydratedPart, dcid: string): DatasetParsedEntry { const restatement = part.part?.restatement; - const candidate = part.part?.no_data?.ambiguous_entities - ?.find((group) => group.text === 'Congo') - ?.candidates.find((option) => option.ref === dcid); - assert.ok(restatement && candidate); + assert.ok(restatement); - const picked = buildAmbiguousPickPart(restatement, 'Congo', candidate); + const entities = (restatement.entities ?? []).map((entity) => + entity.text === 'Congo' ? { ...entity, dcid } : entity, + ); - assert.ok(picked); - return { entry_kind: 'parsed', part: picked }; + return { entry_kind: 'parsed', part: { ...restatement, entities } }; } test('buildDatasetEntries posts the pick as the parsed entry at its part', () => { @@ -102,6 +101,162 @@ test("buildDatasetEntries carries a part's operations on an edit's override entr assert.deepEqual((entry as DatasetIdentityEntry).operations, parts[0].part?.operations); }); +test("buildDatasetEntries posts a parsed entry's part unchanged, its own operations kept", () => { + const parts = hydratedParts('definite-rank-operation'); + const picked: DatasetParsedEntry = { + entry_kind: 'parsed', + part: { variable_text: 'aid', operations: [] }, + }; + const [entry] = buildDatasetEntries(parts, { index: 0, entry: picked }); + + assert.ok(parts[0].part?.operations?.length); + assert.equal(entry, picked); +}); + +test("buildDatasetEntries echoes a part's assumptions on its identity entry, an edit's included", () => { + const parts = hydratedParts('broad-truncated'); + const assumptions = parts[0].part?.assumptions; + const [own] = buildDatasetEntries(parts) as DatasetIdentityEntry[]; + const override: DatasetIdentityEntry = { ...own, assumptions: undefined }; + const [edited] = buildDatasetEntries(parts, { index: 0, entry: override }); + + assert.ok(assumptions?.length); + assert.deepEqual(own.assumptions, assumptions); + assert.deepEqual((edited as DatasetIdentityEntry).assumptions, assumptions); +}); + +test("an edit that carries its own echo keeps it over the part's list", () => { + const parts = hydratedParts('broad-truncated'); + const [own] = buildDatasetEntries(parts) as DatasetIdentityEntry[]; + const [replayed] = buildDatasetEntries(parts, { + index: 0, + entry: { ...own, assumptions: [] }, + }) as DatasetIdentityEntry[]; + + assert.ok(parts[0].part?.assumptions?.length); + assert.deepEqual(replayed.assumptions, []); +}); + +test('buildDatasetEntries echoes an empty assumptions list, which states none', () => { + const [part] = hydratedParts('definite-single'); + const stated: HydratedPart = { ...part, part: { ...part.part!, assumptions: [] } }; + const [entry] = buildDatasetEntries([stated]) as DatasetIdentityEntry[]; + + assert.deepEqual(entry.assumptions, []); +}); + +test('an identity edit of a no_data part posts no echo, so the edit reads fresh', () => { + const [own] = buildDatasetEntries(hydratedParts('definite-single')) as DatasetIdentityEntry[]; + const [refused] = hydratedParts('entity-ambiguous'); + const stated: HydratedPart = { ...refused, part: { ...refused.part!, assumptions: [] } }; + const override: DatasetIdentityEntry = { ...own, assumptions: undefined }; + const [entry] = buildDatasetEntries([stated], { index: 0, entry: override }); + + assert.equal(stated.part?.status, 'no_data'); + assert.equal((entry as DatasetIdentityEntry).assumptions, undefined); +}); + +test('buildDatasetEntries leaves assumptions off a response without them and off every other entry kind', () => { + const [definite] = buildDatasetEntries(hydratedParts('definite-single')); + const [candidates] = hydratedParts('multi-candidate'); + const assumed: HydratedPart = { + ...candidates, + part: { + ...candidates.part!, + assumptions: [ + { kind: 'place_scope', binding_ref: 'where:', refs: [], text: 'every place' }, + ], + }, + }; + + assert.equal('assumptions' in definite, false); + assert.equal('assumptions' in buildDatasetEntries([assumed])[0], false); +}); + +const everyPlace: Assumption = { + kind: 'place_scope', + binding_ref: 'where:', + refs: [], + text: 'every place', +}; +const everyProvider: Assumption = { + kind: 'open_role', + binding_ref: 'origin:', + refs: [], + text: 'every provider', +}; + +/** The multi-candidate part with `reading_assumptions` as given. The first + * reading reads an open role the second lacks, so the part's own list holds + * only the place scope both share. */ +function candidatesWithReadings( + readings: (specIds: string[]) => Record | undefined, +): HydratedPart { + const [part] = hydratedParts('multi-candidate'); + const specIds = part.part!.candidates!.specs.map((spec) => spec.spec_id); + + return { + ...part, + part: { + ...part.part!, + assumptions: [everyPlace], + reading_assumptions: readings(specIds), + }, + }; +} + +const twoReadings = ([first, second]: string[]) => ({ + [first]: [everyProvider, everyPlace], + [second]: [everyPlace], +}); + +test("a pick echoes the picked reading's own assumptions, not the readings' shared list", () => { + const part = candidatesWithReadings(twoReadings); + const [first, second] = part.part!.candidates!.specs; + + for (const [spec, expected] of [ + [first, [everyProvider, everyPlace]], + [second, [everyPlace]], + ] as const) { + const entry = candidatePickEntry(part, spec); + const [posted] = buildDatasetEntries([part], { index: 0, entry }); + + assert.equal(posted.entry_kind, 'identity'); + assert.deepEqual((posted as DatasetIdentityEntry).identity, spec.identity); + assert.deepEqual((posted as DatasetIdentityEntry).assumptions, expected); + } +}); + +test("a candidates entry echoes each reading's own assumptions on its candidate", () => { + const [entry] = buildDatasetEntries([candidatesWithReadings(twoReadings)]); + + assert.equal(entry.entry_kind, 'candidates'); + assert.deepEqual( + (entry as DatasetCandidatesEntry).candidates.map((candidate) => candidate.assumptions), + [[everyProvider, everyPlace], [everyPlace]], + ); +}); + +test('a reading the response gives no list for posts no echo, on a pick or a candidate', () => { + for (const part of [ + candidatesWithReadings(() => undefined), + candidatesWithReadings(([first]) => ({ [first]: [] })), + ]) { + const specs = part.part!.candidates!.specs; + const pick = candidatePickEntry(part, specs[1]); + const [posted] = buildDatasetEntries([part], { index: 0, entry: pick }); + const [entry] = buildDatasetEntries([part]) as DatasetCandidatesEntry[]; + + assert.equal('assumptions' in posted, false); + assert.equal('assumptions' in entry.candidates[1], false); + } + + const stated = candidatesWithReadings(([first]) => ({ [first]: [] })); + const [entry] = buildDatasetEntries([stated]) as DatasetCandidatesEntry[]; + + assert.deepEqual(entry.candidates[0].assumptions, []); +}); + test('buildDatasetEntries leaves operations off a part that has none', () => { const [entry] = buildDatasetEntries(hydratedParts('definite-single')); @@ -158,7 +313,13 @@ test('a stored share part replays its groups beside the identity, and no member const groups = { origin: { ref: 'africa', label: 'Africa', entity_type: 'Continent' }, }; - const ref = { variableText: 'aid', identity, entityGroups: groups, operations: [] }; + const ref = { + variableText: 'aid', + identity, + entityGroups: groups, + operations: [], + assumptions: null, + }; assert.deepEqual(sharedRefEntry(ref), { entry_kind: 'identity', @@ -182,3 +343,15 @@ test('a stored share part replays its groups beside the identity, and no member operations: [operation], }); }); + +test('a stored share part echoes its assumptions, [] included, and a share stored before them posts none', () => { + const [part] = hydratedParts('broad-truncated'); + const identity = part.hydratedSpec!.identity; + const assumptions = part.part!.assumptions!; + const ref = { variableText: 'aid', identity, entityGroups: null, operations: [] }; + + assert.ok(assumptions.length); + assert.deepEqual(sharedRefEntry({ ...ref, assumptions }).assumptions, assumptions); + assert.deepEqual(sharedRefEntry({ ...ref, assumptions: [] }).assumptions, []); + assert.equal('assumptions' in sharedRefEntry({ ...ref, assumptions: null }), false); +}); diff --git a/src/lib/search/test/departures.test.ts b/src/lib/search/test/departures.test.ts index 076fd4c..3c94392 100644 --- a/src/lib/search/test/departures.test.ts +++ b/src/lib/search/test/departures.test.ts @@ -3,6 +3,7 @@ import test from 'node:test'; import { answerDepartures, + assumptionClauses, departureAction, isClosestMatch, refsStatedBesides, @@ -10,6 +11,8 @@ import { statingTokenIndex, } from '../departures.ts'; import { normalizeResponse } from '../searchClient.ts'; +import { planCoverageText, planSentenceText } from '../sentencePlan.ts'; +import { fixtures } from './fixtures/index.ts'; import { noDataResponse, wireSpec } from './fixtures/no-data.ts'; import { servedAnswer } from './fixtures/served-answer.ts'; import { servedAid } from './fixtures/served-options.ts'; @@ -63,6 +66,193 @@ test('sentenceBlocks joins heading runs a departure separated -- a no_data part assert.deepEqual(sentenceBlocks(null), []); }); +// --------------------------------------------------------------------------- +// The assumptions clause and the mixed-answer opener +// --------------------------------------------------------------------------- + +const token = ( + text: string, + segment: SentenceToken['segment'], + part_index?: number | null, +): SentenceToken => ({ kind: 'text', text, segment, part_index }); + +test('sentenceBlocks closes the plan with the assumptions clause, apart from the heading and byline', () => { + const plan = (fixtures['broad-truncated'] as { sentence_plan: SentenceToken[] }).sentence_plan; + + assert.deepEqual(blockTexts(plan), [ + ['heading', 'Gross ODA disbursements to Kenya from 86 donors, 2015 to 2024.'], + ['byline', 'Current US dollars · OECD Creditor Reporting System (CRS).'], + ['assumption', 'Assumed: every donor.'], + ]); + assert.deepEqual(assumptionClauses(plan, ['ODA']), [ + { key: 0, prefix: null, text: 'Assumed: every donor.' }, + ]); + // The

's accessible name is the heading alone. + assert.equal( + planSentenceText(plan), + 'Gross ODA disbursements to Kenya from 86 donors, 2015 to 2024.', + ); +}); + +test('the clause follows the departures, which the departure list draws', () => { + const plan = [ + token('Aid to Kenya.', 'heading'), + token(' ', 'byline'), + token('OECD.', 'byline'), + token(' The data holds no values for 2024.', 'departure'), + token(' ', 'assumption'), + token('Assumed: ', 'assumption'), + token('official development assistance', 'assumption'), + token('; ', 'assumption'), + token('every provider', 'assumption'), + token('.', 'assumption'), + ]; + + assert.deepEqual( + blockTexts(plan).map(([segment]) => segment), + ['heading', 'byline', 'assumption'], + ); + assert.deepEqual(assumptionClauses(plan, ['aid']), [ + { + key: 0, + prefix: null, + text: 'Assumed: official development assistance; every provider.', + }, + ]); +}); + +test("assumptionClauses names each part's own clause when the plan is written part by part", () => { + const sentence = (partIndex: number, item: string) => [ + token('Aid.', 'heading', partIndex), + token(' ', 'byline', partIndex), + token('OECD.', 'byline', partIndex), + token(' ', 'assumption', partIndex), + token('Assumed: ', 'assumption', partIndex), + token(item, 'assumption', partIndex), + token('.', 'assumption', partIndex), + ]; + const plan = [ + ...sentence(0, 'every provider'), + token(' ', 'heading', null), + ...sentence(1, 'every country'), + ]; + + assert.deepEqual(assumptionClauses(plan, ['aid', 'debt']), [ + { key: 0, prefix: 'aid', text: 'Assumed: every provider.' }, + { key: 1, prefix: 'debt', text: 'Assumed: every country.' }, + ]); + // A part with no clause beside one that states its own. The clause still names its part. + const onlySecond = [ + token('Aid.', 'heading', 0), + token(' ', 'byline', 0), + token('OECD.', 'byline', 0), + token(' ', 'heading', null), + ...sentence(1, 'every country'), + ]; + + assert.deepEqual(assumptionClauses(onlySecond, ['aid', 'debt']), [ + { key: 0, prefix: 'debt', text: 'Assumed: every country.' }, + ]); + // One joined clause over two parts, an item two parts share unowned, takes no prefix. + const joined = [ + token('Aid and debt.', 'heading', null), + token(' ', 'assumption', null), + token('Assumed: ', 'assumption', null), + token('every provider', 'assumption', 0), + token('; ', 'assumption', null), + token('every country', 'assumption', null), + token('.', 'assumption', null), + ]; + + assert.deepEqual(assumptionClauses(joined, ['aid', 'debt']), [ + { key: 0, prefix: null, text: 'Assumed: every provider; every country.' }, + ]); + assert.deepEqual(assumptionClauses(null, []), []); +}); + +test('assumptionClauses labels a clause by the owner of its lead token', () => { + // A definite part beside a candidates part gives one byline and two owned clauses. + const definiteAndCandidates = [ + token('Answered 1 of 2 parts.', 'coverage', null), + token(' ', 'coverage', null), + token('Aid to Kenya', 'heading', 0), + token('.', 'heading', 0), + token(' ', 'byline', 0), + token('OECD.', 'byline', 0), + token(' ', 'assumption', 0), + token('Assumed: ', 'assumption', 0), + token('every provider', 'assumption', 0), + token('.', 'assumption', 0), + token(' ', 'heading', null), + token('debt', 'heading', 1), + token(' ', 'assumption', 1), + token('Assumed: ', 'assumption', 1), + token('every country', 'assumption', 1), + token('.', 'assumption', 1), + ]; + + assert.deepEqual(assumptionClauses(definiteAndCandidates, ['aid', 'debt']), [ + { key: 0, prefix: 'aid', text: 'Assumed: every provider.' }, + { key: 1, prefix: 'debt', text: 'Assumed: every country.' }, + ]); + // Parts 0 and 1 fold into one sentence. The folded clause's lead has no owner, though + // one of its items is part 1's, so only part 2's clause takes a label. + const folded = [ + token('Aid to Kenya.', 'heading', null), + token(' ', 'byline', null), + token('OECD.', 'byline', null), + token(' ', 'assumption', null), + token('Assumed: ', 'assumption', null), + token('official development assistance', 'assumption', null), + token('; ', 'assumption', null), + token('every provider', 'assumption', 1), + token('.', 'assumption', null), + token(' ', 'heading', null), + token('Debt.', 'heading', 2), + token(' ', 'byline', 2), + token('World Bank.', 'byline', 2), + token(' ', 'assumption', 2), + token('Assumed: ', 'assumption', 2), + token('every country', 'assumption', 2), + token('.', 'assumption', 2), + ]; + + assert.deepEqual(assumptionClauses(folded, ['aid', 'aid to Uganda', 'debt']), [ + { + key: 0, + prefix: null, + text: 'Assumed: official development assistance; every provider.', + }, + { key: 1, prefix: 'debt', text: 'Assumed: every country.' }, + ]); +}); + +test("a mixed answer's coverage line is its own block, left out of the heading text", () => { + const plan = [ + token('Answered 1 of 2 parts.', 'coverage', null), + token(' ', 'coverage', null), + { kind: 'slot', text: 'Aid to Kenya', axis: 'what', segment: 'heading', part_index: 0 }, + token('.', 'heading', 0), + token(' ', 'byline', 0), + token('OECD.', 'byline', 0), + ] satisfies SentenceToken[]; + + assert.deepEqual( + sentenceBlocks(plan).map(({ segment, tokens }) => [ + segment, + tokens.map(({ text }) => text).join(''), + ]), + [ + ['coverage', 'Answered 1 of 2 parts.'], + ['heading', 'Aid to Kenya.'], + ['byline', 'OECD.'], + ], + ); + assert.equal(planSentenceText(plan), 'Aid to Kenya.'); + assert.equal(planCoverageText(plan), 'Answered 1 of 2 parts.'); + assert.equal(planCoverageText(plan.slice(2)), ''); +}); + // --------------------------------------------------------------------------- // departureAction and answerDepartures // --------------------------------------------------------------------------- diff --git a/src/lib/search/test/fixtures.test.ts b/src/lib/search/test/fixtures.test.ts index 9889a44..8e2a1f2 100644 --- a/src/lib/search/test/fixtures.test.ts +++ b/src/lib/search/test/fixtures.test.ts @@ -191,15 +191,20 @@ test('definite-rank-operation: the definite part keeps its rank operation and fl assert.ok(part.spec.identity, 'flattenSpec keeps the definition identity'); }); -test('entity-ambiguous: operations default to empty and data_query rides through', () => { +test('entity-ambiguous: operations default to empty and each option rides through', () => { const normalized = normalizeResponse(fixtures['entity-ambiguous']); const [part] = normalized.parts; assert.deepEqual(part.operations, []); assert.equal(part.no_data.reason, 'entity_ambiguous'); assert.deepEqual( - part.no_data.data_query.parts[0].entities.map((entity: any) => entity.dcid ?? null), - [null, 'country/FRA'], + part.no_data.options.map((option: any) => + option.data_query.parts[0].entities.map((entity: any) => entity.dcid ?? null), + ), + [ + ['country/COD', 'country/FRA'], + ['country/COG', 'country/FRA'], + ], ); assert.deepEqual( part.no_data.ambiguous_entities[0].candidates.map((candidate: any) => candidate.ref), diff --git a/src/lib/search/test/fixtures/ambiguous-congo.ts b/src/lib/search/test/fixtures/ambiguous-congo.ts index e26270f..a7b29b1 100644 --- a/src/lib/search/test/fixtures/ambiguous-congo.ts +++ b/src/lib/search/test/fixtures/ambiguous-congo.ts @@ -7,14 +7,32 @@ * [{dcid, label, kind}]}`. The two real Congo dcids (country/COD, * country/COG) and the candidate labels are the ones the captures' own * `rendered_sentence` already names ("Congo [DRC]"/"Congo [Republic]"). + * The "ODA to the Congo" refusal also carries `no_data.options`, one ready + * request per candidate, built the way dfo-apis search/engine/conjoin.py + * `_entity_options` builds them from the restatement. "GDP of Congo" has no + * restatement, so it carries none. */ -import type { NoData, ParsedPart } from '../../types.ts'; +import type { NoData, NoDataOption, ParsedPart } from '../../types.ts'; + +const congoOption = (dcid: string, label: string): NoDataOption => ({ + mentions: ['Congo'], + label, + data_query: { + kind: 'parsed', + parts: [ + { + variable_text: 'official development assistance', + entities: [{ text: 'Congo', role_text: 'to', dcid }], + }, + ], + }, +}); /** "GDP of Congo": a subject-role entity_ambiguous part. `restatement` is * null on this real capture (the leg's direction came from wording alone, - * with no door-supplied role to confirm it), so this fixture exercises the - * "no restatement" fallback. */ + * with no door-supplied role to confirm it), so the refusal carries no + * options and offers no picker. */ export const gdpOfCongoNoData: NoData = { reason: 'entity_ambiguous', entity_terms: ['Congo'], @@ -47,6 +65,10 @@ export const odaToCongoNoData: NoData = { ], }, ], + options: [ + congoOption('country/COD', 'Congo [DRC]'), + congoOption('country/COG', 'Congo [Republic]'), + ], unbound_qualifiers: [], }; diff --git a/src/lib/search/test/fixtures/broad-truncated.ts b/src/lib/search/test/fixtures/broad-truncated.ts index eefc3a1..af23d4c 100644 --- a/src/lib/search/test/fixtures/broad-truncated.ts +++ b/src/lib/search/test/fixtures/broad-truncated.ts @@ -6,6 +6,9 @@ * A single-part `definite` answer over a broad StatVar set: it serves all * 1106 members (`served_count: 1106`). The part carries the * `role_scope_defaulted` and `observations_series_truncated` warnings. + * Its open provider role, the one assumption, was added to the part's + * `assumptions` and as the envelope sentence's closing clause (" Assumed: + * every donor."), in the shape dfo-apis' render/assumptions.py writes. * * Trimmed for size; each body still validates against the API's models * (scripts/check-search-fixtures.mts). `slot_values` and @@ -89,6 +92,14 @@ export const broadTruncated: unknown = { }, "operations": [], "departures": [], + "assumptions": [ + { + "kind": "open_role", + "binding_ref": "origin:", + "refs": [], + "text": "every donor" + } + ], "status": "definite", "spec": { "spec_id": "spec_7c8a3bcf9c627393f5b9c2177d96174f", @@ -4020,7 +4031,7 @@ export const broadTruncated: unknown = { } } ], - "rendered_sentence": "Gross ODA disbursements to Kenya from 86 donors, 2015 to 2024. Current US dollars · OECD Creditor Reporting System (CRS).", + "rendered_sentence": "Gross ODA disbursements to Kenya from 86 donors, 2015 to 2024. Current US dollars · OECD Creditor Reporting System (CRS). Assumed: every donor.", "sentence_plan": [ { "kind": "slot", @@ -4209,6 +4220,34 @@ export const broadTruncated: unknown = { "segment": "byline", "entity_refs": [], "states": [] + }, + { + "kind": "text", + "text": " ", + "segment": "assumption", + "entity_refs": [], + "states": [] + }, + { + "kind": "text", + "text": "Assumed: ", + "segment": "assumption", + "entity_refs": [], + "states": [] + }, + { + "kind": "text", + "text": "every donor", + "segment": "assumption", + "entity_refs": [], + "states": [] + }, + { + "kind": "text", + "text": ".", + "segment": "assumption", + "entity_refs": [], + "states": [] } ], "status": "definite" diff --git a/src/lib/search/test/fixtures/entity-ambiguous.ts b/src/lib/search/test/fixtures/entity-ambiguous.ts index 88ad2ba..45511d4 100644 --- a/src/lib/search/test/fixtures/entity-ambiguous.ts +++ b/src/lib/search/test/fixtures/entity-ambiguous.ts @@ -4,8 +4,8 @@ * one `no_data` part with reason `entity_ambiguous`. * * "Congo" names both the Democratic Republic of the Congo and the Republic - * of the Congo. `no_data.data_query` is the part's parsed form with France - * pinned and Congo's dcid left empty, ready for a pick to fill. The part + * of the Congo. `no_data.options` holds one ready request per candidate: the + * part's parsed form with France pinned and Congo pinned to that candidate. The part * omits `operations`, which the wire allows, so normalization must default * it to empty. */ @@ -69,27 +69,62 @@ export const entityAmbiguous: unknown = { ] } ], - "data_query": { - "kind": "parsed", - "parts": [ - { - "variable_text": "ODA", - "entities": [ + "options": [ + { + "mentions": [ + "Congo" + ], + "label": "Congo [DRC]", + "data_query": { + "kind": "parsed", + "parts": [ { - "text": "Congo", - "role_text": "to" - }, + "variable_text": "ODA", + "entities": [ + { + "text": "Congo", + "role_text": "to", + "dcid": "country/COD" + }, + { + "text": "France", + "role_text": "from", + "dcid": "country/FRA" + } + ], + "time_text": "2022" + } + ] + } + }, + { + "mentions": [ + "Congo" + ], + "label": "Congo [Republic]", + "data_query": { + "kind": "parsed", + "parts": [ { - "text": "France", - "role_text": "from", - "dcid": "country/FRA" + "variable_text": "ODA", + "entities": [ + { + "text": "Congo", + "role_text": "to", + "dcid": "country/COG" + }, + { + "text": "France", + "role_text": "from", + "dcid": "country/FRA" + } + ], + "time_text": "2022" } - ], - "time_text": "2022" + ] } - ], - "raw_query": "ODA to Congo from France in 2022" - }, + } + ], "unbound_qualifiers": [], "nearest_real": null, "flagged_match": null diff --git a/src/lib/search/test/fixtures/world-bank-ambiguous.ts b/src/lib/search/test/fixtures/world-bank-ambiguous.ts index 830dffe..71cfaf4 100644 --- a/src/lib/search/test/fixtures/world-bank-ambiguous.ts +++ b/src/lib/search/test/fixtures/world-bank-ambiguous.ts @@ -87,22 +87,52 @@ export const worldBankAmbiguous: unknown = { } ], "unbound_qualifiers": [], - "data_query": { - "kind": "parsed", - "parts": [ - { - "variable_text": "ODA", - "entities": [ + "options": [ + { + "mentions": [ + "World Bank" + ], + "label": "International Bank for Reconstruction and Development", + "data_query": { + "kind": "parsed", + "parts": [ { - "text": "World Bank", - "role_text": "from" + "variable_text": "ODA", + "entities": [ + { + "text": "World Bank", + "role_text": "from", + "dcid": "InternationalBankForReconstructionAndDevelopment" + } + ], + "operations": [] } - ], - "operations": [] + ] } - ], - "raw_query": "ODA from the World Bank" - } + }, + { + "mentions": [ + "World Bank" + ], + "label": "International Development Association", + "data_query": { + "kind": "parsed", + "parts": [ + { + "variable_text": "ODA", + "entities": [ + { + "text": "World Bank", + "role_text": "from", + "dcid": "InternationalDevelopmentAssociation" + } + ], + "operations": [] + } + ] + } + } + ] } } ], diff --git a/src/lib/search/test/parsedEdit.test.ts b/src/lib/search/test/parsedEdit.test.ts index ad81df7..726afc9 100644 --- a/src/lib/search/test/parsedEdit.test.ts +++ b/src/lib/search/test/parsedEdit.test.ts @@ -3,8 +3,8 @@ import test from 'node:test'; import { applyRestatementEdit, - buildAmbiguousPickPart, - canPinEntityDcid, + entityChoiceGroups, + entityChoicePart, offersAmbiguousPicker, resolveTypedEdit, restatementTargetForToken, @@ -15,7 +15,9 @@ import { odaToCongoNoData, odaToCongoRestatement, } from './fixtures/ambiguous-congo.ts'; -import type { ParsedPart, RoleAxis, SlotAxis } from '../types.ts'; +import { normalizeResponse } from '../searchClient.ts'; +import { fixtures } from './fixtures/index.ts'; +import type { NoData, ParsedPart, RoleAxis, SlotAxis } from '../types.ts'; // --------------------------------------------------------------------------- // restatementTargetForToken / applyRestatementEdit: one restatement field per @@ -196,18 +198,80 @@ test('resolveTypedEdit is null for a how token even with a restatement present', }); // --------------------------------------------------------------------------- -// canPinEntityDcid: the gating rule +// entityChoiceGroups / entityChoicePart: the refusal's own options, by wording // --------------------------------------------------------------------------- -test('canPinEntityDcid is true when the response carried ambiguous_entities', () => { - assert.equal(canPinEntityDcid(odaToCongoNoData), true); +const congoPart = () => { + const [part] = normalizeResponse(fixtures['entity-ambiguous']).parts; + + return part as { no_data: NoData; restatement: ParsedPart }; +}; + +test("entityChoiceGroups groups the refusal's options by the wording each settles", () => { + const { no_data: noData } = congoPart(); + const groups = entityChoiceGroups(noData); + + assert.deepEqual( + groups.map(({ text, options }) => [text, options.map((option) => option.label)]), + [['Congo', ['Congo [DRC]', 'Congo [Republic]']]], + ); + // Each choice posts its option's one part unchanged, the resolved entity pinned too. + const [, republic] = groups[0].options; + + assert.equal(entityChoicePart(republic), noData.options![1].data_query.parts[0]); + assert.deepEqual( + entityChoicePart(republic)?.entities?.map((entity) => entity.dcid), + ['country/COG', 'country/FRA'], + ); +}); + +test("a members option posts the group's registered name, no dcid pinned, unchanged", () => { + const { no_data: noData } = congoPart(); + const members = { + mentions: ['Congo'], + label: 'Congo basin', + data_query: { + kind: 'parsed' as const, + parts: [ + { + variable_text: 'ODA', + entities: [{ text: 'Congo basin', role_text: 'to', members: true }], + }, + ], + }, + }; + const [group] = entityChoiceGroups({ ...noData, options: [members] }); + + assert.equal(entityChoicePart(group.options[0]), members.data_query.parts[0]); }); -test('canPinEntityDcid is false with no ambiguous_entities, absent or empty', () => { - assert.equal(canPinEntityDcid(undefined), false); - assert.equal(canPinEntityDcid(null), false); - assert.equal(canPinEntityDcid({ reason: 'no_observations' }), false); - assert.equal(canPinEntityDcid({ reason: 'entity_ambiguous', ambiguous_entities: [] }), false); +test('entityChoiceGroups lists a wording per mention, in the order the options come', () => { + const option = (mentions: string[], label: string) => ({ + mentions, + label, + data_query: { kind: 'parsed' as const, parts: [{ variable_text: 'aid' }] }, + }); + const noData: NoData = { + reason: 'entity_ambiguous', + options: [ + option(['World Bank'], 'IBRD'), + option(['Georgia'], 'Georgia (country)'), + option(['World Bank'], 'IDA'), + ], + }; + + assert.deepEqual( + entityChoiceGroups(noData).map(({ text, options }) => [ + text, + options.map((choice) => choice.label), + ]), + [ + ['World Bank', ['IBRD', 'IDA']], + ['Georgia', ['Georgia (country)']], + ], + ); + assert.deepEqual(entityChoiceGroups({ ...noData, reason: 'source_ambiguous' }), []); + assert.deepEqual(entityChoiceGroups(null), []); }); // --------------------------------------------------------------------------- @@ -216,7 +280,7 @@ test('canPinEntityDcid is false with no ambiguous_entities, absent or empty', () // generic Studio hand-off panel // --------------------------------------------------------------------------- -test('offersAmbiguousPicker is true for the real ODA-to-Congo capture (restatement present)', () => { +test('offersAmbiguousPicker is true for an entity_ambiguous refusal with options', () => { assert.equal( offersAmbiguousPicker({ no_data: odaToCongoNoData, @@ -226,7 +290,7 @@ test('offersAmbiguousPicker is true for the real ODA-to-Congo capture (restateme ); }); -test('offersAmbiguousPicker is false for the real GDP-of-Congo capture (restatement null)', () => { +test('offersAmbiguousPicker is false for the real GDP-of-Congo capture (no options)', () => { assert.equal( offersAmbiguousPicker({ no_data: gdpOfCongoNoData, @@ -236,14 +300,10 @@ test('offersAmbiguousPicker is false for the real GDP-of-Congo capture (restatem ); }); -test('offersAmbiguousPicker is false with no ambiguous_entities, absent or empty', () => { - assert.equal( - offersAmbiguousPicker({ - no_data: { reason: 'entity_ambiguous', ambiguous_entities: [] }, - restatement: odaToCongoRestatement, - }), - false, - ); +test('offersAmbiguousPicker is false with options absent or empty', () => { + for (const options of [undefined, []]) { + assert.equal(offersAmbiguousPicker({ no_data: { ...odaToCongoNoData, options } }), false); + } }); test('offersAmbiguousPicker is false for a non-object part, undefined, or null', () => { @@ -251,41 +311,3 @@ test('offersAmbiguousPicker is false for a non-object part, undefined, or null', assert.equal(offersAmbiguousPicker(null), false); assert.equal(offersAmbiguousPicker('no_data'), false); }); - -// --------------------------------------------------------------------------- -// buildAmbiguousPickPart: a pick's own parsed part, built from the fixtures -// captured for "GDP of Congo" and "official development assistance to the -// Congo" (ambiguous_entities added by hand -- see the fixture's own doc -// comment) -// --------------------------------------------------------------------------- - -test('buildAmbiguousPickPart pins either candidate onto the matching restatement entity (ODA to the Congo)', () => { - const group = odaToCongoNoData.ambiguous_entities![0]; - const cases = [ - { - candidateIndex: 0, // Congo [DRC] - entities: [{ text: 'Congo [DRC]', role_text: 'to', dcid: 'country/COD' }], - }, - { - candidateIndex: 1, // Congo [Republic] - entities: [{ text: 'Congo [Republic]', role_text: 'to', dcid: 'country/COG' }], - }, - ]; - - for (const { candidateIndex, entities } of cases) { - const candidate = group.candidates[candidateIndex]; - const part = buildAmbiguousPickPart(odaToCongoRestatement, group.text, candidate); - - assert.deepEqual(part, { variable_text: 'official development assistance', entities }); - } -}); - -test('buildAmbiguousPickPart returns null (nothing to post) when no entity text matches the group', () => { - const group = odaToCongoNoData.ambiguous_entities![0]; - const candidate = group.candidates[0]; - const mismatched: ParsedPart = { variable_text: 'aid', entities: [{ text: 'Kenya' }] }; - - const part = buildAmbiguousPickPart(mismatched, group.text, candidate); - - assert.equal(part, null); -}); diff --git a/src/lib/search/test/permalink.test.ts b/src/lib/search/test/permalink.test.ts index b1232a5..4a4c353 100644 --- a/src/lib/search/test/permalink.test.ts +++ b/src/lib/search/test/permalink.test.ts @@ -3,6 +3,21 @@ import test from 'node:test'; import { decodePermalinkState, encodePermalinkState, encodeSearchState } from '../permalink.ts'; import type { PermalinkEdit, PermalinkState } from '../permalink.ts'; +import type { Assumption } from '../types.ts'; + +const declaredDefault: Assumption = { + kind: 'declared_default', + binding_ref: 'how:typeofflow', + refs: ['TypeOfFlow_ODA'], + text: 'official development assistance', +}; + +const openProviders: Assumption = { + kind: 'open_role', + binding_ref: 'origin:', + refs: [], + text: 'every provider', +}; const identityEdit: PermalinkEdit = { part: 0, @@ -58,6 +73,22 @@ test('a PermalinkState round-trips through encode/decode -- a multi-dcid identit edit: multiEntityEdit, }, { description: 'a parsed-only edit', query: 'donor spending', edit: parsedEdit }, + { + description: "a reloaded pick's assumptions echo", + query: 'ODA disbursements', + edit: { + ...identityEdit, + entry: { ...identityEdit.entry, assumptions: [declaredDefault, openProviders] }, + } as PermalinkEdit, + }, + { + description: 'an empty assumptions echo, which states no default', + query: 'ODA disbursements', + edit: { + ...identityEdit, + entry: { ...identityEdit.entry, assumptions: [] }, + } as PermalinkEdit, + }, { description: 'an old identity-only ?s= payload', query: 'ODA disbursements', @@ -115,3 +146,25 @@ test('an identity edit stored with its base and StatVars decodes to the identity assert.deepEqual(decoded, { query: stored.query, edits: [identityEdit] }); }); + +test('an assumptions echo Search would refuse is dropped and the edit still replays', async () => { + const tooMany = Array.from({ length: 68 }, () => declaredDefault); + const refused = [ + { description: 'an unknown kind', echo: [{ ...declaredDefault, kind: 'guess' }] }, + { description: 'a missing text', echo: [{ ...declaredDefault, text: '' }] }, + { description: 'an extra member', echo: [{ ...declaredDefault, extra: 1 }] }, + { description: 'a list past the cap', echo: tooMany }, + { description: 'a list that is not an array', echo: declaredDefault }, + { description: 'a null echo', echo: null }, + ]; + + for (const { description, echo } of refused) { + const stored = { + query: 'ODA disbursements', + edits: [{ ...identityEdit, entry: { ...identityEdit.entry, assumptions: echo } }], + }; + const decoded = await decodePermalinkState(await encodeSearchState(stored)); + + assert.deepEqual(decoded, { query: stored.query, edits: [identityEdit] }, description); + } +}); diff --git a/src/lib/search/test/readings.test.ts b/src/lib/search/test/readings.test.ts index 663aaf3..0917a7e 100644 --- a/src/lib/search/test/readings.test.ts +++ b/src/lib/search/test/readings.test.ts @@ -128,6 +128,37 @@ test('foldReadings falls back to the summary heading and leaves other parts and ); }); +test("foldReadings keeps a candidates part's assumptions clause, and readingsSummary leaves it to the answer block", () => { + const clause = (text: string): SentenceToken => ({ + kind: 'text', + text, + segment: 'assumption', + part_index: 0, + }); + const plan = [ + ...summaryPlan(0), + clause(' '), + clause('Assumed: '), + clause('every provider'), + clause('.'), + ]; + const folded = foldReadings(plan, [ + { status: 'candidates', variable_text: 'poverty reduction grants' }, + ]); + + assert.deepEqual( + folded.map((token) => [token.segment, token.text]), + [ + ['heading', 'poverty reduction grants'], + ['assumption', ' '], + ['assumption', 'Assumed: '], + ['assumption', 'every provider'], + ['assumption', '.'], + ], + ); + assert.equal(readingsSummary(plan), readingsSummary(summaryPlan(0))); +}); + test('foldReadings returns the plan itself when no part is a candidates part', () => { const plan = [heading('Aid to Kenya.', 0)]; diff --git a/src/lib/search/test/resultNotes.test.ts b/src/lib/search/test/resultNotes.test.ts index 2d438a1..5e52216 100644 --- a/src/lib/search/test/resultNotes.test.ts +++ b/src/lib/search/test/resultNotes.test.ts @@ -21,7 +21,7 @@ import { odaToCongoNoData, odaToCongoRestatement, } from './fixtures/ambiguous-congo.ts'; -import type { Warning } from '../types.ts'; +import type { Assumption, Warning } from '../types.ts'; // --------------------------------------------------------------------------- // The warning lines the page draws, by place @@ -74,6 +74,18 @@ const humanitarianWarnings: Warning[] = [ }, ]; +// The part's assumptions on the same answer. They hold the recipient role it +// left open and the flow type its code system's default filled. +const humanitarianAssumptions: Assumption[] = [ + { + kind: 'declared_default', + binding_ref: 'how:crsTypeOfFlow', + refs: ['CRSTypeOfFlow10'], + text: 'official development assistance', + }, + { kind: 'open_role', binding_ref: 'where:', refs: [], text: 'every recipient' }, +]; + // dfo-apis src/search/engine/region/un_system.py un_unavailable_warning, for // a record holding one refused place and one failed read of two variables, // with each variable's label in `variable_labels`. @@ -91,11 +103,21 @@ const unSystemWarning = (variableLabels: string[]): Warning => ({ }); test('aboutAnswerNotes holds the coverage sentence and assumptions -- humanitarian aid from Germany', () => { - assert.deepEqual(aboutAnswerNotes(humanitarianSpec, humanitarianWarnings, {}), { + const spec = { + ...humanitarianSpec, + resolution: { + slot_filters: [ + { key: { axis: 'how', property: { ref: 'crsTypeOfFlow' }, label: 'Type of flow' } }, + ], + }, + }; + + assert.deepEqual(aboutAnswerNotes(spec, humanitarianWarnings, {}, humanitarianAssumptions), { coverage: '2024 is the latest year with data, with 155 values for 61 of the 145 recipients that have data from Germany.', assumptions: [ - 'No recipient was named, so every recipient with data from Germany is included.', + 'No type of flow was named, so the answer uses official development assistance, the default for this measure.', + 'The question left this role open, so the answer covers every recipient from Germany.', 'No year was named, so the counts are for 2024, the latest year with data.', ], variance: [], @@ -211,6 +233,9 @@ test('no warning a part states as a departure, or the API keeps internal, reache 'currency_unmapped_omitted', 'withheld_by_registry', 'slot_defaulted', + // The part's assumptions state these scopes, so the warnings add no line. + 'place_scope_defaulted', + 'role_scope_defaulted', 'family_total_absent', 'date_coverage_partial', ]; diff --git a/src/lib/search/test/searchShareRequest.test.ts b/src/lib/search/test/searchShareRequest.test.ts index ca92689..c70cef3 100644 --- a/src/lib/search/test/searchShareRequest.test.ts +++ b/src/lib/search/test/searchShareRequest.test.ts @@ -95,6 +95,39 @@ test("a part's groups hold role keys only, each a label of 1 to 120 characters a assert.equal(reads({ origin: { label: 'EU', members: ['country/DEU'] } }), false); }); +test('a part reads with its assumptions, [] and null included, and a malformed one is refused', () => { + const reads = (assumptions: unknown) => + readSearchShareRequest({ ...body(1), parts: [{ ...part(0), assumptions }] })?.parts[0]; + const oda = { + kind: 'declared_default', + binding_ref: 'how:flowType', + refs: ['ODA'], + text: 'official development assistance', + }; + const scope = { kind: 'place_scope', binding_ref: 'where:', refs: [], text: 'every country' }; + + assert.deepEqual(reads([oda, scope])?.assumptions, [oda, scope]); + assert.deepEqual(reads([])?.assumptions, []); + assert.equal(reads(null)?.assumptions, null); + assert.ok(reads(undefined)); + + for (const malformed of [ + oda, + [{ ...oda, kind: 'guess' }], + [{ ...oda, refs: [27] }], + [{ ...oda, text: null }], + [{ ...oda, note: 'extra' }], + [{ ...oda, text: '' }], + [{ ...oda, text: 'a'.repeat(201) }], + [{ ...oda, refs: ['ODA', 'OOF'] }], + [{ ...oda, refs: ['r'.repeat(1025)] }], + [{ ...oda, binding_ref: 'b'.repeat(1037) }], + Array.from({ length: 68 }, () => scope), + ]) { + assert.equal(reads(malformed), undefined, JSON.stringify(malformed)); + } +}); + test('a part whose identity holds entity_groups is refused', () => { const withIdentity = (identity: unknown) => ({ ...body(1), @@ -136,6 +169,7 @@ test("a definite part shares its Spec's groups beside the identity, and null whe identity: spec.identity, entityGroups: groups, operations: response.operations, + assumptions: response.assumptions ?? null, definition: spec.definition, }); assert.equal( diff --git a/src/lib/search/test/shareDownload.test.ts b/src/lib/search/test/shareDownload.test.ts index a14cf53..f9505ef 100644 --- a/src/lib/search/test/shareDownload.test.ts +++ b/src/lib/search/test/shareDownload.test.ts @@ -30,6 +30,7 @@ const ref = ( identity: {}, entityGroups: null, operations: [], + assumptions: null, contentVersion, artifactDigest, }) satisfies SearchShareRef; diff --git a/src/lib/search/test/versionLink.test.ts b/src/lib/search/test/versionLink.test.ts index 839376f..c75e4fd 100644 --- a/src/lib/search/test/versionLink.test.ts +++ b/src/lib/search/test/versionLink.test.ts @@ -76,6 +76,7 @@ test('outcomeFromDocument opens a version as a snapshot share dated by its earli identity, entityGroups: null, operations: [], + assumptions: null, contentVersion: 'cv_b2', }, ], diff --git a/src/lib/search/test/view.test.ts b/src/lib/search/test/view.test.ts index 021e4cf..2c105b9 100644 --- a/src/lib/search/test/view.test.ts +++ b/src/lib/search/test/view.test.ts @@ -1681,8 +1681,13 @@ test("a single-part entity_ambiguous plan routes its wording token to that wordi assert.equal(partIndex, 0); assert.equal(wording.role, 'origin'); assert.deepEqual( - ambiguousWordingGroup(wording, response.parts[0].no_data)?.candidates.map((c) => c.ref), - ['InternationalBankForReconstructionAndDevelopment', 'InternationalDevelopmentAssociation'], + ambiguousWordingGroup(wording, response.parts[0].no_data)?.options.map( + (option) => option.label, + ), + [ + 'International Bank for Reconstruction and Development', + 'International Development Association', + ], ); }); @@ -1724,14 +1729,14 @@ test("ambiguousEntityMenuTitle titles the menu by the ambiguous wording's role", test('ambiguousEntityMenuTitle names every ambiguous wording a shared menu offers', () => { const part = normalizeResponse(fixtures['entity-ambiguous']).parts[0]; - const [congo] = part.no_data.ambiguous_entities; + const [congo] = part.no_data.options; const bothAmbiguous = { ...part, no_data: { ...part.no_data, - ambiguous_entities: [ - { ...congo, text: 'World Bank' }, - { ...congo, text: 'Georgia' }, + options: [ + { ...congo, mentions: ['World Bank'] }, + { ...congo, mentions: ['Georgia'] }, ], }, restatement: { diff --git a/src/lib/search/test/why.test.ts b/src/lib/search/test/why.test.ts index e985fd1..2d23ebd 100644 --- a/src/lib/search/test/why.test.ts +++ b/src/lib/search/test/why.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import type { Warning } from '../types.ts'; +import type { Assumption, Warning } from '../types.ts'; import { buildWhySummary, scopeAssumptions } from '../view.ts'; // Trimmed from dev API answers: inflation in Pakistan (one variable) and ODA @@ -64,6 +64,21 @@ const odaWarnings: Warning[] = [ }, ]; +// The assumption the ODA answer carries for the recipient role it left open. +const openRecipients: Assumption = { + kind: 'open_role', + binding_ref: 'where:', + refs: [], + text: 'every recipient', +}; + +const placeScope = (text: string): Assumption => ({ + kind: 'place_scope', + binding_ref: 'where:', + refs: [], + text, +}); + test('one variable: the Measure row carries its code and no Variables group appears', () => { const part = { warnings: [], variable_text: 'inflation' }; const summary = buildWhySummary(inflation, part, inflationMembers); @@ -79,7 +94,7 @@ test('one variable: the Measure row carries its code and no Variables group appe { label: 'Place', value: 'Pakistan' }, ]); assert.equal(summary.variables, null); - assert.deepEqual(scopeAssumptions(inflation, []), []); + assert.deepEqual(scopeAssumptions(inflation, [], []), []); // Until the members door answers, the row names the measure without a code. assert.deepEqual(buildWhySummary(inflation, part).rows[0], { label: 'Measure', @@ -99,8 +114,8 @@ test('many variables: rows name the role scope and the list reads the members do { label: 'Recipients', value: 'All 191 with data' }, ]); // No time_window_defaulted on the wire: the headline's year_named says it. - assert.deepEqual(scopeAssumptions(oda, odaWarnings), [ - 'No recipient was named, so every recipient with data from Germany is included.', + assert.deepEqual(scopeAssumptions(oda, [openRecipients], odaWarnings), [ + 'The question left this role open, so the answer covers every recipient from Germany.', 'No year was named, so the counts are for 2024, the latest year with data.', ]); assert.equal(summary.variables?.count, 1626); @@ -337,8 +352,15 @@ test('one variable of a larger family: laid out as one variable, the family size } }); -test('each scope warning becomes one assumption sentence', () => { - const cases = [ +test('each assumption becomes one sentence, and each scope warning names its row', () => { + const cases: { + description: string; + spec: Record; + codes: string[]; + given: Assumption[]; + row: { label: string; value: string }; + assumptions: string[]; + }[] = [ { description: 'no place named', spec: { @@ -354,10 +376,9 @@ test('each scope warning becomes one assumption sentence', () => { }, }, codes: ['place_scope_defaulted'], + given: [placeScope('every place')], row: { label: 'Places', value: 'All 1,649 with data' }, - assumptions: [ - 'No place was named, so every place with data is included, at every level.', - ], + assumptions: ['No place was named, so the answer covers every place.'], }, { // Trimmed from the dev API's "population" answer, which it @@ -377,8 +398,9 @@ test('each scope warning becomes one assumption sentence', () => { }, }, codes: ['place_scope_defaulted'], + given: [placeScope('every country')], row: { label: 'Countries', value: 'All 216 with data' }, - assumptions: ['No place was named, so every country with data is included.'], + assumptions: ['No place was named, so the answer covers every country.'], }, { description: 'no provider named, the recipient bound', @@ -389,9 +411,10 @@ test('each scope warning becomes one assumption sentence', () => { coverage: { kind: 'breadth', dimensions: [{ label: 'providers', count: 36 }] }, }, codes: ['role_scope_defaulted'], + given: [{ ...openRecipients, binding_ref: 'origin:', text: 'every provider' }], row: { label: 'Providers', value: 'All 36 with data' }, assumptions: [ - 'No provider was named, so every provider with data to Kenya is included.', + 'The question left this role open, so the answer covers every provider to Kenya.', 'No year was named, so the counts are for 2024, the latest year with data.', ], }, @@ -399,6 +422,7 @@ test('each scope warning becomes one assumption sentence', () => { description: 'no year named, a broad answer counting its latest year', spec: oda, codes: ['time_window_defaulted'], + given: [], row: { label: 'Years', value: 'Latest is 2024' }, assumptions: [ 'No year was named, so the counts are for 2024, the latest year with data.', @@ -408,22 +432,24 @@ test('each scope warning becomes one assumption sentence', () => { description: 'no year named, no headline year', spec: inflation, codes: ['time_window_defaulted'], + given: [], row: { label: 'Years', value: '2000 to 2022' }, assumptions: ['No year was named, so every year with data is included.'], }, { - description: 'a role warning with no open role on the spec keeps the API wording', + description: + 'a role warning on a part that assumed nothing adds no sentence of its own', spec: { ...oda, entity_options: [] }, codes: ['role_scope_defaulted'], + given: [], row: { label: 'Provider', value: 'Germany' }, assumptions: [ - 'No recipient was named; the answer was resolved across the 191 recipients with data from Germany.', 'No year was named, so the counts are for 2024, the latest year with data.', ], }, ]; - for (const { description, spec, codes, row, assumptions } of cases) { + for (const { description, spec, codes, given, row, assumptions } of cases) { const warnings: Warning[] = codes.map( (code) => odaWarnings.find((warning) => warning.code === code) ?? { @@ -438,7 +464,7 @@ test('each scope warning becomes one assumption sentence', () => { why.rows.some((r) => r.label === row.label && r.value === row.value), description, ); - assert.deepEqual(scopeAssumptions(spec, warnings), assumptions, description); + assert.deepEqual(scopeAssumptions(spec, given, warnings), assumptions, description); } }); @@ -470,9 +496,9 @@ test('chosen: each facet axis gives its value and why, in source, method, unit o ]); }); -test('chosen: a slot a declared default filled follows the facets; other provenances add nothing', () => { +test('a slot a declared default filled is stated once, as an assumption; no provenance adds a choice', () => { const filter = (provenance: string, label: string, ref: string) => ({ - key: { axis: 'how', label }, + key: { axis: 'how', property: { ref: label.replaceAll(' ', '') }, label }, binding_kind: 'value', refs: [{ ref, label: ref }], provenance, @@ -487,12 +513,20 @@ test('chosen: a slot a declared default filled follows the facets; other provena }, }; - assert.deepEqual(buildWhySummary(spec).chosen, [ - { - label: 'Type of flow', - value: 'ODA', - reason: 'Filled in from the default for this measure.', - }, + const defaulted: Assumption = { + kind: 'declared_default', + binding_ref: 'how:typeofflow', + refs: ['ODA'], + text: 'official development assistance', + }; + + assert.deepEqual(buildWhySummary(spec).chosen, []); + assert.deepEqual(scopeAssumptions(spec, [defaulted]), [ + 'No type of flow was named, so the answer uses official development assistance, the default for this measure.', + ]); + // A ref no slot names keeps the sentence without a label. + assert.deepEqual(scopeAssumptions(spec, [{ ...defaulted, binding_ref: 'how:other' }]), [ + 'The question left this open, so the answer uses official development assistance, the default for this measure.', ]); }); diff --git a/src/lib/search/types.ts b/src/lib/search/types.ts index 1d38bb8..9a63f95 100644 --- a/src/lib/search/types.ts +++ b/src/lib/search/types.ts @@ -66,6 +66,9 @@ export type QuantityForm = 'per_person' | 'per_head_base' | 'per_area' | 'rate' export type NoDataReason = | 'source_not_resolved' | 'source_ambiguous' + | 'source_code_not_held' + | 'source_code_ambiguous' + | 'source_code_shares_variable' | 'entity_not_resolved' | 'entity_ambiguous' | 'entity_set_too_large' @@ -1098,10 +1101,14 @@ export interface WireSpec { // Sentence plan (the engine's rendered, tokenised confirmation sentence) // --------------------------------------------------------------------------- -/** The block of the answer a sentence token belongs to: the heading states - * what the answer is, the byline its unit and source, and a departure - * token carries the text of one `Departure` on the same part. */ -export type SentenceSegment = 'heading' | 'byline' | 'departure'; +/** The block of the answer a sentence token belongs to: a coverage token is + * a word of the "Answered 1 of 2 parts." a mixed answer's envelope sentence + * opens with, the heading states what the answer is, the byline its unit + * and source, a departure token carries the text of one `Departure` on the + * same part, and an assumption token is a word of the closing clause naming + * the part's `assumptions` ("Assumed: official development assistance; + * every provider."). */ +export type SentenceSegment = 'coverage' | 'heading' | 'byline' | 'departure' | 'assumption'; /** The axis a sentence token fills: every slot axis plus the byline's * facet axes no request slot carries: 'unit', the unit of measure, and @@ -1124,8 +1131,8 @@ export interface SentenceToken { * states it */ text: string; /** the block this token belongs to. A Spec's own plan holds heading and - * byline tokens only; departure tokens sit on a part's or the - * envelope's plan. */ + * byline tokens only; departure and assumption tokens sit on a part's + * or the envelope's plan. */ segment: SentenceSegment; /** the grammatical axis this token fills, when kind==="slot" */ axis?: SentenceAxis | null; @@ -1212,6 +1219,34 @@ export interface LateDepartures { departures: Departure[]; } +/** What an `Assumption` fills (dfo-apis vocab.py AssumptionKind): + * 'declared_default', a slot the request left open that its code system's + * or shape's declared default fixed; 'open_role', a flow role no request + * word asked for, read across every entity on it with data, as one series + * per member where the role scope default filled it and the shape declares + * the role, else as the total over them; 'place_scope', a request that named no + * place, read across every place with data. */ +export type AssumptionKind = 'declared_default' | 'open_role' | 'place_scope'; + +/** One value a part's answer filled that its request did not ask for. + * Mirrors dfo-apis src/search/models/assumption.py Assumption. The sentence + * carrying the part closes with every one as one clause of `assumption` + * tokens. */ +export interface Assumption { + kind: AssumptionKind; + /** the binding whose edit replaces it: "how:" for a + * declared default, "origin:" or "where:" for an open role (the + * destination shares the subject's where slot), "where:" for the place + * scope */ + binding_ref: string; + /** the value dcids a declared default bound its slot to; [] for an + * open role and the place scope */ + refs: string[]; + /** the clause's words for it: "official development assistance", + * "every provider", "every country", "the world as a whole" */ + text: string; +} + // --------------------------------------------------------------------------- // Frame (what one part asks; carried by the stream's `frame` event) // --------------------------------------------------------------------------- @@ -1410,6 +1445,20 @@ export interface NoDataDenominator { found?: GraphRef | null; } +/** One answer to the choice a no_data refusal asks for (dfo-apis + * src/search/models/outcomes.py NoDataOption). */ +export interface NoDataOption { + /** the request's own words this option settles, in query order */ + mentions: string[]; + /** the choice in words for a menu ("Republic of the Congo", "France to + * Germany"), for display only */ + label: string; + /** a one-part parsed input settling this choice only, posted unchanged + * to the resolve door; its one part is a `parsed` dataset_resubmit + * entry's part */ + data_query: ParsedInput; +} + /** Outcome when no grounded result can be returned. */ export interface NoData { /** why nothing could be returned (closed enum) */ @@ -1427,23 +1476,24 @@ export interface NoData { ambiguous_entities?: AmbiguousEntityGroup[]; /** The source wordings this refusal is about, in query order: every * included wording that matched no admitted source for - * source_not_resolved, every ambiguous wording for source_ambiguous. + * source_not_resolved, every ambiguous wording for source_ambiguous, + * and the part's source_code for the three source_code_* reasons. * Empty for every other reason. */ source_terms?: string[]; /** Every ambiguous wording behind a source_ambiguous outcome, in query - * order, each with the admitted sources it could name. Empty for every - * other reason. */ + * order, each with the admitted sources it could name, and for + * source_code_ambiguous the part's source_code with the provenances + * holding it. Empty for every other reason. */ ambiguous_sources?: AmbiguousSourceGroup[]; /** - * Set when the reason is entity_ambiguous or source_ambiguous: the parsed - * form of this part. On entity_ambiguous every resolved entity's dcid is - * pinned and the ambiguous entity's dcid left empty; on source_ambiguous - * every resolved source mention carries its ref and each ambiguous - * wording carries none. The site pins a pick onto the part's - * `restatement` instead (parsedEdit.ts's buildAmbiguousPickPart), so it - * reads none of this. + * One ready request per choice of an entity_ambiguous, source_ambiguous, + * source_code_ambiguous, role_not_resolved or unit_not_resolved refusal, + * in query then candidate order; empty for every other reason. It can + * list fewer entries than the candidates. The site's entity picker + * lists these options and posts the chosen one's part unchanged + * (parsedEdit.ts entityChoiceGroups). */ - data_query?: ParsedInput | null; + options?: NoDataOption[]; /** Terms the engine refused to silently drop before answering. Present * only for qualifier_not_bound. */ unbound_qualifiers?: ResolvedQualifier[]; @@ -1580,7 +1630,7 @@ export interface WireNoData { ambiguous_entities?: AmbiguousEntityGroup[]; source_terms?: string[]; ambiguous_sources?: AmbiguousSourceGroup[]; - data_query?: ParsedInput | null; + options?: NoDataOption[]; unbound_qualifiers?: ResolvedQualifier[]; /** optional; real, grounded WireSpecs with data adjacent to the request */ nearest_real?: WireSpec[] | null; @@ -1658,11 +1708,22 @@ export interface WirePart { /** every way this part's answer differs from what its request asked, in * DepartureCode order; [] when the answer is what was asked */ departures: Departure[]; + /** every value the answer filled that the request did not ask for, in + * clause order: declared defaults, open roles, then the place scope; + * on a candidates part, the items every `reading_assumptions` list holds + * alike; [] on no_data and unsupported */ + assumptions?: Assumption[]; status: ResolveStatus | 'unsupported'; /** present iff status === "definite" */ spec?: WireSpec; /** present iff status === "candidates" */ candidates?: WireCandidateSet; + /** candidates only: each shown reading's own assumptions, keyed by its + * Spec's spec_id, as a definite part answering that Spec would carry + * them; [] for a reading that assumes nothing. A pick echoes the picked + * reading's list on its identity entry, and a candidates entry echoes + * each reading's list on its candidate. */ + reading_assumptions?: Record; /** present iff status === "no_data" */ no_data?: WireNoData; /** present iff status === "unsupported" */ @@ -1693,11 +1754,17 @@ export interface Part { * DepartureCode order; [] when the answer is what was asked. The stream's * `done` event can add more (see searchClient.applyDoneEvent). */ departures: Departure[]; + /** every value the answer filled that the request did not ask for, in + * clause order (see WirePart.assumptions) */ + assumptions?: Assumption[]; status: ResolveStatus | 'unsupported'; /** present iff status === "definite" */ spec?: Spec; /** present iff status === "candidates" */ candidates?: CandidateSet; + /** candidates only: each shown reading's own assumptions, keyed by + * spec_id (see WirePart.reading_assumptions) */ + reading_assumptions?: Record; /** present iff status === "no_data" */ no_data?: NoData; /** present iff status === "unsupported" */ @@ -1838,7 +1905,7 @@ export interface RawTextInput { } /** path B: pre-parsed parts, extract skipped. Also the shape of - * `NoData.data_query`. Parts byte-identical in every field collapse to + * `NoDataOption.data_query`. Parts byte-identical in every field collapse to * one before resolution. */ export interface ParsedInput { kind: 'parsed'; @@ -1889,7 +1956,9 @@ export interface ParsedPart { // (datasetEntries.ts) never derives a `parsed` entry from a part's own // status; a typed edit or an entity_ambiguous pick builds one directly and // passes it in as an override. Every entry carries its part's `operations` -// back, and the engine echoes them onto the rebuilt part. +// back, and the engine echoes them onto the rebuilt part. An identity entry +// also carries its part's `assumptions` (a picked reading's own list on a +// candidates part), and each candidate its reading's. // --------------------------------------------------------------------------- /** A definite part's entry: the complete identity the part should resolve @@ -1906,6 +1975,11 @@ export interface DatasetIdentityEntry { identity: BaseIdentity; entity_groups?: Record | null; operations?: Operation[]; + /** the part's `assumptions`, echoed as the response gave them, [] included. + * The engine states the echoed items that still hold after the edit and + * nothing the echo lacks, so [] states none. When absent, it computes + * them as for a fresh request. */ + assumptions?: Assumption[]; } /** One competing identity inside a `candidates` entry, with the @@ -1914,6 +1988,11 @@ export interface DatasetIdentityEntry { export interface DatasetCandidateIdentity { identity: BaseIdentity; entity_groups?: Record | null; + /** the reading's `reading_assumptions` list, echoed as the response gave + * it, [] included, by the rule `DatasetIdentityEntry.assumptions` states. + * When absent, the engine computes the reading's assumptions as for a + * fresh request. */ + assumptions?: Assumption[]; } /** A candidates part's entry: `ordering`/`truncated` echoed verbatim from the diff --git a/src/lib/search/versionLink.ts b/src/lib/search/versionLink.ts index 70c8445..c60915e 100644 --- a/src/lib/search/versionLink.ts +++ b/src/lib/search/versionLink.ts @@ -166,6 +166,7 @@ export const outcomeFromDocument = ( // A dataset document holds the definition alone. entityGroups: null, operations: [], + assumptions: null, contentVersion: link.contentVersion, }, ], diff --git a/src/lib/search/view.ts b/src/lib/search/view.ts index e19c1e5..f5a12d7 100644 --- a/src/lib/search/view.ts +++ b/src/lib/search/view.ts @@ -11,11 +11,13 @@ import { spaceCamel } from '../spaceCamel.ts'; import type { MemberRef, MembersRead } from './definitionMembers.ts'; import { coverageRichness, coverageState, partitionOptions } from './optionCoverage.ts'; import type { OperationProgress } from './operationPoll'; +import { entityChoiceGroups } from './parsedEdit.ts'; +import type { EntityChoiceGroup } from './parsedEdit.ts'; import { entityOptionDcid } from './pickerState.ts'; import type { OptionLike } from './pickerState.ts'; import type { ApiError, EntityOption } from './searchClient'; import type { - AmbiguousEntityGroup, + Assumption, EntityOptions, EntityRoleKey, FacetBasis, @@ -35,7 +37,7 @@ import type { UnitAlternative, Warning, } from './types'; -import { sentenceOf } from './searchClient.ts'; +import { keyRef, sentenceOf } from './searchClient.ts'; import { unitWithBasis } from './unitLabel.ts'; /** Display label of a slot/entity option, wherever the engine put it. */ @@ -1037,7 +1039,7 @@ export type WhySummary = { origin: string | null; licence: string | null; }[]; - /** how each facet axis and each defaulted slot got its value */ + /** how each facet axis got its value */ chosen: WhyChoice[]; }; @@ -1167,62 +1169,67 @@ const openRoles = (spec: any, codes: Set): EntityRoleKey[] => { return open; }; -/** What a definite part's answer assumed, one sentence per scope the - * question left open: the place, an open provider or recipient role, and the - * year. "About this answer" reads this list (through resultNotes's - * aboutAnswerNotes). The year sentence keys on `headline.year_named` - * as well as `time_window_defaulted`, since a role-bearing answer can say - * no year was named without sending that warning. */ -export const scopeAssumptions = (spec: any, warnings?: readonly Warning[] | null): string[] => { - const codes = new Set((warnings ?? []).map((warning) => warning?.code)); - const open = openRoles(spec, codes); - const assumptions: string[] = []; - - if (codes.has('place_scope_defaulted')) { - assumptions.push( - placeScope(spec).noun === 'country' - ? 'No place was named, so every country with data is included.' - : 'No place was named, so every place with data is included, at every level.', - ); - } +// The label of the slot `bindingRef` names on `spec` ("Type of flow"), +// null when neither its trace nor its slots carry one. +const slotLabelOf = (spec: any, bindingRef: string): string | null => + [...(spec.resolution?.slot_filters ?? []), ...(spec.slots ?? [])].find( + (entry: any) => entry?.key && keyRef(entry.key) === bindingRef, + )?.key?.label ?? null; - if (codes.has('role_scope_defaulted')) { - const directional = open.filter((role) => role !== 'observationAbout'); +const lowerFirst = (text: string): string => + /^[A-Z][a-z]/.test(text) ? text.charAt(0).toLowerCase() + text.slice(1) : text; - for (const role of directional) { - const named = entityLabels(spec, role === 'origin' ? 'destination' : 'origin'); - const noun = ROLE_WORDS[role].noun; - const scope = named.length - ? ` ${role === 'destination' ? 'from' : 'to'} ${orList(named)}` - : ''; +// One assumption as the panel states it, in the words of its own `text`. +const assumptionSentence = (spec: any, assumption: Assumption): string => { + const { kind, binding_ref: ref, text } = assumption; - assumptions.push( - `No ${noun} was named, so every ${noun} with data${scope} is included.`, - ); - } + if (kind === 'place_scope') return `No place was named, so the answer covers ${text}.`; - // The warning names no role, so with no open role to read it from, - // the API's own wording stands. - if (directional.length === 0) { - const message = warnings?.find((w) => w?.code === 'role_scope_defaulted')?.message; + if (kind === 'open_role') { + // An open role reads "origin:" or "where:", the slot the destination + // shares with the subject. + const origin = ref === 'origin:'; + const named = entityLabels(spec, origin ? 'destination' : 'origin'); + const scope = named.length ? ` ${origin ? 'to' : 'from'} ${orList(named)}` : ''; - if (message) { - assumptions.push(capitalise(message)); - } - } + return `The question left this role open, so the answer covers ${text}${scope}.`; } - if (codes.has('time_window_defaulted') || spec.headline?.year_named === false) { + const label = slotLabelOf(spec, ref); + + return label + ? `No ${lowerFirst(label)} was named, so the answer uses ${text}, the default for this measure.` + : `The question left this open, so the answer uses ${text}, the default for this measure.`; +}; + +/** What a definite part's answer assumed: one sentence per item of the + * part's `assumptions` (a declared default, an open role, the place scope), + * in the API's order, then the year. "About this answer" reads this list + * (through resultNotes's aboutAnswerNotes). The year sentence keys on + * `headline.year_named` as well as `time_window_defaulted`, since a + * role-bearing answer can say no year was named without sending that + * warning. */ +export const scopeAssumptions = ( + spec: any, + assumptions: readonly Assumption[] | null | undefined, + warnings?: readonly Pick[] | null, +): string[] => { + const sentences = (assumptions ?? []).map((assumption) => assumptionSentence(spec, assumption)); + const yearDefaulted = (warnings ?? []).some( + (warning) => warning?.code === 'time_window_defaulted', + ); + + if (yearDefaulted || spec.headline?.year_named === false) { const year = spec.headline?.year; - assumptions.push( + sentences.push( year != null && spec.headline?.year_named === false ? `No year was named, so the counts are for ${year}, the latest year with data.` : 'No year was named, so every year with data is included.', ); } - return assumptions; + return sentences; }; // The values each axis lists before the rest are counted as "more". @@ -1270,7 +1277,7 @@ const whyMembers = (members: MembersRead | null | undefined): WhyMembers => { }; }; -// Why a facet axis or slot holds its value (ResolvedFacet.basis). +// Why a facet axis holds its value (ResolvedFacet.basis). const CHOICE_REASONS: Record = { bound: 'Set by your request.', declared_default: 'Filled in from the default for this measure.', @@ -1298,35 +1305,17 @@ const facetChoiceValue = (axis: string, facet: any): string | null => { return axis === 'measurementMethod' ? formatMethod(facet) : spaceCamel(raw); }; -// How each facet axis got its value (`resolved.facets[axis].basis`), then -// each slot a declared default filled (`trace.slot_filters[].provenance`). -const whyChoices = (spec: any): WhyChoice[] => { - const facets = FACET_CHOICE_AXES.flatMap(([axis, label]) => { +// How each facet axis got its value (`resolved.facets[axis].basis`). A +// slot a declared default filled is one of the part's `assumptions`, which +// "What we assumed" states (scopeAssumptions). +const whyChoices = (spec: any): WhyChoice[] => + FACET_CHOICE_AXES.flatMap(([axis, label]) => { const facet = spec.facets?.[axis]; const reason: string | undefined = facet && CHOICE_REASONS[facet.basis as FacetBasis]; const value = reason ? facetChoiceValue(axis, facet) : null; return reason && value ? [{ label, value, reason }] : []; }); - const slots = (spec.resolution?.slot_filters ?? []).flatMap((filter: any) => { - const value = (filter?.refs ?? []) - .map((ref: any) => ref?.label ?? ref?.ref) - .filter(Boolean) - .join(', '); - - return filter?.provenance === 'declared_default' && value - ? [ - { - label: capitalise(filter.key?.label ?? filter.key?.axis ?? 'Value'), - value, - reason: CHOICE_REASONS.declared_default, - }, - ] - : []; - }); - - return [...facets, ...slots]; -}; // The licence label of the one source the answer serves, in words. const licenceText = (spec: any): string | null => @@ -1334,7 +1323,7 @@ const licenceText = (spec: any): string | null => /** The "About this answer" summary for one definite part: what the search * matched (rows), the variables when there is more than one, the sources - * with their licence, and how each facet and defaulted slot was chosen. The + * with their licence, and how each facet was chosen. The * scope assumptions come from resultNotes's aboutAnswerNotes. * * The resolved membership decides the layout (`servedMembership`). @@ -2288,6 +2277,11 @@ export const flattenPartsRows = ( const NO_DATA_REASON_TEXT: Record = { source_not_resolved: "We couldn't match the data source named in that query.", source_ambiguous: 'More than one data source matches that wording. Try naming it in full.', + source_code_not_held: 'No source we search publishes that code for this question.', + source_code_ambiguous: + 'More than one of the sources named publishes that code. Name the one you mean.', + source_code_shares_variable: + "That code's series also answer other codes of its source, so it can't be shown alone.", entity_not_resolved: "We couldn't match the place in that query.", entity_ambiguous: 'More than one place matches that wording. Try being more specific.', entity_set_too_large: 'That place has too many members for us to check them all at once.', @@ -2483,7 +2477,7 @@ const ambiguousEntityNoun = (restatement: ParsedPart | null | undefined, text: s export const ambiguousEntityMenuTitle = ( part: { no_data?: NoData | null; restatement?: ParsedPart | null } | null | undefined, ): string => { - const groups = part?.no_data?.ambiguous_entities ?? []; + const groups = entityChoiceGroups(part?.no_data); if (groups.length === 0) { return 'Select Place'; @@ -2497,27 +2491,27 @@ export const ambiguousEntityMenuTitle = ( }; /** - * The ambiguous wording an entity_ambiguous part's slot token renders, or - * null for any other token. The engine marks each wording as a `slot` token - * with no `entity_refs` or `binding_ref`, the role the request gave it, and - * text equal to one `no_data.ambiguous_entities` group's text (dfo-apis - * search/render/no_data.py). The part's own variable slot is never one. + * The ambiguous wording an entity_ambiguous part's slot token renders, with + * the options that settle it (entityChoiceGroups), or null for any other + * token. The engine marks each wording as a `slot` token with no + * `entity_refs` or `binding_ref`, the role the request gave it, and text + * equal to the wording (dfo-apis search/render/no_data.py). The part's own + * variable slot is never one. */ export const ambiguousWordingGroup = ( token: SentenceToken, noData: NoData | null | undefined, -): AmbiguousEntityGroup | null => { +): EntityChoiceGroup | null => { if ( token.kind !== 'slot' || token.axis === 'what' || token.binding_ref || - token.entity_refs?.length || - noData?.reason !== 'entity_ambiguous' + token.entity_refs?.length ) { return null; } - return noData.ambiguous_entities?.find((group) => group.text === token.text) ?? null; + return entityChoiceGroups(noData).find((group) => group.text === token.text) ?? null; }; // Values render exactly as the engine sent them — US-style separators,