diff --git a/bench/README.md b/bench/README.md index 697584a8..e6a68bf5 100644 --- a/bench/README.md +++ b/bench/README.md @@ -220,8 +220,10 @@ and deletes every browser. gone resolves to `refs/pull//head`, and the report marks it. Leave all three off a new intent: it describes the user's task, and the report shows which tools it used. - `{{name}}` in `prompt` and `rules` comes from `fixture`. `{"days_from_start": n}` resolves to a - date when the run starts. A fixed date that has passed skips the intent, or stops the run if - the intent was selected by name; move dates tied to a film's run forward before each run. + date when the run starts, and with `"format": "month_day"` to that date as a page shows it + without its year (`Dec 10`), for a check's rule on the page text. A fixed date that has passed + skips the intent, or stops the run if the intent was selected by name; move dates tied to a + film's run forward before each run. - `rules` apply to every value a path resolves to (`[]` steps into arrays): `equals` and `in` (case- and whitespace-insensitive), `matches` (case-insensitive regex; fixture values match literally), `range`, `count`, `time` (`HH:MM`), `date` (`YYYY-MM-DD`) and `unique`. diff --git a/bench/lib/intents.mjs b/bench/lib/intents.mjs index a93962a8..5d2d8993 100644 --- a/bench/lib/intents.mjs +++ b/bench/lib/intents.mjs @@ -54,13 +54,16 @@ export function fillRules(rules = [], fixture = {}) { }); } -const relativeDate = (value) => value !== null && typeof value === "object" && Number.isInteger(value.days_from_start); +// How a site shows a date without its year, such as "Dec 10", for rules on its page text. +const MONTH_DAY = new Intl.DateTimeFormat("en-US", { month: "short", day: "numeric", timeZone: "UTC" }); +const relativeDate = (value) => value !== null && typeof value === "object" && Number.isInteger(value.days_from_start) + && (value.format === undefined || value.format === "month_day"); export function resolveFixture(fixture = {}, start) { return Object.fromEntries(Object.entries(fixture).map(([key, value]) => { if (!relativeDate(value)) return [key, value]; const date = new Date(start.getTime() + value.days_from_start * 86_400_000); - return [key, date.toISOString().slice(0, 10)]; + return [key, value.format === "month_day" ? MONTH_DAY.format(date) : date.toISOString().slice(0, 10)]; })); } @@ -138,7 +141,7 @@ export function validateIntent(intent, benchDir) { need(agreement === undefined || (typeof agreement?.path === "string" && (best(agreement) || same(agreement))), "agreement needs a path and either better (lower or higher) with an optional tolerance of at least 0, or same: true"); for (const [key, value] of Object.entries(intent.fixture ?? {})) { - need(typeof value === "string" || typeof value === "number" || relativeDate(value), `fixture ${key} must be a string, a number or {"days_from_start": n}`); + need(typeof value === "string" || typeof value === "number" || relativeDate(value), `fixture ${key} must be a string, a number or {"days_from_start": n}, optionally with "format": "month_day"`); } const text = JSON.stringify([intent.prompt, intent.rules ?? [], check?.rules ?? []]); for (const [, key] of text.matchAll(/\{\{(\w+)\}\}/g)) need(Object.hasOwn(intent.fixture ?? {}, key), `{{${key}}} has no fixture value`); diff --git a/bench/tasks/delta.com/delta-cart-main-classic.json b/bench/tasks/delta.com/delta-cart-main-classic.json index 5ca3d2da..4e7ecf52 100644 --- a/bench/tasks/delta.com/delta-cart-main-classic.json +++ b/bench/tasks/delta.com/delta-cart-main-classic.json @@ -6,7 +6,8 @@ "fixture": { "origin": "Atlanta (ATL)", "destination": "Orlando (MCO)", - "date": {"days_from_start": 28} + "date": {"days_from_start": 28}, + "date_shown": {"days_from_start": 28, "format": "month_day"} }, "start_url": "https://www.delta.com/", "steps": [ @@ -25,6 +26,7 @@ "rules": [ {"path": "url", "matches": "^https://www\\.delta\\.com/(purchase/flights|completepurchase)/trip-summary"}, {"path": "text", "matches": "\\bATL - MCO\\s*\\n\\s*One Way\\b"}, + {"path": "text", "matches": "Journey date\\s*[^\\n]*\\b{{date_shown}}\\b"}, {"path": "extracted.cabin", "equals": "Main Classic"}, {"path": "extracted.passengers", "equals": 1}, {"path": "extracted.amount_due", "range": [1, 3000]} diff --git a/bench/test/page.test.mjs b/bench/test/page.test.mjs index 9370724c..02d9bcc2 100644 --- a/bench/test/page.test.mjs +++ b/bench/test/page.test.mjs @@ -1,5 +1,8 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { loadSchema, resolveFixture } from "../lib/intents.mjs"; import { extractValues, stepName, stepTimeline } from "../lib/page.mjs"; import { gradeAnswer } from "../grade.mjs"; @@ -66,3 +69,25 @@ test("a failing rule on long page text shows only the start of it", () => { const grade = gradeAnswer(intent, schema, "```json\n{}\n```", { output: { url: "u", text: "x".repeat(50_000) } }); assert.ok(grade.problems[0].length < 200, grade.problems[0].length); }); + +test("the Delta cart check needs the flight in the cart to be on the date the task asked for", () => { + const benchDir = fileURLToPath(new URL("..", import.meta.url)); + const original = JSON.parse(readFileSync(new URL("../tasks/delta.com/delta-cart-main-classic.json", import.meta.url), "utf8")); + // Trip Summary's visible text for DL 750 on Dec 10, as delta.com showed it. + const summary = "ATL - MCO\nOne Way\nDec 10\n1 Passenger\nModify\nTrip Summary\nATL\nto\nMCO\nChange Flight\nJourney date\nThu, Dec 10\nFlight number\nDL750\n" + + "Departure time\n6:05am - \nArrival time\n7:40am\nFlight duration\nNonstop, 1h 35m\nCabin class\nMain Classic\nChangeable / Nonrefundable\n" + + "Trip Total\nCurrency Calculator\n1 Passenger\nFlights\n$123.72\nTaxes, Fees & Charges\n$24.68\nAmount Due\nView Details\n$148.40 USD"; + const grade = (start, text = summary) => { + const intent = { ...original, fixture: resolveFixture(original.fixture, new Date(start)) }; + const page = { output: { url: "https://www.delta.com/purchase/flights/trip-summary", title: "Trip Summary", text, fields: {}, missing: [], invalid: [] } }; + return gradeAnswer(intent, loadSchema(benchDir, intent), '{"flight_numbers": ["DL 750"], "amount_due": 148.4}', page); + }; + assert.equal(grade("2026-11-12T16:00:00Z").outcome, "correct"); + assert.equal(grade("2026-11-12T16:00:00Z", summary.replace("Journey date\nThu, Dec 10", "Journey date Thu, Dec 10, 2026")).outcome, "correct"); + // The task asked for Dec 11 or Dec 1, and the cart holds a flight on Dec 10. + for (const start of ["2026-11-13T16:00:00Z", "2026-11-03T16:00:00Z"]) { + const wrong = grade(start); + assert.equal(wrong.outcome, "wrong"); + assert.match(wrong.problems.join(), /does not match \/Journey date/); + } +}); diff --git a/bench/test/tasks.test.mjs b/bench/test/tasks.test.mjs index f3d1ba17..81a61d7a 100644 --- a/bench/test/tasks.test.mjs +++ b/bench/test/tasks.test.mjs @@ -71,6 +71,8 @@ test("validateIntent reports missing fixture values and bad rules", () => { assert.ok(problems.some(p => p.includes("{{ip}} has no fixture value"))); assert.ok(problems.some(p => p.startsWith("rule "))); assert.deepEqual(validateIntent({ ...intent, fixture: { ip: { days_from_start: 3 } } }, benchDir), []); + assert.deepEqual(validateIntent({ ...intent, fixture: { ip: { days_from_start: 3, format: "month_day" } } }, benchDir), []); + assert.ok(validateIntent({ ...intent, fixture: { ip: { days_from_start: 3, format: "long" } } }, benchDir).some(p => p.startsWith("fixture ip must be"))); assert.ok(validateIntent({ ...intent, fixture: { ip: { days: 3 } } }, benchDir).some(p => p.startsWith("fixture ip"))); assert.deepEqual(validateIntent({ ...intent, rules: [{ path: "ip", matches: "^{{ip}}$" }] }, benchDir), []); assert.ok(validateIntent({ ...intent, rules: [{ path: "ip", matches: "(" }] }, benchDir).some(p => p.endsWith("needs a valid regex"))); @@ -116,6 +118,9 @@ test("relative fixture dates resolve from the run's start", () => { const start = new Date("2026-10-07T23:30:00Z"); assert.deepEqual(resolveFixture({ date: { days_from_start: 28 }, zip: "10001", n: 3 }, start), { date: "2026-11-04", zip: "10001", n: 3 }); assert.deepEqual(resolveFixture(undefined, start), {}); + // A date as the site's page shows it, for a rule on the page's text. + assert.deepEqual(resolveFixture({ date: { days_from_start: 28 }, shown: { days_from_start: 28, format: "month_day" } }, start), { date: "2026-11-04", shown: "Nov 4" }); + assert.deepEqual(resolveFixture({ shown: { days_from_start: 64, format: "month_day" } }, start), { shown: "Dec 10" }); }); test("URL patterns match the way Kernel's do", () => {