From 37be61922fb60e8bc2c643e4ca4769a1bb794374 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:36:12 +0000 Subject: [PATCH] Check the date of the flight in the Delta cart The delta-cart-main-classic check passed a cart holding the right route, cabin and total on the wrong day. It now needs Trip Summary's journey date to be the date the task asked for. Delta shows the date without its year, so a relative fixture date can resolve to that form with "format": "month_day". --- bench/README.md | 6 +++-- bench/lib/intents.mjs | 9 ++++--- .../delta.com/delta-cart-main-classic.json | 4 ++- bench/test/page.test.mjs | 25 +++++++++++++++++++ bench/test/tasks.test.mjs | 5 ++++ 5 files changed, 43 insertions(+), 6 deletions(-) diff --git a/bench/README.md b/bench/README.md index 697584a8..e6a68bf5 100644 --- a/bench/README.md +++ b/bench/README.md @@ -220,8 +220,10 @@ and deletes every browser. gone resolves to `refs/pull//head`, and the report marks it. Leave all three off a new intent: it describes the user's task, and the report shows which tools it used. - `{{name}}` in `prompt` and `rules` comes from `fixture`. `{"days_from_start": n}` resolves to a - date when the run starts. A fixed date that has passed skips the intent, or stops the run if - the intent was selected by name; move dates tied to a film's run forward before each run. + date when the run starts, and with `"format": "month_day"` to that date as a page shows it + without its year (`Dec 10`), for a check's rule on the page text. A fixed date that has passed + skips the intent, or stops the run if the intent was selected by name; move dates tied to a + film's run forward before each run. - `rules` apply to every value a path resolves to (`[]` steps into arrays): `equals` and `in` (case- and whitespace-insensitive), `matches` (case-insensitive regex; fixture values match literally), `range`, `count`, `time` (`HH:MM`), `date` (`YYYY-MM-DD`) and `unique`. diff --git a/bench/lib/intents.mjs b/bench/lib/intents.mjs index a93962a8..5d2d8993 100644 --- a/bench/lib/intents.mjs +++ b/bench/lib/intents.mjs @@ -54,13 +54,16 @@ export function fillRules(rules = [], fixture = {}) { }); } -const relativeDate = (value) => value !== null && typeof value === "object" && Number.isInteger(value.days_from_start); +// How a site shows a date without its year, such as "Dec 10", for rules on its page text. +const MONTH_DAY = new Intl.DateTimeFormat("en-US", { month: "short", day: "numeric", timeZone: "UTC" }); +const relativeDate = (value) => value !== null && typeof value === "object" && Number.isInteger(value.days_from_start) + && (value.format === undefined || value.format === "month_day"); export function resolveFixture(fixture = {}, start) { return Object.fromEntries(Object.entries(fixture).map(([key, value]) => { if (!relativeDate(value)) return [key, value]; const date = new Date(start.getTime() + value.days_from_start * 86_400_000); - return [key, date.toISOString().slice(0, 10)]; + return [key, value.format === "month_day" ? MONTH_DAY.format(date) : date.toISOString().slice(0, 10)]; })); } @@ -138,7 +141,7 @@ export function validateIntent(intent, benchDir) { need(agreement === undefined || (typeof agreement?.path === "string" && (best(agreement) || same(agreement))), "agreement needs a path and either better (lower or higher) with an optional tolerance of at least 0, or same: true"); for (const [key, value] of Object.entries(intent.fixture ?? {})) { - need(typeof value === "string" || typeof value === "number" || relativeDate(value), `fixture ${key} must be a string, a number or {"days_from_start": n}`); + need(typeof value === "string" || typeof value === "number" || relativeDate(value), `fixture ${key} must be a string, a number or {"days_from_start": n}, optionally with "format": "month_day"`); } const text = JSON.stringify([intent.prompt, intent.rules ?? [], check?.rules ?? []]); for (const [, key] of text.matchAll(/\{\{(\w+)\}\}/g)) need(Object.hasOwn(intent.fixture ?? {}, key), `{{${key}}} has no fixture value`); diff --git a/bench/tasks/delta.com/delta-cart-main-classic.json b/bench/tasks/delta.com/delta-cart-main-classic.json index 5ca3d2da..4e7ecf52 100644 --- a/bench/tasks/delta.com/delta-cart-main-classic.json +++ b/bench/tasks/delta.com/delta-cart-main-classic.json @@ -6,7 +6,8 @@ "fixture": { "origin": "Atlanta (ATL)", "destination": "Orlando (MCO)", - "date": {"days_from_start": 28} + "date": {"days_from_start": 28}, + "date_shown": {"days_from_start": 28, "format": "month_day"} }, "start_url": "https://www.delta.com/", "steps": [ @@ -25,6 +26,7 @@ "rules": [ {"path": "url", "matches": "^https://www\\.delta\\.com/(purchase/flights|completepurchase)/trip-summary"}, {"path": "text", "matches": "\\bATL - MCO\\s*\\n\\s*One Way\\b"}, + {"path": "text", "matches": "Journey date\\s*[^\\n]*\\b{{date_shown}}\\b"}, {"path": "extracted.cabin", "equals": "Main Classic"}, {"path": "extracted.passengers", "equals": 1}, {"path": "extracted.amount_due", "range": [1, 3000]} diff --git a/bench/test/page.test.mjs b/bench/test/page.test.mjs index 9370724c..02d9bcc2 100644 --- a/bench/test/page.test.mjs +++ b/bench/test/page.test.mjs @@ -1,5 +1,8 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { loadSchema, resolveFixture } from "../lib/intents.mjs"; import { extractValues, stepName, stepTimeline } from "../lib/page.mjs"; import { gradeAnswer } from "../grade.mjs"; @@ -66,3 +69,25 @@ test("a failing rule on long page text shows only the start of it", () => { const grade = gradeAnswer(intent, schema, "```json\n{}\n```", { output: { url: "u", text: "x".repeat(50_000) } }); assert.ok(grade.problems[0].length < 200, grade.problems[0].length); }); + +test("the Delta cart check needs the flight in the cart to be on the date the task asked for", () => { + const benchDir = fileURLToPath(new URL("..", import.meta.url)); + const original = JSON.parse(readFileSync(new URL("../tasks/delta.com/delta-cart-main-classic.json", import.meta.url), "utf8")); + // Trip Summary's visible text for DL 750 on Dec 10, as delta.com showed it. + const summary = "ATL - MCO\nOne Way\nDec 10\n1 Passenger\nModify\nTrip Summary\nATL\nto\nMCO\nChange Flight\nJourney date\nThu, Dec 10\nFlight number\nDL750\n" + + "Departure time\n6:05am - \nArrival time\n7:40am\nFlight duration\nNonstop, 1h 35m\nCabin class\nMain Classic\nChangeable / Nonrefundable\n" + + "Trip Total\nCurrency Calculator\n1 Passenger\nFlights\n$123.72\nTaxes, Fees & Charges\n$24.68\nAmount Due\nView Details\n$148.40 USD"; + const grade = (start, text = summary) => { + const intent = { ...original, fixture: resolveFixture(original.fixture, new Date(start)) }; + const page = { output: { url: "https://www.delta.com/purchase/flights/trip-summary", title: "Trip Summary", text, fields: {}, missing: [], invalid: [] } }; + return gradeAnswer(intent, loadSchema(benchDir, intent), '{"flight_numbers": ["DL 750"], "amount_due": 148.4}', page); + }; + assert.equal(grade("2026-11-12T16:00:00Z").outcome, "correct"); + assert.equal(grade("2026-11-12T16:00:00Z", summary.replace("Journey date\nThu, Dec 10", "Journey date Thu, Dec 10, 2026")).outcome, "correct"); + // The task asked for Dec 11 or Dec 1, and the cart holds a flight on Dec 10. + for (const start of ["2026-11-13T16:00:00Z", "2026-11-03T16:00:00Z"]) { + const wrong = grade(start); + assert.equal(wrong.outcome, "wrong"); + assert.match(wrong.problems.join(), /does not match \/Journey date/); + } +}); diff --git a/bench/test/tasks.test.mjs b/bench/test/tasks.test.mjs index f3d1ba17..81a61d7a 100644 --- a/bench/test/tasks.test.mjs +++ b/bench/test/tasks.test.mjs @@ -71,6 +71,8 @@ test("validateIntent reports missing fixture values and bad rules", () => { assert.ok(problems.some(p => p.includes("{{ip}} has no fixture value"))); assert.ok(problems.some(p => p.startsWith("rule "))); assert.deepEqual(validateIntent({ ...intent, fixture: { ip: { days_from_start: 3 } } }, benchDir), []); + assert.deepEqual(validateIntent({ ...intent, fixture: { ip: { days_from_start: 3, format: "month_day" } } }, benchDir), []); + assert.ok(validateIntent({ ...intent, fixture: { ip: { days_from_start: 3, format: "long" } } }, benchDir).some(p => p.startsWith("fixture ip must be"))); assert.ok(validateIntent({ ...intent, fixture: { ip: { days: 3 } } }, benchDir).some(p => p.startsWith("fixture ip"))); assert.deepEqual(validateIntent({ ...intent, rules: [{ path: "ip", matches: "^{{ip}}$" }] }, benchDir), []); assert.ok(validateIntent({ ...intent, rules: [{ path: "ip", matches: "(" }] }, benchDir).some(p => p.endsWith("needs a valid regex"))); @@ -116,6 +118,9 @@ test("relative fixture dates resolve from the run's start", () => { const start = new Date("2026-10-07T23:30:00Z"); assert.deepEqual(resolveFixture({ date: { days_from_start: 28 }, zip: "10001", n: 3 }, start), { date: "2026-11-04", zip: "10001", n: 3 }); assert.deepEqual(resolveFixture(undefined, start), {}); + // A date as the site's page shows it, for a rule on the page's text. + assert.deepEqual(resolveFixture({ date: { days_from_start: 28 }, shown: { days_from_start: 28, format: "month_day" } }, start), { date: "2026-11-04", shown: "Nov 4" }); + assert.deepEqual(resolveFixture({ shown: { days_from_start: 64, format: "month_day" } }, start), { shown: "Dec 10" }); }); test("URL patterns match the way Kernel's do", () => {