From 93813778a117abf9064b69174756fad14fea5568 Mon Sep 17 00:00:00 2001 From: Stefan Bauer Date: Mon, 16 Nov 2026 07:39:36 +0100 Subject: [PATCH 1/3] feat(storefront): search with Pagefind pd build indexes each page's own content (its rendered body, never the layout's markup) with the project's own Pagefind and writes the index to _pd/search/ under the site's path, with section, tag and category filters. storefront.search: left out, on when the project has Pagefind; true stops a build without it; false turns it off. The 404 page, tag and category pages and pages with index: false are left out. pd build --browser serves the index from memory and uses the site's search field once per viewport. pd dev builds no index and says so at start. --- package-lock.json | 118 +++++++++++++ package.json | 1 + packages/config/src/index.ts | 6 + packages/config/src/validate.spec.ts | 7 + packages/config/src/validate.ts | 2 +- packages/core/src/index.ts | 3 + packages/dev-server/src/index.ts | 9 + packages/dev-server/src/routes.ts | 4 + packages/dev-server/src/server-config.ts | 3 + packages/dev-server/src/static-build.ts | 43 ++++- .../src/storefront/browser-checks.ts | 72 ++++++-- packages/dev-server/src/storefront/page.ts | 21 ++- packages/dev-server/src/storefront/search.ts | 134 ++++++++++++++ .../dev-server/test/startup-addresses.spec.ts | 8 + .../dev-server/test/storefront-search.spec.ts | 164 ++++++++++++++++++ storefront-bench/e2e/build-browser.spec.ts | 20 ++- storefront-bench/e2e/playwright.config.ts | 2 +- storefront-bench/e2e/search.spec.ts | 161 +++++++++++++++++ storefront-bench/e2e/static.spec.ts | 12 ++ storefront-bench/pattern-dump.config.ts | 1 + 20 files changed, 775 insertions(+), 16 deletions(-) create mode 100644 packages/dev-server/src/storefront/search.ts create mode 100644 packages/dev-server/test/storefront-search.spec.ts create mode 100644 storefront-bench/e2e/search.spec.ts diff --git a/package-lock.json b/package-lock.json index 4e12f8c..a17d6cd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,6 +39,7 @@ "handlebars": "^4.7.8", "husky": "^9.1.7", "nx": "^19.0.0", + "pagefind": "^1.6.0", "postcss": "^8.4.0", "sass": "^1.77.0", "stylelint": "^17.12.0", @@ -2363,6 +2364,104 @@ "node": ">= 10" } }, + "node_modules/@pagefind/darwin-arm64": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@pagefind/darwin-arm64/-/darwin-arm64-1.6.0.tgz", + "integrity": "sha512-611gl64aqN12UcMyKPtZ8+vRWBKlJFSh2JgO8uScxa1Eyjo1dXz9r4wZuRno0yCUE/KN0FauEoKZ3KEwUYHCCg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@pagefind/darwin-x64": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@pagefind/darwin-x64/-/darwin-x64-1.6.0.tgz", + "integrity": "sha512-rd3ZbtuxyYqjASKZAPuIV7eeQQHJ7VYpoNKKG8CvglaD3g30eT5Nb7XGNLYwVReBsCccpkURwrGwWjSwr8nyQQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@pagefind/freebsd-x64": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@pagefind/freebsd-x64/-/freebsd-x64-1.6.0.tgz", + "integrity": "sha512-d+sXqsaThszmPWGVDSRKqIz9GpZcj/bAksNZfg19asSxU0ogNPOMND4kV4giQvOjPFMsltUFf6hQJfoHQ6+hSQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@pagefind/linux-arm64": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@pagefind/linux-arm64/-/linux-arm64-1.6.0.tgz", + "integrity": "sha512-SLB2I+XqCZma8NvIGIH8nyKblgKIxw5PmqsVtofWOKxNHhEtOEsddbqU+hsrNcT3EV0faM6UmKDHVIxcPYq29w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@pagefind/linux-x64": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@pagefind/linux-x64/-/linux-x64-1.6.0.tgz", + "integrity": "sha512-IbMSx/+Vr/Xk0TTBlLkjgK+rvnbOXGmmb2/MKxQu99AR5DsqQoV3NKEA9PQz2tAhuCZjeD5xJBp4w5N5DHIC7Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@pagefind/windows-arm64": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@pagefind/windows-arm64/-/windows-arm64-1.6.0.tgz", + "integrity": "sha512-wUrniW3REr7VWJHz6xE6F/p/AQgHDFpGZiY9sbWT71KfEijDE5Jnc37V8zc7cilV9h/QkwJGXoEP2+ZLwDmhtw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@pagefind/windows-x64": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@pagefind/windows-x64/-/windows-x64-1.6.0.tgz", + "integrity": "sha512-cUA2+yXfbymdJllyMjY5YTBwuOgh0/H53Pn5fYKjDnDjk154Ft4U5RVj3TenWySPL/jG6zJfARUWSlyUyXHlZw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@parcel/watcher": { "version": "2.5.6", "resolved": "https://registry.npmjs.org/@parcel/watcher/-/watcher-2.5.6.tgz", @@ -7325,6 +7424,25 @@ "resolved": "package-bench", "link": true }, + "node_modules/pagefind": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/pagefind/-/pagefind-1.6.0.tgz", + "integrity": "sha512-yeOd9EA0TUE1PUfbQIluBxzPadBG5/0BV48FAxwMVLkhZcOvwCz6TX4/a/Ofz97pF9/FwwT7CVbTC97BfmI+LQ==", + "dev": true, + "license": "MIT", + "bin": { + "pagefind": "lib/runner/bin.cjs" + }, + "optionalDependencies": { + "@pagefind/darwin-arm64": "1.6.0", + "@pagefind/darwin-x64": "1.6.0", + "@pagefind/freebsd-x64": "1.6.0", + "@pagefind/linux-arm64": "1.6.0", + "@pagefind/linux-x64": "1.6.0", + "@pagefind/windows-arm64": "1.6.0", + "@pagefind/windows-x64": "1.6.0" + } + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", diff --git a/package.json b/package.json index 896aaff..c57d798 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,7 @@ "handlebars": "^4.7.8", "husky": "^9.1.7", "nx": "^19.0.0", + "pagefind": "^1.6.0", "postcss": "^8.4.0", "sass": "^1.77.0", "stylelint": "^17.12.0", diff --git a/packages/config/src/index.ts b/packages/config/src/index.ts index b062997..894203f 100644 --- a/packages/config/src/index.ts +++ b/packages/config/src/index.ts @@ -255,6 +255,12 @@ export interface StorefrontConfig { /** A page per category and an index of them, as tagPages (`layouts/category`, `layouts/categories`; default path * `categories`, title "Categories"). */ categoryPages?: boolean | StorefrontTermPages; + /** + * Search with Pagefind: pd build indexes each page's own content and writes the index to `_pd/search/` in the site. + * Pagefind is the project's own (`npm install --save-dev pagefind`). Left out: on when the project has Pagefind. + * `true`: on, and a build without Pagefind stops. `false`: off. The search UI is a component of the skin's. + */ + search?: boolean; } /** storefront.tagPages and storefront.categoryPages as an object. */ diff --git a/packages/config/src/validate.spec.ts b/packages/config/src/validate.spec.ts index 3665717..1b3d094 100644 --- a/packages/config/src/validate.spec.ts +++ b/packages/config/src/validate.spec.ts @@ -258,6 +258,13 @@ describe("configProblems — storefront", () => { ]); }); + it("takes search: true or false", () => { + const base = { pagesDir: "./pages", outDir: "./www" }; + expect(storefront({ ...base, search: true })).toEqual([]); + expect(storefront({ ...base, search: false })).toEqual([]); + expect(storefront({ ...base, search: "pagefind" })).toEqual([`"storefront.search" must be true or false`]); + }); + it("takes tagPages and categoryPages: true, false, or { path, title }", () => { const base = { pagesDir: "./pages", outDir: "./www" }; for (const ok of [true, false, {}, { path: "topics" }, { title: "Topics" }, { path: "t2", title: "All tags" }]) { diff --git a/packages/config/src/validate.ts b/packages/config/src/validate.ts index c7206eb..c719e73 100644 --- a/packages/config/src/validate.ts +++ b/packages/config/src/validate.ts @@ -63,7 +63,7 @@ const SHAPE: Shape = { keys: { pagesDir: "string", outDir: "string", host: "host", workbenchUrl: "workbench-url", site: "plain-object", url: "site-url", lang: "lang", index: "boolean", image: "asset-path", images: IMAGES, head: "hbs-file", - categories: "categories", tags: "tags", tagPages: "term-pages", categoryPages: "term-pages", + categories: "categories", tags: "tags", tagPages: "term-pages", categoryPages: "term-pages", search: "boolean", }, required: ["pagesDir", "outDir"], }, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 43729f2..feb27d0 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -65,6 +65,9 @@ export async function runBuild( console.log(`[pd build] images: ${n ? `${n} with a variant that doesn't fit:` : "every variant fits its box"}`); for (const w of report.images.warnings) console.log(`[pd build] warning: ${w}`); } + if (report.search) { + console.log(`[pd build] search: ${report.search.pages} pages indexed → ${report.search.files} files in _pd/search/ (${(report.search.bytes / 1024).toFixed(0)} KB)`); + } if (report.drafts) { console.log(`[pd build] drafts: ${report.drafts.length} left out (${report.drafts.join(", ")})`); } diff --git a/packages/dev-server/src/index.ts b/packages/dev-server/src/index.ts index 6e0c7eb..13041c0 100644 --- a/packages/dev-server/src/index.ts +++ b/packages/dev-server/src/index.ts @@ -64,6 +64,7 @@ import { surfaceAssetRoots, surfaceCollisions, surfaceDefaultTheme, surfaceDocum export { runDiagnosticsAudit } from "./audit.js"; export { buildStaticSite } from "./static-build.js"; export { withBasePath } from "./snapshot-writer.js"; +export { loadProjectPagefind, PAGEFIND_INSTALL } from "./storefront/search.js"; export type { StaticBuildOptions, StaticBuildReport } from "./static-build.js"; // Folder overview gallery — public leaves under a group node. // Component detail. Widest fan-out in the package — nothing may import it back. @@ -89,6 +90,7 @@ import { watchedRoots } from "./watch-roots.js"; import { startWatching, type WatchHandle } from "./watch.js"; import { createGitDates } from "./storefront/git-dates.js"; import { readPages } from "./storefront/pages.js"; +import { projectPagefindEntry, type SearchRecord } from "./storefront/search.js"; // The style index — class ownership attributed from compiled CSS + source maps, and the SCSS // module graph. Substrate for the style and style-modules graphs. import { buildAllStyleData } from "./style-index.js"; @@ -141,6 +143,8 @@ export interface DevServerOptions { /** `pd build` only: receives each note a storefront page gives (a suffixed heading anchor), for the build * report. Without it, notes are printed to the terminal. */ onStorefrontNote?: (note: string) => void; + /** `pd build` only: receives what each storefront page gives the search index (search.ts), as it is served. */ + onSearchRecord?: (record: SearchRecord) => void; /** `pd dev` only: the config file, watched, and how to load it again (core owns config loading). A change * to `storefront.site` alone is taken live and every storefront page reloads; any other change asks for a * restart. Without them the config is read once, at start. */ @@ -338,6 +342,7 @@ export async function startDevServer( ...(live ? { generation: () => hotreloader.generation() } : {}), ...(opts.storefrontWorkbenchUrl ? { storefrontWorkbenchUrl: opts.storefrontWorkbenchUrl } : {}), ...(opts.onStorefrontNote ? { onStorefrontNote: opts.onStorefrontNote } : {}), + ...(opts.onSearchRecord ? { onSearchRecord: opts.onSearchRecord } : {}), }; const httpServer = createServer((req, res) => { void handleRequest(req, res, routeCtx); }); @@ -356,6 +361,10 @@ export async function startDevServer( if (live) { if (cfg.storefront) console.log(`[dev-server] Website: http://${cfg.storefront.host}:${actualPort}${cfg.storefront.basePath}`); console.log(`[dev-server] Workbench: http://${cfg.storefront ? PD_HOSTS.workbench : "localhost"}:${actualPort}/`); + // Search is built by pd build only: an index on every save would slow it down. + if (cfg.storefront && cfg.storefront.search !== false && (cfg.storefront.search === true || projectPagefindEntry(cfg.cwd) !== undefined)) { + console.log(`[dev-server] Search: the index is built by pd build, so the site's search finds nothing in pd dev`); + } } const startupDiscovery = await cachedDiscovery(state); const componentCount = startupDiscovery ? [...startupDiscovery.discoveryMap.keys()].filter(id => !id.includes("~")).length : 0; diff --git a/packages/dev-server/src/routes.ts b/packages/dev-server/src/routes.ts index 8bcc716..80e455e 100644 --- a/packages/dev-server/src/routes.ts +++ b/packages/dev-server/src/routes.ts @@ -62,6 +62,7 @@ import { checkPageImage } from "./storefront/page-image.js"; import { sitemapEntries, sitemapXml, storefrontRobotsTxt, workbenchRobotsTxt } from "./storefront/site-files.js"; import { redirectStub } from "./storefront/redirects.js"; import { assemblePage } from "./storefront/page.js"; +import type { SearchRecord } from "./storefront/search.js"; import { pageStylesheets } from "./storefront/page-head.js"; import type { RenderingHandle } from "./rendering.js"; import type { SiteValuesHandle } from "./storefront/site-values.js"; @@ -108,6 +109,8 @@ export interface RouteContext { readonly storefrontWorkbenchUrl?: string; /** Set by `pd build`: where a storefront page's notes go. Without it they go to the terminal. */ readonly onStorefrontNote?: (note: string) => void; + /** Set by `pd build`: where what a storefront page gives the search index goes. */ + readonly onSearchRecord?: (record: SearchRecord) => void; /** The storefront pages' dates from git, when there is a storefront (git-dates.ts). */ readonly dates?: GitDates; /** TEST ONLY: set when PD_TEST_HOOKS=1 on a live server; resolves once every change so far is handled. */ @@ -696,6 +699,7 @@ async function routeRequest( ...(cfg.storefront.image !== undefined ? { siteImage: cfg.storefront.image } : {}), ...(ctx.images ? { images: ctx.images } : {}), }); + if (page.search && ctx.onSearchRecord) ctx.onSearchRecord(page.search); if (page.problems.length) console.error(`[storefront] ${route}:\n${page.problems.map(p => ` - ${p}`).join("\n")}`); for (const note of page.notes) { if (ctx.onStorefrontNote) { ctx.onStorefrontNote(note); continue; } diff --git a/packages/dev-server/src/server-config.ts b/packages/dev-server/src/server-config.ts index 5542c00..fc14f6d 100644 --- a/packages/dev-server/src/server-config.ts +++ b/packages/dev-server/src/server-config.ts @@ -107,6 +107,8 @@ export interface ResolvedConfig { /** storefront.tagPages and storefront.categoryPages, as the config writes them (terms.ts reads them). */ readonly tagPages: boolean | StorefrontTermPages | undefined; readonly categoryPages: boolean | StorefrontTermPages | undefined; + /** storefront.search as the config writes it (search.ts reads it). */ + readonly search: boolean | undefined; } | null; readonly overviewsEnabled: boolean; @@ -250,6 +252,7 @@ export function resolveServerConfig( tags: config.storefront.tags, tagPages: config.storefront.tagPages, categoryPages: config.storefront.categoryPages, + search: config.storefront.search, } : null, images, overviewsEnabled: config.overviews !== false, diff --git a/packages/dev-server/src/static-build.ts b/packages/dev-server/src/static-build.ts index 0a108b3..4382664 100644 --- a/packages/dev-server/src/static-build.ts +++ b/packages/dev-server/src/static-build.ts @@ -11,7 +11,7 @@ // - the shell: one shell page per route, carrying the static ShellRuntime instead of a port. import { cp, mkdir, readdir, rm, stat, writeFile } from "node:fs/promises"; -import { basename, join, relative, resolve, sep } from "node:path"; +import { basename, dirname, join, relative, resolve, sep } from "node:path"; import { createImageCache } from "@pattern-dump/build"; import { sameHostWorkbenchPath, type PatternDumpConfig } from "@pattern-dump/config"; import { @@ -33,6 +33,7 @@ import { loadProjectPlaywright, runBrowserChecks, VIEWPORTS, type PlaywrightModu import { runPageA11y, type PageA11yFinding } from "@pattern-dump/a11y"; import { workbenchRobotsTxt } from "./storefront/site-files.js"; import { withNoindex } from "./storefront/base-path.js"; +import { buildSearchIndex, SEARCH_PATH, searchPagefind, type PagefindModule, type SearchRecord } from "./storefront/search.js"; export interface StaticBuildOptions { /** Absolute output directory. Emptied first. */ @@ -53,6 +54,8 @@ export interface StaticBuildOptions { browser?: boolean; /** Where `browser` gets Playwright from. Default: the project's own (loadProjectPlaywright). */ loadPlaywright?: (root: string) => Promise; + /** Where search gets Pagefind from. Default: the project's own (loadProjectPagefind); undefined for none. */ + loadPagefind?: (root: string) => Promise; } /** Not a test's watcher barrier (`.__testOnly_barrier`, which tests and the settle hook write into watched @@ -78,6 +81,9 @@ export interface StaticBuildReport { /** The storefront's accessibility check (axe in jsdom, and one
and one

per page): the site pages read, * and one line per rule that only warns, with its pages. An error stops the build instead. */ accessibility?: { pages: number; warnings: string[]; browser?: boolean }; + /** The storefront's search index (search.ts), written to its _pd/search/: the pages in it, and its files. Absent + * when the site has no search. */ + search?: { pages: number; files: number; bytes: number }; /** `--browser`: one line per image whose chosen variant doesn't fit the box it is laid out in. */ images?: { warnings: string[] }; /** The storefront's drafts left out of the build (a draft, and each page under a draft folder's index.md), by @@ -89,6 +95,16 @@ export interface StaticBuildReport { ms: number; } +/** A word --browser searches for: the longest word of the first indexed page's title (by route) that has one of three + * letters or more, so it finds that page. */ +export function searchWord(records: readonly SearchRecord[]): string { + for (const r of [...records].sort((a, b) => a.route < b.route ? -1 : 1)) { + const words = r.title.match(/\p{L}{3,}/gu); + if (words) return [...words].sort((a, b) => b.length - a.length)[0]!; + } + return "search"; +} + export async function buildStaticSite( config: PatternDumpConfig, root: string, @@ -141,6 +157,10 @@ export async function buildStaticSite( // --browser: the project's Playwright, loaded before anything is built, so a missing one stops the build at once. const playwright = opts.browser && storefront ? await (opts.loadPlaywright ?? loadProjectPlaywright)(project) : undefined; + // Search: the project's Pagefind, also before anything is built, so a missing one the site asks for stops at once. + const pagefind = storefront ? await searchPagefind(storefront.search, project, opts.loadPagefind) : undefined; + /** What each page gives the index, by route, as the crawl serves it. */ + const searchRecords = new Map(); const fixed = opts.now; const now = fixed ? (): Date => new Date(fixed.getTime()) : (): Date => new Date(); @@ -151,6 +171,7 @@ export async function buildStaticSite( const server = await startDevServer(config, root, { port: 0, live: false, now, today, ...(storefront?.workbenchUrl ? { storefrontWorkbenchUrl: storefront.workbenchUrl } : {}), onStorefrontNote: note => { storefrontNotes.add(note); }, + ...(pagefind ? { onSearchRecord: (r: SearchRecord) => { searchRecords.set(r.route, r); } } : {}), }); log("… server ready"); @@ -169,6 +190,12 @@ export async function buildStaticSite( throw new Error(`pd build: the storefront has problems, so nothing was written:\n${sfCrawl.problems.map(p => ` - ${p}`).join("\n")}`); } if (sfCrawl) log(`… storefront checked: ${sfCrawl.urls.length} files`); + // The search index, from the pages the crawl served, in memory: checked in Chromium with --browser, written with the + // site. + const searchFiles = pagefind && sfCrawl + ? await buildSearchIndex([...searchRecords.values()], pagefind, sfBase).catch(async (e: unknown) => { await server.close(); throw e; }) + : undefined; + if (searchFiles) log(`… search index: ${searchRecords.size} pages`); const origin = `http://127.0.0.1:${server.port}`; // The link check (links.ts), also before anything is written: every reference on every site page (its pages and @@ -239,7 +266,7 @@ export async function buildStaticSite( let imageWarnings: string[] | undefined; const browserWarnings: string[] = []; if (playwright) { - const checked = await runBrowserChecks({ pages, get: sfGet, playwright, basePath: sfBase }).catch(async (e: unknown) => { await server.close(); throw e; }); + const checked = await runBrowserChecks({ pages, get: sfGet, playwright, basePath: sfBase, ...(searchFiles ? { search: { files: searchFiles, word: searchWord([...searchRecords.values()]) } } : {}) }).catch(async (e: unknown) => { await server.close(); throw e; }); if (checked.errors.length) { await server.close(); throw new Error(`pd build: the storefront has problems in a browser, so nothing was written:\n${checked.errors.map(e => ` - ${e}`).join("\n")}`); @@ -390,6 +417,18 @@ export async function buildStaticSite( } await writeFile(join(sfOut, ".nojekyll"), ""); counts.files++; + if (searchFiles) { + let bytes = 0; + for (const [path, content] of searchFiles) { + const file = join(sfOut, ...SEARCH_PATH.split("/").filter(Boolean), ...path.split("/")); + await mkdir(dirname(file), { recursive: true }); + await writeFile(file, content); + bytes += content.byteLength; + } + counts.files += searchFiles.size; + counts.bytes += bytes; + report.search = { pages: searchRecords.size, files: searchFiles.size, bytes }; + } const stubs = new Set((await (await sfGet(PD_ROUTES.pages)).json() as StorefrontPagesResponse).pages.flatMap(p => p.redirectFrom ?? [])); const pages = sfCrawl.urls.filter(u => !/^\/(?:_pd|assets|preview)\//.test(u) && !sfFiles.includes(u) && !stubs.has(u)).length; report.storefront = { outDir: sfOut, basePath: sfBase, pages, files: counts.files, bytes: counts.bytes }; diff --git a/packages/dev-server/src/storefront/browser-checks.ts b/packages/dev-server/src/storefront/browser-checks.ts index a5b52eb..23bbe3a 100644 --- a/packages/dev-server/src/storefront/browser-checks.ts +++ b/packages/dev-server/src/storefront/browser-checks.ts @@ -11,6 +11,11 @@ // - on a site published under a path: every request the page makes to the site's own host outside that path is an // error. The base-path pass puts the path in front of every reference in the markup and CSS, so such a request // comes from a script that builds its URL itself (it can read ). +// - with search (search.ts): the index, served from memory at _pd/search/. On the first page with a search field +// (input[type=search]), at each viewport, a word of an indexed page's title is typed and submitted: the page must +// load a result (a fragment of the index) within 5 s, and is checked with axe again showing it. Once per viewport: +// the search component is the skin's, the same on every page. A request under _pd/search/ the index doesn't have +// is an error. // // Playwright is the project's own (`npm install --save-dev playwright`); Pattern Dump does not depend on it. Without // it, or without its Chromium, the build stops with a named error before anything is built. @@ -18,6 +23,12 @@ import { createRequire } from "node:module"; import { join } from "node:path"; import { pathToFileURL } from "node:url"; import { axeScript, pageRuleTiers, severityOf } from "@pattern-dump/a11y"; +import { SEARCH_PATH } from "./search.js"; + +/** A page's search field. */ +const FIELD = `input[type="search"]`; +const typeOf = (path: string): string => path.endsWith(".js") ? "text/javascript" : path.endsWith(".json") ? "application/json" + : "application/octet-stream"; /** The part of Playwright this uses. */ export interface PlaywrightModule { @@ -32,6 +43,9 @@ interface PwContext { interface PwRoute { request(): { url(): string }; fulfill(o: { status: number; headers?: Record; body: Buffer }): Promise; abort(): Promise } interface PwPage { goto(url: string, o?: { waitUntil?: "load" }): Promise; + fill(selector: string, value: string): Promise; + press(selector: string, key: string): Promise; + waitForTimeout(ms: number): Promise; addScriptTag(o: { content: string }): Promise; evaluate(fn: (arg: A) => R | Promise, arg: A): Promise; } @@ -78,6 +92,8 @@ export async function runBrowserChecks(o: { get: (url: string) => Promise; playwright: PlaywrightModule; basePath?: string; + /** The site's search index, by path under _pd/search/, and a word to search for that finds a page. */ + search?: { files: ReadonlyMap; word: string }; }): Promise { const base = o.basePath ?? "/"; let browser: PwBrowser; @@ -97,6 +113,8 @@ export async function runBrowserChecks(o: { }; /** The page being checked: the requests it makes are named by it. */ let current = ""; + /** The results the pages loaded from the search index (its fragments). */ + let fragments = 0; try { for (const vp of VIEWPORTS) { const context = await browser.newContext({ viewport: vp.viewport, deviceScaleFactor: vp.dpr }); @@ -104,6 +122,19 @@ export async function runBrowserChecks(o: { await context.route("**/*", async route => { const url = new URL(route.request().url()); if (url.origin !== origin) { await route.abort(); return; } + const searchAt = `${base}${SEARCH_PATH.slice(1)}`; + if (o.search && url.pathname.startsWith(searchAt)) { + const file = o.search.files.get(url.pathname.slice(searchAt.length)); + if (file === undefined) { + const text = `${current}: a request to ${url.pathname} is under ${searchAt}, and the search index has no such file`; + note("error", text, text, vp.name); + await route.fulfill({ status: 404, body: Buffer.from("") }); + return; + } + if (url.pathname.startsWith(`${searchAt}fragment/`)) fragments++; + await route.fulfill({ status: 200, headers: { "content-type": typeOf(url.pathname) }, body: Buffer.from(file) }); + return; + } if (!url.pathname.startsWith(base)) { // The browser's own favicon request is not the page's. if (url.pathname !== "/favicon.ico") { @@ -121,20 +152,24 @@ export async function runBrowserChecks(o: { await route.fulfill({ status: res.status, headers, body: Buffer.from(await res.arrayBuffer()) }); }); const page = await context.newPage(); + let searched = false; for (const path of o.pages) { current = path; await page.goto(`${origin}${base}${path.slice(1)}`, { waitUntil: "load" }); await page.addScriptTag({ content: axe }); - const violations = await page.evaluate(async (rules: string[]) => { - const r = await (window as unknown as { axe: { run(c: unknown, o: unknown): Promise<{ violations: unknown[] }> } }).axe - .run(document, { runOnly: { type: "rule", values: rules }, resultTypes: ["violations"], iframes: false }); - return r.violations; - }, pageRuleTiers.browser) as AxeViolation[]; - for (const v of violations) { - const where = v.nodes.map(n => n.target.map(String).join(" ")).join(", "); - const text = `${path}: ${v.id} (${v.impact ?? "minor"}): ${v.help}, on ${where} (${v.helpUrl})`; - note(severityOf(v.impact, v.tags) === "error" ? "error" : "a11y", `a11y ${text}`, text, vp.name); - } + const runAxe = async (state: string) => { + const violations = await page.evaluate(async (rules: string[]) => { + const r = await (window as unknown as { axe: { run(c: unknown, o: unknown): Promise<{ violations: unknown[] }> } }).axe + .run(document, { runOnly: { type: "rule", values: rules }, resultTypes: ["violations"], iframes: false }); + return r.violations; + }, pageRuleTiers.browser) as AxeViolation[]; + for (const v of violations) { + const where = v.nodes.map(n => n.target.map(String).join(" ")).join(", "); + const text = `${path}${state}: ${v.id} (${v.impact ?? "minor"}): ${v.help}, on ${where} (${v.helpUrl})`; + note(severityOf(v.impact, v.tags) === "error" ? "error" : "a11y", `a11y ${text}`, text, vp.name); + } + }; + await runAxe(""); // Each image with variants, in view (a lazy one loads), then its chosen candidate against its box. const images = await page.evaluate(async () => { const seen: ImageSeen[] = []; @@ -155,6 +190,23 @@ export async function runBrowserChecks(o: { } return seen; }, undefined) as ImageSeen[]; + // The search field, used (its images are read already): a word that finds a page, submitted, must load a result. + if (o.search && !searched && await page.evaluate((field: string) => document.querySelector(field) !== null, FIELD)) { + searched = true; + const before = fragments; + await page.fill(FIELD, o.search.word); + await page.press(FIELD, "Enter"); + for (let waited = 0; fragments === before && waited < 5000; waited += 100) await page.waitForTimeout(100); + if (fragments === before) { + const text = `${path}: searching for "${o.search.word}" in its search field (${FIELD}) showed no result within 5 s, ` + + `though the index has a page with it: the search component loads ${SEARCH_PATH}pagefind.js under the site's path ` + + `() and shows each result's data()`; + note("error", text, text, vp.name); + } else { + await page.addScriptTag({ content: axe }); + await runAxe(` (showing the results for "${o.search.word}")`); + } + } for (const img of images) { if (img.chosen === null || img.box === 0) continue; const need = Math.round(img.box * vp.dpr); diff --git a/packages/dev-server/src/storefront/page.ts b/packages/dev-server/src/storefront/page.ts index ceed142..fb1d802 100644 --- a/packages/dev-server/src/storefront/page.ts +++ b/packages/dev-server/src/storefront/page.ts @@ -16,6 +16,7 @@ import { createHeightFitter, generationBehind, PD_ROUTES, type DiscoveryMap, type ImageLookup, type Render, type StorefrontNavItem } from "@pattern-dump/contracts"; import { frontmatterBody, frontmatterBodyOffset, readFrontmatterStrict } from "../frontmatter.js"; import { escapeAttr, escapeHtml } from "../http-utils.js"; +import type { SearchRecord } from "./search.js"; import { isHelperNameProblem, itemsWrittenAsText, readProseSlot, renderPlainTemplate } from "@pattern-dump/renderer"; import { basename } from "node:path"; import { renderDoc, type ProseImages } from "../markdown.js"; @@ -96,6 +97,9 @@ export interface AssembledPage { problems: string[]; /** Notes about a page that rendered, one line each: a heading whose anchor was suffixed (anchors.ts). */ notes: string[]; + /** What the page gives the site's search index (search.ts): a page of the site that rendered, other than the 404 + * page, a tag or category page, and a page with `index: false`. */ + search?: SearchRecord; } /** The fitter's glue: a frame with a declared height (its `height` attribute, from the component's @@ -766,5 +770,20 @@ export async function assemblePage(route: string, model: PageModel, a: PageAssem // One main landmark and one

(page-basics.ts): errors in pd build, warnings here. notes.push(...checkPageBasics(html).map(p => `warning: ${page!.path}: ${p}`)); } - return { status: isNotFound ? 404 : 200, html: document(head, `${html}\n${fitterScript()}${html.includes(`class="shiki-container"`) ? `\n${copyScript()}` : ""}`, a.live, a.lang, a.basePath ?? "/"), problems: [], notes }; + // The page's own content for the search index: its body, never the layout's markup (search.ts). + const entry = model.entries.get(route); + const search: SearchRecord | undefined = isNotFound || generated || checked.build.index === false ? undefined : { + route, title, body: finalBody, section: collectionOf(route, model), tags: (entry?.tags ?? []).map(t => t.label), + categories: (entry?.categories ?? []).map(c => c.label), teaser: entry?.description ?? null, lang: a.lang ?? "en", + }; + return { status: isNotFound ? 404 : 200, html: document(head, `${html}\n${fitterScript()}${html.includes(`class="shiki-container"`) ? `\n${copyScript()}` : ""}`, a.live, a.lang, a.basePath ?? "/"), problems: [], notes, + ...(search ? { search } : {}) }; +} + +/** The label of the collection `route` is in: the folder that lists it (a folder's index page is its own), by its + * index page's title; null for a page at the top. */ +export function collectionOf(route: string, model: PageModel): string | null { + const folder = route !== "/" && model.folders.has(route) ? route + : [...model.folders].find(([r, f]) => r !== "/" && f.children.includes(route))?.[0]; + return folder === undefined ? null : model.pages.find(p => p.route === folder)?.title ?? null; } diff --git a/packages/dev-server/src/storefront/search.ts b/packages/dev-server/src/storefront/search.ts new file mode 100644 index 0000000..17462af --- /dev/null +++ b/packages/dev-server/src/storefront/search.ts @@ -0,0 +1,134 @@ +// Search (step 5): pd build indexes the site with Pagefind and writes the index into the site, at /_pd/search/. No +// server: the skin's search component loads /_pd/search/pagefind.js (under the site's path, which +// names) and searches in the browser. +// +// - What is indexed is each page's own content, which Pattern Dump knows: its rendered body (markdown and the +// components it places), with its title. Never the layout's markup: header, footer, navigation, listings and +// related pages (other pages' titles and teasers) and the copy buttons (added by script) are never in the index. +// The written pages carry no Pagefind attributes; each page is given to Pagefind as a document of its own. +// - Left out: the 404 page (no address of its own), tag and category pages and their indexes (other pages' labels +// and teasers; the filters do their job), redirect stubs, and a page with `index: false`. Drafts and scheduled +// pages are never built. +// - Filters: section (the label of the page's collection), tag and category (their labels). Meta: title, teaser. +// - storefront.search: true asks for it (no Pagefind stops the build); false turns it off; left out, it is on when +// the project has Pagefind. Pagefind is the project's own (`npm install --save-dev pagefind`), as Playwright is +// for --browser: Pattern Dump does not depend on it. +// - pd dev builds no index (a save stays as fast as it is): search works on the built site. + +import { existsSync, readFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { escapeAttr, escapeHtml } from "../http-utils.js"; + +/** Where the index is written, under the site's path. */ +export const SEARCH_PATH = "/_pd/search/"; + +/** What a page gives the index. */ +export interface SearchRecord { + route: string; + title: string; + /** The page's rendered body, without the layout. */ + body: string; + /** The label of the page's collection (the folder that lists it), or null for a page at the top. */ + section: string | null; + tags: string[]; + categories: string[]; + /** Its listing teaser: its description, else its first paragraph. */ + teaser: string | null; + lang: string; +} + +/** The part of Pagefind this uses. */ +export interface PagefindModule { + createIndex(config?: Record): Promise<{ errors: string[]; index?: PagefindIndex }>; + close(): Promise; +} +interface PagefindIndex { + addHTMLFile(file: { url: string; content: string }): Promise<{ errors: string[] }>; + getFiles(): Promise<{ errors: string[]; files: Array<{ path: string; content: Uint8Array }> }>; +} + +/** Search is asked for and Pagefind couldn't be loaded: what to install. */ +export class SearchUnavailableError extends Error {} + +export const PAGEFIND_INSTALL = "npm install --save-dev pagefind"; + +/** + * The entry of the project's Pagefind: its package in node_modules, from the project's root up (Pagefind exports + * only an `import` condition, so require.resolve can't find it); undefined when it has none. + */ +export function projectPagefindEntry(root: string): string | undefined { + for (let dir = resolve(root); ; dir = dirname(dir)) { + const pkg = join(dir, "node_modules", "pagefind", "package.json"); + if (existsSync(pkg)) { + const json = JSON.parse(readFileSync(pkg, "utf8")) as { main?: string; exports?: { ".": { import?: string } } }; + return join(dirname(pkg), json.exports?.["."]?.import ?? json.main ?? "lib/index.js"); + } + if (dirname(dir) === dir) return undefined; + } +} + +/** The project's Pagefind; undefined when it has none. */ +export async function loadProjectPagefind(root: string): Promise { + const entry = projectPagefindEntry(root); + if (entry === undefined) return undefined; + const mod = await import(pathToFileURL(entry).href) as Partial; + return typeof mod.createIndex === "function" && typeof mod.close === "function" ? mod as PagefindModule : undefined; +} + +/** The Pagefind to index with, by storefront.search: undefined when there is no search; a stop when it is asked for + * and the project has none. */ +export async function searchPagefind(setting: boolean | undefined, root: string, + load: (root: string) => Promise = loadProjectPagefind): Promise { + if (setting === false) return undefined; + const pagefind = await load(root); + if (!pagefind && setting === true) { + throw new SearchUnavailableError(`pd build: storefront.search is on, and this project has no Pagefind to build the index with: ${PAGEFIND_INSTALL} ` + + `(or set storefront.search to false)`); + } + return pagefind; +} + +/** The document Pagefind is given for a page: its title, its filters and meta outside the indexed part, and its body, + * with the title as its

when the body has none. */ +export function searchDocument(r: SearchRecord): string { + const filter = (name: string, values: readonly string[]) => values.map(v => `${escapeHtml(v)}`).join(""); + const meta = (name: string, value: string | null) => value === null ? "" : `${escapeHtml(value)}`; + const h1 = /]/i.test(r.body) ? "" : `

${escapeHtml(r.title)}

\n`; + return `\n${escapeHtml(r.title)}\n` + + `${meta("title", r.title)}${meta("teaser", r.teaser)}${filter("section", r.section === null ? [] : [r.section])}` + + `${filter("tag", r.tags)}${filter("category", r.categories)}\n` + + `
\n${h1}${r.body}\n
\n`; +} + +/** Pagefind's own UIs: the skin has its own, so they are not written. */ +const UI_FILE = /^pagefind-(?:ui|modular-ui|component-ui|highlight)\.(?:js|css)$/; + +/** + * The index of `records`, as files by their path under SEARCH_PATH. `basePath`: the path the site is published under, + * in front of each result's URL. Sorted by path, so a build writes them in one order. + */ +export async function buildSearchIndex(records: readonly SearchRecord[], pagefind: PagefindModule, basePath: string): Promise> { + try { + return await indexWith(records, pagefind, basePath); + } finally { + await pagefind.close(); + } +} + +async function indexWith(records: readonly SearchRecord[], pagefind: PagefindModule, basePath: string): Promise> { + const created = await pagefind.createIndex({}); + if (!created.index) throw new Error(`pd build: Pagefind couldn't start: ${created.errors.join("; ")}`); + const index = created.index; + const errors: string[] = []; + for (const r of [...records].sort((a, b) => a.route < b.route ? -1 : a.route > b.route ? 1 : 0)) { + const added = await index.addHTMLFile({ url: `${basePath}${r.route.slice(1)}`, content: searchDocument(r) }); + errors.push(...added.errors.map(e => `${r.route}: ${e}`)); + } + const got = await index.getFiles(); + errors.push(...got.errors); + if (errors.length) throw new Error(`pd build: Pagefind couldn't index the site:\n${errors.map(e => ` - ${e}`).join("\n")}`); + const files = new Map(); + for (const f of [...got.files].sort((a, b) => a.path < b.path ? -1 : 1)) if (!UI_FILE.test(f.path)) files.set(f.path, f.content); + return files; +} diff --git a/packages/dev-server/test/startup-addresses.spec.ts b/packages/dev-server/test/startup-addresses.spec.ts index bea074a..4f07900 100644 --- a/packages/dev-server/test/startup-addresses.spec.ts +++ b/packages/dev-server/test/startup-addresses.spec.ts @@ -46,6 +46,14 @@ describe("pd dev's startup addresses", () => { expect(lines.at(-1)).toMatch(/^\[dev-server\] Ready in \d+ms, \d+ components$/); }, 60_000); + it("says that search works in pd build only: with Pagefind (the repository's), not with search: false", async () => { + const note = "[dev-server] Search: the index is built by pd build, so the site's search finds nothing in pd dev"; + const on = await startupLines(benchConfig, storefrontBench, { port: 0, wsPort: 0 }); + expect(on.lines).toContain(note); + const off = validateConfig({ ...benchConfig, storefront: { ...benchConfig.storefront!, search: false } }, "test"); + expect((await startupLines(off, storefrontBench, { port: 0, wsPort: 0 })).lines).not.toContain(note); + }, 60_000); + it("without a storefront: one workbench line, on localhost", async () => { const { lines, port } = await startupLines(plain, testBench, { port: 0, wsPort: 0 }); expect(lines.filter(l => /\] (Website|Workbench):/.test(l))).toEqual([`[dev-server] Workbench: http://localhost:${port}/`]); diff --git a/packages/dev-server/test/storefront-search.spec.ts b/packages/dev-server/test/storefront-search.spec.ts new file mode 100644 index 0000000..ddf339f --- /dev/null +++ b/packages/dev-server/test/storefront-search.spec.ts @@ -0,0 +1,164 @@ +// Search (search.ts): what a page gives the index (its own content, never the layout's), which pages give nothing, +// the document Pagefind is given, the index's files, and how storefront.search finds Pagefind. +import { afterAll, describe, expect, it } from "vitest"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { StorefrontCategory } from "@pattern-dump/config"; +import { buildDiscoveryMap } from "@pattern-dump/discovery"; +import { createRenderer } from "@pattern-dump/renderer"; +import { readPages } from "../src/storefront/pages.js"; +import { buildCompositionRegistry } from "../src/composition.js"; +import { assemblePage, type PageAssembly } from "../src/storefront/page.js"; +import { buildSearchIndex, loadProjectPagefind, searchDocument, searchPagefind, SearchUnavailableError, type PagefindModule, type SearchRecord } from "../src/storefront/search.js"; +import { save } from "../../../test-support/save.js"; + +const roots: string[] = []; +afterAll(async () => { await Promise.all(roots.map(r => rm(r, { recursive: true, force: true }))); }); + +const page = (title: string, extra = "", body = `Body of ${title}.`): string => `---\ntitle: ${title}\n${extra}---\n\n${body}\n`; + +// A layout with text of its own around the body, and a related-pages listing of other pages' titles and teasers. +const PAGE_LAYOUT = "

{{title}}

{{{body}}}" + + "
Layoutfooter
"; +const PAGE_SCHEMA = { properties: { title: { type: "string" }, description: { type: "string" }, page: { type: "object", properties: { + related: { "x-pd-query": { collections: ["articles"], limit: 5, view: "list" } }, +} } } }; +const VIEW = { hbs: "
    {{#each entries}}
  • {{title}}: {{description}}
  • {{/each}}
", schema: { properties: { entries: { type: "array", "x-pd-card": "cards/entry", items: {} } } } }; +const CARD = { hbs: "{{title}}", schema: { properties: { title: { type: "string" }, description: { type: ["string", "null"] } } } }; + +async function site(files: Record, o: { categories?: Record } = {}) { + const root = await mkdtemp(join(tmpdir(), "pd-search-")); + roots.push(root); + const patterns = join(root, "patterns"); + const all: Record = { + "layouts/page/page.hbs": PAGE_LAYOUT, + "layouts/page/page.schema.json": JSON.stringify(PAGE_SCHEMA), + "layouts/tag/tag.hbs": "

{{title}}

{{{page.pages.listing}}}
", + "layouts/tag/tag.schema.json": JSON.stringify({ properties: { title: { type: "string" }, page: { type: "object", properties: { pages: { "x-pd-query": { tags: "same" } } } } } }), + "views/list/list.hbs": VIEW.hbs, "views/list/list.schema.json": JSON.stringify(VIEW.schema), + "cards/entry/entry.hbs": CARD.hbs, "cards/entry/entry.schema.json": JSON.stringify(CARD.schema), + }; + for (const [rel, body] of Object.entries(all)) await save(join(patterns, ...rel.split("/")), body); + for (const [rel, body] of Object.entries(files)) await save(join(root, "pages", ...rel.split("/")), body); + const { discoveryMap, handleMap } = await buildDiscoveryMap(patterns); + const render = createRenderer({ patternsDir: patterns, discoveryMap, handleMap }); + const model = await readPages(join(root, "pages"), { + categories: o.categories, layouts: new Set([...discoveryMap.keys()].filter(id => id.startsWith("layouts/")).map(id => id.slice("layouts/".length))), + drafts: "leave-out", scheduled: "leave-out", today: "2026-11-16", + }); + const composition = await buildCompositionRegistry(patterns, handleMap, discoveryMap); + const assemble = (route: string, extra: Partial = {}) => assemblePage(route, model, { + discoveryMap, handleMap, composition, site: undefined, workbenchUrl: "https://wb.example/", shown: x => x, head: async () => "", + image: async () => ({ ok: false, message: "no images here" }), render, lang: "en", ...extra, + } as PageAssembly); + return { model, assemble }; +} + +const ARTICLES = { + "index.md": page("Home"), + "about.md": page("About us"), + "articles/index.md": page("Articles", "", "All the articles."), + "articles/grids.md": page("Grids", "description: Laying out with grids\ntags: [css, layout]\ncategories: [guides]\n", "Zebracrossing in a grid.\n\n## Subgrid\n\nMore."), + "articles/colour.md": page("Colour", "tags: [css]\nindex: false\n", "Colour body."), +}; + +describe("what a page gives the index", () => { + it("its own body and title, its collection's label, its tags' and categories' labels, its teaser: not the layout's text", async () => { + const s = await site(ARTICLES, { categories: { guides: { label: "How-to guides" } } }); + const p = await s.assemble("/articles/grids/"); + expect(p.problems).toEqual([]); + // The page itself shows the layout's text and the related pages. + expect(p.html).toContain("Layoutheader"); + expect(p.html).toContain("Colour"); + expect(p.search).toMatchObject({ + route: "/articles/grids/", title: "Grids", section: "Articles", tags: ["css", "layout"], categories: ["How-to guides"], + teaser: "Laying out with grids", lang: "en", + } satisfies Partial); + expect(p.search!.body).toContain("Zebracrossing in a grid."); + expect(p.search!.body).toContain(`id="subgrid"`); + for (const word of ["Layoutheader", "Layoutfooter", "Colour", " { + const s = await site(ARTICLES); + expect((await s.assemble("/articles/")).search).toMatchObject({ section: "Articles", body: expect.stringContaining("All the articles.") }); + expect((await s.assemble("/about/")).search).toMatchObject({ section: null }); + }); + + it("nothing from the 404 page, a tag page, or a page with index: false", async () => { + const s = await site(ARTICLES); + const notFound = await s.assemble("/404.html"); + expect(notFound.status).toBe(404); + expect(notFound.search).toBeUndefined(); + const tag = await s.assemble("/tags/css/"); + expect(tag.problems).toEqual([]); + expect(tag.search).toBeUndefined(); + const hidden = await s.assemble("/articles/colour/"); + expect(hidden.problems).toEqual([]); + expect(hidden.search).toBeUndefined(); + }); +}); + +const RECORD: SearchRecord = { route: "/a/", title: "A & B", body: "

Text

", section: "Articles", tags: ["Design, Inc"], categories: [], teaser: null, lang: "de" }; + +describe("the document Pagefind is given", () => { + it("title, meta and filters outside the indexed body; the title as

when the body has none", () => { + const d = searchDocument(RECORD); + expect(d).toContain(``); + expect(d).toContain(`A & B`); + expect(d).not.toContain(`data-pagefind-meta="teaser"`); + expect(d).toContain(`ArticlesDesign, Inc`); + expect(d).toContain(`
\n

A & B

\n

Text

\n
`); + expect(searchDocument({ ...RECORD, body: `

Own

` })).toContain(`
\n

Own

\n
`); + }); +}); + +describe("the index", () => { + it("each page at its URL under the site's path, in route order", async () => { + const added: string[] = []; + let closed = false; + const fake: PagefindModule = { + createIndex: async () => ({ errors: [], index: { + addHTMLFile: async f => { added.push(f.url); return { errors: [] }; }, + getFiles: async () => ({ errors: [], files: [{ path: "pagefind.js", content: new Uint8Array() }] }), + } }), + close: async () => { closed = true; }, + }; + await buildSearchIndex([{ ...RECORD, route: "/b/" }, { ...RECORD, route: "/" }, RECORD], fake, "/site/"); + expect(added).toEqual(["/site/", "/site/a/", "/site/b/"]); + expect(closed).toBe(true); + }); + + it("with the project's Pagefind: its search files, not its own UIs", async () => { + const pagefind = await loadProjectPagefind(process.cwd()); + expect(pagefind).toBeDefined(); + const files = await buildSearchIndex([RECORD, { ...RECORD, route: "/b/", lang: "en" }], pagefind!, "/"); + const names = [...files.keys()]; + expect(names).toContain("pagefind.js"); + expect(names).toContain("pagefind-entry.json"); + expect(names.some(n => n.startsWith("fragment/"))).toBe(true); + expect(names.filter(n => /^pagefind-(?:ui|modular-ui|component-ui|highlight)\./.test(n))).toEqual([]); + }); +}); + +describe("storefront.search", () => { + const none = async () => undefined; + it("true without Pagefind stops, naming the install", async () => { + const e = await searchPagefind(true, "/nowhere", none).catch((x: unknown) => x); + expect(e).toBeInstanceOf(SearchUnavailableError); + expect((e as Error).message).toBe("pd build: storefront.search is on, and this project has no Pagefind to build the index with: " + + "npm install --save-dev pagefind (or set storefront.search to false)"); + }); + + it("left out: Pagefind when the project has it, else no search and no stop; false never loads it", async () => { + expect(await searchPagefind(undefined, "/nowhere", none)).toBeUndefined(); + const found = { createIndex: async () => ({ errors: [] }), close: async () => null } as PagefindModule; + expect(await searchPagefind(undefined, "/x", async () => found)).toBe(found); + expect(await searchPagefind(false, "/x", async () => { throw new Error("loaded"); })).toBeUndefined(); + }); + + it("a project without Pagefind has none", async () => { + expect(await loadProjectPagefind(await mkdtemp(join(tmpdir(), "pd-no-pf-")))).toBeUndefined(); + }); +}); diff --git a/storefront-bench/e2e/build-browser.spec.ts b/storefront-bench/e2e/build-browser.spec.ts index 727d21b..909b49f 100644 --- a/storefront-bench/e2e/build-browser.spec.ts +++ b/storefront-bench/e2e/build-browser.spec.ts @@ -11,7 +11,7 @@ import { join } from "node:path"; import { crc32, deflateSync } from "node:zlib"; import { fileURLToPath } from "node:url"; import { defineConfig, validateConfig } from "@pattern-dump/config"; -import { buildStaticSite } from "@pattern-dump/dev-server"; +import { buildStaticSite, loadProjectPagefind } from "@pattern-dump/dev-server"; import benchConfig from "../pattern-dump.config.js"; import { save } from "../../test-support/save.js"; @@ -208,3 +208,21 @@ test("the bench passes in a browser: no accessibility finding, every variant fit expect(report.accessibility).toMatchObject({ browser: true, warnings: [] }); expect(report.images).toEqual({ warnings: [] }); }); + +test("with search, a search field that shows no result for a word the index has stops the build, and nothing is written", async () => { + const root = await project("search-dead", { + // A search field with no script behind it: submitting it reloads the page, and no result is ever loaded. + "src/patterns/layouts/page/page.hbs": `
` + + `

{{title}}

{{{body}}}

F

\n`, + "pages/index.md": "---\ntitle: Home\n---\n\nText.\n", + "pages/harbour.md": "---\ntitle: Harbour\n---\n\nBoats.\n", + }); + const cfg = validateConfig({ ...config, storefront: { ...config.storefront!, search: true } }, "test"); + const message = await buildStaticSite(cfg, root, { + outDir: join(root, ".out", "wb"), pdVersion: "0.0.0-test", browser: true, loadPlaywright, loadPagefind: () => loadProjectPagefind(bench), + }).then(() => "built", (e: Error) => e.message); + expect(message).toContain("pd build: the storefront has problems in a browser, so nothing was written:"); + expect(message).toContain(` - /harbour/: searching for "Home" in its search field (input[type="search"]) showed no result within 5 s, though the index has a page with it: ` + + `the search component loads /_pd/search/pagefind.js under the site's path () and shows each result's data() (at 1000×720 and 390×844)`); + expect(existsSync(join(root, ".out"))).toBe(false); +}); diff --git a/storefront-bench/e2e/playwright.config.ts b/storefront-bench/e2e/playwright.config.ts index 5ed3f70..fb550d6 100644 --- a/storefront-bench/e2e/playwright.config.ts +++ b/storefront-bench/e2e/playwright.config.ts @@ -32,7 +32,7 @@ export default defineConfig({ use: { trace: "on-first-retry", video: "retain-on-failure" }, projects: [ // pd dev, on the storefront host. dom-parity also reads the static root. - { name: "storefront", testMatch: /\b(pages|js-off|workbench-links|hmr|live-handshake|add-page|new-files|workbench|dom-parity|placed|config-reload|derived-page|headings|position|dates|navigations|seo|not-found|redirects|example-height|example-height-check|asset-stamps|image-slots|prose-images|build-browser|collections|collections-reload|query|base-path|scripts|copy|categories|terms)\.spec\.ts$/, + { name: "storefront", testMatch: /\b(pages|js-off|workbench-links|hmr|live-handshake|add-page|new-files|workbench|dom-parity|placed|config-reload|derived-page|headings|position|dates|navigations|seo|not-found|redirects|example-height|example-height-check|asset-stamps|image-slots|prose-images|build-browser|collections|collections-reload|query|base-path|scripts|copy|categories|terms|search)\.spec\.ts$/, use: { ...devices["Desktop Chrome"], baseURL: `http://live.localhost:${DEV_PORT}` } }, // The built storefront root, from a static host. { name: "storefront-static", testMatch: /\b(pages|js-off|workbench-links|static|placed|headings|position|dates|navigations|seo|not-found|redirects|example-height|example-height-check|asset-stamps|image-slots|prose-images|collections|query|scripts|copy|categories|terms)\.spec\.ts$/, diff --git a/storefront-bench/e2e/search.spec.ts b/storefront-bench/e2e/search.spec.ts new file mode 100644 index 0000000..2165bab --- /dev/null +++ b/storefront-bench/e2e/search.spec.ts @@ -0,0 +1,161 @@ +// Search in the built site (search.ts, the starter's components/search): a copy of the bench, with the starter's search +// component on a search page and a draft, built with Pagefind and --browser, then searched in Chromium from its files. +// A word on one page finds that page; the section filter narrows the results; a word a listing shows in a teaser finds +// the page, not the listing; a draft's words find nothing; under a path, results link under it. Pagefind and +// Playwright here are the repository's, as a project's own would be. +import { test, expect, chromium, type BrowserContext } from "@playwright/test"; +import { execFileSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { cp, mkdir, mkdtemp, readdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { validateConfig, type PatternDumpConfig } from "@pattern-dump/config"; +import { buildStaticSite, loadProjectPagefind } from "@pattern-dump/dev-server"; +import benchConfig from "../pattern-dump.config.js"; +import { save } from "../../test-support/save.js"; + +test.setTimeout(180_000); +const repo = fileURLToPath(new URL("../../", import.meta.url)); +const starter = join(repo, "packages", "cli", "templates", "storefront"); +const loadPlaywright = async () => ({ chromium }); +const loadPagefind = () => loadProjectPagefind(repo); + +let tmp = ""; +test.beforeAll(async () => { tmp = await mkdtemp(join(tmpdir(), "pd-search-")); }); +test.afterAll(async () => { if (tmp) await rm(tmp, { recursive: true, force: true }); }); + +/** The bench as committed, with the starter's search component and its script, a search page and a draft. */ +async function benchWithSearch(name: string): Promise { + const root = join(tmp, name); + for (const file of execFileSync("git", ["ls-files", "storefront-bench"], { cwd: repo, encoding: "utf8" }).split("\n").filter(Boolean)) { + const to = join(root, file.slice("storefront-bench/".length)); + await mkdir(dirname(to), { recursive: true }); + await cp(join(repo, file), to); + } + await cp(join(starter, "src", "patterns", "components", "search"), join(root, "src", "patterns", "components", "search"), { recursive: true }); + await cp(join(starter, "src", "scripts", "site-search.ts"), join(root, "src", "scripts", "site-search.ts")); + await save(join(root, "pages", "search.md"), "---\ntitle: Search\norder: 9\n---\n\n\n"); + await save(join(root, "pages", "articles", "zz-draft.md"), "---\ntitle: Not yet\ndate: 2026-11-01\ndraft: true\n---\n\nA quokka, written ahead.\n"); + return root; +} + +const config = (over: Partial> = {}): PatternDumpConfig => + validateConfig({ ...benchConfig, storefront: { ...benchConfig.storefront!, ...over } }, "test"); + +/** A context that serves the built site's files at http://site.localhost/. */ +async function served(context: BrowserContext, site: string, base = "/"): Promise { + await context.route("http://site.localhost/**", async route => { + const path = new URL(route.request().url()).pathname; + if (!path.startsWith(base)) { await route.fulfill({ status: 404, body: "" }); return; } + const rel = path.slice(base.length); + const file = join(site, rel === "" || rel.endsWith("/") ? `${rel}index.html` : rel); + await (existsSync(file) ? route.fulfill({ path: file }) : route.fulfill({ status: 404, body: "" })); + }); +} + +test.describe("search on the built bench", () => { + let site = ""; + test.beforeAll(async () => { + const root = await benchWithSearch("bench"); + const report = await buildStaticSite(config(), root, { + outDir: join(root, ".out", "workbench"), pdVersion: "0.0.0-test", browser: true, loadPlaywright, loadPagefind, + }); + // Every page of the site but the 404 page and the tag and category pages; never the draft. + expect(report.search).toMatchObject({ pages: 8 }); + expect(report.drafts).toEqual(["articles/zz-draft.md"]); + site = join(root, ".out", "storefront"); + // Only Pagefind's search files: its own UIs are not written. + const files = await readdir(join(site, "_pd", "search")); + expect(files).toContain("pagefind.js"); + expect(files.filter(f => f.startsWith("pagefind-ui"))).toEqual([]); + }); + + const results = (context: BrowserContext) => async (query: string) => { + const page = await context.newPage(); + await page.goto(`http://site.localhost/search/?${query}`); + const status = page.getByRole("status"); + await expect(status).toHaveText(/result/); + const titles = await page.locator(".search-result-title a").allTextContents(); + return { page, status: await status.textContent(), titles }; + }; + + test("a word on one page finds that page, with its section and the passage", async ({ browser }) => { + const context = await browser.newContext(); + await served(context, site); + const { page, status, titles } = await results(context)("q=photograph"); + expect(titles).toEqual(["A harbour of cable cars"]); + expect(status).toBe(`1 result for "photograph".`); + const result = page.locator(".search-result").first(); + await expect(result.locator(".search-result-section")).toHaveText("Articles"); + await expect(result.locator("mark")).toHaveText("photograph"); + await expect(result.locator("a")).toHaveAttribute("href", "/articles/harbour/"); + await context.close(); + }); + + test("the section filter narrows the results", async ({ browser }) => { + const context = await browser.newContext(); + await served(context, site); + const all = await results(context)("q=skin"); + expect(all.titles.length).toBeGreaterThan(1); + expect(all.titles).toContain("Plain on purpose"); + // Chosen in the form's section list, which the script adds once the index is loaded. + await all.page.getByLabel("Section").selectOption("Articles"); + await expect(all.page.getByRole("status")).toHaveText(`1 result for "skin".`); + expect(await all.page.locator(".search-result-title a").allTextContents()).toEqual(["Plain on purpose"]); + expect(new URL(all.page.url()).searchParams.get("section")).toBe("Articles"); + await context.close(); + }); + + test("a word a listing shows in its teaser finds the page, not the listing; a draft's words find nothing", async ({ browser }) => { + const context = await browser.newContext(); + await served(context, site); + // "markup" is in the harbour's first paragraph, its teaser, which the articles page lists. + expect((await results(context)("q=markup")).titles).toEqual(["A harbour of cable cars"]); + const draft = await results(context)("q=quokka"); + expect(draft.status).toBe(`No results for "quokka".`); + expect(draft.titles).toEqual([]); + await context.close(); + }); + + test("typed into the field: results as you type, announced, and the words kept in the address", async ({ browser }) => { + const context = await browser.newContext(); + await served(context, site); + const page = await context.newPage(); + await page.goto("http://site.localhost/search/"); + await page.getByLabel("Search this site").fill("photograph"); + await expect(page.getByRole("status")).toHaveText(`1 result for "photograph".`); + await expect(page.locator(".search-result-title a")).toHaveText(["A harbour of cable cars"]); + expect(new URL(page.url()).searchParams.get("q")).toBe("photograph"); + await context.close(); + }); +}); + +test("under a path: the index is under it, and results link under it", async ({ browser }) => { + const root = await benchWithSearch("bench-path"); + await buildStaticSite(config({ url: "https://bench.example.test/sub/" }), root, { + outDir: join(root, ".out", "workbench"), pdVersion: "0.0.0-test", loadPagefind, + }); + const context = await browser.newContext(); + await served(context, join(root, ".out", "storefront"), "/sub/"); + const page = await context.newPage(); + await page.goto("http://site.localhost/sub/search/?q=photograph"); + await expect(page.getByRole("status")).toHaveText(`1 result for "photograph".`); + await expect(page.locator(".search-result-title a")).toHaveAttribute("href", "/sub/articles/harbour/"); + await context.close(); +}); + +test("without Pagefind the component says search works on the built site", async ({ browser }) => { + const root = await benchWithSearch("bench-none"); + const report = await buildStaticSite(config(), root, { + outDir: join(root, ".out", "workbench"), pdVersion: "0.0.0-test", loadPagefind: async () => undefined, + }); + expect(report.search).toBeUndefined(); + expect(existsSync(join(root, ".out", "storefront", "_pd", "search"))).toBe(false); + const context = await browser.newContext(); + await served(context, join(root, ".out", "storefront")); + const page = await context.newPage(); + await page.goto("http://site.localhost/search/?q=photograph"); + await expect(page.getByRole("status")).toHaveText("Search works on the built site: this one has no search index."); + await context.close(); +}); diff --git a/storefront-bench/e2e/static.spec.ts b/storefront-bench/e2e/static.spec.ts index fd09bb7..f0bc75d 100644 --- a/storefront-bench/e2e/static.spec.ts +++ b/storefront-bench/e2e/static.spec.ts @@ -21,6 +21,18 @@ test.describe("the built storefront on a static host", () => { }); } + // storefront.search is left out in the bench's config, so the repository's Pagefind turns it on. + test("serves the search index pd build wrote, and a search in it finds a page", async ({ page }) => { + const res = await page.request.get("/_pd/search/pagefind.js", { headers: NO_CACHE }); + expect(res.status()).toBe(200); + await page.goto("/"); + const found = await page.evaluate(async () => { + const pf = await import("/_pd/search/pagefind.js") as { search(t: string): Promise<{ results: Array<{ data(): Promise<{ url: string }> }> }> }; + return Promise.all((await pf.search("photograph")).results.map(r => r.data().then(d => d.url))); + }); + expect(found).toEqual(["/articles/harbour/"]); + }); + test("stamps each served file it references with the content hash of that file as written", async () => { const files = (await readdir(ROOT, { recursive: true, withFileTypes: true })).filter(e => e.isFile()).map(e => join(e.parentPath, e.name)); const stamps: Array<{ in: string; path: string; v: string }> = []; diff --git a/storefront-bench/pattern-dump.config.ts b/storefront-bench/pattern-dump.config.ts index 96c8002..165bda8 100644 --- a/storefront-bench/pattern-dump.config.ts +++ b/storefront-bench/pattern-dump.config.ts @@ -31,6 +31,7 @@ export default defineConfig({ images: { label: "Images", description: "Pictures, and how pages show them", aliases: ["pictures"] }, layout: { label: "Layout" }, }, + // Search is left out (storefront.search): pd build indexes the site because the repository has Pagefind. // The link-preview image of every page that names none of its own. image: "/assets/images/share.png", site: { From 84b32a8163b8013c85f4d3bf636213e736d90f11 Mon Sep 17 00:00:00 2001 From: Stefan Bauer Date: Mon, 16 Nov 2026 07:39:41 +0100 Subject: [PATCH 2/3] feat(cli): a search page and component in the storefront starter --- packages/cli/src/init/run.spec.ts | 3 +- packages/cli/src/init/run.ts | 1 + packages/cli/src/init/starter.spec.ts | 8 +- .../cli/templates/storefront-fills/config.txt | 2 + .../cli/templates/storefront/pages/about.md | 2 + .../cli/templates/storefront/pages/search.md | 7 ++ .../src/patterns/components/search/search.hbs | 12 ++ .../patterns/components/search/search.json | 3 + .../src/patterns/components/search/search.md | 23 ++++ .../components/search/search.schema.json | 9 ++ .../storefront/src/scripts/site-search.ts | 119 ++++++++++++++++++ .../src/styles/components/search.scss | 15 +++ 12 files changed, 202 insertions(+), 2 deletions(-) create mode 100644 packages/cli/templates/storefront/pages/search.md create mode 100644 packages/cli/templates/storefront/src/patterns/components/search/search.hbs create mode 100644 packages/cli/templates/storefront/src/patterns/components/search/search.json create mode 100644 packages/cli/templates/storefront/src/patterns/components/search/search.md create mode 100644 packages/cli/templates/storefront/src/patterns/components/search/search.schema.json create mode 100644 packages/cli/templates/storefront/src/scripts/site-search.ts create mode 100644 packages/cli/templates/storefront/src/styles/components/search.scss diff --git a/packages/cli/src/init/run.spec.ts b/packages/cli/src/init/run.spec.ts index 87dcf33..5fc062b 100644 --- a/packages/cli/src/init/run.spec.ts +++ b/packages/cli/src/init/run.spec.ts @@ -87,7 +87,8 @@ describe("runInit", () => { expect(await runInit(init(join(root, "ds"), { storefront: true, install: false, git: false }), io)).toBe(0); expect(asked).not.toContain("Include a website (storefront)"); expect(existsSync(join(root, "ds", ".github", "workflows", "pages.yml"))).toBe(true); - expect(logs.at(-1)).toBe(" npm run dev # the website at http://live.localhost:4000/, the workbench at http://pd.localhost:4000/"); + expect(logs.slice(-2)).toEqual([" npm run dev # the website at http://live.localhost:4000/, the workbench at http://pd.localhost:4000/", + " npm install --save-dev pagefind # for the site's search: pd build then writes its index"]); }); it("--yes leaves the website out, and the next steps name the workbench only", async () => { diff --git a/packages/cli/src/init/run.ts b/packages/cli/src/init/run.ts index d6678f6..d983b31 100644 --- a/packages/cli/src/init/run.ts +++ b/packages/cli/src/init/run.ts @@ -136,6 +136,7 @@ export async function runInit(cmd: InitCommand, io: InitIO): Promise { const dev = `npm run ${JSON.parse(plan.packageJson.content).scripts["pd:dev"] ? "pd:dev" : "dev"}`; if (storefront) { io.log(` ${dev} # the website at http://live.localhost:4000/, the workbench at http://pd.localhost:4000/`); + io.log(" npm install --save-dev pagefind # for the site's search: pd build then writes its index"); } else { io.log(` ${dev} # http://localhost:4000`); } diff --git a/packages/cli/src/init/starter.spec.ts b/packages/cli/src/init/starter.spec.ts index 21243c8..469216c 100644 --- a/packages/cli/src/init/starter.spec.ts +++ b/packages/cli/src/init/starter.spec.ts @@ -174,7 +174,7 @@ describe("the starter project with a website (--storefront, no examples)", () => const { graph } = await runDiagnosticsAudit(config, siteDir); expect(graph.occurrences).toEqual([]); expect(graph.components.map(c => [c.id, c.render]).sort()).toEqual( - [["components/compact-card", "ok"], ["components/feature-card", "ok"], ["components/grid-card", "ok"], ["components/listing-card", "ok"], ["components/note", "ok"], + [["components/compact-card", "ok"], ["components/feature-card", "ok"], ["components/grid-card", "ok"], ["components/listing-card", "ok"], ["components/note", "ok"], ["components/search", "ok"], ["layouts/article", "ok"], ["layouts/categories", "ok"], ["layouts/category", "ok"], ["layouts/home", "ok"], ["layouts/listing", "ok"], ["layouts/page", "ok"], ["layouts/tag", "ok"], ["layouts/tags", "ok"], ["partials/base", "ok"], ["partials/head", "ok"], ["views/compact", "ok"], ["views/featured", "ok"], ["views/grid", "ok"], ["views/list", "ok"]]); @@ -207,6 +207,12 @@ describe("the starter project with a website (--storefront, no examples)", () => // Each tag by its label, linked to its tag page. expect(hello.body).toContain(``); // The tag and category pages, and their indexes, through the starter's plain layouts. + // The search page: the search component, its script, and a link to it in the navigation. + const search = await onHost(server.port, "live.localhost", "/search/"); + expect(search.status).toBe(200); + expect(search.body).toMatch(/\s*
\s*