diff --git a/src/layouts/Api.astro b/src/layouts/Api.astro index 54552b1bbf..df7c0d35eb 100644 --- a/src/layouts/Api.astro +++ b/src/layouts/Api.astro @@ -99,7 +99,7 @@ const searchIndex = searchIndexAttributes(Astro.url.pathname, frontmatter); /* Copy as markdown temporarily disabled until we can get it working with the API docs. Deliberately no "Edit on GitHub" because these are generated and should not be hand edited */ } -
+
diff --git a/src/layouts/Default.astro b/src/layouts/Default.astro index a5711494fd..3d4f62a740 100644 --- a/src/layouts/Default.astro +++ b/src/layouts/Default.astro @@ -100,7 +100,7 @@ const searchIndex = searchIndexAttributes(Astro.url.pathname, frontmatter); lang={lang} />
-
+
diff --git a/src/lib/searchIndexing.ts b/src/lib/searchIndexing.ts index b21aa20b17..5e9917453e 100644 --- a/src/lib/searchIndexing.ts +++ b/src/lib/searchIndexing.ts @@ -13,53 +13,41 @@ type ArticleAttributes = { 'data-pagefind-default-meta'?: string; }; -/** - * A second filter, on an element inside the article rather than on the article - * itself: Pagefind reads one `key:value` per `data-pagefind-filter`, and a - * comma-separated pair is taken as a single value. - */ -type ContentAttributes = { - 'data-pagefind-filter'?: string; -}; - type IndexAttributes = { article: ArticleAttributes; - content: ContentAttributes; }; /** - * How shallow a page has to be to count as one a reader might name. Two segments - * past `/docs/`, which covers `/docs/deployments/` and - * `/docs/infrastructure/deployment-targets/` but not the pages inside them. - */ -const LANDING_DEPTH = 3; - -/** - * The `data-pagefind-*` attributes for a page: `article` spreads onto the - * `
`, `content` onto the page content inside it. + * Whether a page is in the search index at all. * * `navSearch` rather than `PostFiltering.showInSearch`, which also hides a page * with a future `pubDate`, a `draft: true` and a `listable: false`: a page that * is built and served is a page worth finding. No docs page carries any of the * three today, so this is the same set either way — decide again if one starts * being used to hold a page back. + * + * Exported because `/docs/search-titles.json` has to describe the same set. A + * title in that list with no page behind it in the index would promote a row the + * search itself has no answer for. + */ +export function isSearchIndexable( + pathname: string, + frontmatter: Frontmatter +): boolean { + return frontmatter.navSearch !== false && !isUnderConstructionUrl(pathname); +} + +/** + * The `data-pagefind-*` attributes for a page, to spread onto the `
`. */ export function searchIndexAttributes( pathname: string, frontmatter: Frontmatter ): IndexAttributes { - const indexable = - frontmatter.navSearch !== false && !isUnderConstructionUrl(pathname); - // `all` rather than the default `index`: a bare ignore still lets Pagefind // read a title or metadata out of the block. - if (!indexable) - return { article: { 'data-pagefind-ignore': 'all' }, content: {} }; - - // Marks the pages the overlay's second, narrowed search looks through. Only - // the shallow pages carry it, so the filter chunk stays small and that search - // has a few hundred candidates rather than the whole site. - const isLanding = pathname.split('/').filter(Boolean).length <= LANDING_DEPTH; + if (!isSearchIndexable(pathname, frontmatter)) + return { article: { 'data-pagefind-ignore': 'all' } }; return { article: { @@ -75,6 +63,5 @@ export function searchIndexAttributes( ? { 'data-pagefind-default-meta': `title:${frontmatter.title}` } : {}), }, - content: isLanding ? { 'data-pagefind-filter': 'landing:true' } : {}, }; } diff --git a/src/pages/docs/search-titles.json.ts b/src/pages/docs/search-titles.json.ts new file mode 100644 index 0000000000..9a7df5d298 --- /dev/null +++ b/src/pages/docs/search-titles.json.ts @@ -0,0 +1,60 @@ +// Every indexed page's URL and title, for the search overlay to find the page a +// query names. +// +// `claimsName` in the overlay compares a query against a page's title and the +// last segment of its URL. Pagefind holds both, but only inside a result's +// fragment — one fetch per result — so a page ranked past the rows the overlay +// draws could not be reached at all. Both are known here at build time, from the +// same glob llms.txt.ts and sitemap.xml.ts walk. + +import { accelerator } from '@lib/accelerator'; +import { flattenGeneratedApiPath } from '@lib/generatedApiPaths'; +import { isSearchIndexable } from '@lib/searchIndexing'; + +const allPages = import.meta.glob(['./**/*.md', './**/*.mdx']); + +async function getData() { + // Pairs rather than objects: this is a row per page across the whole site, and + // repeated keys would be most of the bytes. No description — a promoted row + // shows no excerpt, which the overlay's CSS already collapses, and carrying one + // per page would roughly triple this file. + const pages: [url: string, title: string][] = []; + + for (const path in allPages) { + const article: any = await allPages[path](); + const frontmatter = article.frontmatter ?? {}; + + // A redirect stub is built and served but has no article, so Pagefind drops + // it and a promoted row for one would lead somewhere the search itself + // cannot go. + if (frontmatter.redirect) continue; + + const address = accelerator.urlFormatter.formatAddress( + flattenGeneratedApiPath(article.url ?? '') + ); + if (!address) continue; + if (!isSearchIndexable(address, frontmatter)) continue; + + // Pagefind returns every URL with a trailing slash and `formatAddress` + // returns none, and these are compared against each other by string: an + // unslashed URL here silently matches no result and is promoted a second + // time when its own row is reached. + const url = address.endsWith('/') ? address : `${address}/`; + + // Pagefind takes a result's title from the first heading in the body and + // only falls back to this one, so the two can differ. `claimsName` also + // matches on the URL's last segment, which covers the pages where they do. + const title = + typeof frontmatter.title === 'string' ? frontmatter.title.trim() : ''; + if (!title) continue; + + pages.push([url, title]); + } + + return new Response(JSON.stringify(pages), { + status: 200, + headers: { 'Content-Type': 'application/json; charset=utf-8' }, + }); +} + +export const GET = getData; diff --git a/src/scripts/search-engine-pagefind.ts b/src/scripts/search-engine-pagefind.ts index e6b902add9..66cbdae7da 100644 --- a/src/scripts/search-engine-pagefind.ts +++ b/src/scripts/search-engine-pagefind.ts @@ -28,10 +28,12 @@ const PAGE_SIZE = 30; // 79%. A mash landing on the gentler message costs nothing; both offer no rows. const COMMON_TERM_SHARE = 0.8; -// How many shallow pages the named-page lookup looks through. Measured over 18 -// section queries: three finds the page for 16, five for 17, and twenty finds no -// more than five does. -const LANDING_CANDIDATES = 5; +/** + * Every indexed page's url and title, built from the site's own markdown. See + * `src/pages/docs/search-titles.json.ts` for why this exists rather than being + * read out of Pagefind. + */ +type TitleList = [url: string, title: string][]; type PagefindSubResult = { title: string; @@ -197,7 +199,11 @@ function byNameThenDepth< /** A stub's score paired with its fetched fragment, which is where the URL is. */ type Hit = { - fragment: PagefindFragment; + /** + * Absent on a page promoted from the title list, which is not one of the + * results and so has no stub to fetch a fragment from. + */ + fragment?: PagefindFragment; score: number; url: string; title: string; @@ -242,11 +248,14 @@ function rows(hits: Hit[], term: string, from: number): SearchResult[] { url: hit.url, title: hit.title, // Already carries around the hits, and Pagefind escapes the - // surrounding text itself. - excerpt: hit.fragment.excerpt, + // surrounding text itself. Empty for a promoted page, which the overlay + // renders as a row without an excerpt line. + excerpt: hit.fragment?.excerpt ?? '', breadcrumb: breadcrumbFrom(hit.url), sections: - from + rank < ROWS_WITH_SECTIONS ? sectionsOf(hit.fragment) : undefined, + hit.fragment && from + rank < ROWS_WITH_SECTIONS + ? sectionsOf(hit.fragment) + : undefined, ...classify(hit.url), })); } @@ -256,25 +265,58 @@ function rows(hits: Hit[], term: string, from: number): SearchResult[] { * * `byNameThenDepth` can only promote what has been fetched, and a section's own * page can rank far below the pages inside it: `/docs/infrastructure/ - * deployment-targets/` is 36th for "deployment targets", six places past the - * page size. These stubs come from a search narrowed to the shallow pages alone, - * where the page a query names sits near the top of a few hundred candidates. + * deployment-targets/` is 36th for "deployment targets", past the page size. + * `claimsName` reads only a url and a title, and the title list carries both for + * every indexed page, so the page a query names is found without fetching + * anything and from any depth. * - * Only called when nothing already fetched names the query, so the extra - * fragments are paid for by the queries that need them and no others. + * Only called when nothing already fetched names the query, and it returns at + * most one page: `claimsName` is an exact match on a title or a slug, so a + * second page answering to the same name is a page the reader could not have + * meant either way. */ -async function namedPage( - stubs: PagefindResultStub[], +function namedPage( + titles: TitleList, term: string, - already: Hit[] -): Promise { + already: Hit[], + facet: string +): Hit | null { const seen = new Set(already.map((hit) => hit.url)); - const candidates = await hydrate(stubs.slice(0, LANDING_CANDIDATES)); - return ( - candidates.find((hit) => claimsName(hit, term) && !seen.has(hit.url)) ?? - null - ); + for (const [url, title] of titles) { + if (seen.has(url)) continue; + if (!claimsName({ url, title }, term)) continue; + // The list is the whole site, so a tab showing one section has to reject a + // page from another. The search this promotion joins was narrowed by the + // tab; this lookup was not. + if (facet !== 'all' && classify(url).facet !== facet) continue; + + // `byNameThenDepth` sorts on `claimsName` before it looks at a score, and + // this page claims the name, so the score below it is never reached. + return { score: 0, url, title }; + } + + return null; +} + +/** + * The title list, fetched once alongside the index. + * + * A failure leaves it null and the overlay keeps working: every result still + * ranks, and only the promotion of a page from past the drawn rows is lost. + */ +async function loadTitles(url: string): Promise { + try { + const response = await fetch(url); + if (!response.ok) throw new Error(`${response.status}`); + return (await response.json()) as TitleList; + } catch (error) { + console.error( + '[docs-search] could not load the title list; a page ranked past the drawn rows will not be promoted', + error + ); + return null; + } } export function pagefindEngine(bundlePath: string): SearchEngine { @@ -295,6 +337,10 @@ export function pagefindEngine(bundlePath: string): SearchEngine { let searches = 0; // Pages in the index, read off the filter counts when the index loads. let corpus = 0; + // Every indexed page's url and title, fetched once with the index. + let titles: TitleList | null = null; + // A sibling of the index directory: `/docs/pagefind/` leaves `/docs/`. + const titlesUrl = bundlePath.replace(/pagefind\/?$/, 'search-titles.json'); function load() { loading ??= import(/* @vite-ignore */ `${bundlePath}pagefind.js`) @@ -311,7 +357,14 @@ export function pagefindEngine(bundlePath: string): SearchEngine { // The filter index is a separate chunk, and a search returns empty filter // counts until it has been pulled down. Its section totals also add up to // the size of the corpus, which is what `COMMON_TERM_SHARE` is a share of. - const filters = await api.filters(); + const [filters] = await Promise.all([ + api.filters(), + // Alongside the filters rather than after them: both are wanted before + // the first search and neither depends on the other. + loadTitles(titlesUrl).then((list) => { + titles = list; + }), + ]); corpus = Object.values(filters.section ?? {}).reduce( (total, count) => total + count, 0 @@ -358,16 +411,13 @@ export function pagefindEngine(bundlePath: string): SearchEngine { const filters = facet && facet !== 'all' ? { section: [facet] } : undefined; - // Three searches at once, because a second await here would sit in front - // of every fragment fetch below it. The first supplies the rows; the - // second says whether the query has any answer at all, and only runs - // while a tab is narrowing the first; the third is the shallow-page - // shortlist `namedPage` draws on, which costs nothing until its - // fragments are fetched. - const [response, wholeCorpus, landing] = await Promise.all([ + // Together, because a second await here would sit in front of every + // fragment fetch below it. The first supplies the rows; the second says + // whether the query has any answer at all, and only runs while a tab is + // narrowing the first. + const [response, wholeCorpus] = await Promise.all([ api.search(query, { filters }), filters ? api.search(query) : null, - api.search(query, { filters: { ...filters, landing: ['true'] } }), ]); const unfiltered = wholeCorpus ?? response; @@ -397,12 +447,13 @@ export function pagefindEngine(bundlePath: string): SearchEngine { const hits = await hydrate(response.results.slice(0, PAGE_SIZE)); - const named = hits.some((hit) => claimsName(hit, query)) - ? null - : await namedPage(landing.results, query, hits); + const named = + !titles || hits.some((hit) => claimsName(hit, query)) + ? null + : namedPage(titles, query, hits, facet ?? 'all'); if (named) hits.push(named); - // A promoted page was fetched precisely because its own stub ranks past + // A promoted page was promoted precisely because its own stub ranks past // `PAGE_SIZE`, so that stub is still ahead of `more()` and has to be // skipped there rather than drawn a second time. settle({ @@ -413,8 +464,11 @@ export function pagefindEngine(bundlePath: string): SearchEngine { }); return { - // The promoted page is already one of these stubs, so counting them - // is counting the rows the query has in all. + // The promoted page is one of these stubs in every case seen: the + // query is its own title or slug, and the title is in the indexed + // body. The list is the whole site rather than this query's matches, + // though, so that is an assumption and not a guarantee — a page whose + // slug answers to a query its body does not would leave this one short. results: rows(hits, query, 0), counts, total: response.results.length, diff --git a/tests/docs-search.spec.ts b/tests/docs-search.spec.ts index 18ef10ffdf..e1debc321e 100644 --- a/tests/docs-search.spec.ts +++ b/tests/docs-search.spec.ts @@ -377,8 +377,9 @@ test('results stay on the host that served them', async ({ page }) => { // A section's own page can rank far below the pages inside it on raw score - // /docs/infrastructure/deployment-targets/ is 36th for this query - so the -// overlay runs a second search over the shallow pages alone and puts the page the -// query names at the top. Without it the first result is a getting-started page. +// overlay looks the query up in the title list at /docs/search-titles.json and +// puts the page it names at the top. Without it the first result is a +// getting-started page. test('the page a query names comes first', async ({ page }) => { await page.goto('/docs');