diff --git a/src/integrations/pagefind-index.ts b/src/integrations/pagefind-index.ts index a61f83438b..9eae86f530 100644 --- a/src/integrations/pagefind-index.ts +++ b/src/integrations/pagefind-index.ts @@ -10,6 +10,8 @@ import type { AstroIntegration } from 'astro'; import { fileURLToPath } from 'node:url'; import * as path from 'node:path'; +import { gunzipSync } from 'node:zlib'; +import { readdir, readFile, writeFile } from 'node:fs/promises'; // Statically, because `astro:build:done` fires after Vite's module runner has // closed and a dynamic import from inside the hook cannot be resolved. import { createIndex, close } from 'pagefind'; @@ -81,6 +83,11 @@ export default function pagefindIndex(): AstroIntegration { // Files scanned, not pages indexed: the redirect stubs are counted // here and then dropped for having no article. logger.info(`scanned ${added.page_count} pages into docs/pagefind`); + + const titles = await writeTitleMap( + path.join(distDir, 'docs', 'pagefind') + ); + logger.info(`wrote ${titles.pages} titles to ${titles.file}`); } finally { await close(); } @@ -88,3 +95,93 @@ export default function pagefindIndex(): AstroIntegration { }, }; } + +/** + * The map's name, carrying the index's own hash. Pagefind hashes its chunks so a + * cache cannot serve one build's index against another's, and a map read against + * the wrong index joins against nothing. The client reads the hash out of + * `pagefind-entry.json` to build the same name. + * + * A module, so that Front Door's static-content rule caches it for a week as + * immutable and compressed. That rule lists extensions, and a `.json` name falls + * through to `no-cache`, where an ETag-less revalidation costs a full + * re-download — paid on every page that loads the overlay. + */ +const titleMapName = (hash: string) => `docs-titles.${hash}.js`; + +// Pagefind prefixes every decompressed chunk with this before the JSON. +const FRAGMENT_MAGIC = 'pagefind_dcd'; + +/** + * Writes what the overlay needs to rank a result without fetching it. + * + * Ranking needs a URL and a title, and Pagefind keeps both in the per-page + * fragment — so ranking thirty results meant fetching thirty files, and a + * landing page ranked past that could not be reached at all. A search result + * stub carries the id of its own fragment, so one map from id to url and title + * lets the whole result set be ranked from a single file. + * + * Read back out of the fragments rather than collected during indexing, because + * the ids are assigned by Pagefind as it writes them. + */ +async function writeTitleMap( + pagefindDir: string +): Promise<{ pages: number; file: string }> { + const entry = JSON.parse( + await readFile(path.join(pagefindDir, 'pagefind-entry.json'), 'utf8') + ) as { languages: Record }; + + const languages = Object.keys(entry.languages ?? {}); + // One map covers every fragment, so it can only carry one language's hash. A + // second language would need one map each, keyed the way Pagefind keys its own + // chunks — worth failing loudly over rather than shipping a map the client + // looks for under the wrong name. + if (languages.length !== 1) { + throw new Error( + `expected one indexed language, found ${languages.length || 'none'}: the title map is named after the index hash and cannot cover several` + ); + } + + const file = titleMapName(entry.languages[languages[0]].hash); + const dir = path.join(pagefindDir, 'fragment'); + const names = (await readdir(dir)).filter((name) => + name.endsWith('.pf_fragment') + ); + + const map: Record = {}; + + for (const name of names) { + const raw = gunzipSync(await readFile(path.join(dir, name))).toString( + 'utf8' + ); + + // Checked before parsing: a Pagefind release that changes the chunk format + // has to fail the build here, rather than write a map the overlay silently + // cannot join against. + if (!raw.startsWith(FRAGMENT_MAGIC)) { + throw new Error( + `unexpected fragment format in ${name}: Pagefind's own prefix is missing, so the title map cannot be trusted` + ); + } + + const fragment = JSON.parse(raw.slice(raw.indexOf('{'))) as { + url: string; + meta?: Record; + }; + + // The stub's `id` is the filename without its extension, which is the join. + map[path.basename(name, '.pf_fragment')] = [ + fragment.url, + fragment.meta?.title ?? '', + ]; + } + + await writeFile( + path.join(pagefindDir, file), + `export default ${JSON.stringify(map)}; +`, + 'utf8' + ); + + return { pages: names.length, file }; +} diff --git a/src/layouts/Api.astro b/src/layouts/Api.astro index 54552b1bbf..df7c0d35eb 100644 --- a/src/layouts/Api.astro +++ b/src/layouts/Api.astro @@ -99,7 +99,7 @@ const searchIndex = searchIndexAttributes(Astro.url.pathname, frontmatter); /* Copy as markdown temporarily disabled until we can get it working with the API docs. Deliberately no "Edit on GitHub" because these are generated and should not be hand edited */ } -
+
diff --git a/src/layouts/Default.astro b/src/layouts/Default.astro index a5711494fd..3d4f62a740 100644 --- a/src/layouts/Default.astro +++ b/src/layouts/Default.astro @@ -100,7 +100,7 @@ const searchIndex = searchIndexAttributes(Astro.url.pathname, frontmatter); lang={lang} />
-
+
diff --git a/src/lib/searchIndexing.ts b/src/lib/searchIndexing.ts index b21aa20b17..ea46993cdb 100644 --- a/src/lib/searchIndexing.ts +++ b/src/lib/searchIndexing.ts @@ -13,30 +13,12 @@ type ArticleAttributes = { 'data-pagefind-default-meta'?: string; }; -/** - * A second filter, on an element inside the article rather than on the article - * itself: Pagefind reads one `key:value` per `data-pagefind-filter`, and a - * comma-separated pair is taken as a single value. - */ -type ContentAttributes = { - 'data-pagefind-filter'?: string; -}; - type IndexAttributes = { article: ArticleAttributes; - content: ContentAttributes; }; /** - * How shallow a page has to be to count as one a reader might name. Two segments - * past `/docs/`, which covers `/docs/deployments/` and - * `/docs/infrastructure/deployment-targets/` but not the pages inside them. - */ -const LANDING_DEPTH = 3; - -/** - * The `data-pagefind-*` attributes for a page: `article` spreads onto the - * `
`, `content` onto the page content inside it. + * The `data-pagefind-*` attributes for a page, to spread onto the `
`. * * `navSearch` rather than `PostFiltering.showInSearch`, which also hides a page * with a future `pubDate`, a `draft: true` and a `listable: false`: a page that @@ -53,13 +35,7 @@ export function searchIndexAttributes( // `all` rather than the default `index`: a bare ignore still lets Pagefind // read a title or metadata out of the block. - if (!indexable) - return { article: { 'data-pagefind-ignore': 'all' }, content: {} }; - - // Marks the pages the overlay's second, narrowed search looks through. Only - // the shallow pages carry it, so the filter chunk stays small and that search - // has a few hundred candidates rather than the whole site. - const isLanding = pathname.split('/').filter(Boolean).length <= LANDING_DEPTH; + if (!indexable) return { article: { 'data-pagefind-ignore': 'all' } }; return { article: { @@ -75,6 +51,5 @@ export function searchIndexAttributes( ? { 'data-pagefind-default-meta': `title:${frontmatter.title}` } : {}), }, - content: isLanding ? { 'data-pagefind-filter': 'landing:true' } : {}, }; } diff --git a/src/scripts/search-engine-pagefind.ts b/src/scripts/search-engine-pagefind.ts index e6b902add9..ba5c94b5de 100644 --- a/src/scripts/search-engine-pagefind.ts +++ b/src/scripts/search-engine-pagefind.ts @@ -12,10 +12,17 @@ import { type SearchResult, } from './search-engine'; -// Rows fetched at a time. Pagefind's own UI shows five and offers the rest on -// demand; thirty, because `byNameThenDepth` reorders within a page and needs -// enough of the list to have something to reorder. -const PAGE_SIZE = 30; +// Rows fetched at a time. The panel shows about five, and the rest arrive as it +// is scrolled. Ranking no longer needs them, so this is only a drawing budget: +// ten covers the first screen with room to scroll into. +const PAGE_SIZE = 10; + +/** + * The map of result id to url and title, written beside the index at build and + * named after the index's own hash. Reading it means reading that hash first, out + * of the entry file Pagefind publishes for the same purpose. + */ +const titleMapName = (hash: string) => `docs-titles.${hash}.js`; // Above this share of the corpus, a query is too general to rank rather than // unanswerable, and the overlay says so instead of reporting nothing found. @@ -28,11 +35,6 @@ const PAGE_SIZE = 30; // 79%. A mash landing on the gentler message costs nothing; both offer no rows. const COMMON_TERM_SHARE = 0.8; -// How many shallow pages the named-page lookup looks through. Measured over 18 -// section queries: three finds the page for 16, five for 17, and twenty finds no -// more than five does. -const LANDING_CANDIDATES = 5; - type PagefindSubResult = { title: string; /** Carries the heading's `#anchor` when the match is below the page title. */ @@ -171,8 +173,10 @@ function claimsName(hit: { url: string; title: string }, term: string) { * order for none of them; `data-pagefind-weight` on the h1 measured as no change * at all. * - * Reaches only the results already fetched, which is what `namedPage` below is - * for: a page ranked past `PAGE_SIZE` on raw score cannot be rescued here. + * Runs over every result, because `rank` supplies url and title from the title + * map and nothing here has to be fetched. So a page the query names wins from + * anywhere in the list — `/docs/infrastructure/deployment-targets/` is 36th on + * raw score for "deployment targets" and still comes first. */ function byNameThenDepth< T extends { score: number; url: string; title: string }, @@ -195,22 +199,70 @@ function byNameThenDepth< .map((entry) => entry.hit); } -/** A stub's score paired with its fetched fragment, which is where the URL is. */ -type Hit = { - fragment: PagefindFragment; +/** Everything ranking needs, and nothing that has to be fetched to get it. */ +type Ranked = { + stub: PagefindResultStub; score: number; url: string; title: string; }; +type TitleMap = Record; + /** - * Fetches the fragment for each stub. A fragment that fails takes its own row - * out rather than the whole result set. + * Pairs each result with its url and title from the map, so the whole set can be + * ranked before anything is fetched. + * + * A result the map does not know is dropped from ranking. That only happens when + * the map and the index disagree, which means a stale deploy of one of them; the + * caller falls back to ranking what it fetches. */ -async function hydrate(stubs: PagefindResultStub[]): Promise { - const settled = await Promise.allSettled(stubs.map((stub) => stub.data())); +function rank( + stubs: PagefindResultStub[], + titles: TitleMap, + term: string, + prefix: string +): Ranked[] { + const known = stubs.flatMap((stub) => { + const entry = titles[stub.id]; + if (!entry) return []; + // The map holds urls as the index does, relative to the indexed directory. + // Pagefind applies the same prefix to the urls it returns from `data()`. + const [path, title] = entry; + const url = prefix + path; + return [{ stub, score: stub.score, url, title: title || url }]; + }); + + // Partly stale is worse than wholly stale, because it drops results quietly + // and the fallback never fires. Say so rather than answering with a hole in + // the list. + if (known.length < stubs.length) { + console.warn( + `[docs-search] ${stubs.length - known.length} of ${stubs.length} results are missing from the title map and were dropped` + ); + } - return settled.flatMap((outcome, at) => { + return byNameThenDepth(known, term); +} + +/** + * Draws a slice of the ranked list, fetching a fragment for each row in it. The + * fragment supplies the excerpt and the matched headings; the order was settled + * before any of it was asked for. + * + * `from` is how many rows already precede these, which keeps the headings on the + * leading rows of the list rather than the leading rows of every batch. + */ +async function draw( + ranked: Ranked[], + from: number, + count: number, + reorder?: string +): Promise { + const slice = ranked.slice(from, from + count); + const settled = await Promise.allSettled(slice.map((hit) => hit.stub.data())); + + const drawn = settled.flatMap((outcome, at) => { if (outcome.status === 'rejected') { console.error( '[docs-search] dropped a result whose fragment failed', @@ -219,26 +271,28 @@ async function hydrate(stubs: PagefindResultStub[]): Promise { return []; } + const hit = slice[at]; const fragment = outcome.value; + // Without the title map `rank` had no url or title to give, so the fragment + // is the only source for both. const { pathname } = new URL(fragment.url, window.location.origin); return [ { fragment, - score: stubs[at].score, - url: pathname, - title: fragment.meta?.title ?? pathname, + score: hit.score, + url: hit.url || pathname, + title: hit.title || fragment.meta?.title || pathname, }, ]; }); -} -/** - * One page of rows, ordered within itself. `from` is how many rows already - * precede them, which is what keeps the headings on the leading pages of the - * list rather than on the leading rows of every page. - */ -function rows(hits: Hit[], term: string, from: number): SearchResult[] { - return byNameThenDepth(hits, term).map((hit, rank) => ({ + // Ranked already, unless the map was unusable and this slice arrived in + // Pagefind's own order. Ordering these few is worse than ordering the whole + // set, and far better than leaving a bare BM25 list: the reorder is what puts + // a section's own page above the pages inside it. + const ordered = reorder ? byNameThenDepth(drawn, reorder) : drawn; + + return ordered.map((hit, at) => ({ url: hit.url, title: hit.title, // Already carries around the hits, and Pagefind escapes the @@ -246,50 +300,61 @@ function rows(hits: Hit[], term: string, from: number): SearchResult[] { excerpt: hit.fragment.excerpt, breadcrumb: breadcrumbFrom(hit.url), sections: - from + rank < ROWS_WITH_SECTIONS ? sectionsOf(hit.fragment) : undefined, + from + at < ROWS_WITH_SECTIONS ? sectionsOf(hit.fragment) : undefined, ...classify(hit.url), })); } -/** - * The page the query names, when the first page of results missed it. - * - * `byNameThenDepth` can only promote what has been fetched, and a section's own - * page can rank far below the pages inside it: `/docs/infrastructure/ - * deployment-targets/` is 36th for "deployment targets", six places past the - * page size. These stubs come from a search narrowed to the shallow pages alone, - * where the page a query names sits near the top of a few hundred candidates. - * - * Only called when nothing already fetched names the query, so the extra - * fragments are paid for by the queries that need them and no others. - */ -async function namedPage( - stubs: PagefindResultStub[], - term: string, - already: Hit[] -): Promise { - const seen = new Set(already.map((hit) => hit.url)); - const candidates = await hydrate(stubs.slice(0, LANDING_CANDIDATES)); - - return ( - candidates.find((hit) => claimsName(hit, term) && !seen.has(hit.url)) ?? - null - ); -} - export function pagefindEngine(bundlePath: string): SearchEngine { let loading: Promise | null = null; - // Everything the last search matched, and how much of it has been handed over. - // A stub is a score and a promise of its fragment, so holding a thousand of - // them costs nothing and saves searching again to show row thirty-one. - // `promoted` is the page `namedPage` pulled forward, whose own stub is still - // waiting further down `stubs`. + // The last search's whole result set, already ranked, and how much of it has + // been drawn. Ranking the tail costs nothing because it needs no fetches, so + // `more()` only has to draw the next slice. let page: { term: string; - stubs: PagefindResultStub[]; + ranked: Ranked[]; at: number; - promoted: string | null; + // Set when the list was never ranked, so each batch is ordered as it is drawn. + reorder?: string; } | null = null; + // The map from result id to url and title, fetched once with the index. + let titles: TitleMap | null = null; + // What Pagefind prepends to the urls it returns, and therefore what the map's + // own relative urls need. `/docs/pagefind/` leaves `/docs`. + const urlPrefix = bundlePath.replace(/\/?pagefind\/?$/, ''); + + /** + * The map, found by the hash the index publishes for itself. Two requests, + * because the name cannot be known without the first — the alternative is an + * unhashed name a cache can serve from the wrong build. Both happen while the + * index is warming, off the path of any search. + */ + async function loadTitles(): Promise { + try { + const entry = await fetch(`${bundlePath}pagefind-entry.json`); + if (!entry.ok) throw new Error(`entry file: ${entry.status}`); + + const languages: Record = (await entry.json()) + .languages; + // One language, the same assumption the build makes when it names the file. + const hash = Object.values(languages ?? {})[0]?.hash; + if (!hash) throw new Error('no index hash in the entry file'); + + // Imported rather than fetched: the map is a module so that Front Door's + // static-content rule caches it, and the browser keeps parsed modules. + const map: { default: TitleMap } = await import( + /* @vite-ignore */ `${bundlePath}${titleMapName(hash)}` + ); + + return map.default; + } catch (error) { + console.error( + '[docs-search] could not load the title map; ranking falls back to the rows it draws', + error + ); + return null; + } + } // Which search owns `page`. Searches run concurrently and can settle out of // order, and an overtaken one must not leave its stubs behind for `more()`. let searches = 0; @@ -311,7 +376,15 @@ export function pagefindEngine(bundlePath: string): SearchEngine { // The filter index is a separate chunk, and a search returns empty filter // counts until it has been pulled down. Its section totals also add up to // the size of the corpus, which is what `COMMON_TERM_SHARE` is a share of. - const filters = await api.filters(); + const [filters] = await Promise.all([ + api.filters(), + // Ranking reads url and title out of this, so it has to be here before + // the first search returns. A failure leaves it null and ranking falls + // back to ordering the rows it draws. + loadTitles().then((map) => { + titles = map; + }), + ]); corpus = Object.values(filters.section ?? {}).reduce( (total, count) => total + count, 0 @@ -358,16 +431,13 @@ export function pagefindEngine(bundlePath: string): SearchEngine { const filters = facet && facet !== 'all' ? { section: [facet] } : undefined; - // Three searches at once, because a second await here would sit in front - // of every fragment fetch below it. The first supplies the rows; the - // second says whether the query has any answer at all, and only runs - // while a tab is narrowing the first; the third is the shallow-page - // shortlist `namedPage` draws on, which costs nothing until its - // fragments are fetched. - const [response, wholeCorpus, landing] = await Promise.all([ + // Together, because a second await here would sit in front of every + // fragment fetch below it. The first supplies the rows; the second says + // whether the query has any answer at all, and only runs while a tab is + // narrowing the first. + const [response, wholeCorpus] = await Promise.all([ api.search(query, { filters }), filters ? api.search(query) : null, - api.search(query, { filters: { ...filters, landing: ['true'] } }), ]); const unfiltered = wholeCorpus ?? response; @@ -395,29 +465,42 @@ export function pagefindEngine(bundlePath: string): SearchEngine { : empty; } - const hits = await hydrate(response.results.slice(0, PAGE_SIZE)); - - const named = hits.some((hit) => claimsName(hit, query)) - ? null - : await namedPage(landing.results, query, hits); - if (named) hits.push(named); - - // A promoted page was fetched precisely because its own stub ranks past - // `PAGE_SIZE`, so that stub is still ahead of `more()` and has to be - // skipped there rather than drawn a second time. - settle({ - term: query, - stubs: response.results, - at: PAGE_SIZE, - promoted: named?.url ?? null, - }); + // Every match is ranked here, whether it will be drawn or not. A page + // the query names wins from anywhere in the list, which is what the + // shallow-page search used to be for. + const ranked = titles + ? rank(response.results, titles, query, urlPrefix) + : []; + + // The map was missing, or disagreed with the index so completely that + // nothing joined. These carry no url or title, so `draw` reads both off + // the fragments and orders each batch as it draws it — which is what the + // engine did before the map existed. + const fallback = + ranked.length === 0 && response.results.length > 0 + ? response.results.map((stub) => ({ + stub, + score: stub.score, + url: '', + title: '', + })) + : null; + if (fallback) { + console.error( + `[docs-search] ranking without the title map: ${response.results.length} results, none of them in it` + ); + } + + const ordered = fallback ?? ranked; + // Every batch of an unranked list has to be ordered as it is drawn, + // including the ones `more()` fetches later. + const reorder = fallback ? query : undefined; + settle({ term: query, ranked: ordered, at: PAGE_SIZE, reorder }); return { - // The promoted page is already one of these stubs, so counting them - // is counting the rows the query has in all. - results: rows(hits, query, 0), + results: await draw(ordered, 0, PAGE_SIZE, reorder), counts, - total: response.results.length, + total: ordered.length, }; } catch (error) { // The search itself failed, rather than one row of it. Rejecting would @@ -433,18 +516,14 @@ export function pagefindEngine(bundlePath: string): SearchEngine { // Read off before the await: a search landing in the meantime replaces // `page`, and these rows still belong to the query that asked for them. - const { term, promoted } = page; - const slice = page.stubs.slice(page.at, page.at + PAGE_SIZE); - if (slice.length === 0) return []; - + const { ranked, reorder } = page; const from = page.at; - page.at += slice.length; + if (from >= ranked.length) return []; + + page.at = Math.min(from + PAGE_SIZE, ranked.length); try { - const hits = (await hydrate(slice)).filter( - (hit) => hit.url !== promoted - ); - return rows(hits, term, from); + return await draw(ranked, from, PAGE_SIZE, reorder); } catch (error) { // The rows already on screen are still good, so this fails quietly and // leaves them alone. diff --git a/tests/docs-search.spec.ts b/tests/docs-search.spec.ts index 18ef10ffdf..77bc177f09 100644 --- a/tests/docs-search.spec.ts +++ b/tests/docs-search.spec.ts @@ -430,14 +430,11 @@ test('scrolling to the end of the results loads more', async ({ page }) => { expect(new Set(ids).size, 'every option needs its own id').toBe(ids.length); }); -// The two features meeting: a page pulled onto the first screen for naming the -// query still has its own stub further down the list, because ranking past -// PAGE_SIZE is why it had to be pulled forward at all. Paging has to skip it. -// -// `deployment targets` rather than a query with more results: the promotion only -// happens when nothing on the first page already names the query, and -// /docs/infrastructure/deployment-targets/ ranks 36th on raw score. -test('a page pulled forward is not listed again further down', async ({ +// Ranking and paging meeting. The whole result set is ranked before anything is +// drawn, and paging walks that one list, so a page promoted from deep in it — +// /docs/infrastructure/deployment-targets/ ranks 36th for this query on raw +// score — must not come round again when its own position is reached. +test('a page promoted from deep in the list is drawn only once', async ({ page, }) => { await page.goto('/docs');