From 399b5a1693e4d41fc10c23fe8e8f95ebef1d7bef Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Fri, 21 Aug 2026 11:07:26 +1200 Subject: [PATCH 1/5] Rank from a title map, and fetch only the rows on screen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ranking needed a url and a title, and Pagefind keeps both in the per-page fragment — so ordering thirty results meant fetching thirty files before the panel could draw, and a page ranked past thirty could not be reached at all. The build now writes a map of result id to url and title beside the index, which is the join Pagefind's own result stub already carries. So the whole result set is ranked before anything is fetched, and fragments are fetched only for the rows being drawn: ten per batch, against thirty to thirty-five for every settled query before. On Slow 4G with the map served uncompressed, first results arrive in 7.2s against 8.9s; the map is 28 KB gzipped, so most of that 122 KB is transfer a CDN removes. The shallow-page search this replaces is gone with it — the second Pagefind query, the landing filter, LANDING_DEPTH and the attribute it needed on every page's content div. A page the query names now wins from anywhere in the list rather than from a shortlist of 227. Relevance holds on both traffic-weighted sets: real-searches 57% w-S@1 and 84% w-S@5, top-pages 90% and 98%, unchanged either side. Co-Authored-By: Claude Opus 5 (1M context) --- src/integrations/pagefind-index.ts | 68 +++++++ src/layouts/Api.astro | 2 +- src/layouts/Default.astro | 2 +- src/lib/searchIndexing.ts | 29 +-- src/scripts/search-engine-pagefind.ts | 244 ++++++++++++++------------ tests/docs-search.spec.ts | 13 +- 6 files changed, 206 insertions(+), 152 deletions(-) diff --git a/src/integrations/pagefind-index.ts b/src/integrations/pagefind-index.ts index a61f83438b..3c574bfdd2 100644 --- a/src/integrations/pagefind-index.ts +++ b/src/integrations/pagefind-index.ts @@ -10,6 +10,8 @@ import type { AstroIntegration } from 'astro'; import { fileURLToPath } from 'node:url'; import * as path from 'node:path'; +import { gunzipSync } from 'node:zlib'; +import { readdir, readFile, writeFile } from 'node:fs/promises'; // Statically, because `astro:build:done` fires after Vite's module runner has // closed and a dynamic import from inside the hook cannot be resolved. import { createIndex, close } from 'pagefind'; @@ -81,6 +83,11 @@ export default function pagefindIndex(): AstroIntegration { // Files scanned, not pages indexed: the redirect stubs are counted // here and then dropped for having no article. logger.info(`scanned ${added.page_count} pages into docs/pagefind`); + + const titles = await writeTitleMap( + path.join(distDir, 'docs', 'pagefind') + ); + logger.info(`wrote ${titles} titles to ${TITLE_MAP}`); } finally { await close(); } @@ -88,3 +95,64 @@ export default function pagefindIndex(): AstroIntegration { }, }; } + +/** Where the map lands, and the name the client fetches it by. */ +const TITLE_MAP = 'docs-titles.json'; + +// Pagefind prefixes every decompressed chunk with this before the JSON. +const FRAGMENT_MAGIC = 'pagefind_dcd'; + +/** + * Writes what the overlay needs to rank a result without fetching it. + * + * Ranking needs a URL and a title, and Pagefind keeps both in the per-page + * fragment — so ranking thirty results meant fetching thirty files, and a + * landing page ranked past that could not be reached at all. A search result + * stub carries the id of its own fragment, so one map from id to url and title + * lets the whole result set be ranked from a single file. + * + * Read back out of the fragments rather than collected during indexing, because + * the ids are assigned by Pagefind as it writes them. + */ +async function writeTitleMap(pagefindDir: string): Promise { + const dir = path.join(pagefindDir, 'fragment'); + const files = (await readdir(dir)).filter((name) => + name.endsWith('.pf_fragment') + ); + + const map: Record = {}; + + for (const file of files) { + const raw = gunzipSync(await readFile(path.join(dir, file))).toString( + 'utf8' + ); + + // Checked before parsing: a Pagefind release that changes the chunk format + // has to fail the build here, rather than write a map the overlay silently + // cannot join against. + if (!raw.startsWith(FRAGMENT_MAGIC)) { + throw new Error( + `unexpected fragment format in ${file}: Pagefind's own prefix is missing, so ${TITLE_MAP} cannot be trusted` + ); + } + + const fragment = JSON.parse(raw.slice(raw.indexOf('{'))) as { + url: string; + meta?: Record; + }; + + // The stub's `id` is the filename without its extension, which is the join. + map[path.basename(file, '.pf_fragment')] = [ + fragment.url, + fragment.meta?.title ?? '', + ]; + } + + await writeFile( + path.join(pagefindDir, TITLE_MAP), + JSON.stringify(map), + 'utf8' + ); + + return files.length; +} diff --git a/src/layouts/Api.astro b/src/layouts/Api.astro index 54552b1bbf..df7c0d35eb 100644 --- a/src/layouts/Api.astro +++ b/src/layouts/Api.astro @@ -99,7 +99,7 @@ const searchIndex = searchIndexAttributes(Astro.url.pathname, frontmatter); /* Copy as markdown temporarily disabled until we can get it working with the API docs. Deliberately no "Edit on GitHub" because these are generated and should not be hand edited */ } -
+
diff --git a/src/layouts/Default.astro b/src/layouts/Default.astro index a5711494fd..3d4f62a740 100644 --- a/src/layouts/Default.astro +++ b/src/layouts/Default.astro @@ -100,7 +100,7 @@ const searchIndex = searchIndexAttributes(Astro.url.pathname, frontmatter); lang={lang} />
-
+
diff --git a/src/lib/searchIndexing.ts b/src/lib/searchIndexing.ts index b21aa20b17..ea46993cdb 100644 --- a/src/lib/searchIndexing.ts +++ b/src/lib/searchIndexing.ts @@ -13,30 +13,12 @@ type ArticleAttributes = { 'data-pagefind-default-meta'?: string; }; -/** - * A second filter, on an element inside the article rather than on the article - * itself: Pagefind reads one `key:value` per `data-pagefind-filter`, and a - * comma-separated pair is taken as a single value. - */ -type ContentAttributes = { - 'data-pagefind-filter'?: string; -}; - type IndexAttributes = { article: ArticleAttributes; - content: ContentAttributes; }; /** - * How shallow a page has to be to count as one a reader might name. Two segments - * past `/docs/`, which covers `/docs/deployments/` and - * `/docs/infrastructure/deployment-targets/` but not the pages inside them. - */ -const LANDING_DEPTH = 3; - -/** - * The `data-pagefind-*` attributes for a page: `article` spreads onto the - * `
`, `content` onto the page content inside it. + * The `data-pagefind-*` attributes for a page, to spread onto the `
`. * * `navSearch` rather than `PostFiltering.showInSearch`, which also hides a page * with a future `pubDate`, a `draft: true` and a `listable: false`: a page that @@ -53,13 +35,7 @@ export function searchIndexAttributes( // `all` rather than the default `index`: a bare ignore still lets Pagefind // read a title or metadata out of the block. - if (!indexable) - return { article: { 'data-pagefind-ignore': 'all' }, content: {} }; - - // Marks the pages the overlay's second, narrowed search looks through. Only - // the shallow pages carry it, so the filter chunk stays small and that search - // has a few hundred candidates rather than the whole site. - const isLanding = pathname.split('/').filter(Boolean).length <= LANDING_DEPTH; + if (!indexable) return { article: { 'data-pagefind-ignore': 'all' } }; return { article: { @@ -75,6 +51,5 @@ export function searchIndexAttributes( ? { 'data-pagefind-default-meta': `title:${frontmatter.title}` } : {}), }, - content: isLanding ? { 'data-pagefind-filter': 'landing:true' } : {}, }; } diff --git a/src/scripts/search-engine-pagefind.ts b/src/scripts/search-engine-pagefind.ts index e6b902add9..16a60e5c99 100644 --- a/src/scripts/search-engine-pagefind.ts +++ b/src/scripts/search-engine-pagefind.ts @@ -12,10 +12,13 @@ import { type SearchResult, } from './search-engine'; -// Rows fetched at a time. Pagefind's own UI shows five and offers the rest on -// demand; thirty, because `byNameThenDepth` reorders within a page and needs -// enough of the list to have something to reorder. -const PAGE_SIZE = 30; +// Rows fetched at a time. The panel shows about five, and the rest arrive as it +// is scrolled. Ranking no longer needs them, so this is only a drawing budget: +// ten covers the first screen with room to scroll into. +const PAGE_SIZE = 10; + +/** The map of result id to url and title, written beside the index at build. */ +const TITLE_MAP = 'docs-titles.json'; // Above this share of the corpus, a query is too general to rank rather than // unanswerable, and the overlay says so instead of reporting nothing found. @@ -28,11 +31,6 @@ const PAGE_SIZE = 30; // 79%. A mash landing on the gentler message costs nothing; both offer no rows. const COMMON_TERM_SHARE = 0.8; -// How many shallow pages the named-page lookup looks through. Measured over 18 -// section queries: three finds the page for 16, five for 17, and twenty finds no -// more than five does. -const LANDING_CANDIDATES = 5; - type PagefindSubResult = { title: string; /** Carries the heading's `#anchor` when the match is below the page title. */ @@ -171,8 +169,10 @@ function claimsName(hit: { url: string; title: string }, term: string) { * order for none of them; `data-pagefind-weight` on the h1 measured as no change * at all. * - * Reaches only the results already fetched, which is what `namedPage` below is - * for: a page ranked past `PAGE_SIZE` on raw score cannot be rescued here. + * Runs over every result, because `rank` supplies url and title from the title + * map and nothing here has to be fetched. So a page the query names wins from + * anywhere in the list — `/docs/infrastructure/deployment-targets/` is 36th on + * raw score for "deployment targets" and still comes first. */ function byNameThenDepth< T extends { score: number; url: string; title: string }, @@ -196,19 +196,58 @@ function byNameThenDepth< } /** A stub's score paired with its fetched fragment, which is where the URL is. */ -type Hit = { - fragment: PagefindFragment; +/** Everything ranking needs, and nothing that has to be fetched to get it. */ +type Ranked = { + stub: PagefindResultStub; score: number; url: string; title: string; }; +type TitleMap = Record; + +/** + * Pairs each result with its url and title from the map, so the whole set can be + * ranked before anything is fetched. + * + * A result the map does not know is dropped from ranking. That only happens when + * the map and the index disagree, which means a stale deploy of one of them; the + * caller falls back to ranking what it fetches. + */ +function rank( + stubs: PagefindResultStub[], + titles: TitleMap, + term: string, + prefix: string +): Ranked[] { + const known = stubs.flatMap((stub) => { + const entry = titles[stub.id]; + if (!entry) return []; + // The map holds urls as the index does, relative to the indexed directory. + // Pagefind applies the same prefix to the urls it returns from `data()`. + const [path, title] = entry; + const url = prefix + path; + return [{ stub, score: stub.score, url, title: title || url }]; + }); + + return byNameThenDepth(known, term); +} + /** - * Fetches the fragment for each stub. A fragment that fails takes its own row - * out rather than the whole result set. + * Draws a slice of the ranked list, fetching a fragment for each row in it. The + * fragment supplies the excerpt and the matched headings; the order was settled + * before any of it was asked for. + * + * `from` is how many rows already precede these, which keeps the headings on the + * leading rows of the list rather than the leading rows of every batch. */ -async function hydrate(stubs: PagefindResultStub[]): Promise { - const settled = await Promise.allSettled(stubs.map((stub) => stub.data())); +async function draw( + ranked: Ranked[], + from: number, + count: number +): Promise { + const slice = ranked.slice(from, from + count); + const settled = await Promise.allSettled(slice.map((hit) => hit.stub.data())); return settled.flatMap((outcome, at) => { if (outcome.status === 'rejected') { @@ -219,77 +258,38 @@ async function hydrate(stubs: PagefindResultStub[]): Promise { return []; } + const hit = slice[at]; const fragment = outcome.value; + // The fragment is the fallback for both: without the title map, `rank` has + // no url or title to give and the fragment is the only source. const { pathname } = new URL(fragment.url, window.location.origin); return [ { - fragment, - score: stubs[at].score, - url: pathname, - title: fragment.meta?.title ?? pathname, + url: hit.url || pathname, + title: hit.title || fragment.meta?.title || pathname, + // Already carries around the hits, and Pagefind escapes the + // surrounding text itself. + excerpt: fragment.excerpt, + breadcrumb: breadcrumbFrom(hit.url || pathname), + sections: + from + at < ROWS_WITH_SECTIONS ? sectionsOf(fragment) : undefined, + ...classify(hit.url || pathname), }, ]; }); } -/** - * One page of rows, ordered within itself. `from` is how many rows already - * precede them, which is what keeps the headings on the leading pages of the - * list rather than on the leading rows of every page. - */ -function rows(hits: Hit[], term: string, from: number): SearchResult[] { - return byNameThenDepth(hits, term).map((hit, rank) => ({ - url: hit.url, - title: hit.title, - // Already carries around the hits, and Pagefind escapes the - // surrounding text itself. - excerpt: hit.fragment.excerpt, - breadcrumb: breadcrumbFrom(hit.url), - sections: - from + rank < ROWS_WITH_SECTIONS ? sectionsOf(hit.fragment) : undefined, - ...classify(hit.url), - })); -} - -/** - * The page the query names, when the first page of results missed it. - * - * `byNameThenDepth` can only promote what has been fetched, and a section's own - * page can rank far below the pages inside it: `/docs/infrastructure/ - * deployment-targets/` is 36th for "deployment targets", six places past the - * page size. These stubs come from a search narrowed to the shallow pages alone, - * where the page a query names sits near the top of a few hundred candidates. - * - * Only called when nothing already fetched names the query, so the extra - * fragments are paid for by the queries that need them and no others. - */ -async function namedPage( - stubs: PagefindResultStub[], - term: string, - already: Hit[] -): Promise { - const seen = new Set(already.map((hit) => hit.url)); - const candidates = await hydrate(stubs.slice(0, LANDING_CANDIDATES)); - - return ( - candidates.find((hit) => claimsName(hit, term) && !seen.has(hit.url)) ?? - null - ); -} - export function pagefindEngine(bundlePath: string): SearchEngine { let loading: Promise | null = null; - // Everything the last search matched, and how much of it has been handed over. - // A stub is a score and a promise of its fragment, so holding a thousand of - // them costs nothing and saves searching again to show row thirty-one. - // `promoted` is the page `namedPage` pulled forward, whose own stub is still - // waiting further down `stubs`. - let page: { - term: string; - stubs: PagefindResultStub[]; - at: number; - promoted: string | null; - } | null = null; + // The last search's whole result set, already ranked, and how much of it has + // been drawn. Ranking the tail costs nothing because it needs no fetches, so + // `more()` only has to draw the next slice. + let page: { term: string; ranked: Ranked[]; at: number } | null = null; + // The map from result id to url and title, fetched once with the index. + let titles: TitleMap | null = null; + // What Pagefind prepends to the urls it returns, and therefore what the map's + // own relative urls need. `/docs/pagefind/` leaves `/docs`. + const urlPrefix = bundlePath.replace(/\/?pagefind\/?$/, ''); // Which search owns `page`. Searches run concurrently and can settle out of // order, and an overtaken one must not leave its stubs behind for `more()`. let searches = 0; @@ -311,7 +311,20 @@ export function pagefindEngine(bundlePath: string): SearchEngine { // The filter index is a separate chunk, and a search returns empty filter // counts until it has been pulled down. Its section totals also add up to // the size of the corpus, which is what `COMMON_TERM_SHARE` is a share of. - const filters = await api.filters(); + const [filters] = await Promise.all([ + api.filters(), + // Ranking reads url and title out of this, so it has to be here before + // the first search returns. A failure leaves it null and ranking falls + // back to ordering the rows it draws. + fetch(`${bundlePath}${TITLE_MAP}`) + .then((response) => (response.ok ? response.json() : null)) + .then((map: TitleMap | null) => { + titles = map; + }) + .catch((error) => { + console.error(`[docs-search] could not load ${TITLE_MAP}`, error); + }), + ]); corpus = Object.values(filters.section ?? {}).reduce( (total, count) => total + count, 0 @@ -358,16 +371,13 @@ export function pagefindEngine(bundlePath: string): SearchEngine { const filters = facet && facet !== 'all' ? { section: [facet] } : undefined; - // Three searches at once, because a second await here would sit in front - // of every fragment fetch below it. The first supplies the rows; the - // second says whether the query has any answer at all, and only runs - // while a tab is narrowing the first; the third is the shallow-page - // shortlist `namedPage` draws on, which costs nothing until its - // fragments are fetched. - const [response, wholeCorpus, landing] = await Promise.all([ + // Together, because a second await here would sit in front of every + // fragment fetch below it. The first supplies the rows; the second says + // whether the query has any answer at all, and only runs while a tab is + // narrowing the first. + const [response, wholeCorpus] = await Promise.all([ api.search(query, { filters }), filters ? api.search(query) : null, - api.search(query, { filters: { ...filters, landing: ['true'] } }), ]); const unfiltered = wholeCorpus ?? response; @@ -395,29 +405,37 @@ export function pagefindEngine(bundlePath: string): SearchEngine { : empty; } - const hits = await hydrate(response.results.slice(0, PAGE_SIZE)); - - const named = hits.some((hit) => claimsName(hit, query)) - ? null - : await namedPage(landing.results, query, hits); - if (named) hits.push(named); - - // A promoted page was fetched precisely because its own stub ranks past - // `PAGE_SIZE`, so that stub is still ahead of `more()` and has to be - // skipped there rather than drawn a second time. - settle({ - term: query, - stubs: response.results, - at: PAGE_SIZE, - promoted: named?.url ?? null, - }); + // Every match is ranked here, whether it will be drawn or not. A page + // the query names wins from anywhere in the list, which is what the + // shallow-page search used to be for. + const ranked = titles + ? rank(response.results, titles, query, urlPrefix) + : []; + + // The map was missing or disagreed with the index. Ranking what gets + // drawn is worse than ranking everything, and it still answers. + const fallback = + ranked.length === 0 && response.results.length > 0 + ? response.results.map((stub) => ({ + stub, + score: stub.score, + url: '', + title: '', + })) + : null; + if (fallback) { + console.error( + `[docs-search] ranking without ${TITLE_MAP}: ${response.results.length} results, none of them in the map` + ); + } + + const ordered = fallback ?? ranked; + settle({ term: query, ranked: ordered, at: PAGE_SIZE }); return { - // The promoted page is already one of these stubs, so counting them - // is counting the rows the query has in all. - results: rows(hits, query, 0), + results: await draw(ordered, 0, PAGE_SIZE), counts, - total: response.results.length, + total: ordered.length, }; } catch (error) { // The search itself failed, rather than one row of it. Rejecting would @@ -433,18 +451,14 @@ export function pagefindEngine(bundlePath: string): SearchEngine { // Read off before the await: a search landing in the meantime replaces // `page`, and these rows still belong to the query that asked for them. - const { term, promoted } = page; - const slice = page.stubs.slice(page.at, page.at + PAGE_SIZE); - if (slice.length === 0) return []; - + const { ranked } = page; const from = page.at; - page.at += slice.length; + if (from >= ranked.length) return []; + + page.at = Math.min(from + PAGE_SIZE, ranked.length); try { - const hits = (await hydrate(slice)).filter( - (hit) => hit.url !== promoted - ); - return rows(hits, term, from); + return await draw(ranked, from, PAGE_SIZE); } catch (error) { // The rows already on screen are still good, so this fails quietly and // leaves them alone. diff --git a/tests/docs-search.spec.ts b/tests/docs-search.spec.ts index 18ef10ffdf..77bc177f09 100644 --- a/tests/docs-search.spec.ts +++ b/tests/docs-search.spec.ts @@ -430,14 +430,11 @@ test('scrolling to the end of the results loads more', async ({ page }) => { expect(new Set(ids).size, 'every option needs its own id').toBe(ids.length); }); -// The two features meeting: a page pulled onto the first screen for naming the -// query still has its own stub further down the list, because ranking past -// PAGE_SIZE is why it had to be pulled forward at all. Paging has to skip it. -// -// `deployment targets` rather than a query with more results: the promotion only -// happens when nothing on the first page already names the query, and -// /docs/infrastructure/deployment-targets/ ranks 36th on raw score. -test('a page pulled forward is not listed again further down', async ({ +// Ranking and paging meeting. The whole result set is ranked before anything is +// drawn, and paging walks that one list, so a page promoted from deep in it — +// /docs/infrastructure/deployment-targets/ ranks 36th for this query on raw +// score — must not come round again when its own position is reached. +test('a page promoted from deep in the list is drawn only once', async ({ page, }) => { await page.goto('/docs'); From 80efca3acf6413028bd8c2b5554b62d83a75a9a9 Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Wed, 26 Aug 2026 17:04:04 +1200 Subject: [PATCH 2/5] Rank the fallback list, and say when the map only half matches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two things found reviewing the branch before publishing it. The fallback for a missing or mismatched title map drew results in Pagefind's own order and never ranked them, while its comment claimed otherwise. That is the order the reorder exists to correct — a bare BM25 list puts a getting-started page above the section it belongs to. `draw` now orders each batch as it draws it when the list reached it unranked, which is what the engine did before the map existed, and `more()` carries the same flag so later batches match. Verified by blocking the map: three of four sample queries still land on the right page, and the fourth is one whose answer ranks past the batch. A map that matches the index only partly dropped the unmatched results quietly, because the fallback fires only when nothing joins at all. It now warns with the counts. Also removed a stale doc comment left on the Ranked type. Relevance unchanged: real-searches holds at 57% w-S@1 and 84% w-S@5. Co-Authored-By: Claude Opus 5 (1M context) --- src/scripts/search-engine-pagefind.ts | 71 ++++++++++++++++++++------- 1 file changed, 52 insertions(+), 19 deletions(-) diff --git a/src/scripts/search-engine-pagefind.ts b/src/scripts/search-engine-pagefind.ts index 16a60e5c99..b47b2e496f 100644 --- a/src/scripts/search-engine-pagefind.ts +++ b/src/scripts/search-engine-pagefind.ts @@ -195,7 +195,6 @@ function byNameThenDepth< .map((entry) => entry.hit); } -/** A stub's score paired with its fetched fragment, which is where the URL is. */ /** Everything ranking needs, and nothing that has to be fetched to get it. */ type Ranked = { stub: PagefindResultStub; @@ -230,6 +229,15 @@ function rank( return [{ stub, score: stub.score, url, title: title || url }]; }); + // Partly stale is worse than wholly stale, because it drops results quietly + // and the fallback never fires. Say so rather than answering with a hole in + // the list. + if (known.length < stubs.length) { + console.warn( + `[docs-search] ${stubs.length - known.length} of ${stubs.length} results are missing from ${TITLE_MAP} and were dropped` + ); + } + return byNameThenDepth(known, term); } @@ -244,12 +252,13 @@ function rank( async function draw( ranked: Ranked[], from: number, - count: number + count: number, + reorder?: string ): Promise { const slice = ranked.slice(from, from + count); const settled = await Promise.allSettled(slice.map((hit) => hit.stub.data())); - return settled.flatMap((outcome, at) => { + const drawn = settled.flatMap((outcome, at) => { if (outcome.status === 'rejected') { console.error( '[docs-search] dropped a result whose fragment failed', @@ -260,23 +269,36 @@ async function draw( const hit = slice[at]; const fragment = outcome.value; - // The fragment is the fallback for both: without the title map, `rank` has - // no url or title to give and the fragment is the only source. + // Without the title map `rank` had no url or title to give, so the fragment + // is the only source for both. const { pathname } = new URL(fragment.url, window.location.origin); return [ { + fragment, + score: hit.score, url: hit.url || pathname, title: hit.title || fragment.meta?.title || pathname, - // Already carries around the hits, and Pagefind escapes the - // surrounding text itself. - excerpt: fragment.excerpt, - breadcrumb: breadcrumbFrom(hit.url || pathname), - sections: - from + at < ROWS_WITH_SECTIONS ? sectionsOf(fragment) : undefined, - ...classify(hit.url || pathname), }, ]; }); + + // Ranked already, unless the map was unusable and this slice arrived in + // Pagefind's own order. Ordering these few is worse than ordering the whole + // set, and far better than leaving a bare BM25 list: the reorder is what puts + // a section's own page above the pages inside it. + const ordered = reorder ? byNameThenDepth(drawn, reorder) : drawn; + + return ordered.map((hit, at) => ({ + url: hit.url, + title: hit.title, + // Already carries around the hits, and Pagefind escapes the + // surrounding text itself. + excerpt: hit.fragment.excerpt, + breadcrumb: breadcrumbFrom(hit.url), + sections: + from + at < ROWS_WITH_SECTIONS ? sectionsOf(hit.fragment) : undefined, + ...classify(hit.url), + })); } export function pagefindEngine(bundlePath: string): SearchEngine { @@ -284,7 +306,13 @@ export function pagefindEngine(bundlePath: string): SearchEngine { // The last search's whole result set, already ranked, and how much of it has // been drawn. Ranking the tail costs nothing because it needs no fetches, so // `more()` only has to draw the next slice. - let page: { term: string; ranked: Ranked[]; at: number } | null = null; + let page: { + term: string; + ranked: Ranked[]; + at: number; + // Set when the list was never ranked, so each batch is ordered as it is drawn. + reorder?: string; + } | null = null; // The map from result id to url and title, fetched once with the index. let titles: TitleMap | null = null; // What Pagefind prepends to the urls it returns, and therefore what the map's @@ -412,8 +440,10 @@ export function pagefindEngine(bundlePath: string): SearchEngine { ? rank(response.results, titles, query, urlPrefix) : []; - // The map was missing or disagreed with the index. Ranking what gets - // drawn is worse than ranking everything, and it still answers. + // The map was missing, or disagreed with the index so completely that + // nothing joined. These carry no url or title, so `draw` reads both off + // the fragments and orders each batch as it draws it — which is what the + // engine did before the map existed. const fallback = ranked.length === 0 && response.results.length > 0 ? response.results.map((stub) => ({ @@ -430,10 +460,13 @@ export function pagefindEngine(bundlePath: string): SearchEngine { } const ordered = fallback ?? ranked; - settle({ term: query, ranked: ordered, at: PAGE_SIZE }); + // Every batch of an unranked list has to be ordered as it is drawn, + // including the ones `more()` fetches later. + const reorder = fallback ? query : undefined; + settle({ term: query, ranked: ordered, at: PAGE_SIZE, reorder }); return { - results: await draw(ordered, 0, PAGE_SIZE), + results: await draw(ordered, 0, PAGE_SIZE, reorder), counts, total: ordered.length, }; @@ -451,14 +484,14 @@ export function pagefindEngine(bundlePath: string): SearchEngine { // Read off before the await: a search landing in the meantime replaces // `page`, and these rows still belong to the query that asked for them. - const { ranked } = page; + const { ranked, reorder } = page; const from = page.at; if (from >= ranked.length) return []; page.at = Math.min(from + PAGE_SIZE, ranked.length); try { - return await draw(ranked, from, PAGE_SIZE); + return await draw(ranked, from, PAGE_SIZE, reorder); } catch (error) { // The rows already on screen are still good, so this fails quietly and // leaves them alone. From 3cf428583c54ff5abb9fd7fa622febddee75976e Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Wed, 26 Aug 2026 17:29:56 +1200 Subject: [PATCH 3/5] Name the title map after the index hash An unhashed name lets a cache serve one build's map against another build's index. Pagefind hashes its own chunks for that reason, and the map has to be versioned with them: read against the wrong index it joins against nothing. The build now reads the hash out of pagefind-entry.json and writes docs-titles..json; the client reads the same file to build the same name. Two requests where there was one, both while the index is warming and neither on the path of a search. It also means the map can carry an immutable cache header, which an unhashed name could not. The build fails if the index has more than one language, because one map cannot carry two hashes and the client would look under a name that does not exist. Co-Authored-By: Claude Opus 5 (1M context) --- src/integrations/pagefind-index.ts | 49 +++++++++++++++++-------- src/scripts/search-engine-pagefind.ts | 53 +++++++++++++++++++++------ 2 files changed, 75 insertions(+), 27 deletions(-) diff --git a/src/integrations/pagefind-index.ts b/src/integrations/pagefind-index.ts index 3c574bfdd2..1f391978d2 100644 --- a/src/integrations/pagefind-index.ts +++ b/src/integrations/pagefind-index.ts @@ -87,7 +87,7 @@ export default function pagefindIndex(): AstroIntegration { const titles = await writeTitleMap( path.join(distDir, 'docs', 'pagefind') ); - logger.info(`wrote ${titles} titles to ${TITLE_MAP}`); + logger.info(`wrote ${titles.pages} titles to ${titles.file}`); } finally { await close(); } @@ -96,8 +96,13 @@ export default function pagefindIndex(): AstroIntegration { }; } -/** Where the map lands, and the name the client fetches it by. */ -const TITLE_MAP = 'docs-titles.json'; +/** + * The map's name, carrying the index's own hash. Pagefind hashes its chunks so a + * cache cannot serve one build's index against another's, and a map read against + * the wrong index joins against nothing. The client reads the hash out of + * `pagefind-entry.json` to build the same name. + */ +const titleMapName = (hash: string) => `docs-titles.${hash}.json`; // Pagefind prefixes every decompressed chunk with this before the JSON. const FRAGMENT_MAGIC = 'pagefind_dcd'; @@ -114,16 +119,34 @@ const FRAGMENT_MAGIC = 'pagefind_dcd'; * Read back out of the fragments rather than collected during indexing, because * the ids are assigned by Pagefind as it writes them. */ -async function writeTitleMap(pagefindDir: string): Promise { +async function writeTitleMap( + pagefindDir: string +): Promise<{ pages: number; file: string }> { + const entry = JSON.parse( + await readFile(path.join(pagefindDir, 'pagefind-entry.json'), 'utf8') + ) as { languages: Record }; + + const languages = Object.keys(entry.languages ?? {}); + // One map covers every fragment, so it can only carry one language's hash. A + // second language would need one map each, keyed the way Pagefind keys its own + // chunks — worth failing loudly over rather than shipping a map the client + // looks for under the wrong name. + if (languages.length !== 1) { + throw new Error( + `expected one indexed language, found ${languages.length || 'none'}: the title map is named after the index hash and cannot cover several` + ); + } + + const file = titleMapName(entry.languages[languages[0]].hash); const dir = path.join(pagefindDir, 'fragment'); - const files = (await readdir(dir)).filter((name) => + const names = (await readdir(dir)).filter((name) => name.endsWith('.pf_fragment') ); const map: Record = {}; - for (const file of files) { - const raw = gunzipSync(await readFile(path.join(dir, file))).toString( + for (const name of names) { + const raw = gunzipSync(await readFile(path.join(dir, name))).toString( 'utf8' ); @@ -132,7 +155,7 @@ async function writeTitleMap(pagefindDir: string): Promise { // cannot join against. if (!raw.startsWith(FRAGMENT_MAGIC)) { throw new Error( - `unexpected fragment format in ${file}: Pagefind's own prefix is missing, so ${TITLE_MAP} cannot be trusted` + `unexpected fragment format in ${name}: Pagefind's own prefix is missing, so the title map cannot be trusted` ); } @@ -142,17 +165,13 @@ async function writeTitleMap(pagefindDir: string): Promise { }; // The stub's `id` is the filename without its extension, which is the join. - map[path.basename(file, '.pf_fragment')] = [ + map[path.basename(name, '.pf_fragment')] = [ fragment.url, fragment.meta?.title ?? '', ]; } - await writeFile( - path.join(pagefindDir, TITLE_MAP), - JSON.stringify(map), - 'utf8' - ); + await writeFile(path.join(pagefindDir, file), JSON.stringify(map), 'utf8'); - return files.length; + return { pages: names.length, file }; } diff --git a/src/scripts/search-engine-pagefind.ts b/src/scripts/search-engine-pagefind.ts index b47b2e496f..4975fbf34e 100644 --- a/src/scripts/search-engine-pagefind.ts +++ b/src/scripts/search-engine-pagefind.ts @@ -17,8 +17,12 @@ import { // ten covers the first screen with room to scroll into. const PAGE_SIZE = 10; -/** The map of result id to url and title, written beside the index at build. */ -const TITLE_MAP = 'docs-titles.json'; +/** + * The map of result id to url and title, written beside the index at build and + * named after the index's own hash. Reading it means reading that hash first, out + * of the entry file Pagefind publishes for the same purpose. + */ +const titleMapName = (hash: string) => `docs-titles.${hash}.json`; // Above this share of the corpus, a query is too general to rank rather than // unanswerable, and the overlay says so instead of reporting nothing found. @@ -234,7 +238,7 @@ function rank( // the list. if (known.length < stubs.length) { console.warn( - `[docs-search] ${stubs.length - known.length} of ${stubs.length} results are missing from ${TITLE_MAP} and were dropped` + `[docs-search] ${stubs.length - known.length} of ${stubs.length} results are missing from the title map and were dropped` ); } @@ -318,6 +322,36 @@ export function pagefindEngine(bundlePath: string): SearchEngine { // What Pagefind prepends to the urls it returns, and therefore what the map's // own relative urls need. `/docs/pagefind/` leaves `/docs`. const urlPrefix = bundlePath.replace(/\/?pagefind\/?$/, ''); + + /** + * The map, found by the hash the index publishes for itself. Two requests, + * because the name cannot be known without the first — the alternative is an + * unhashed name a cache can serve from the wrong build. Both happen while the + * index is warming, off the path of any search. + */ + async function loadTitles(): Promise { + try { + const entry = await fetch(`${bundlePath}pagefind-entry.json`); + if (!entry.ok) throw new Error(`entry file: ${entry.status}`); + + const languages: Record = (await entry.json()) + .languages; + // One language, the same assumption the build makes when it names the file. + const hash = Object.values(languages ?? {})[0]?.hash; + if (!hash) throw new Error('no index hash in the entry file'); + + const map = await fetch(`${bundlePath}${titleMapName(hash)}`); + if (!map.ok) throw new Error(`${titleMapName(hash)}: ${map.status}`); + + return (await map.json()) as TitleMap; + } catch (error) { + console.error( + '[docs-search] could not load the title map; ranking falls back to the rows it draws', + error + ); + return null; + } + } // Which search owns `page`. Searches run concurrently and can settle out of // order, and an overtaken one must not leave its stubs behind for `more()`. let searches = 0; @@ -344,14 +378,9 @@ export function pagefindEngine(bundlePath: string): SearchEngine { // Ranking reads url and title out of this, so it has to be here before // the first search returns. A failure leaves it null and ranking falls // back to ordering the rows it draws. - fetch(`${bundlePath}${TITLE_MAP}`) - .then((response) => (response.ok ? response.json() : null)) - .then((map: TitleMap | null) => { - titles = map; - }) - .catch((error) => { - console.error(`[docs-search] could not load ${TITLE_MAP}`, error); - }), + loadTitles().then((map) => { + titles = map; + }), ]); corpus = Object.values(filters.section ?? {}).reduce( (total, count) => total + count, @@ -455,7 +484,7 @@ export function pagefindEngine(bundlePath: string): SearchEngine { : null; if (fallback) { console.error( - `[docs-search] ranking without ${TITLE_MAP}: ${response.results.length} results, none of them in the map` + `[docs-search] ranking without the title map: ${response.results.length} results, none of them in it` ); } From a8801080b3298b5301b8fa50c179269e1c55e8f6 Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Thu, 27 Aug 2026 08:52:39 +1200 Subject: [PATCH 4/5] Ship the title map as a module so the CDN caches it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Front Door caches `.js` for a week as immutable and compressed, and gives anything it does not recognise `no-cache`. It also strips ETags, so revalidating is a full re-download — and the map loads on every page that opens the overlay. A `.json` name meant paying 29 KB per page view. The map is now `docs-titles..js`, an `export default`, loaded with the same dynamic import the engine already uses for `pagefind.js`. It lands in the existing static-content rule with no infrastructure change, and the hash in the name is what makes a week of immutable caching safe. `pagefind-entry.json` stays on the default no-cache rule, which is correct: it changes every build and the client reads the hash out of it. Co-Authored-By: Claude Opus 5 (1M context) --- src/integrations/pagefind-index.ts | 13 +++++++++++-- src/scripts/search-engine-pagefind.ts | 11 +++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/integrations/pagefind-index.ts b/src/integrations/pagefind-index.ts index 1f391978d2..3706b20028 100644 --- a/src/integrations/pagefind-index.ts +++ b/src/integrations/pagefind-index.ts @@ -101,8 +101,12 @@ export default function pagefindIndex(): AstroIntegration { * cache cannot serve one build's index against another's, and a map read against * the wrong index joins against nothing. The client reads the hash out of * `pagefind-entry.json` to build the same name. + * + * A module rather than JSON, because Front Door caches `.js` for a week as + * immutable and compressed, and gives anything unrecognised `no-cache`. An + * ETag-less revalidation is a full re-download, and this loads on every page. */ -const titleMapName = (hash: string) => `docs-titles.${hash}.json`; +const titleMapName = (hash: string) => `docs-titles.${hash}.js`; // Pagefind prefixes every decompressed chunk with this before the JSON. const FRAGMENT_MAGIC = 'pagefind_dcd'; @@ -171,7 +175,12 @@ async function writeTitleMap( ]; } - await writeFile(path.join(pagefindDir, file), JSON.stringify(map), 'utf8'); + await writeFile( + path.join(pagefindDir, file), + `export default ${JSON.stringify(map)}; +`, + 'utf8' + ); return { pages: names.length, file }; } diff --git a/src/scripts/search-engine-pagefind.ts b/src/scripts/search-engine-pagefind.ts index 4975fbf34e..ba5c94b5de 100644 --- a/src/scripts/search-engine-pagefind.ts +++ b/src/scripts/search-engine-pagefind.ts @@ -22,7 +22,7 @@ const PAGE_SIZE = 10; * named after the index's own hash. Reading it means reading that hash first, out * of the entry file Pagefind publishes for the same purpose. */ -const titleMapName = (hash: string) => `docs-titles.${hash}.json`; +const titleMapName = (hash: string) => `docs-titles.${hash}.js`; // Above this share of the corpus, a query is too general to rank rather than // unanswerable, and the overlay says so instead of reporting nothing found. @@ -340,10 +340,13 @@ export function pagefindEngine(bundlePath: string): SearchEngine { const hash = Object.values(languages ?? {})[0]?.hash; if (!hash) throw new Error('no index hash in the entry file'); - const map = await fetch(`${bundlePath}${titleMapName(hash)}`); - if (!map.ok) throw new Error(`${titleMapName(hash)}: ${map.status}`); + // Imported rather than fetched: the map is a module so that Front Door's + // static-content rule caches it, and the browser keeps parsed modules. + const map: { default: TitleMap } = await import( + /* @vite-ignore */ `${bundlePath}${titleMapName(hash)}` + ); - return (await map.json()) as TitleMap; + return map.default; } catch (error) { console.error( '[docs-search] could not load the title map; ranking falls back to the rows it draws', From c780f165f9c5c0faf8d9de1bc016aa65ddd99a12 Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Fri, 28 Aug 2026 14:05:12 +1200 Subject: [PATCH 5/5] Spell the comment the way the dictionary does Co-Authored-By: Claude Opus 5 (1M context) --- src/integrations/pagefind-index.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/integrations/pagefind-index.ts b/src/integrations/pagefind-index.ts index 3706b20028..9eae86f530 100644 --- a/src/integrations/pagefind-index.ts +++ b/src/integrations/pagefind-index.ts @@ -102,9 +102,10 @@ export default function pagefindIndex(): AstroIntegration { * the wrong index joins against nothing. The client reads the hash out of * `pagefind-entry.json` to build the same name. * - * A module rather than JSON, because Front Door caches `.js` for a week as - * immutable and compressed, and gives anything unrecognised `no-cache`. An - * ETag-less revalidation is a full re-download, and this loads on every page. + * A module, so that Front Door's static-content rule caches it for a week as + * immutable and compressed. That rule lists extensions, and a `.json` name falls + * through to `no-cache`, where an ETag-less revalidation costs a full + * re-download — paid on every page that loads the overlay. */ const titleMapName = (hash: string) => `docs-titles.${hash}.js`;