From d1710b2c72c0879dd625171ac8614d8894b70ef1 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 13 Aug 2026 21:41:54 +0800 Subject: [PATCH 1/6] fix(performance): harden crawler and overlay safeguards --- README.md | 3 ++ scripts/lighthouse-measurement-outcome.mjs | 12 +++-- scripts/run-lighthouse-budget.mjs | 13 ++++- src/app/layout.tsx | 2 + src/app/robots.ts | 8 +-- .../document-search-results.tsx | 19 +++++-- src/components/ui/tooltip.tsx | 23 +++++--- src/lib/crawler-policy.ts | 29 ++++++++++ tests/check-lighthouse-budget.test.ts | 6 ++- tests/crawler-policy.test.ts | 31 +++++++++++ .../document-search-record-fault.dom.test.tsx | 54 ++++++++++++++++++- tests/ui-v2-components.dom.test.tsx | 38 +++++++++++++ 12 files changed, 215 insertions(+), 23 deletions(-) create mode 100644 src/lib/crawler-policy.ts create mode 100644 tests/crawler-policy.test.ts diff --git a/README.md b/README.md index 18c30e1236..d98cb30d62 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,9 @@ local credentials or enabling live provider access. - `SUPABASE_PROJECT_REF` must stay `sjrfecxgysukkwxsowpy` for the live `Clinical KB Database` project. - Documents and extracted images are stored in private Supabase buckets. +- Application routes are intentionally excluded from search discovery: `/robots.txt` + disallows crawling, root metadata emits `noindex`/`nofollow`, and the app does not + publish an XML sitemap. `docs/site-map.md` is an internal route inventory only. - Initial assumptions are guideline/reference documents only, not patient identifiable records. - OpenAI receives extracted document text/images for embeddings, image captions, diff --git a/scripts/lighthouse-measurement-outcome.mjs b/scripts/lighthouse-measurement-outcome.mjs index 8c4b46ca96..3c0bee432a 100644 --- a/scripts/lighthouse-measurement-outcome.mjs +++ b/scripts/lighthouse-measurement-outcome.mjs @@ -36,11 +36,10 @@ * measured something real (including something real and slow). */ export function measurementFailureReason(exitCode, reportText) { - if (typeof exitCode !== "number" || exitCode !== 0) { - return `lighthouse exited ${exitCode ?? "without a status"}`; - } if (reportText === null || reportText === undefined || reportText === "") { - return "no report file was written"; + return typeof exitCode !== "number" || exitCode !== 0 + ? `lighthouse exited ${exitCode ?? "without a status"}` + : "no report file was written"; } let parsed; @@ -56,5 +55,10 @@ export function measurementFailureReason(exitCode, reportText) { // cell that produced no comparable numbers. if (typeof code === "string" && code.length > 0) return `lighthouse runtimeError ${code}`; + // Chrome can finish the audit, write a complete report, then fail to remove its + // temporary profile on Windows with EPERM. The report is still the evidence: the + // grader independently rejects missing metrics, runtime errors, wrong pages and + // unverified browser identities. Do not discard that evidence based only on the + // wrapper's cleanup exit status. return null; } diff --git a/scripts/run-lighthouse-budget.mjs b/scripts/run-lighthouse-budget.mjs index 4c21b77823..08fa3bca2c 100644 --- a/scripts/run-lighthouse-budget.mjs +++ b/scripts/run-lighthouse-budget.mjs @@ -21,7 +21,7 @@ * live workflow, which is grading a flaky public network): a route that produced no * report is incomplete evidence, and the grader fails closed on it. * - * A cell that produced no measurement AT ALL — a non-zero exit, no report file, or a + * A cell that produced no measurement AT ALL — no report file, invalid JSON, or a * report carrying only a `runtimeError` such as Lighthouse's own `NO_NAVSTART` — gets * exactly ONE announced retry first (`lighthouse-measurement-outcome.mjs` draws that * line). That is not a softening: a run that never started measured nothing about this @@ -426,6 +426,12 @@ try { let result = await measure(strategy, route, output, firstAttemptTimeout); let reason = measurementFailureReason(childProcessExitCode(result), readIfPresent(output)); + if (!reason && childProcessExitCode(result) !== 0) { + console.log( + `::warning::lighthouse ${cell} wrote a parseable report but ${childProcessFailureSummary(result)} after measurement; grading the report`, + ); + } + if (reason) { // ONE retry, and it is announced. A cell that never started measured nothing // about this diff, so treating it as a regression is wrong — but so is @@ -449,6 +455,11 @@ try { removePathSync(output); result = await measure(strategy, route, output, retryTimeout); const after = measurementFailureReason(childProcessExitCode(result), readIfPresent(output)); + if (!after && childProcessExitCode(result) !== 0) { + console.log( + `::warning::lighthouse ${cell} wrote a parseable report on retry but ${childProcessFailureSummary(result)} after measurement; grading the report`, + ); + } retried.push(`${cell} (${reason}${after ? ` -> still ${after}` : " -> recovered"})`); reason = after; } diff --git a/src/app/layout.tsx b/src/app/layout.tsx index 817658f0be..f5ade6de91 100644 --- a/src/app/layout.tsx +++ b/src/app/layout.tsx @@ -10,6 +10,7 @@ import { APP_THEME_COLORS, THEME_BOOTSTRAP_SCRIPT, THEME_COOKIE_NAME } from "@/l import { MobileKeyboardProvider } from "@/components/use-mobile-keyboard"; import { AppAnnouncements } from "@/components/app-announcements"; import { OverlayRoot } from "@/components/ui/overlay-root"; +import { PRIVATE_APP_ROBOTS_METADATA } from "@/lib/crawler-policy"; import "./globals.css"; /** @@ -55,6 +56,7 @@ const baseMetadata: Metadata = { applicationName: "Clinical KB", title: "Clinical KB", description: "Private medical guideline RAG knowledge base", + robots: PRIVATE_APP_ROBOTS_METADATA, appleWebApp: { capable: true, title: "Clinical KB", diff --git a/src/app/robots.ts b/src/app/robots.ts index eab62fe240..99a2b06d18 100644 --- a/src/app/robots.ts +++ b/src/app/robots.ts @@ -1,10 +1,6 @@ import type { MetadataRoute } from "next"; +import { PRIVATE_APP_ROBOTS_TXT } from "@/lib/crawler-policy"; export default function robots(): MetadataRoute.Robots { - return { - rules: { - userAgent: "*", - disallow: ["/mockups/"], - }, - }; + return PRIVATE_APP_ROBOTS_TXT; } diff --git a/src/components/clinical-dashboard/document-search-results.tsx b/src/components/clinical-dashboard/document-search-results.tsx index bd07efd780..0977e9e97b 100644 --- a/src/components/clinical-dashboard/document-search-results.tsx +++ b/src/components/clinical-dashboard/document-search-results.tsx @@ -312,6 +312,16 @@ function DocumentResultMoreMenu({ useEffect(() => { if (!open) return; + let positionFrame: number | null = null; + const scheduleMenuPositionUpdate = () => { + if (positionFrame !== null) return; + positionFrame = window.requestAnimationFrame(() => { + positionFrame = null; + updateMenuPosition(); + }); + }; + const scrollOptions: AddEventListenerOptions = { capture: true, passive: true }; + function closeOutside(event: PointerEvent) { if (menuRef.current?.contains(event.target as Node) || buttonRef.current?.contains(event.target as Node)) return; setOpen(false); @@ -325,14 +335,15 @@ function DocumentResultMoreMenu({ window.document.addEventListener("pointerdown", closeOutside); window.addEventListener("keydown", closeOnEscape); - window.addEventListener("resize", updateMenuPosition); - window.addEventListener("scroll", updateMenuPosition, true); + window.addEventListener("resize", scheduleMenuPositionUpdate); + window.addEventListener("scroll", scheduleMenuPositionUpdate, scrollOptions); updateMenuPosition(); return () => { + if (positionFrame !== null) window.cancelAnimationFrame(positionFrame); window.document.removeEventListener("pointerdown", closeOutside); window.removeEventListener("keydown", closeOnEscape); - window.removeEventListener("resize", updateMenuPosition); - window.removeEventListener("scroll", updateMenuPosition, true); + window.removeEventListener("resize", scheduleMenuPositionUpdate); + window.removeEventListener("scroll", scheduleMenuPositionUpdate, scrollOptions); }; }, [open, updateMenuPosition]); diff --git a/src/components/ui/tooltip.tsx b/src/components/ui/tooltip.tsx index a219f5fc80..b67233b110 100644 --- a/src/components/ui/tooltip.tsx +++ b/src/components/ui/tooltip.tsx @@ -80,13 +80,24 @@ export function Tooltip({ children, content, placement = "top", className }: Too useEffect(() => { if (!open) return; - const frame = window.requestAnimationFrame(updatePosition); - window.addEventListener("resize", updatePosition); - window.addEventListener("scroll", updatePosition, true); + + let frame: number | null = null; + const schedulePositionUpdate = () => { + if (frame !== null) return; + frame = window.requestAnimationFrame(() => { + frame = null; + updatePosition(); + }); + }; + const scrollOptions: AddEventListenerOptions = { capture: true, passive: true }; + + schedulePositionUpdate(); + window.addEventListener("resize", schedulePositionUpdate); + window.addEventListener("scroll", schedulePositionUpdate, scrollOptions); return () => { - window.cancelAnimationFrame(frame); - window.removeEventListener("resize", updatePosition); - window.removeEventListener("scroll", updatePosition, true); + if (frame !== null) window.cancelAnimationFrame(frame); + window.removeEventListener("resize", schedulePositionUpdate); + window.removeEventListener("scroll", schedulePositionUpdate, scrollOptions); }; }, [open, updatePosition]); diff --git a/src/lib/crawler-policy.ts b/src/lib/crawler-policy.ts new file mode 100644 index 0000000000..2f69c5b193 --- /dev/null +++ b/src/lib/crawler-policy.ts @@ -0,0 +1,29 @@ +import type { Metadata, MetadataRoute } from "next"; + +/** + * Clinical KB is a private application, not a public content catalogue. Keep its + * routes out of search results even when a crawler reaches a URL without first + * consulting robots.txt. + */ +export const PRIVATE_APP_ROBOTS_METADATA = { + index: false, + follow: false, + nocache: true, + googleBot: { + index: false, + follow: false, + noimageindex: true, + nosnippet: true, + }, +} satisfies Metadata["robots"]; + +/** + * Do not advertise a sitemap for private application routes, and ask compliant + * crawlers not to fetch them. + */ +export const PRIVATE_APP_ROBOTS_TXT = { + rules: { + userAgent: "*", + disallow: "/", + }, +} satisfies MetadataRoute.Robots; diff --git a/tests/check-lighthouse-budget.test.ts b/tests/check-lighthouse-budget.test.ts index 66a9d6545a..9074337424 100644 --- a/tests/check-lighthouse-budget.test.ts +++ b/tests/check-lighthouse-budget.test.ts @@ -469,10 +469,14 @@ describe("measurementFailureReason", () => { expect(measurementFailureReason(0, report())).toBeNull(); }); - it("flags a non-zero exit", () => { + it("flags a non-zero exit that wrote no report", () => { expect(measurementFailureReason(1, null)).toContain("exited 1"); }); + it("grades a parseable report even when post-measurement cleanup exits non-zero", () => { + expect(measurementFailureReason(1, report())).toBeNull(); + }); + it("flags a run that was killed without a status", () => { expect(measurementFailureReason(null, null)).toContain("without a status"); }); diff --git a/tests/crawler-policy.test.ts b/tests/crawler-policy.test.ts new file mode 100644 index 0000000000..f4901c4ee4 --- /dev/null +++ b/tests/crawler-policy.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; + +import robots from "../src/app/robots"; +import { PRIVATE_APP_ROBOTS_METADATA, PRIVATE_APP_ROBOTS_TXT } from "../src/lib/crawler-policy"; + +describe("private application crawler policy", () => { + it("asks crawlers not to fetch any application route and advertises no sitemap", () => { + expect(robots()).toEqual(PRIVATE_APP_ROBOTS_TXT); + expect(PRIVATE_APP_ROBOTS_TXT).toEqual({ + rules: { + userAgent: "*", + disallow: "/", + }, + }); + expect(PRIVATE_APP_ROBOTS_TXT).not.toHaveProperty("sitemap"); + }); + + it("keeps fetched routes and their images out of search results", () => { + expect(PRIVATE_APP_ROBOTS_METADATA).toMatchObject({ + index: false, + follow: false, + nocache: true, + googleBot: { + index: false, + follow: false, + noimageindex: true, + nosnippet: true, + }, + }); + }); +}); diff --git a/tests/document-search-record-fault.dom.test.tsx b/tests/document-search-record-fault.dom.test.tsx index f566546edc..6688a1bed8 100644 --- a/tests/document-search-record-fault.dom.test.tsx +++ b/tests/document-search-record-fault.dom.test.tsx @@ -1,4 +1,4 @@ -import { fireEvent, render, screen, within } from "@testing-library/react"; +import { act, fireEvent, render, screen, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; @@ -134,6 +134,58 @@ describe("document search record path fault reporting", () => { fireEvent.click(within(menu).getByRole("menuitem", { name: "Search only this source" })); expect(baseProps.onScopeDocument).toHaveBeenCalledWith(lithiumMatch.document_id); }); + + it("coalesces more-menu viewport events into one animation-frame measurement", () => { + const pendingFrames: Array<(time: number) => void> = []; + const raf = vi.spyOn(window, "requestAnimationFrame").mockImplementation((callback) => { + pendingFrames.push(callback as (time: number) => void); + return pendingFrames.length; + }); + const rect = vi.spyOn(HTMLElement.prototype, "getBoundingClientRect").mockReturnValue({ + x: 20, + y: 20, + left: 20, + top: 20, + right: 120, + bottom: 60, + width: 100, + height: 40, + toJSON: () => ({}), + } as DOMRect); + + try { + render( + , + ); + + const resultCard = screen.getByTestId("document-result-card"); + fireEvent.click(within(resultCard).getByRole("button", { name: `More actions for ${lithiumMatch.title}` })); + expect(screen.getByTestId("document-result-more-menu")).toBeInTheDocument(); + expect(pendingFrames).toHaveLength(1); + act(() => pendingFrames.shift()?.(0)); + const measuredCalls = rect.mock.calls.length; + + act(() => { + window.dispatchEvent(new Event("scroll")); + window.dispatchEvent(new Event("scroll")); + window.dispatchEvent(new Event("resize")); + }); + + expect(pendingFrames).toHaveLength(1); + expect(rect).toHaveBeenCalledTimes(measuredCalls); + act(() => pendingFrames.shift()?.(16)); + expect(rect).toHaveBeenCalledTimes(measuredCalls + 2); + } finally { + raf.mockRestore(); + rect.mockRestore(); + } + }); }); describe("forms record subtitle compaction", () => { diff --git a/tests/ui-v2-components.dom.test.tsx b/tests/ui-v2-components.dom.test.tsx index c5e356b005..701189f36b 100644 --- a/tests/ui-v2-components.dom.test.tsx +++ b/tests/ui-v2-components.dom.test.tsx @@ -792,6 +792,44 @@ describe("Tooltip", () => { } }); + it("coalesces capture scroll events into one passive animation-frame measurement", async () => { + const pendingFrames: Array<(time: number) => void> = []; + const raf = vi.spyOn(window, "requestAnimationFrame").mockImplementation((callback) => { + pendingFrames.push(callback as (time: number) => void); + return pendingFrames.length; + }); + const rect = vi.spyOn(HTMLElement.prototype, "getBoundingClientRect"); + + try { + render( + + + , + ); + + screen.getByRole("button", { name: "Batched trigger" }).focus(); + await screen.findByTestId("tooltip"); + expect(pendingFrames).toHaveLength(1); + act(() => pendingFrames.shift()?.(0)); + await waitFor(() => expect(screen.getByRole("tooltip")).toBeVisible()); + const measuredCalls = rect.mock.calls.length; + + act(() => { + window.dispatchEvent(new Event("scroll")); + window.dispatchEvent(new Event("scroll")); + window.dispatchEvent(new Event("resize")); + }); + + expect(pendingFrames).toHaveLength(1); + expect(rect).toHaveBeenCalledTimes(measuredCalls); + act(() => pendingFrames.shift()?.(16)); + await waitFor(() => expect(rect.mock.calls.length).toBe(measuredCalls + 2)); + } finally { + raf.mockRestore(); + rect.mockRestore(); + } + }); + it("composes over existing child event handlers instead of replacing them", async () => { const onFocus = vi.fn(); const onKeyDown = vi.fn(); From fc22f0fa16accfaf76754e6d9c2a452060b8feb5 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 14 Aug 2026 01:02:40 +0800 Subject: [PATCH 2/6] fix: correct crawler policy --- src/lib/crawler-policy.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lib/crawler-policy.ts b/src/lib/crawler-policy.ts index 2f69c5b193..522b450c72 100644 --- a/src/lib/crawler-policy.ts +++ b/src/lib/crawler-policy.ts @@ -18,12 +18,13 @@ export const PRIVATE_APP_ROBOTS_METADATA = { } satisfies Metadata["robots"]; /** - * Do not advertise a sitemap for private application routes, and ask compliant - * crawlers not to fetch them. + * Let compliant crawlers fetch application routes so they can observe the global + * `noindex` metadata. Search exclusion is not access control; private content must + * remain protected by authentication. Do not advertise an XML sitemap. */ export const PRIVATE_APP_ROBOTS_TXT = { rules: { userAgent: "*", - disallow: "/", + allow: "/", }, } satisfies MetadataRoute.Robots; From 5d7a24067552817b6e9cba3554033e9a2f9744fe Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 14 Aug 2026 01:04:29 +0800 Subject: [PATCH 3/6] fix: align crawler expectation --- tests/crawler-policy.test.ts | 31 ++++--------------------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/tests/crawler-policy.test.ts b/tests/crawler-policy.test.ts index f4901c4ee4..758ea59def 100644 --- a/tests/crawler-policy.test.ts +++ b/tests/crawler-policy.test.ts @@ -1,31 +1,8 @@ -import { describe, expect, it } from "vitest"; +import { expect, it } from "vitest"; import robots from "../src/app/robots"; -import { PRIVATE_APP_ROBOTS_METADATA, PRIVATE_APP_ROBOTS_TXT } from "../src/lib/crawler-policy"; -describe("private application crawler policy", () => { - it("asks crawlers not to fetch any application route and advertises no sitemap", () => { - expect(robots()).toEqual(PRIVATE_APP_ROBOTS_TXT); - expect(PRIVATE_APP_ROBOTS_TXT).toEqual({ - rules: { - userAgent: "*", - disallow: "/", - }, - }); - expect(PRIVATE_APP_ROBOTS_TXT).not.toHaveProperty("sitemap"); - }); - - it("keeps fetched routes and their images out of search results", () => { - expect(PRIVATE_APP_ROBOTS_METADATA).toMatchObject({ - index: false, - follow: false, - nocache: true, - googleBot: { - index: false, - follow: false, - noimageindex: true, - nosnippet: true, - }, - }); - }); +it("serves crawler rules", () => { + expect(robots()).toEqual({ rules: { userAgent: "*", allow: "/" } }); + expect(robots()).not.toHaveProperty("sitemap"); }); From 084184dfa313f0556a92b354e3bc86169a460b0b Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 14 Aug 2026 01:05:19 +0800 Subject: [PATCH 4/6] test: preserve crawler metadata coverage --- tests/crawler-policy.test.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/crawler-policy.test.ts b/tests/crawler-policy.test.ts index 758ea59def..ab28a4fb59 100644 --- a/tests/crawler-policy.test.ts +++ b/tests/crawler-policy.test.ts @@ -1,8 +1,20 @@ import { expect, it } from "vitest"; import robots from "../src/app/robots"; +import { PRIVATE_APP_ROBOTS_METADATA } from "../src/lib/crawler-policy"; -it("serves crawler rules", () => { +it("serves restrictive search metadata through crawlable routes", () => { expect(robots()).toEqual({ rules: { userAgent: "*", allow: "/" } }); expect(robots()).not.toHaveProperty("sitemap"); + expect(PRIVATE_APP_ROBOTS_METADATA).toMatchObject({ + index: false, + follow: false, + nocache: true, + googleBot: { + index: false, + follow: false, + noimageindex: true, + nosnippet: true, + }, + }); }); From 17246d45eab217ab3a7b1028db06a30ce16894dd Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 14 Aug 2026 01:05:33 +0800 Subject: [PATCH 5/6] docs: record PR 1919 review --- ...b8a7fe6997dde55a64c29e794a34c2cb241ddbd3fa512a3b604.record.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/branch-review-records/78ad337f2fcd7b8a7fe6997dde55a64c29e794a34c2cb241ddbd3fa512a3b604.record.md diff --git a/docs/branch-review-records/78ad337f2fcd7b8a7fe6997dde55a64c29e794a34c2cb241ddbd3fa512a3b604.record.md b/docs/branch-review-records/78ad337f2fcd7b8a7fe6997dde55a64c29e794a34c2cb241ddbd3fa512a3b604.record.md new file mode 100644 index 0000000000..bd4a9e9be2 --- /dev/null +++ b/docs/branch-review-records/78ad337f2fcd7b8a7fe6997dde55a64c29e794a34c2cb241ddbd3fa512a3b604.record.md @@ -0,0 +1 @@ +| 2026-08-13 | codex/sitemap-dom-fixes (PR #1919) | 4076d8034b61fbb3420efa398fab11d9e243b193 | PR #1919 heavy review and fix | Fixed crawler policy and added regression coverage; no other blocking finding | Focused contract and source checks passed | From 99056610a0e645297940c1e218683b3f4623db6a Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 14 Aug 2026 01:10:12 +0800 Subject: [PATCH 6/6] docs: remove stale crawler description --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index d98cb30d62..18c30e1236 100644 --- a/README.md +++ b/README.md @@ -129,9 +129,6 @@ local credentials or enabling live provider access. - `SUPABASE_PROJECT_REF` must stay `sjrfecxgysukkwxsowpy` for the live `Clinical KB Database` project. - Documents and extracted images are stored in private Supabase buckets. -- Application routes are intentionally excluded from search discovery: `/robots.txt` - disallows crawling, root metadata emits `noindex`/`nofollow`, and the app does not - publish an XML sitemap. `docs/site-map.md` is an internal route inventory only. - Initial assumptions are guideline/reference documents only, not patient identifiable records. - OpenAI receives extracted document text/images for embeddings, image captions,