From 3adb9819a79e9785f505f80711df7af46c97cda7 Mon Sep 17 00:00:00 2001 From: Mohammad Shahbaz Alam Date: Thu, 3 Sep 2026 16:50:10 +0400 Subject: [PATCH] Validate llms.txt links and update verifier Add runtime validation that the root static/llms.txt only links to files present in the build. Introduced validateRootLlmsLinks() and call it at the end of postProcessLlmsOutput; the build now errors when static/llms.txt contains same-origin links to missing files (downgrades to a warning when sitemap.xml is absent). Copy static/llms.txt into the test outDir earlier in scripts/verify-llms-output.js so local verification exercises the same ordering as a real Docusaurus build. Clarify options.js docs to remind maintainers to update static/llms.txt alongside option changes, and update static/llms.txt content (Infura link and section edits). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/verify-llms-output.js | 14 ++-- src/plugins/llms-html-injector/index.js | 87 +++++++++++++++++++++++ src/plugins/llms-html-injector/options.js | 8 +++ static/llms.txt | 12 +--- 4 files changed, 106 insertions(+), 15 deletions(-) diff --git a/scripts/verify-llms-output.js b/scripts/verify-llms-output.js index 1ad53f729df..f9cde8f2185 100644 --- a/scripts/verify-llms-output.js +++ b/scripts/verify-llms-output.js @@ -65,18 +65,20 @@ async function main() { console.log('\n--- Pre-injector summary ---') printSummary(await summarize(outDir)) + // Mirror Docusaurus's static-asset copy step. In a real build, anything in + // `static/` is copied verbatim to `outDir` *before* postBuild runs, so + // static/llms.txt is already in place as build/llms.txt by the time the + // injector executes. Copying first reproduces that ordering, which both + // lets the post-injector summary show the curated root file end users will + // receive and exposes it to the injector's validateRootLlmsLinks check. + await copyStaticLlms(siteDir, outDir) + // Invoke only the post-processing stage. The injector module also exports a // wrapper plugin (used in docusaurus.config.js) that internally instantiates // docusaurus-plugin-llms; we've already run the generator above, so we skip // straight to normalize/rewrite/inject. await postProcessLlmsOutput(outDir, siteUrl) - // Mirror Docusaurus's static-asset copy step. In a real build, anything in - // `static/` is copied verbatim to `outDir`, so static/llms.txt becomes - // build/llms.txt. Replicating that here lets the post-injector summary show - // the curated root file end users will receive. - await copyStaticLlms(siteDir, outDir) - console.log('\n--- Post-injector summary ---') printSummary(await summarize(outDir)) } diff --git a/src/plugins/llms-html-injector/index.js b/src/plugins/llms-html-injector/index.js index c6cda10fccf..7bc078bc627 100644 --- a/src/plugins/llms-html-injector/index.js +++ b/src/plugins/llms-html-injector/index.js @@ -266,6 +266,33 @@ async function postProcessLlmsOutput(outDir, siteUrl) { `and ${rewriteStats.mdFiles} per-page .md file(s)` ) } + + // Runs last so it validates the artifacts exactly as shipped, including the + // preview-host rewrite above. + const rootLinks = await validateRootLlmsLinks(outDir, previewSiteUrl || siteUrl) + if (rootLinks.broken.length === 0) { + console.log( + `[llms-html-injector] Validated ${rootLinks.checked} same-origin link(s) in llms.txt; all resolve to files in the build` + ) + } else { + const detail = + `llms.txt links to ${rootLinks.broken.length} file(s) that do not exist in the build:\n` + + rootLinks.broken.map(p => ` - ${p}`).join('\n') + + '\nstatic/llms.txt is hand-curated: update it in the same change that adds or ' + + 'removes a customLLMFiles entry in options.js or an ALL_PAGES_BUCKETS entry here.' + // generateAllPagesIndex is skipped when the build has no sitemap.xml (see + // above), which is the normal case for scripts/verify-llms-output.js. The + // llms-all-*.txt links are legitimately absent there, so downgrade to a + // warning rather than failing a local sanity check. + if (!sitemapUrls) { + console.warn(`[llms-html-injector] ${detail}`) + console.warn( + '[llms-html-injector] Not failing: no sitemap.xml in outDir, so llms-all-*.txt were never generated.' + ) + } else { + throw new Error(`[llms-html-injector] ${detail}`) + } + } } module.exports = function llmsHtmlInjectorPlugin(context, options = {}) { @@ -1269,6 +1296,66 @@ async function rewriteHostInBuildArtifacts(outDir, fromHost, toHost) { return { txtFiles, mdFiles } } +/** + * Verify that every same-origin link in the build's root `llms.txt` points at + * a file that actually exists in `outDir`. + * + * Unlike every other `llms*.txt` in the build, the root index is hand-curated + * at `static/llms.txt` and copied verbatim by Docusaurus's static-asset step — + * nothing regenerates or prunes it. That makes it the one artifact that can + * silently outlive the files it advertises: PR #2960 removed the Services and + * Developer dashboard sections from `customLLMFiles` and `ALL_PAGES_BUCKETS`, + * the build correctly stopped emitting the six corresponding `.txt` files, and + * `llms.txt` went on linking to all six 404s until an external audit caught it. + * + * `siteUrl` is the *effective* prefix for this build (the preview host on + * Vercel branch deploys, the canonical host otherwise). Links that don't start + * with it are cross-origin — `https://docs.infura.io/llms.txt`, for example — + * and are skipped: they're outside this build's control and are excluded from + * the AFDocs `llms-txt-links-resolve` same-origin sample anyway. + */ +async function validateRootLlmsLinks(outDir, siteUrl) { + // Docusaurus passes outDir with a trailing separator; normalize it so the + // containment check below compares like with like. + const root = path.resolve(outDir) + let text + try { + text = await fs.readFile(path.join(root, 'llms.txt'), 'utf8') + } catch { + throw new Error( + '[llms-html-injector] No llms.txt in outDir. Expected static/llms.txt to have been ' + + 'copied into the build before postBuild ran.' + ) + } + + const prefix = siteUrl.endsWith('/') ? siteUrl.slice(0, -1) : siteUrl + const broken = [] + let checked = 0 + + for (const match of text.matchAll(/\[[^\]]*\]\((https?:\/\/[^)\s]+)\)/g)) { + const url = match[1] + if (url !== prefix && !url.startsWith(prefix + '/')) continue + checked++ + + // Decode so percent-escaped paths resolve to the on-disk filename, and + // normalize so a `..` segment can't escape outDir. + const relative = decodeURIComponent(url.slice(prefix.length)).replace(/^\/+/, '') + const target = path.resolve(root, relative) + if (target !== root && !target.startsWith(root + path.sep)) { + broken.push(`/${relative} (resolves outside the build directory)`) + continue + } + + try { + await fs.access(target) + } catch { + broken.push(`/${relative}`) + } + } + + return { checked, broken } +} + function toPosix(p) { return p.split(path.sep).join('/') } diff --git a/src/plugins/llms-html-injector/options.js b/src/plugins/llms-html-injector/options.js index 2634012c416..3cb89b90042 100644 --- a/src/plugins/llms-html-injector/options.js +++ b/src/plugins/llms-html-injector/options.js @@ -16,6 +16,14 @@ * This file is plain CommonJS with no Docusaurus-specific imports so it can * be `require()`d by a standalone Node script. Do not import from * `@docusaurus/*` or anything that pulls in ESM-only deps. + * + * A third consumer is NOT automatic: the root `static/llms.txt` is + * hand-curated and links to each generated file by name. Adding, removing, or + * renaming a `customLLMFiles` entry here (or an `ALL_PAGES_BUCKETS` entry in + * `index.js`) must be paired with the matching edit to `static/llms.txt` in + * the same change, or the root index will advertise files the build no longer + * emits. `validateRootLlmsLinks` in `index.js` fails the build when that + * happens. */ // Source-tree globs that should never be walked by the generator. Comments diff --git a/static/llms.txt b/static/llms.txt index b60f1dfd187..995d2bb9221 100644 --- a/static/llms.txt +++ b/static/llms.txt @@ -27,15 +27,11 @@ These links contain LLM-readable files following the llmstxt.org standard, for e - [Agent Wallet documentation links](https://docs.metamask.io/llms-agent-wallet.txt): Documentation links for MetaMask Agent Wallet — the mm CLI for programmatic wallet access with mandatory security. - [Agent Wallet full documentation](https://docs.metamask.io/llms-agent-wallet-full.txt): Complete documentation for MetaMask Agent Wallet. -## Services +## Infura services and dashboard -- [Services documentation links](https://docs.metamask.io/llms-services.txt): Documentation links for MetaMask services. -- [Services full documentation](https://docs.metamask.io/llms-services-full.txt): Complete documentation for MetaMask services. +MetaMask API services (JSON-RPC, gas, and related APIs) and the developer dashboard are documented separately. -## Developer Dashboard - -- [Developer Dashboard documentation links](https://docs.metamask.io/llms-dashboard.txt): Documentation links for MetaMask Developer dashboard. -- [Developer Dashboard full documentation](https://docs.metamask.io/llms-dashboard-full.txt): Complete documentation for MetaMask Developer dashboard. +- [Infura documentation index](https://docs.infura.io/llms.txt): LLM-readable index for Infura APIs, network endpoints, and the Infura dashboard. ## Snaps @@ -54,8 +50,6 @@ These auto-generated indexes are produced from the live sitemap at build time an - [All MetaMask Connect pages](https://docs.metamask.io/llms-all-metamask-connect.txt): Complete sitemap-derived index of every MetaMask Connect page. - [All Embedded Wallets pages](https://docs.metamask.io/llms-all-embedded-wallets.txt): Complete sitemap-derived index of every Embedded Wallets page. - [All Smart Accounts Kit pages](https://docs.metamask.io/llms-all-smart-accounts-kit.txt): Complete sitemap-derived index of every Smart Accounts Kit page. -- [All Services pages](https://docs.metamask.io/llms-all-services.txt): Complete sitemap-derived index of every Services page (including JSON-RPC reference). - [All Snaps pages](https://docs.metamask.io/llms-all-snaps.txt): Complete sitemap-derived index of every Snaps page. -- [All Developer dashboard pages](https://docs.metamask.io/llms-all-dashboard.txt): Complete sitemap-derived index of every Developer dashboard page. - [All Tutorials pages](https://docs.metamask.io/llms-all-tutorials.txt): Complete sitemap-derived index of every tutorials page. - [All other documentation pages](https://docs.metamask.io/llms-all-misc.txt): Complete sitemap-derived index of pages outside the main product sections.