From 3697a32aa5e8fb9973543d0de2a5adb267a286ce Mon Sep 17 00:00:00 2001 From: Christopher Tauchen Date: Wed, 8 Jul 2026 10:57:42 +0100 Subject: [PATCH 1/3] docs-2923: add PR-scoped link-check mechanism Add the pieces that let the crawler check only the pages a PR changes: - New plugin docusaurus-plugin-link-check-routes writes build/link-check-routes.json (source file -> built URL) when LINK_CHECK_ROUTES=true. - crawler.test.js gains a scoped mode: PAGES_TO_CHECK seeds only those pages and turns off link-following; URLS_TO_CHECK optionally narrows checks to given links. - scripts/changed-pages.js maps the PR diff to the built page URLs via the manifest. Co-Authored-By: Claude Opus 4.8 (1M context) --- __tests__/crawler.test.js | 38 +++++++--- docusaurus.config.js | 2 + scripts/changed-pages.js | 70 +++++++++++++++++++ .../index.js | 58 +++++++++++++++ 4 files changed, 159 insertions(+), 9 deletions(-) create mode 100644 scripts/changed-pages.js create mode 100644 src/plugins/docusaurus-plugin-link-check-routes/index.js diff --git a/__tests__/crawler.test.js b/__tests__/crawler.test.js index 222db64e33..41ce8ed461 100644 --- a/__tests__/crawler.test.js +++ b/__tests__/crawler.test.js @@ -17,6 +17,17 @@ test('Crawl the docs and execute tests', async () => { const validityTest = process.env.VALIDITY_TEST ? process.env.VALIDITY_TEST.split(',') : []; const validityTestFiles = process.env.VALIDITY_TEST_FILES ? process.env.VALIDITY_TEST_FILES.split(',') : []; const isDeepCrawl = process.env.DEEP_CRAWL ? process.env.DEEP_CRAWL === 'true' : false; + // PR-scoped mode: when PAGES_TO_CHECK is set, crawl only those pages and do not follow links. + // URLS_TO_CHECK optionally narrows checking to specific links (line-level); empty means check all. + const parseScopeList = (v) => (v ? v.split(/[\n,]/).map((s) => s.trim()).filter(Boolean) : []); + const toPageUrl = (p) => { + let u = /^https?:\/\//i.test(p) ? p : `${DOCS}${p.startsWith('/') ? '' : '/'}${p}`; + if (isLocalHost) u = u.replace(PROD_REGEX, DOCS); + return u; + }; + const pagesToCheck = parseScopeList(process.env.PAGES_TO_CHECK).map(toPageUrl); + const urlsToCheck = new Set(parseScopeList(process.env.URLS_TO_CHECK)); + const isScoped = pagesToCheck.length > 0; const fileRegex = /https?:\/\/[-a-zA-Z0-9()@:%._+~#?&/=]+?\.(ya?ml|zip|ps1|tgz|sh|exe|bat|json)/gi; const varRegex = /\{\{[ \t]*[-\w\[\]]+[ \t]*}}/g; const varSkipList = ['{{end}}']; @@ -249,6 +260,7 @@ test('Crawl the docs and execute tests', async () => { const testUrl = url.replace(PROD_REGEX, DOCS); if (request.url === testUrl) url = testUrl; } + if (urlsToCheck.size > 0 && !urlsToCheck.has(url)) continue; checkAndUseLinkChecker(request.url, url); } @@ -258,11 +270,14 @@ test('Crawl the docs and execute tests', async () => { testLiquid(allText, request.url); - await enqueueLinks({ - strategy: EnqueueStrategy.All, - transformRequestFunction: transformRequest, - userData: { origin: request.url }, - }); + // In PR-scoped mode, do not follow links: check only the seeded pages. + if (!isScoped) { + await enqueueLinks({ + strategy: EnqueueStrategy.All, + transformRequestFunction: transformRequest, + userData: { origin: request.url }, + }); + } }, }); } @@ -599,10 +614,15 @@ test('Crawl the docs and execute tests', async () => { } const crawler = getCrawler(); - await processSiteMap(SITEMAP_URL); - const urls = [...urlCache.keys()].filter((url) => !url.endsWith(SITEMAP)); - await crawler.addRequests([DOCS]); - await crawler.addRequests(urls); + if (isScoped) { + console.log(`PR-scoped mode: checking ${pagesToCheck.length} page(s) only.`); + await crawler.addRequests(pagesToCheck); + } else { + await processSiteMap(SITEMAP_URL); + const urls = [...urlCache.keys()].filter((url) => !url.endsWith(SITEMAP)); + await crawler.addRequests([DOCS]); + await crawler.addRequests(urls); + } console.log(`Crawling the docs (${DOCS}) and executing tests.`); console.log(`Localhost mode is ${isLocalHost ? 'ON' : 'OFF'}.`); diff --git a/docusaurus.config.js b/docusaurus.config.js index 18b90e6221..f8ec969bf6 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -411,6 +411,8 @@ export default async function createAsyncConfig() { }, plugins: [ 'docusaurus-plugin-sass', + // Writes build/link-check-routes.json when LINK_CHECK_ROUTES=true. Used by the PR link check. + './src/plugins/docusaurus-plugin-link-check-routes', [ '@docusaurus/plugin-content-docs', /** @type {import('@docusaurus/plugin-content-docs').Options} */ diff --git a/scripts/changed-pages.js b/scripts/changed-pages.js new file mode 100644 index 0000000000..10fa62d1e5 --- /dev/null +++ b/scripts/changed-pages.js @@ -0,0 +1,70 @@ +#!/usr/bin/env node +/** + * changed-pages.js + * + * Print the built page URLs for the docs files changed in a PR, one per line. + * The PR link-check workflow feeds this list to the crawler as PAGES_TO_CHECK. + * + * Usage: + * node scripts/changed-pages.js [baseRef] [manifestPath] + * + * Defaults: baseRef = origin/main, manifestPath = build/link-check-routes.json. + * The manifest is written by the docusaurus-plugin-link-check-routes plugin + * during a build run with LINK_CHECK_ROUTES=true. + * + * Note: only files registered as docs pages map to a URL. Changes to partials, + * includes, or components are reported as "unmatched" and are not checked here. + */ + +const { execFileSync } = require('child_process'); +const fs = require('fs'); + +const base = process.argv[2] || process.env.BASE_REF || 'origin/main'; +const manifestPath = process.argv[3] || 'build/link-check-routes.json'; + +const DOC_RE = /\.mdx?$/; + +function changedDocFiles(baseRef) { + // execFile with an argument array: no shell, so baseRef cannot inject commands. + const out = execFileSync('git', ['diff', '--name-only', `${baseRef}...HEAD`], { encoding: 'utf8' }); + return out + .split('\n') + .map((s) => s.trim()) + .filter(Boolean) + .filter((f) => DOC_RE.test(f)); +} + +let manifest; +try { + manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); +} catch (e) { + console.error(`[changed-pages] cannot read manifest ${manifestPath}: ${e.message}`); + process.exit(2); +} + +const files = changedDocFiles(base); +const permalinks = new Set(); +const unmatched = []; + +for (const f of files) { + const links = manifest[f]; + if (links && links.length) { + links.forEach((l) => permalinks.add(l)); + } else { + unmatched.push(f); + } +} + +// Pages go to stdout (consumed by the workflow); diagnostics go to stderr. +for (const p of [...permalinks].sort()) { + console.log(p); +} + +console.error( + `[changed-pages] base=${base} changed-docs=${files.length} pages=${permalinks.size} unmatched=${unmatched.length}` +); +if (unmatched.length) { + console.error( + `[changed-pages] unmatched (partials/includes/components, not checked): ${unmatched.slice(0, 10).join(', ')}${unmatched.length > 10 ? '...' : ''}` + ); +} diff --git a/src/plugins/docusaurus-plugin-link-check-routes/index.js b/src/plugins/docusaurus-plugin-link-check-routes/index.js new file mode 100644 index 0000000000..3d9695bdba --- /dev/null +++ b/src/plugins/docusaurus-plugin-link-check-routes/index.js @@ -0,0 +1,58 @@ +/** + * docusaurus-plugin-link-check-routes + * + * A Docusaurus postBuild plugin that writes a map of source file to built URL. + * The PR link check uses this map to turn "files changed in a PR" into "pages to check". + * + * Output: /link-check-routes.json, shaped as: + * { "calico/getting-started/install.mdx": ["/calico/latest/getting-started/install/"], ... } + * + * Gated by LINK_CHECK_ROUTES=true — no-op on regular builds, so it does not change normal output. + */ + +import fs from 'fs/promises'; +import path from 'path'; + +const PLUGIN_NAME = 'docusaurus-plugin-link-check-routes'; +const LOG_PREFIX = '[link-check-routes]'; +const OUTPUT_FILE = 'link-check-routes.json'; + +export default function linkCheckRoutesPlugin() { + return { + name: PLUGIN_NAME, + + async postBuild({ outDir, plugins }) { + if (process.env.LINK_CHECK_ROUTES !== 'true') { + return; + } + + // sourcePath (repo-relative) -> array of permalinks (one source can map to more than one). + const routes = {}; + const add = (source, permalink) => { + if (!source || !permalink) return; + const rel = source.replace(/^@site\//, ''); + if (!routes[rel]) routes[rel] = []; + if (!routes[rel].includes(permalink)) routes[rel].push(permalink); + }; + + const docsPlugins = plugins.filter( + (p) => p.name === 'docusaurus-plugin-content-docs' + ); + + for (const dp of docsPlugins) { + const versions = dp.content?.loadedVersions || []; + for (const version of versions) { + for (const doc of version.docs || []) { + add(doc.source, doc.permalink); + } + } + } + + const outPath = path.join(outDir, OUTPUT_FILE); + await fs.writeFile(outPath, JSON.stringify(routes, null, 2)); + console.log( + `${LOG_PREFIX} Wrote ${outPath} (${Object.keys(routes).length} source files).` + ); + }, + }; +} From 4edcf7b00cc9ee1ffe508ba3afcb631997a5b23d Mon Sep 17 00:00:00 2001 From: Christopher Tauchen Date: Wed, 8 Jul 2026 10:57:42 +0100 Subject: [PATCH 2/3] docs-2923: add blocking PR link-check workflow Add .github/workflows/link-check-pr.yml. On a docs PR it builds the site, maps the changed files to pages, and runs the crawler against only those pages. A broken link fails the check and blocks the merge. It comments the report on the PR and is a no-op when no docs pages change. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/link-check-pr.yml | 105 ++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 .github/workflows/link-check-pr.yml diff --git a/.github/workflows/link-check-pr.yml b/.github/workflows/link-check-pr.yml new file mode 100644 index 0000000000..c756d9e474 --- /dev/null +++ b/.github/workflows/link-check-pr.yml @@ -0,0 +1,105 @@ +name: PR link check + +on: + pull_request: + paths: + - 'calico/**' + - 'calico-enterprise/**' + - 'calico-cloud/**' + - 'use-cases/**' + - 'calico_versioned_docs/**' + - 'calico-enterprise_versioned_docs/**' + - 'calico-cloud_versioned_docs/**' + +# Cancel an older run when the PR gets a new push. +concurrency: + group: pr-link-check-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + +jobs: + link-check: + name: Check links on changed pages + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Enable Corepack + run: corepack enable + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'yarn' + + - name: Build the site and the route map + run: make build + env: + LINK_CHECK_ROUTES: 'true' + NODE_OPTIONS: '--max-old-space-size=6000' + DOCUSAURUS_IGNORE_SSG_WARNINGS: 'true' + + - name: Find the changed pages + id: pages + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + node scripts/changed-pages.js "$BASE_SHA" build/link-check-routes.json > pages.txt || true + COUNT=$(grep -c . pages.txt || true) + echo "count=$COUNT" >> "$GITHUB_OUTPUT" + echo "Changed pages ($COUNT):" + cat pages.txt + + - name: Install Playwright Chrome + if: steps.pages.outputs.count != '0' + run: yarn playwright install --with-deps chrome + + - name: Serve the site and check the changed pages + if: steps.pages.outputs.count != '0' + env: + CI: 'true' + run: | + export PAGES_TO_CHECK="$(cat pages.txt)" + make test 2>&1 | tee link-check.log + + - name: Upload the link-check log + if: always() && steps.pages.outputs.count != '0' + uses: actions/upload-artifact@v4 + with: + name: link-check-log + path: link-check.log + if-no-files-found: ignore + + - name: Comment on the PR when the check fails + if: failure() && steps.pages.outputs.count != '0' + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + BODY=comment.md + { + echo "## Link check failed" + echo "" + echo "The link check found broken links on the pages this PR changes." + echo "" + echo "Pages checked:" + echo '```' + cat pages.txt + echo '```' + echo "" + echo "Report (last 200 lines, full log is in the run artifact \"link-check-log\"):" + echo '```' + tail -n 200 link-check.log 2>/dev/null || echo "(no log captured)" + echo '```' + } > "$BODY" + gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" --body-file "$BODY" From c0cd9f68652f1ba99de400c0bc4879b17061804b Mon Sep 17 00:00:00 2001 From: Christopher Tauchen Date: Wed, 8 Jul 2026 12:12:37 +0100 Subject: [PATCH 3/3] docs-2923: fail on scoping errors and pass pages robustly Address Copilot review: - Remove '|| true' on changed-pages.js so a real failure (bad manifest, git diff error) fails the job instead of silently skipping the scoped crawl. The script already exits 0 when no docs pages changed. - Join pages with commas when setting PAGES_TO_CHECK so the list reliably reaches the crawler as separate entries. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/link-check-pr.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/link-check-pr.yml b/.github/workflows/link-check-pr.yml index c756d9e474..4e65f6e8c5 100644 --- a/.github/workflows/link-check-pr.yml +++ b/.github/workflows/link-check-pr.yml @@ -53,7 +53,7 @@ jobs: env: BASE_SHA: ${{ github.event.pull_request.base.sha }} run: | - node scripts/changed-pages.js "$BASE_SHA" build/link-check-routes.json > pages.txt || true + node scripts/changed-pages.js "$BASE_SHA" build/link-check-routes.json > pages.txt COUNT=$(grep -c . pages.txt || true) echo "count=$COUNT" >> "$GITHUB_OUTPUT" echo "Changed pages ($COUNT):" @@ -68,7 +68,8 @@ jobs: env: CI: 'true' run: | - export PAGES_TO_CHECK="$(cat pages.txt)" + # Join with commas so the list survives as one env var; the crawler splits on commas/newlines. + export PAGES_TO_CHECK="$(tr '\n' ',' < pages.txt)" make test 2>&1 | tee link-check.log - name: Upload the link-check log