fix(docs): preserve negotiated response cache headers

Author: waleedlatif1

.github/workflows/docs-negotiation.yml

@@ -0,0 +1,60 @@
+name: docs content negotiation
+
+on:
+  pull_request:
+    paths:
+      - 'apps/docs/**'
+      - 'patches/next@*.patch'
+      - 'package.json'
+      - 'bun.lock'
+      - '.github/workflows/docs-negotiation.yml'
+  push:
+    branches: [staging]
+    paths:
+      - 'apps/docs/**'
+      - 'patches/next@*.patch'
+      - 'package.json'
+      - 'bun.lock'
+      - '.github/workflows/docs-negotiation.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: docs-negotiation-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  http:
+    name: docs HTTP
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'linux-x64-8-core' }}
+    timeout-minutes: 15
+    env:
+      DATABASE_URL: postgresql://docs_test:docs_test@127.0.0.1:1/docs_test
+      DOCS_E2E_BASE_URL: http://127.0.0.1:3001
+      DOCS_NEGOTIATION_REPORT_PATH: ${{ github.workspace }}/test-results/docs-negotiation.json
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
+      - name: Setup workspace
+        uses: ./.github/actions/setup
+        with:
+          provider: ${{ vars.CI_PROVIDER }}
+      - name: Build production docs
+        run: bun run --cwd apps/docs build
+      - name: Verify HTTP content negotiation
+        run: |
+          mkdir -p test-results
+          bun run --cwd apps/docs start --port 3001 >test-results/docs-server.log 2>&1 &
+          DOCS_SERVER_PID=$!
+          trap 'kill "$DOCS_SERVER_PID" 2>/dev/null || true' EXIT
+          bun run --cwd apps/docs test:negotiation:e2e
+      - name: Upload HTTP regression report
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: docs-negotiation-report
+          path: test-results/
+          if-no-files-found: warn
+          retention-days: 14

apps/docs/package.json

@@ -11,6 +11,7 @@
     "start": "next start",
     "postinstall": "fumadocs-mdx",
     "test": "vitest run",
+    "test:negotiation:e2e": "bun scripts/test-negotiation-e2e.ts",
     "type-check": "fumadocs-mdx && tsc --noEmit",
     "lint": "biome check --write --unsafe .",
     "lint:check": "biome check .",

apps/docs/scripts/test-negotiation-e2e.ts

@@ -0,0 +1,87 @@
+import { mkdir, writeFile } from 'node:fs/promises'
+import { dirname } from 'node:path'
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { sleep } from '@sim/utils/helpers'
+
+const logger = createLogger('DocsNegotiationE2E')
+const origin = process.env.DOCS_E2E_BASE_URL
+const reportPath = process.env.DOCS_NEGOTIATION_REPORT_PATH
+if (!origin || !reportPath) {
+  throw new Error('DOCS_E2E_BASE_URL and DOCS_NEGOTIATION_REPORT_PATH are required')
+}
+const url = new URL('/introduction', origin)
+if (!['localhost', '127.0.0.1', '[::1]'].includes(url.hostname)) {
+  throw new Error('The documentation server must use a loopback origin')
+}
+
+interface CheckResult {
+  name: string
+  status: 'passed' | 'failed'
+  durationMs: number
+  contentType?: string
+  vary?: string
+  error?: string
+}
+
+const cases = [
+  { name: 'HTML advertises its representation and router cache dimensions', accept: 'text/html' },
+  { name: 'Markdown negotiates at the same URL', accept: 'text/markdown' },
+  { name: 'HTML remains available after a Markdown request', accept: 'text/html' },
+] as const
+
+for (let attempt = 0; attempt < 20; attempt += 1) {
+  try {
+    const response = await fetch(url, { signal: AbortSignal.timeout(1000) })
+    await response.body?.cancel()
+    if (response.ok) break
+  } catch {}
+  await sleep(250)
+}
+
+const checks: CheckResult[] = []
+for (const entry of cases) {
+  const started = performance.now()
+  const result: CheckResult = { name: entry.name, status: 'failed', durationMs: 0 }
+  try {
+    const response = await fetch(url, {
+      headers: { Accept: entry.accept },
+      signal: AbortSignal.timeout(10000),
+    })
+    await response.body?.cancel()
+    result.contentType = response.headers.get('Content-Type') ?? ''
+    result.vary = response.headers.get('Vary') ?? ''
+    const fields = result.vary.split(',').map((field) => field.trim().toLowerCase())
+    if (!response.ok) throw new Error(`Unexpected HTTP status ${response.status}`)
+    if (!result.contentType.startsWith(entry.accept)) {
+      throw new Error(`Expected ${entry.accept}; received ${result.contentType}`)
+    }
+    if (!fields.includes('accept')) throw new Error('The response lost Vary: Accept')
+    if (entry.accept === 'text/html') {
+      for (const field of [
+        'rsc',
+        'next-router-state-tree',
+        'next-router-prefetch',
+        'next-router-segment-prefetch',
+      ]) {
+        if (!fields.includes(field)) throw new Error(`The response lost Vary: ${field}`)
+      }
+    }
+    if (fields.length !== new Set(fields).size) throw new Error('Vary contains duplicate fields')
+    result.status = 'passed'
+  } catch (error) {
+    result.error = getErrorMessage(error)
+  }
+  result.durationMs = Math.round(performance.now() - started)
+  checks.push(result)
+}
+
+await mkdir(dirname(reportPath), { recursive: true })
+await writeFile(reportPath, `${JSON.stringify({ checks }, null, 2)}\n`)
+const failures = checks.filter((check) => check.status === 'failed')
+if (failures.length) {
+  logger.error('Documentation negotiation failed', { failures })
+  process.exitCode = 1
+} else {
+  logger.info('Documentation negotiation passed', { checks: checks.length })
+}

bun.lock

@@ -212,6 +212,7 @@
     "drizzle-kit@0.31.10": "patches/drizzle-kit@0.31.10.patch",
     "postgres@3.4.9": "patches/postgres@3.4.9.patch",
     "oracledb@7.0.1": "patches/oracledb@7.0.1.patch",
-    "echarts@6.1.0": "patches/echarts@6.1.0.patch"
+    "echarts@6.1.0": "patches/echarts@6.1.0.patch",
+    "next@16.4.0": "patches/next@16.4.0.patch"
   }
 }

package.json

@@ -89,3 +89,39 @@ rich-text mode, preserving the chart document's untrusted-markup boundary.
 `apps/sim/lib/charts/tooltip.test.ts` exercises actual ECharts tooltip rendering,
 font overrides, generated tokens, and box sizing. Remove this patch when an
 upstream release provides equivalent font handling and these tests pass.
+
+# Next.js application Vary preservation
+
+`next@16.4.0` replaces application `Vary` values when the Node App Page handler
+sets its router cache dimensions. The docs proxy's `Vary: Accept` therefore
+vanishes from the final HTML response, allowing downstream caches to confuse
+HTML with negotiated Markdown.
+
+The pinned patch applies the merging semantics of [upstream PR #98104](https://github.com/vercel/next.js/pull/98104)
+at Next's native Node outgoing-header merger, which is used before the bundled
+App Page handler. Both published CommonJS and ESM entry points preserve existing
+Vary fields and add later fields without case-sensitive duplicates. Wildcards
+remain intact, and native header validation still rejects invalid values.
+
+Vary writes become cumulative across Node server responses in both the docs and
+Sim workspaces; explicit header removal, cookie handling, other headers, router
+selection, cache lifetimes, and authentication are unchanged. Edge SSR and direct
+handlers outside Next's Node server are outside this patch's scope.
+
+The `docs content negotiation` CI workflow builds and serves the actual docs,
+then runs `apps/docs/scripts/test-negotiation-e2e.ts`. Its HTTP checks verify
+HTML → Markdown → HTML at one URL, `Vary: Accept` on both representations, and
+preservation of Next's router dimensions. It uploads a JSON report and server
+log. The HTML checks fail with the unpatched dependency even when the proxy
+already sets `Vary: Accept`.
+
+To repeat against a running production docs server:
+
+```bash
+DOCS_E2E_BASE_URL=http://127.0.0.1:3001 \
+DOCS_NEGOTIATION_REPORT_PATH=/tmp/docs-negotiation.json \
+bun run --cwd apps/docs test:negotiation:e2e
+```
+
+Remove the patch when upgrading to a release with equivalent Vary preservation,
+and retain the HTTP regression check.

patches/README.md

@@ -0,0 +1,75 @@
+diff --git a/dist/esm/server/lib/patch-set-header.js b/dist/esm/server/lib/patch-set-header.js
+index f2236f7ae222afcb8554267ed295ada59dceee6a..3be146fc8fa48196254d93a8a21c2ceac2596b09 100644
+--- a/dist/esm/server/lib/patch-set-header.js
++++ b/dist/esm/server/lib/patch-set-header.js
+@@ -1,3 +1,4 @@
++import { validateHeaderValue } from 'node:http';
+ import { getRequestMeta } from '../request-meta';
+ const PATCHED_SET_HEADER = Symbol('next.patchSetHeaderWithCookieSupport');
+ /**
+@@ -20,6 +21,26 @@ const PATCHED_SET_HEADER = Symbol('next.patchSetHeaderWithCookieSupport');
+         if ('headersSent' in res && res.headersSent) {
+             return res;
+         }
++        if (name.toLowerCase() === 'vary' && (typeof value === 'string' || Array.isArray(value))) {
++            validateHeaderValue(name, value);
++            const fields = [];
++            const seen = new Set();
++            const collect = (candidate)=>{
++                const values = Array.isArray(candidate) ? candidate : typeof candidate === 'string' ? [candidate] : [];
++                for (const entry of values){
++                    for (const field of entry.split(',')){
++                        const trimmed = field.trim();
++                        const normalized = trimmed.toLowerCase();
++                        if (!trimmed || seen.has(normalized)) continue;
++                        seen.add(normalized);
++                        fields.push(trimmed);
++                    }
++                }
++            };
++            collect(res.getHeader(name));
++            collect(value);
++            value = seen.has('*') ? '*' : fields.join(', ');
++        }
+         if (name.toLowerCase() === 'set-cookie') {
+             const middlewareValue = getRequestMeta(req, 'middlewareCookie');
+             if (!middlewareValue || !Array.isArray(value) || !value.every((item, idx)=>item === middlewareValue[idx])) {
+diff --git a/dist/server/lib/patch-set-header.js b/dist/server/lib/patch-set-header.js
+index 16cdc4703f3e9957a16c68c962cfaf5ee5d04561..a1c3c7061bc917284e01ed9947cd914818cfd708 100644
+--- a/dist/server/lib/patch-set-header.js
++++ b/dist/server/lib/patch-set-header.js
+@@ -8,6 +8,7 @@ Object.defineProperty(exports, "patchSetHeaderWithCookieSupport", {
+         return patchSetHeaderWithCookieSupport;
+     }
+ });
++const _nodehttp = require("node:http");
+ const _requestmeta = require("../request-meta");
+ const PATCHED_SET_HEADER = Symbol('next.patchSetHeaderWithCookieSupport');
+ function patchSetHeaderWithCookieSupport(req, res) {
+@@ -24,6 +25,26 @@ function patchSetHeaderWithCookieSupport(req, res) {
+         if ('headersSent' in res && res.headersSent) {
+             return res;
+         }
++        if (name.toLowerCase() === 'vary' && (typeof value === 'string' || Array.isArray(value))) {
++            (0, _nodehttp.validateHeaderValue)(name, value);
++            const fields = [];
++            const seen = new Set();
++            const collect = (candidate)=>{
++                const values = Array.isArray(candidate) ? candidate : typeof candidate === 'string' ? [candidate] : [];
++                for (const entry of values){
++                    for (const field of entry.split(',')){
++                        const trimmed = field.trim();
++                        const normalized = trimmed.toLowerCase();
++                        if (!trimmed || seen.has(normalized)) continue;
++                        seen.add(normalized);
++                        fields.push(trimmed);
++                    }
++                }
++            };
++            collect(res.getHeader(name));
++            collect(value);
++            value = seen.has('*') ? '*' : fields.join(', ');
++        }
+         if (name.toLowerCase() === 'set-cookie') {
+             const middlewareValue = (0, _requestmeta.getRequestMeta)(req, 'middlewareCookie');
+             if (!middlewareValue || !Array.isArray(value) || !value.every((item, idx)=>item === middlewareValue[idx])) {