From 00dc05e2896176feec12b8900030f8d963aee9a3 Mon Sep 17 00:00:00 2001 From: isaaclombardssw Date: Tue, 22 Sep 2026 10:37:16 +1000 Subject: [PATCH 1/3] docs: add Markdown content negotiation guide --- .../guides/markdown-content-negotiation.mdx | 242 ++++++++++++++++++ .../navigation-bar/docs-navigation-bar.json | 5 + 2 files changed, 247 insertions(+) create mode 100644 content/docs/guides/markdown-content-negotiation.mdx diff --git a/content/docs/guides/markdown-content-negotiation.mdx b/content/docs/guides/markdown-content-negotiation.mdx new file mode 100644 index 00000000..24735cb0 --- /dev/null +++ b/content/docs/guides/markdown-content-negotiation.mdx @@ -0,0 +1,242 @@ +--- +title: Serve Markdown to AI Agents +seo: + title: Serve Markdown to AI Agents | TinaCMS Docs + description: Configure content negotiation for TinaCMS pages in Next.js, Astro, and Vite. +--- + +AI agents can consume documentation more reliably when they receive Markdown instead of rendered HTML. Because TinaCMS stores content as Markdown or MDX, a server can return the source file when a client sends `Accept: text/markdown` while continuing to serve HTML to browsers at the same URL. + +This requires request-time routing. A static host cannot serve two representations from one URL unless its CDN or edge layer performs the negotiation. + +The examples use a simple `text/markdown` match. If your application needs full HTTP negotiation, parse media-type quality values and do not select Markdown when its quality is `q=0`. + + + +## Next.js App Router + +Create a static route handler that returns the source MDX. This example assumes documents live in `content/docs` and that `/docs` maps to `content/docs/index.mdx`. + +```typescript +// src/app/api/markdown/[...slug]/route.ts +import { readFile, readdir } from "node:fs/promises"; +import path from "node:path"; + +const docsDirectory = path.join(process.cwd(), "content", "docs"); + +export const dynamic = "force-static"; +export const dynamicParams = false; + +export async function generateStaticParams() { + const files = await readdir(docsDirectory, { recursive: true }); + + return files + .filter((file) => file.endsWith(".mdx")) + .map((file) => ({ + slug: file.replace(/\.mdx$/, "").split(path.sep), + })); +} + +export async function GET( + _request: Request, + { params }: { params: Promise<{ slug: string[] }> }, +) { + const { slug } = await params; + const filePath = path.join(docsDirectory, `${slug.join(path.sep)}.mdx`); + + if (!filePath.startsWith(`${docsDirectory}${path.sep}`)) { + return new Response(null, { status: 404 }); + } + + try { + return new Response(await readFile(filePath, "utf8"), { + headers: { "Content-Type": "text/markdown; charset=utf-8" }, + }); + } catch { + return new Response(null, { status: 404 }); + } +} +``` + +Use header-based rewrites to keep the public URL unchanged. Omit them when creating a static export. + +```javascript +// next.config.js +const isStatic = process.env.EXPORT_MODE === "static"; + +/** @type {import("next").NextConfig} */ +module.exports = { + ...(!isStatic && { + async rewrites() { + const acceptsMarkdown = [ + { + type: "header", + key: "accept", + value: "(.*)text/markdown(.*)", + }, + ]; + + return { + beforeFiles: [ + { + source: "/docs", + destination: "/api/markdown/index", + has: acceptsMarkdown, + }, + { + source: "/docs/:path+", + destination: "/api/markdown/:path+", + has: acceptsMarkdown, + }, + ], + }; + }, + }), +}; +``` + +Next.js App Router currently overwrites custom `Vary` values with its internal routing headers. Vercel includes `Accept` in its cache key automatically. For other CDNs and reverse proxies, configure the cache key to include `Accept`; otherwise HTML and Markdown can share a cache entry. + +## Astro + +This approach requires an Astro server adapter and on-demand rendering. Create a server endpoint for the source MDX: + +```typescript +// src/pages/_markdown/[...slug].ts +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import type { APIRoute } from "astro"; + +export const prerender = false; + +const docsDirectory = path.join(process.cwd(), "content", "docs"); + +export const GET: APIRoute = async ({ params }) => { + const slug = params.slug || "index"; + const filePath = path.join(docsDirectory, `${slug}.mdx`); + + if (!filePath.startsWith(`${docsDirectory}${path.sep}`)) { + return new Response(null, { status: 404 }); + } + + try { + return new Response(await readFile(filePath, "utf8"), { + headers: { "Content-Type": "text/markdown; charset=utf-8" }, + }); + } catch { + return new Response(null, { status: 404 }); + } +}; +``` + +Then add middleware that rewrites Markdown requests to the endpoint and adds `Vary: Accept` to both representations: + +```typescript +// src/middleware.ts +import { defineMiddleware } from "astro:middleware"; + +export const onRequest = defineMiddleware(async (context, next) => { + const pathname = context.url.pathname.replace(/\/$/, ""); + const isDocsPage = pathname === "/docs" || pathname.startsWith("/docs/"); + + if (!isDocsPage) { + return next(); + } + + const acceptsMarkdown = context.request.headers + .get("accept") + ?.includes("text/markdown"); + const slug = pathname.slice("/docs".length).replace(/^\//, "") || "index"; + const response = acceptsMarkdown + ? await next(`/_markdown/${slug}`) + : await next(); + const headers = new Headers(response.headers); + headers.append("Vary", "Accept"); + + return new Response(response.body, { + status: response.status, + statusText: response.statusText, + headers, + }); +}); +``` + +Astro static output cannot inspect request headers. For a static deployment, publish separate `.md` files or configure negotiation in the hosting layer. + +## Plain Vite + +Vite provides a development server and a preview server, but its production output is static files. The following plugin adds negotiation during `vite` and `vite preview`: + +```typescript +// vite.config.ts +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { defineConfig, type Plugin, type PreviewServer, type ViteDevServer } from "vite"; + +function markdownNegotiation(): Plugin { + const docsDirectory = path.join(process.cwd(), "content", "docs"); + + const install = (server: ViteDevServer | PreviewServer) => { + server.middlewares.use(async (request, response, next) => { + const pathname = new URL(request.url || "/", "http://localhost").pathname; + + if (pathname !== "/docs" && !pathname.startsWith("/docs/")) { + return next(); + } + + response.setHeader("Vary", "Accept"); + + if (!request.headers.accept?.includes("text/markdown")) { + return next(); + } + + const slug = pathname.slice("/docs".length).replace(/^\//, "") || "index"; + const filePath = path.join(docsDirectory, `${slug}.mdx`); + + if (!filePath.startsWith(`${docsDirectory}${path.sep}`)) { + response.statusCode = 404; + return response.end(); + } + + try { + response.setHeader("Content-Type", "text/markdown; charset=utf-8"); + response.end(await readFile(filePath, "utf8")); + } catch { + response.statusCode = 404; + response.end(); + } + }); + }; + + return { + name: "markdown-content-negotiation", + configureServer: install, + configurePreviewServer: install, + }; +} + +export default defineConfig({ + plugins: [markdownNegotiation()], +}); +``` + +`vite preview` is for local verification, not production. In production, run equivalent middleware in your application server or configure your CDN/edge platform to route requests based on `Accept`. A static-only deployment should expose generated `.md` files at separate URLs instead. + +## Test the result + +Verify both representations at the same URL: + +```bash +curl -i https://example.com/docs/getting-started +curl -i -H 'Accept: text/markdown' https://example.com/docs/getting-started +``` + +The first response should use `Content-Type: text/html`. The second should use `Content-Type: text/markdown`. If a CDN or proxy caches the responses, confirm that its cache key includes `Accept` or that the response includes `Vary: Accept`. + +## Related resources + +- [Markdown for Agents](https://developers.cloudflare.com/fundamentals/reference/markdown-for-agents/) +- [Next.js rewrites](https://nextjs.org/docs/15/app/api-reference/config/next-config-js/rewrites) +- [Astro middleware](https://docs.astro.build/en/guides/middleware/) +- [Astro endpoints](https://docs.astro.build/en/guides/endpoints/) +- [Vite plugin API](https://vite.dev/guide/api-plugin) diff --git a/content/navigation-bar/docs-navigation-bar.json b/content/navigation-bar/docs-navigation-bar.json index 997ca85f..4fc610fc 100644 --- a/content/navigation-bar/docs-navigation-bar.json +++ b/content/navigation-bar/docs-navigation-bar.json @@ -917,6 +917,11 @@ "slug": "content/docs/editing/referenced-files.mdx", "_template": "item" }, + { + "slug": "content/docs/guides/markdown-content-negotiation.mdx", + "title": "Serve Markdown to AI Agents", + "_template": "item" + }, { "slug": "content/docs/guides/react-data-fetching.mdx", "title": "Querying TinaCMS Content at Runtime", From f0e540319a6ba66fe5957224b7f0f0b6db7b4690 Mon Sep 17 00:00:00 2001 From: isaaclombardssw Date: Fri, 9 Oct 2026 12:51:22 +1100 Subject: [PATCH 2/3] docs: tighten Markdown content negotiation prose --- .../docs/guides/markdown-content-negotiation.mdx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/content/docs/guides/markdown-content-negotiation.mdx b/content/docs/guides/markdown-content-negotiation.mdx index 24735cb0..c9bb3d86 100644 --- a/content/docs/guides/markdown-content-negotiation.mdx +++ b/content/docs/guides/markdown-content-negotiation.mdx @@ -5,11 +5,11 @@ seo: description: Configure content negotiation for TinaCMS pages in Next.js, Astro, and Vite. --- -AI agents can consume documentation more reliably when they receive Markdown instead of rendered HTML. Because TinaCMS stores content as Markdown or MDX, a server can return the source file when a client sends `Accept: text/markdown` while continuing to serve HTML to browsers at the same URL. +Serve TinaCMS source Markdown or MDX to AI agents when they request `Accept: text/markdown`. Browsers can continue to receive HTML at the same URL. -This requires request-time routing. A static host cannot serve two representations from one URL unless its CDN or edge layer performs the negotiation. +To serve both formats at one URL, route requests through a server, CDN, or edge layer that handles content negotiation. A static host alone cannot do this. -The examples use a simple `text/markdown` match. If your application needs full HTTP negotiation, parse media-type quality values and do not select Markdown when its quality is `q=0`. +These examples match `text/markdown` in the request header. For full HTTP negotiation, parse media-type quality values and exclude Markdown when its quality is `q=0`. @@ -99,7 +99,7 @@ Next.js App Router currently overwrites custom `Vary` values with its internal r ## Astro -This approach requires an Astro server adapter and on-demand rendering. Create a server endpoint for the source MDX: +Use an Astro server adapter with on-demand rendering. Create a server endpoint for the source MDX: ```typescript // src/pages/_markdown/[...slug].ts @@ -129,7 +129,7 @@ export const GET: APIRoute = async ({ params }) => { }; ``` -Then add middleware that rewrites Markdown requests to the endpoint and adds `Vary: Accept` to both representations: +Add middleware to rewrite Markdown requests to the endpoint and append `Vary: Accept` to both HTML and Markdown responses: ```typescript // src/middleware.ts @@ -165,7 +165,7 @@ Astro static output cannot inspect request headers. For a static deployment, pub ## Plain Vite -Vite provides a development server and a preview server, but its production output is static files. The following plugin adds negotiation during `vite` and `vite preview`: +Vite produces static files for production. Use this plugin to add content negotiation to its development and preview servers (`vite` and `vite preview`): ```typescript // vite.config.ts @@ -220,7 +220,7 @@ export default defineConfig({ }); ``` -`vite preview` is for local verification, not production. In production, run equivalent middleware in your application server or configure your CDN/edge platform to route requests based on `Accept`. A static-only deployment should expose generated `.md` files at separate URLs instead. +Use `vite preview` only for local verification. In production, run equivalent middleware in your application server or configure your CDN or edge platform to route requests based on `Accept`. For a static-only deployment, publish generated `.md` files at separate URLs. ## Test the result From 56c4164769dec9ba680e70f9fc19cc956f79c045 Mon Sep 17 00:00:00 2001 From: isaaclombardssw Date: Mon, 12 Oct 2026 16:19:52 +1100 Subject: [PATCH 3/3] Align Markdown negotiation guide with TinaDocs --- .../guides/markdown-content-negotiation.mdx | 114 +++++++++++------- 1 file changed, 68 insertions(+), 46 deletions(-) diff --git a/content/docs/guides/markdown-content-negotiation.mdx b/content/docs/guides/markdown-content-negotiation.mdx index c9bb3d86..9717c988 100644 --- a/content/docs/guides/markdown-content-negotiation.mdx +++ b/content/docs/guides/markdown-content-negotiation.mdx @@ -9,7 +9,14 @@ Serve TinaCMS source Markdown or MDX to AI agents when they request `Accept: tex To serve both formats at one URL, route requests through a server, CDN, or edge layer that handles content negotiation. A static host alone cannot do this. -These examples match `text/markdown` in the request header. For full HTTP negotiation, parse media-type quality values and exclude Markdown when its quality is `q=0`. +These examples use the same `negotiator` library as [TinaDocs](https://github.com/tinacms/tina-docs) to select between HTML and Markdown. It respects quality values (`q`), wildcards, and case-insensitive media types. Markdown with `q=0` is never selected. Missing headers and generic wildcards default to HTML; if neither format is acceptable, these examples fall back to HTML, matching TinaDocs. + +Install the library and its TypeScript types in your project: + +```bash +pnpm add negotiator +pnpm add -D @types/negotiator +``` @@ -58,48 +65,40 @@ export async function GET( } ``` -Use header-based rewrites to keep the public URL unchanged. Omit them when creating a static export. - -```javascript -// next.config.js -const isStatic = process.env.EXPORT_MODE === "static"; - -/** @type {import("next").NextConfig} */ -module.exports = { - ...(!isStatic && { - async rewrites() { - const acceptsMarkdown = [ - { - type: "header", - key: "accept", - value: "(.*)text/markdown(.*)", - }, - ]; - - return { - beforeFiles: [ - { - source: "/docs", - destination: "/api/markdown/index", - has: acceptsMarkdown, - }, - { - source: "/docs/:path+", - destination: "/api/markdown/:path+", - has: acceptsMarkdown, - }, - ], - }; - }, - }), -}; +Use middleware to negotiate the response and keep the public URL unchanged. This matches TinaDocs' Next.js 15 implementation. Merge it into any existing middleware and matcher configuration. + +```typescript +// src/middleware.ts +import Negotiator from "negotiator"; +import { type NextRequest, NextResponse } from "next/server"; + +export function middleware(request: NextRequest) { + const type = new Negotiator({ + headers: { accept: request.headers.get("accept") ?? "*/*" }, + }).mediaType(["text/html", "text/markdown"]); + const url = request.nextUrl.clone(); + url.pathname = `/api/markdown${url.pathname.slice("/docs".length) || "/index"}`; + + const response = + process.env.EXPORT_MODE !== "static" && type === "text/markdown" + ? NextResponse.rewrite(url) + : NextResponse.next(); + response.headers.append("Vary", "Accept"); + return response; +} + +export const config = { matcher: "/docs/:path*" }; ``` -Next.js App Router currently overwrites custom `Vary` values with its internal routing headers. Vercel includes `Accept` in its cache key automatically. For other CDNs and reverse proxies, configure the cache key to include `Accept`; otherwise HTML and Markdown can share a cache entry. +Static exports do not run middleware. TinaDocs uses `EXPORT_MODE=static` to select its static-export configuration; a static host still serves HTML at the docs URL regardless of `Accept`. + +The middleware appends `Vary: Accept` to both representations, but Next.js or the hosting layer can replace this header. [Vercel includes `Accept` in its cache key](https://vercel.com/docs/caching/cdn-cache) automatically. For other CDNs and reverse proxies, configure separate cache entries by `Accept` or bypass caching for docs routes; otherwise HTML and Markdown can share a cache entry. ## Astro -Use an Astro server adapter with on-demand rendering. Create a server endpoint for the source MDX: +Use an Astro adapter for a Node.js server with on-demand rendering. Set `output: "server"`, or set `export const prerender = false` on your docs pages as well as the endpoint below, so middleware can inspect each request. Make `content/docs` available to the deployed server: this example reads source MDX from disk at request time. See [Astro middleware](https://docs.astro.build/en/guides/middleware/). + +Create a server endpoint for the source MDX: ```typescript // src/pages/_markdown/[...slug].ts @@ -134,6 +133,7 @@ Add middleware to rewrite Markdown requests to the endpoint and append `Vary: Ac ```typescript // src/middleware.ts import { defineMiddleware } from "astro:middleware"; +import Negotiator from "negotiator"; export const onRequest = defineMiddleware(async (context, next) => { const pathname = context.url.pathname.replace(/\/$/, ""); @@ -143,13 +143,14 @@ export const onRequest = defineMiddleware(async (context, next) => { return next(); } - const acceptsMarkdown = context.request.headers - .get("accept") - ?.includes("text/markdown"); + const type = new Negotiator({ + headers: { accept: context.request.headers.get("accept") ?? "*/*" }, + }).mediaType(["text/html", "text/markdown"]); const slug = pathname.slice("/docs".length).replace(/^\//, "") || "index"; - const response = acceptsMarkdown - ? await next(`/_markdown/${slug}`) - : await next(); + const response = + type === "text/markdown" + ? await next(`/_markdown/${slug}`) + : await next(); const headers = new Headers(response.headers); headers.append("Vary", "Accept"); @@ -172,6 +173,7 @@ Vite produces static files for production. Use this plugin to add content negoti import { readFile } from "node:fs/promises"; import path from "node:path"; import { defineConfig, type Plugin, type PreviewServer, type ViteDevServer } from "vite"; +import Negotiator from "negotiator"; function markdownNegotiation(): Plugin { const docsDirectory = path.join(process.cwd(), "content", "docs"); @@ -184,9 +186,12 @@ function markdownNegotiation(): Plugin { return next(); } - response.setHeader("Vary", "Accept"); + response.appendHeader("Vary", "Accept"); + const type = new Negotiator({ + headers: { accept: request.headers.accept ?? "*/*" }, + }).mediaType(["text/html", "text/markdown"]); - if (!request.headers.accept?.includes("text/markdown")) { + if (type !== "text/markdown") { return next(); } @@ -233,9 +238,26 @@ curl -i -H 'Accept: text/markdown' https://example.com/docs/getting-started The first response should use `Content-Type: text/html`. The second should use `Content-Type: text/markdown`. If a CDN or proxy caches the responses, confirm that its cache key includes `Accept` or that the response includes `Vary: Accept`. +Also test negotiation at both `/docs` and a nested docs URL: + +| `Accept` header | Expected response | +| --- | --- | +| Missing, `*/*`, or `text/*` | HTML | +| `text/markdown` or `Text/Markdown` | Markdown | +| `text/html;q=1, text/markdown;q=0.1` | HTML | +| `text/html;q=0.1, text/markdown;q=1` | Markdown | +| `text/html, text/markdown;q=0` | HTML | +| `text/markdown;q=0.000, text/html` | HTML | +| `text/markdown; charset=utf-8; q=0, text/html` | HTML | +| `text/markdown;q=0.5` | Markdown | +| `text/html;q=0, text/markdown` | Markdown | + +Repeat requests in HTML → Markdown → HTML → Markdown order to check that the deployed cache keeps the representations separate. Check that Markdown responses contain source MDX, including frontmatter, rather than rendered HTML. On a static-only deployment, expect HTML at the docs URL for every header above. + ## Related resources - [Markdown for Agents](https://developers.cloudflare.com/fundamentals/reference/markdown-for-agents/) +- [Negotiator API](https://github.com/jshttp/negotiator#api) - [Next.js rewrites](https://nextjs.org/docs/15/app/api-reference/config/next-config-js/rewrites) - [Astro middleware](https://docs.astro.build/en/guides/middleware/) - [Astro endpoints](https://docs.astro.build/en/guides/endpoints/)