feat(docs): publish a Markdown copy of every docs page

.gitignore

@@ -12,6 +12,8 @@ tsconfig.tsbuildinfo
 
 # Build-time generated
 src/github-stars.json
+public/docs.md
+public/docs/**/*.md
 
 # System files
 .DS_Store

package.json

@@ -3,9 +3,9 @@
   "version": "0.1.0",
   "private": true,
   "scripts": {
-    "predev": "node scripts/fetch-github-stars.mjs",
+    "predev": "node scripts/fetch-github-stars.mjs && node scripts/generate-docs-markdown.mjs",
     "dev": "next dev",
-    "prebuild": "node scripts/fetch-github-stars.mjs",
+    "prebuild": "node scripts/fetch-github-stars.mjs && node scripts/generate-docs-markdown.mjs",
     "build": "next build",
     "start": "next start",
     "postinstall": "fumadocs-mdx"

scripts/generate-docs-markdown.mjs

@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Emits a Markdown copy of every docs page next to its HTML route, so an agent
+ * can fetch `/docs/server/security.md` instead of parsing the rendered page.
+ *
+ * The site is a static export served by ASF infrastructure, so it cannot do
+ * Accept-header content negotiation; a sibling file is the version of this that
+ * needs no server. Output goes to public/, is gitignored, and is rebuilt from
+ * content/docs on every build.
+ */
+
+import { readdirSync, readFileSync, writeFileSync, mkdirSync, rmSync, statSync } from "fs";
+import { join, relative, dirname, basename } from "path";
+
+const CONTENT = "content/docs";
+const PUBLIC = "public";
+const SITE_URL = "https://iggy.apache.org";
+const REPO_BLOB = "https://github.com/apache/iggy-website/blob/main";
+
+function walk(dir) {
+  return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
+    const path = join(dir, entry.name);
+    if (entry.isDirectory()) return walk(path);
+    return entry.name.endsWith(".mdx") ? [path] : [];
+  });
+}
+
+/** content/docs/server/security.mdx -> public/docs/server/security.md
+ *  content/docs/binary-protocol/index.mdx -> public/docs/binary-protocol.md
+ *  content/docs/index.mdx -> public/docs.md */
+function outputPath(source) {
+  const rel = relative(CONTENT, source).replace(/\.mdx$/, "");
+  const stem = basename(rel) === "index" ? dirname(rel) : rel;
+  return join(PUBLIC, "docs" + (stem === "." ? "" : `/${stem}`) + ".md");
+}
+
+/** Frontmatter is YAML, but only two scalar keys are read here, so it is not
+ *  worth a parser dependency. */
+function splitFrontmatter(raw) {
+  if (!raw.startsWith("---\n")) return { meta: {}, body: raw };
+  const end = raw.indexOf("\n---", 3);
+  if (end === -1) return { meta: {}, body: raw };
+  const meta = {};
+  for (const line of raw.slice(4, end).split("\n")) {
+    const match = /^(title|description):\s*(.*)$/.exec(line);
+    if (match) meta[match[1]] = match[2].trim().replace(/^["']|["']$/g, "");
+  }
+  return { meta, body: raw.slice(end + 4).replace(/^\n+/, "") };
+}
+
+function render(source, raw) {
+  const { meta, body } = splitFrontmatter(raw);
+  const url = `${SITE_URL}/${outputPath(source).replace(/^public\//, "").replace(/\.md$/, "")}/`;
+  const head = [
+    `# ${meta.title ?? basename(source, ".mdx")}`,
+    ...(meta.description ? [`> ${meta.description}`] : []),
+    `Rendered page: ${url}`,
+    `Source: ${REPO_BLOB}/${source}`,
+  ].join("\n\n");
+  // The body keeps its MDX as authored: custom components appear as tags, which
+  // is honest about what the page contains and needs no MDX pipeline here.
+  return `${head}\n\n${body.trimEnd()}\n`;
+}
+
+/** Everything this script wrote last time. Nothing else under public/ is .md. */
+function clean(dir) {
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const path = join(dir, entry.name);
+    if (entry.isDirectory()) clean(path);
+    else if (entry.name.endsWith(".md")) rmSync(path);
+  }
+}
+
+function main() {
+  const sources = walk(CONTENT).sort();
+  try {
+    if (statSync(join(PUBLIC, "docs")).isDirectory()) clean(join(PUBLIC, "docs"));
+  } catch {
+    // No public/docs yet; nothing to clean.
+  }
+  rmSync(join(PUBLIC, "docs.md"), { force: true });
+
+  for (const source of sources) {
+    const target = outputPath(source);
+    mkdirSync(dirname(target), { recursive: true });
+    writeFileSync(target, render(source, readFileSync(source, "utf8")));
+  }
+  console.log(`Generated ${sources.length} Markdown copies of docs pages.`);
+}
+
+main();

src/app/(home)/page.tsx

@@ -25,7 +25,7 @@ import { BenchmarkSection } from "@/components/benchmark-chart";
 import { LATENCY_MS } from "@/lib/benchmark";
 
 export const metadata: Metadata = {
-  title: "Apache Iggy | Hyper-Efficient Message Streaming written in Rust.",
+  title: "Hyper-Efficient Message Streaming written in Rust",
   description:
     "Apache Iggy is a high-performance, persistent message streaming platform written in Rust, capable of processing millions of messages per second with ultra-low latency.",
 };
@@ -359,9 +359,9 @@ export default function HomePage() {
                     {item.step}
                   </div>
                   <div className="min-w-0">
-                    <h4 className="text-base font-semibold text-[#fffaeb] mb-1">
+                    <h3 className="text-base font-semibold text-[#fffaeb] mb-1">
                       {item.title}
-                    </h4>
+                    </h3>
                     <p className="text-sm font-light leading-relaxed text-[#aeb5bd] m-0">
                       {item.desc}
                     </p>

src/app/(site)/blogs/[...slug]/page.tsx

@@ -148,8 +148,18 @@ export async function generateMetadata(props: {
   const page = findPostByDateSlug(params.slug);
   if (!page) notFound();
 
+  const date = new Date(page.date);
+  const year = date.getFullYear();
+  const month = String(date.getMonth() + 1).padStart(2, "0");
+  const day = String(date.getDate()).padStart(2, "0");
+
   return {
     title: page.title,
     description: page.description,
+    // Set explicitly: the inherited relative canonical loses the trailing slash
+    // on this catch-all route, so it would not match the URL actually served.
+    alternates: {
+      canonical: `/blogs/${year}/${month}/${day}/${getSlug(page.info.path)}/`,
+    },
   };
 }

src/app/docs/[[...slug]]/page.tsx

@@ -79,6 +79,12 @@ export async function generateMetadata(props: {
   return {
     title: page.data.title,
     description: page.data.description,
+    alternates: {
+      // Set both: defining alternates here replaces the root's, canonical included.
+      canonical: `${page.url}/`,
+      // The Markdown copy written by scripts/generate-docs-markdown.mjs.
+      types: { "text/markdown": `${page.url}.md` },
+    },
   };
 }
 

src/app/layout.tsx

@@ -21,22 +21,100 @@ import { RootProvider } from "fumadocs-ui/provider/next";
 import type { Metadata } from "next";
 import Script from "next/script";
 import { Matomo } from "@/components/matomo";
+import { SITE_DESCRIPTION, SITE_URL } from "@/lib/site";
 import "./global.css";
 
+const OG_IMAGE = "/img/apache-iggy-color-darkbg0.5x.png";
+
 export const metadata: Metadata = {
+  metadataBase: new URL(SITE_URL),
   title: {
     default: "Apache Iggy",
     template: "%s | Apache Iggy",
   },
-  description:
-    "Apache Iggy is a persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second.",
+  description: SITE_DESCRIPTION,
+  // Relative values resolve against the current route, so every page gets its own canonical.
+  alternates: { canonical: "./" },
+  // No title/description here on purpose: Next fills og:title and og:description
+  // from each page's own metadata, so per-page titles survive.
+  openGraph: {
+    type: "website",
+    siteName: "Apache Iggy",
+    url: "./",
+    images: [
+      {
+        url: OG_IMAGE,
+        width: 1951,
+        height: 652,
+        alt: "Apache Iggy",
+      },
+    ],
+  },
+  twitter: {
+    card: "summary_large_image",
+    images: [OG_IMAGE],
+  },
   icons: { icon: "/img/favicon.png" },
 };
 
+// Structured data. Kept deliberately small: only claims the site already makes elsewhere.
+const structuredData = {
+  "@context": "https://schema.org",
+  "@graph": [
+    {
+      "@type": "Organization",
+      "@id": `${SITE_URL}/#organization`,
+      name: "The Apache Software Foundation",
+      url: "https://www.apache.org/",
+      logo: `${SITE_URL}/img/asf_logo.svg`,
+      contactPoint: {
+        "@type": "ContactPoint",
+        contactType: "technical support",
+        email: "dev@iggy.apache.org",
+        url: `${SITE_URL}/community/`,
+      },
+    },
+    {
+      "@type": "WebSite",
+      "@id": `${SITE_URL}/#website`,
+      name: "Apache Iggy",
+      url: `${SITE_URL}/`,
+      description: SITE_DESCRIPTION,
+      inLanguage: "en",
+      publisher: { "@id": `${SITE_URL}/#organization` },
+    },
+    {
+      "@type": "SoftwareApplication",
+      "@id": `${SITE_URL}/#software`,
+      name: "Apache Iggy",
+      applicationCategory: "DeveloperApplication",
+      description: SITE_DESCRIPTION,
+      url: `${SITE_URL}/`,
+      license: "https://www.apache.org/licenses/LICENSE-2.0",
+      sameAs: ["https://github.com/apache/iggy"],
+      publisher: { "@id": `${SITE_URL}/#organization` },
+    },
+  ],
+};
+
+/**
+ * JSON-LD has to be injected as raw text: React would HTML-escape it as a child
+ * and the quotes would end up as entities. The object above is built from
+ * constants in this file, so nothing here is user input, but JSON.stringify does
+ * not escape "<" -- so escape it, and a stray "</script>" can never close the tag.
+ */
+function serializeJsonLd(data: unknown): string {
+  return JSON.stringify(data).replace(/</g, "\\u003c");
+}
+
 export default function Layout({ children }: { children: React.ReactNode }) {
   return (
     <html lang="en" suppressHydrationWarning>
       <body className="flex min-h-screen flex-col font-sans antialiased">
+        <script
+          type="application/ld+json"
+          dangerouslySetInnerHTML={{ __html: serializeJsonLd(structuredData) }}
+        />
         <RootProvider
           search={{ options: { type: "static" as const } }}
           theme={{ defaultTheme: "dark" }}

src/app/llms.txt/route.ts

@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { llms } from "fumadocs-core/source/llms";
+import { source } from "@/lib/source";
+import { publishedPosts } from "@/lib/blog";
+import { SITE_DESCRIPTION, SITE_URL, absoluteUrl } from "@/lib/site";
+
+export const dynamic = "force-static";
+
+const INTRO = `# Apache Iggy
+
+> ${SITE_DESCRIPTION}
+
+Apache Iggy is a message streaming server, not a hosted service: there is no
+API on this domain to call. This site carries the documentation, blog and
+community pages. The source, releases and issue tracker live at
+https://github.com/apache/iggy.
+
+## When to use this site
+
+- Running or configuring the server, including storage, networking and clustering: ${absoluteUrl("/docs/server")} and ${absoluteUrl("/docs/clustering")}
+- Writing a producer or consumer in a given language: ${absoluteUrl("/docs/sdk")}
+- Talking to the server directly over QUIC, TCP or HTTP: ${absoluteUrl("/docs/binary-protocol")}
+- Moving data in or out of Iggy without writing code: ${absoluteUrl("/docs/connectors")}
+- Command line and web administration: ${absoluteUrl("/docs/cli")} and ${absoluteUrl("/docs/web_ui")}
+- Downloading a release: ${absoluteUrl("/downloads")}
+- Contributing, or reaching the project's mailing lists and Discord: ${absoluteUrl("/community")}
+
+Every documentation page has a Markdown copy at the same path with a .md
+suffix, so ${absoluteUrl("/docs/server/security")} is also served as
+${SITE_URL}/docs/server/security.md.
+`;
+
+export function GET(): Response {
+  const posts = publishedPosts()
+    .slice(0, 10)
+    .map(({ post, href, date }) => {
+      const day = date.toISOString().slice(0, 10);
+      return `- [${post.title}](${absoluteUrl(href)}): ${day}`;
+    })
+    .join("\n");
+
+  // The helper emits site-relative links; llms.txt consumers want absolute ones.
+  const docsIndex = llms(source)
+    .index()
+    .replace(/\]\((\/[^)\s]*)\)/g, (_match, path: string) => `](${absoluteUrl(path)})`)
+    // One H1 per file: the helper's top-level heading becomes a section.
+    .replace(/^# /gm, "## ");
+
+  const body = [
+    INTRO,
+    docsIndex,
+    `## Blog\n\nThe ten most recent posts. The full list is at ${absoluteUrl("/blogs")}.\n\n${posts}\n`,
+  ].join("\n");
+
+  return new Response(body, {
+    headers: { "Content-Type": "text/plain; charset=utf-8" },
+  });
+}