fix: sitemap advertises only linked routes and includes general sections

changelog.d/fixed-sitemap-routes.md

@@ -0,0 +1 @@
+- `sitemap.xml` now lists the routes the docs navigation actually links to (including the community general section) instead of file-path routes that were never linked

src/__tests__/sitemap.test.ts

@@ -23,20 +23,33 @@ describe("sitemap", () => {
     expect(urls).toEqual(
       expect.arrayContaining([
         `${SITE_URL}/docs/hive`,
-        `${SITE_URL}/docs/hive/architecture`,
-        `${SITE_URL}/docs/hive/adr/0001-record-architecture-decisions`,
+        `${SITE_URL}/docs/hive/overview/architecture`,
+        `${SITE_URL}/docs/hive/reference/architecture-decision-records/0001-record-architecture-decisions`,
         `${SITE_URL}/docs/hotshot`,
-        `${SITE_URL}/docs/hotshot/windows`,
         `${SITE_URL}/docs/pluk`,
         `${SITE_URL}/docs/rationguard`,
         `${SITE_URL}/docs/promptargs`,
         `${SITE_URL}/docs/dibs`,
         `${SITE_URL}/docs/spektacular`,
-        `${SITE_URL}/docs/spektacular/getting-started`,
       ])
     );
   });
 
+  it("includes shared general sections once and omits unlinked file-path routes", () => {
+    const urls = sitemap().map(entry => entry.url);
+
+    expect(urls).toEqual(
+      expect.arrayContaining([
+        `${SITE_URL}/docs/community/meetings`,
+        `${SITE_URL}/docs/community/join-hive-commons`,
+        `${SITE_URL}/docs/community/what-is-hive-commons`,
+      ])
+    );
+    expect(new Set(urls).size).toBe(urls.length);
+    expect(urls).not.toContain(`${SITE_URL}/docs/hive/architecture`);
+    expect(urls).not.toContain(`${SITE_URL}/docs/hive/community/meetings`);
+  });
+
   it("does not include shared partials or image assets as docs pages", () => {
     const urls = sitemap().map(entry => entry.url);
 
@@ -66,32 +79,25 @@ describe("sitemap generated from fixture content", () => {
     }
   });
 
-  it("maps markdown files to routes, priorities, frequencies, and mtimes", () => {
+  it("maps navigation-linked files to routes, priorities, frequencies, and mtimes", () => {
     fixtureRoot = fs.mkdtempSync(path.join(realCwd, ".sitemap-fixture-"));
-    const setupPath = write("docs/content/hive/guides/setup.md");
-    write("docs/content/hive/guides/advanced/scaling.mdx");
-    write("docs/content/pluk/cli/index.md");
+    const archPath = write("docs/content/hive/architecture.md");
+    write("docs/content/hive/unlinked/page.md");
     vi.spyOn(process, "cwd").mockReturnValue(fixtureRoot);
 
     const entries = sitemap();
     const byUrl = new Map(entries.map(entry => [entry.url, entry]));
+    const arch = byUrl.get(`${SITE_URL}/docs/hive/overview/architecture`);
 
     expect(byUrl.get(SITE_URL)?.priority).toBe(1.0);
     expect(byUrl.get(`${SITE_URL}/en`)?.changeFrequency).toBe("monthly");
     expect(byUrl.get(`${SITE_URL}/docs`)?.priority).toBe(0.9);
     expect(byUrl.get(`${SITE_URL}/docs/hive`)?.priority).toBe(0.9);
-    expect(byUrl.get(`${SITE_URL}/docs/hive/guides/setup`)?.priority).toBe(0.7);
-    expect(
-      byUrl.get(`${SITE_URL}/docs/hive/guides/setup`)?.changeFrequency
-    ).toBe("weekly");
-    expect(
-      byUrl.get(`${SITE_URL}/docs/hive/guides/setup`)?.lastModified
-    ).toEqual(fs.statSync(setupPath).mtime);
-    expect(byUrl.has(`${SITE_URL}/docs/hive/guides/advanced/scaling`)).toBe(
-      true
-    );
-    expect(byUrl.has(`${SITE_URL}/docs/pluk/cli`)).toBe(true);
-    expect(byUrl.has(`${SITE_URL}/docs/pluk/cli/index`)).toBe(false);
+    expect(arch?.priority).toBe(0.7);
+    expect(arch?.changeFrequency).toBe("weekly");
+    expect(arch?.lastModified).toEqual(fs.statSync(archPath).mtime);
+    expect(byUrl.has(`${SITE_URL}/docs/hive/architecture`)).toBe(false);
+    expect(byUrl.has(`${SITE_URL}/docs/hive/unlinked/page`)).toBe(false);
   });
 
   it("omits generated routes for missing content while retaining project roots", () => {
@@ -100,7 +106,9 @@ describe("sitemap generated from fixture content", () => {
 
     const entries = sitemap();
 
-    expect(entries).toHaveLength(10);
+    expect(entries.some(entry => entry.url.includes("/docs/hive/"))).toBe(
+      false
+    );
     expect(entries.map(entry => entry.url)).toEqual(
       expect.arrayContaining([
         SITE_URL,

src/app/docs/page-map.ts

@@ -437,10 +437,11 @@ export function buildPageMap(projectId: ProjectId = 'hive') {
     }
   }
 
-  // Add top-level meta - only include our defined navigation structure
+  // Add top-level meta - only categories that produced a folder, since
+  // normalizePageMap rejects meta keys that point at a missing page.
   const meta: Record<string, string> = {}
-  for (const category of navStructure) {
-    meta[category.title] = category.title
+  for (const node of _pageMap) {
+    if (node.kind === 'Folder') meta[node.name] = node.name
   }
   _pageMap.unshift({ kind: 'Meta', data: meta })
 

src/app/sitemap.ts

@@ -2,6 +2,7 @@ import type { MetadataRoute } from "next";
 import fs from "fs";
 import path from "path";
 import { PROJECTS, type ProjectId } from "@/config/versions";
+import { buildPageMap, docsContentPath } from "@/app/docs/page-map";
 
 const SITE_URL = "https://docs.hivecommons.dev";
 
@@ -19,63 +20,22 @@ const DOCS_ROOT_PRIORITY = 0.9;
 /** Priority for individual docs pages */
 const DOCS_PAGE_PRIORITY = 0.7;
 
-/**
- * Recursively find all .md and .mdx files in a directory.
- * Returns paths relative to the given base directory.
- */
-function findMarkdownFiles(dir: string, baseDir: string = dir): string[] {
-  const files: string[] = [];
-
-  if (!fs.existsSync(dir)) {
-    return files;
-  }
-
-  const entries = fs.readdirSync(dir, { withFileTypes: true });
-
-  for (const entry of entries) {
-    const fullPath = path.join(dir, entry.name);
-
-    if (entry.isDirectory()) {
-      // Skip hidden directories, node_modules, common-subs (partials), and images
-      if (
-        !entry.name.startsWith(".") &&
-        !entry.name.startsWith("_") &&
-        entry.name !== "node_modules" &&
-        entry.name !== "common-subs" &&
-        entry.name !== "images"
-      ) {
-        files.push(...findMarkdownFiles(fullPath, baseDir));
-      }
-    } else if (
-      entry.isFile() &&
-      (entry.name.endsWith(".md") || entry.name.endsWith(".mdx")) &&
-      !entry.name.startsWith("_")
-    ) {
-      const relativePath = path.relative(baseDir, fullPath).replace(/\\/g, "/");
-      files.push(relativePath);
+type PageMapNode = {
+  route?: string;
+  children?: PageMapNode[];
+  kind?: string;
+};
+
+/** Collect routes of real pages (not folders/meta) that the navigation links to. */
+function collectPageRoutes(nodes: PageMapNode[], routes: string[] = []) {
+  for (const node of nodes) {
+    if (node.children) {
+      collectPageRoutes(node.children, routes);
+    } else if (node.kind === "MdxPage" && node.route?.startsWith("/docs/")) {
+      routes.push(node.route);
     }
   }
-
-  return files;
-}
-
-/**
- * Convert a markdown file path to its URL route for a given project.
- * Uses the same slug logic as page-map.ts navigation structures.
- */
-function filePathToRoute(filePath: string, projectId: ProjectId): string {
-  // Remove file extension
-  let route = filePath.replace(/\.(md|mdx)$/i, "");
-
-  // Remove trailing /index (index files map to the parent folder route)
-  route = route.replace(/\/index$/, "");
-
-  // For project sub-paths (a2a, kubeflex, etc.), strip the project prefix
-  // since content is already scoped to the project directory
-  const project = PROJECTS[projectId];
-  const projectBase = project.basePath ? `/docs/${project.basePath}` : "/docs";
-
-  return `${projectBase}/${route}`;
+  return routes;
 }
 
 /**
@@ -92,7 +52,7 @@ function getLastModified(filePath: string): Date {
 
 export default function sitemap(): MetadataRoute.Sitemap {
   const entries: MetadataRoute.Sitemap = [];
-  const contentRoot = path.join(process.cwd(), "docs", "content");
+  const seen = new Set<string>();
 
   // --- Homepage ---
   entries.push({
@@ -134,11 +94,6 @@ export default function sitemap(): MetadataRoute.Sitemap {
   ];
 
   for (const projectId of projectIds) {
-    const projectContentPath = path.join(
-      contentRoot,
-      PROJECTS[projectId].basePath
-    );
-
     // Add project root entry
     entries.push({
       url: `${SITE_URL}/docs/${PROJECTS[projectId].basePath}`,
@@ -147,15 +102,25 @@ export default function sitemap(): MetadataRoute.Sitemap {
       priority: DOCS_ROOT_PRIORITY,
     });
 
-    const projectFiles = findMarkdownFiles(projectContentPath);
+    const { pageMap, routeMap, contentPath } = buildPageMap(projectId);
+    const projectPrefix = `/docs/${PROJECTS[projectId].basePath}/`;
 
-    for (const file of projectFiles) {
-      const fullPath = path.join(projectContentPath, file);
-      const route = filePathToRoute(file, projectId);
+    for (const route of collectPageRoutes(pageMap as PageMapNode[])) {
+      const isProjectRoute = route.startsWith(projectPrefix);
+      const key = route.slice(
+        isProjectRoute ? projectPrefix.length : "/docs/".length
+      );
+      const file = routeMap[key];
+      const url = `${SITE_URL}${route}`;
+      // General sections appear in every project's nav; list them once.
+      if (!file || seen.has(url)) continue;
+      seen.add(url);
 
       entries.push({
-        url: `${SITE_URL}${route}`,
-        lastModified: getLastModified(fullPath),
+        url,
+        lastModified: getLastModified(
+          path.join(isProjectRoute ? contentPath : docsContentPath, file)
+        ),
         changeFrequency: DOCS_CHANGE_FREQ,
         priority: DOCS_PAGE_PRIORITY,
       });