fix(seo): resolve docs slugs through the catalog, keep API pages negotiable, and harden the crawler suite

Author: waleedlatif1

apps/docs/app/[[...slug]]/page.tsx

@@ -1,5 +1,6 @@
 import type React from 'react'
 import { Children, cloneElement, isValidElement } from 'react'
+import { LANDING_SLUG_BY_DOCS_SLUG } from '@sim/deployment-config/integration-docs'
 import { INTEGRATION_METADATA } from '@sim/deployment-config/integration-metadata'
 import { highlight } from 'fumadocs-core/highlight'
 import { findNeighbour } from 'fumadocs-core/page-tree'
@@ -25,17 +26,21 @@ import { DOCS_BASE_URL, SIM_SITE_URL } from '@/lib/urls'
 
 const BASE_URL = DOCS_BASE_URL
 
-/**
- * Landing page for each docs integration page. Docs pages are keyed by block type without its
- * version suffix (`gmail_v2` → `/integrations/gmail`); landing slugs are hyphenated catalog slugs.
- */
-const INTEGRATION_LANDING_PAGES = new Map(
-  INTEGRATION_METADATA.map((integration) => [
-    integration.type.replace(/_v\d+$/, ''),
-    { name: integration.name, url: `${SIM_SITE_URL}/integrations/${integration.slug}` },
-  ])
+const INTEGRATION_NAME_BY_LANDING_SLUG = new Map(
+  INTEGRATION_METADATA.map((integration) => [integration.slug, integration.name])
 )
 
+/** The landing page a docs integration page (`/integrations/<docs slug>`) links to, if any. */
+function getIntegrationLandingPage(slug: string[] | undefined) {
+  if (slug?.length !== 2 || slug[0] !== 'integrations') return undefined
+  const landingSlug = LANDING_SLUG_BY_DOCS_SLUG.get(slug[1])
+  if (!landingSlug) return undefined
+  return {
+    name: INTEGRATION_NAME_BY_LANDING_SLUG.get(landingSlug) ?? landingSlug,
+    url: `${SIM_SITE_URL}/integrations/${landingSlug}`,
+  }
+}
+
 /**
  * Most pages close with a `## Next` / `## Next steps` grid of onward links.
  * That heading is navigation, not content, so it is kept out of the table of
@@ -134,10 +139,7 @@ export default async function Page(props: { params: Promise<{ slug?: string[] }>
   }
   const isOpenAPI = '_openapi' in data && data._openapi != null
   const isApiReference = slug?.some((s) => s === 'api-reference') ?? false
-  const integrationLandingPage =
-    slug?.length === 2 && slug[0] === 'integrations'
-      ? INTEGRATION_LANDING_PAGES.get(slug[1])
-      : undefined
+  const integrationLandingPage = getIntegrationLandingPage(slug)
   const isCli = slug?.[0] === 'cli'
   const isMcp = slug?.[0] === 'mcp'
 

apps/docs/app/llms.txt/route.ts

@@ -45,7 +45,7 @@ Sim is the open-source AI workspace where teams build, deploy, and manage AI age
 
 ## Documentation Overview
 
-This file provides an overview of our documentation. For full content of all pages, see [llms-full.txt](${baseUrl}/llms-full.txt).
+This file provides an overview of our documentation. For the full content of every guide and integration page, see [llms-full.txt](${baseUrl}/llms-full.txt). API reference operations are not in it: each one is served as markdown at its page URL with \`.md\` appended, e.g. ${baseUrl}/api-reference/tables/countTableRows.md.
 
 ## Main Sections
 
@@ -61,8 +61,8 @@ ${Object.entries(sections)
 
 ## Additional Resources
 
-- [Full documentation content](${baseUrl}/llms-full.txt)
-- Individual page content: ${baseUrl}/llms.mdx/[page-path]
+- [Full documentation content, excluding API reference operations](${baseUrl}/llms-full.txt)
+- Individual page content as markdown: ${baseUrl}/[page-path].md
 - [API documentation](${baseUrl}/api-reference/getting-started)
 - [Integrations](${baseUrl}/integrations)
 

apps/docs/app/robots.txt/route.ts

@@ -23,8 +23,8 @@ Sitemap: ${baseUrl}/sitemap.xml
 # See https://github.com/AnswerDotAI/llms-txt for more info
 # LLM-friendly content:
 #   Manifest: ${baseUrl}/llms.txt
-#   Full content: ${baseUrl}/llms-full.txt
-#   Individual pages: ${baseUrl}/llms.mdx/[page-path]`
+#   Full content (excluding API reference operations): ${baseUrl}/llms-full.txt
+#   Individual pages: ${baseUrl}/[page-path].md`
 
   return new Response(robotsTxt, {
     headers: {

apps/docs/lib/llms.ts

@@ -5,11 +5,13 @@ import type { PageData, source } from '@/lib/source'
 
 const TABLE_ROW = /^\|.*\|$/
 const TABLE_DELIMITER_CELL = /^\s*(:?)-+(:?)\s*$/
+const UNESCAPED_PIPE = /(?<!\\)\|/
 
 /**
  * The processed-markdown serializer pads every table cell to its column's widest cell, so one long
  * enum (a parameter listing every timezone) pads every row of its table and roughly doubles the
- * integration reference. Collapse the padding; the tables stay valid GFM.
+ * integration reference. Drop that trailing padding; a cell's own leading indentation (the `↳`
+ * nesting in output tables) and inline code are kept, and the tables stay valid GFM.
  */
 function compactTables(markdown: string): string {
   let inFence = false
@@ -18,11 +20,11 @@ function compactTables(markdown: string): string {
     .map((line) => {
       if (line.startsWith('```')) inFence = !inFence
       if (inFence || !TABLE_ROW.test(line)) return line
-      const cells = line.slice(1, -1).split('|')
+      const cells = line.slice(1, -1).split(UNESCAPED_PIPE)
       if (cells.every((cell) => TABLE_DELIMITER_CELL.test(cell))) {
         return `|${cells.map((cell) => cell.replace(TABLE_DELIMITER_CELL, ' $1---$2 ')).join('|')}|`
       }
-      return line.replace(/ {2,}/g, ' ')
+      return `|${cells.map((cell) => ` ${cell.replace(/^ /, '').trimEnd()} `).join('|')}|`
     })
     .join('\n')
 }

apps/docs/proxy.ts

@@ -4,16 +4,21 @@ import { type NextRequest, NextResponse } from 'next/server'
 /** Docs pages live at the root (`/introduction`), so every page path maps to its markdown route. */
 const { rewrite: rewriteLLM } = rewritePath('/*path', '/llms.mdx/*path')
 
-/** `.md`/`.mdx` URLs already resolve to markdown through the `next.config.ts` rewrites. */
-const MARKDOWN_SUFFIX = /\.mdx?$/
+/**
+ * Docs page paths never carry an extension, so a final segment with one is a file or route
+ * (`/openapi.json`, `/introduction.md`) that must not be rewritten to a markdown page.
+ */
+const FILE_EXTENSION = /\.[^/]+$/
 
 export default function proxy(request: NextRequest) {
   const { pathname } = request.nextUrl
-  if (isMarkdownPreferred(request) && !MARKDOWN_SUFFIX.test(pathname)) {
+  if (isMarkdownPreferred(request) && !FILE_EXTENSION.test(pathname)) {
     const result = rewriteLLM(pathname)
 
     if (result) {
-      return NextResponse.rewrite(new URL(result, request.nextUrl))
+      const response = NextResponse.rewrite(new URL(result, request.nextUrl))
+      response.headers.set('Vary', 'Accept')
+      return response
     }
   }
 
@@ -22,6 +27,6 @@ export default function proxy(request: NextRequest) {
 
 export const config = {
   matcher: [
-    '/((?!api|llms.mdx|_next/static|_next/image|favicon|icon.svg|static|robots.txt|sitemap.xml|llms.txt|llms-full.txt).*)',
+    '/((?!api/|api$|llms.mdx|_next/static|_next/image|favicon|icon.svg|static|robots.txt|sitemap.xml|llms.txt|llms-full.txt).*)',
   ],
 }

apps/docs/scripts/fetch-git-history.ts

@@ -13,14 +13,21 @@ import { getErrorMessage } from '@sim/utils/errors'
 
 const logger = createLogger('DocsGitHistory')
 
-function git(args: string[]): string {
-  return execFileSync('git', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim()
+/** Bounds the fetch so a stalled remote costs at most this much of the deployment, never all of it. */
+const FETCH_TIMEOUT_MS = 120_000
+
+function git(args: string[], timeout?: number): string {
+  return execFileSync('git', args, {
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'pipe'],
+    timeout,
+  }).trim()
 }
 
 if (process.env.VERCEL) {
   try {
     if (git(['rev-parse', '--is-shallow-repository']) === 'true') {
-      git(['fetch', '--unshallow', '--filter=blob:none', '--no-tags', 'origin'])
+      git(['fetch', '--unshallow', '--filter=blob:none', '--no-tags', 'origin'], FETCH_TIMEOUT_MS)
       logger.info('Fetched full commit history for docs last-modified dates')
     }
   } catch (error) {

apps/sim/proxy.ts

@@ -1,3 +1,7 @@
+import {
+  LANDING_INTEGRATION_SLUGS,
+  LANDING_SLUG_BY_DOCS_SLUG,
+} from '@sim/deployment-config/integration-docs'
 import { createLogger } from '@sim/logger'
 import { getSessionCookie } from 'better-auth/cookies'
 import { type NextRequest, NextResponse } from 'next/server'
@@ -328,15 +332,17 @@ function handleSecurityFiltering(request: NextRequest): NextResponse | null {
 }
 
 /**
- * Landing integration slugs are hyphenated (`google-drive`) while docs slugs use underscores
- * (`google_drive`), so underscore URLs reach the landing site through hand-written and
- * search-indexed links. Redirect them to the hyphenated page instead of serving a 404.
+ * Docs integration slugs (`google_drive`, `microsoft_ad`) are not landing slugs (`google-drive`,
+ * `azure-ad`), yet reach the landing site through hand-written and search-indexed links. Redirect a
+ * known docs slug to its landing page through the catalog mapping instead of serving a 404.
  */
 function handleIntegrationSlugRedirect(request: NextRequest): NextResponse | null {
-  const slug = /^\/integrations\/([^/]*_[^/]*)\/?$/.exec(request.nextUrl.pathname)?.[1]
-  if (!slug) return null
+  const slug = /^\/integrations\/([^/]+)\/?$/.exec(request.nextUrl.pathname)?.[1]
+  if (!slug || LANDING_INTEGRATION_SLUGS.has(slug)) return null
+  const landingSlug = LANDING_SLUG_BY_DOCS_SLUG.get(slug)
+  if (!landingSlug) return null
   const url = request.nextUrl.clone()
-  url.pathname = `/integrations/${slug.replace(/_/g, '-')}`
+  url.pathname = `/integrations/${landingSlug}`
   return NextResponse.redirect(url, 308)
 }
 
@@ -409,7 +415,7 @@ export function proxy(request: NextRequest) {
   }
 
   const integrationSlugRedirect = handleIntegrationSlugRedirect(request)
-  if (integrationSlugRedirect) return integrationSlugRedirect
+  if (integrationSlugRedirect) return applyIndexingPolicy(request, integrationSlugRedirect)
 
   const invitationRedirect = handleInvitationRedirects(request, hasActiveSession)
   if (invitationRedirect) return applyIndexingPolicy(request, invitationRedirect)

apps/sim/scripts/test-landing-seo-e2e.ts

@@ -1,6 +1,10 @@
 import assert from 'node:assert/strict'
 import { mkdir, writeFile } from 'node:fs/promises'
 import { dirname, resolve } from 'node:path'
+import {
+  LANDING_INTEGRATION_SLUGS,
+  LANDING_SLUG_BY_DOCS_SLUG,
+} from '@sim/deployment-config/integration-docs'
 import integrationsJson from '@sim/deployment-config/integrations.json'
 import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
@@ -104,15 +108,18 @@ async function readSitemap(): Promise<Map<string, string | undefined>> {
 }
 
 async function run() {
-  const sitemap = await readSitemap()
+  let sitemap = new Map<string, string | undefined>()
+  await check('sitemap.xml is served', async () => {
+    sitemap = await readSitemap()
+  })
   const sitemapPaths = [...sitemap.keys()]
   const firstPath = (pattern: RegExp) => {
     const path = sitemapPaths.find((candidate) => pattern.test(candidate))
     assert(path, `The sitemap lists no page matching ${pattern}`)
     return path
   }
 
-  await check('unknown slugs return 404 with a single noindex directive', async () => {
+  await check('unknown slugs return 404 and every robots directive is noindex', async () => {
     const unknown = [
       '/integrations/not-a-real-integration',
       '/library/not-a-real-post',
@@ -126,7 +133,8 @@ async function run() {
     for (const path of unknown) {
       const { status, body } = await get(path)
       const robots = robotsDirectives(body)
-      const indexable = robots.some((directive) => !directive.includes('noindex'))
+      const indexable =
+        robots.length === 0 || robots.some((directive) => !/\bnoindex\b/.test(directive))
       if (status !== 404 || indexable) {
         failures.push(`${path} → ${status}, robots ${JSON.stringify(robots)}`)
       }
@@ -154,10 +162,17 @@ async function run() {
     assert.deepEqual(failures, [])
   })
 
-  await check('underscore integration slugs redirect to the hyphenated page', async () => {
-    const { status, location } = await get('/integrations/google_drive')
-    assert.equal(status, 308)
-    assert.equal(location && new URL(location, baseUrl).pathname, '/integrations/google-drive')
+  await check('docs integration slugs redirect to their landing page', async () => {
+    const failures: string[] = []
+    for (const [docsSlug, landingSlug] of LANDING_SLUG_BY_DOCS_SLUG) {
+      if (docsSlug === landingSlug || LANDING_INTEGRATION_SLUGS.has(docsSlug)) continue
+      const { status, location } = await get(`/integrations/${docsSlug}`)
+      const target = location ? new URL(location, baseUrl).pathname : null
+      if (status !== 308 || target !== `/integrations/${landingSlug}`) {
+        failures.push(`${docsSlug} → ${status} ${target}`)
+      }
+    }
+    assert.deepEqual(failures, [])
   })
 
   await check('integration sitemap dates are per integration, not one catalog date', async () => {
@@ -235,7 +250,11 @@ async function run() {
     }
     assert.deepEqual(missing, [])
   })
+}
 
+try {
+  await run()
+} finally {
   await mkdir(dirname(reportPath), { recursive: true })
   await writeFile(
     reportPath,
@@ -252,8 +271,6 @@ async function run() {
     )
   )
 }
-
-await run()
 const failures = checks.filter((result) => result.status === 'failed')
 logger.info('Landing SEO checks complete', {
   passed: checks.length - failures.length,

packages/deployment-config/package.json

@@ -18,6 +18,10 @@
       "types": "./src/integration-availability.ts",
       "default": "./src/integration-availability.ts"
     },
+    "./integration-docs": {
+      "types": "./src/integration-docs.ts",
+      "default": "./src/integration-docs.ts"
+    },
     "./integration-metadata": {
       "types": "./src/integration-metadata.ts",
       "default": "./src/integration-metadata.ts"

packages/deployment-config/src/integration-docs.ts

@@ -0,0 +1,19 @@
+import { INTEGRATION_METADATA } from '@sim/deployment-config/integration-metadata'
+
+/**
+ * Docs integration pages are keyed by block type without its version suffix (`gmail_v2` →
+ * `/integrations/gmail`), while landing pages use the hyphenated catalog slug, and the two differ
+ * in more than spelling (`microsoft_ad` documents the `azure-ad` integration). Maps each docs slug
+ * to its landing slug, so docs and landing links never guess one from the other.
+ */
+export const LANDING_SLUG_BY_DOCS_SLUG: ReadonlyMap<string, string> = new Map(
+  INTEGRATION_METADATA.map((integration) => [
+    integration.type.replace(/_v\d+$/, ''),
+    integration.slug,
+  ])
+)
+
+/** Every landing integration slug, i.e. every `/integrations/<slug>` page on the landing site. */
+export const LANDING_INTEGRATION_SLUGS: ReadonlySet<string> = new Set(
+  INTEGRATION_METADATA.map((integration) => integration.slug)
+)