improvement(seo): fix soft 404s, per-integration freshness, and docs markdown for AI crawlers

.github/scripts/http-e2e.sh

@@ -236,6 +236,10 @@ case "$group" in
     MOBILE_E2E_AUTH_SECRET="$BETTER_AUTH_SECRET" \
     MOBILE_E2E_REPORT_PATH="$report_dir/mobile-e2e-report.json" \
       bun run test:mobile:e2e
+    # The same production build answers as a crawler sees it: statuses, robots, sitemap, JSON-LD.
+    LANDING_SEO_E2E_BASE_URL="$NEXT_PUBLIC_APP_URL" \
+    LANDING_SEO_E2E_REPORT_PATH="$report_dir/landing-seo-e2e-report.json" \
+      bun run test:landing-seo:e2e
     ;;
 
   *)

apps/docs/app/[[...slug]]/page.tsx

@@ -17,6 +17,7 @@ import { APIExampleSelector } from '@/components/ui/api-example-selector'
 import { CodeBlock } from '@/components/ui/code-block'
 import { Heading } from '@/components/ui/heading'
 import { ResponseSection } from '@/components/ui/response-section'
+import { getIntegrationLandingPage, INTEGRATION_LANDING_LABEL } from '@/lib/integration-landing'
 import { getApiSpecContent, getAuthenticatedCodeSamples, openapi } from '@/lib/openapi'
 import { simShikiOptions } from '@/lib/shiki-theme'
 import { type PageData, source } from '@/lib/source'
@@ -122,6 +123,7 @@ export default async function Page(props: { params: Promise<{ slug?: string[] }>
   }
   const isOpenAPI = '_openapi' in data && data._openapi != null
   const isApiReference = slug?.some((s) => s === 'api-reference') ?? false
+  const integrationLandingPage = getIntegrationLandingPage(slug)
   const isCli = slug?.[0] === 'cli'
   const isMcp = slug?.[0] === 'mcp'
 
@@ -266,6 +268,12 @@ export default async function Page(props: { params: Promise<{ slug?: string[] }>
           <PageNavigationArrows previous={neighbours?.previous} next={neighbours?.next} />
         </PageHeader>
         <DocsBody>
+          {integrationLandingPage && (
+            <p>
+              {INTEGRATION_LANDING_LABEL}{' '}
+              <a href={integrationLandingPage.url}>{integrationLandingPage.title}</a>
+            </p>
+          )}
           <MDX
             components={{
               ...defaultMdxComponents,
@@ -367,6 +375,7 @@ export async function generateMetadata(props: { params: Promise<{ slug?: string[
     },
     alternates: {
       canonical: fullUrl,
+      types: { 'text/markdown': `${fullUrl}.md` },
     },
   }
 }

apps/docs/app/layout.tsx

@@ -32,6 +32,7 @@ const structuredData = {
   url: DOCS_BASE_URL,
   publisher: {
     '@type': 'Organization',
+    '@id': `${SIM_SITE_URL}#organization`,
     name: 'Sim',
     url: SIM_SITE_URL,
     logo: {

apps/docs/app/llms.txt/route.ts

@@ -45,7 +45,7 @@ Sim is the open-source AI workspace where teams build, deploy, and manage AI age
 
 ## Documentation Overview
 
-This file provides an overview of our documentation. For full content of all pages, see [llms-full.txt](${baseUrl}/llms-full.txt).
+This file provides an overview of our documentation. For the full content of every guide and integration page, see [llms-full.txt](${baseUrl}/llms-full.txt). API reference operations are not in it: each one is served as markdown at its page URL with \`.md\` appended, e.g. ${baseUrl}/api-reference/tables/countTableRows.md.
 
 ## Main Sections
 
@@ -61,8 +61,8 @@ ${Object.entries(sections)
 
 ## Additional Resources
 
-- [Full documentation content](${baseUrl}/llms-full.txt)
-- Individual page content: ${baseUrl}/llms.mdx/[page-path]
+- [Full documentation content, excluding API reference operations](${baseUrl}/llms-full.txt)
+- Individual page content as markdown: ${baseUrl}/[page-path].md
 - [API documentation](${baseUrl}/api-reference/getting-started)
 - [Integrations](${baseUrl}/integrations)
 

apps/docs/app/robots.txt/route.ts

@@ -23,8 +23,8 @@ Sitemap: ${baseUrl}/sitemap.xml
 # See https://github.com/AnswerDotAI/llms-txt for more info
 # LLM-friendly content:
 #   Manifest: ${baseUrl}/llms.txt
-#   Full content: ${baseUrl}/llms-full.txt
-#   Individual pages: ${baseUrl}/llms.mdx/[page-path]`
+#   Full content (excluding API reference operations): ${baseUrl}/llms-full.txt
+#   Individual pages: ${baseUrl}/[page-path].md`
 
   return new Response(robotsTxt, {
     headers: {

apps/docs/components/footer/footer.tsx

@@ -34,7 +34,7 @@ const PRODUCT_LINKS: FooterItem[] = [
 
 const RESOURCES_LINKS: FooterItem[] = [
   { label: 'Blog', href: `${SIM_SITE_URL}/blog`, external: true },
-  { label: 'Compare', href: `${SIM_SITE_URL}/comparison`, external: true },
+  { label: 'Compare', href: `${SIM_SITE_URL}/comparisons`, external: true },
   { label: 'Careers', href: `${SIM_SITE_URL}/careers`, external: true },
   { label: 'Changelog', href: `${SIM_SITE_URL}/changelog`, external: true },
   { label: 'Contact', href: `${SIM_SITE_URL}/contact`, external: true },

apps/docs/components/structured-data.tsx

@@ -1,6 +1,5 @@
-import { INTEGRATION_COUNT_LABEL } from '@sim/deployment-config/integration-claims'
 import { serializeJsonLd } from '@/lib/json-ld'
-import { DOCS_BASE_URL } from '@/lib/urls'
+import { DOCS_BASE_URL, SIM_SITE_URL } from '@/lib/urls'
 
 interface StructuredDataProps {
   title: string
@@ -29,13 +28,15 @@ export function StructuredData({
     ...(dateModified && { dateModified }),
     author: {
       '@type': 'Organization',
-      name: 'Sim Team',
-      url: baseUrl,
+      '@id': `${SIM_SITE_URL}#organization`,
+      name: 'Sim',
+      url: SIM_SITE_URL,
     },
     publisher: {
       '@type': 'Organization',
+      '@id': `${SIM_SITE_URL}#organization`,
       name: 'Sim',
-      url: baseUrl,
+      url: SIM_SITE_URL,
       logo: {
         '@type': 'ImageObject',
         url: `${baseUrl}/static/logo.png`,
@@ -68,35 +69,6 @@ export function StructuredData({
     })),
   }
 
-  const softwareStructuredData = {
-    '@context': 'https://schema.org',
-    '@type': 'SoftwareApplication',
-    name: 'Sim',
-    applicationCategory: 'BusinessApplication',
-    applicationSubCategory: 'AI Workspace',
-    operatingSystem: 'Any',
-    description: `Sim is the open-source AI workspace where teams build, deploy, and manage AI agents. Connect ${INTEGRATION_COUNT_LABEL} integrations and every major LLM to create agents that automate real work.`,
-    url: baseUrl,
-    author: {
-      '@type': 'Organization',
-      name: 'Sim Team',
-    },
-    offers: {
-      '@type': 'Offer',
-      category: 'Developer Tools',
-    },
-    featureList: [
-      'AI workspace for teams',
-      'Chat — natural language agent creation',
-      'Visual workflow builder',
-      `${INTEGRATION_COUNT_LABEL} integrations`,
-      'LLM orchestration (OpenAI, Anthropic, Google, xAI, Mistral, Perplexity)',
-      'Knowledge base creation',
-      'Table creation',
-      'Document creation',
-    ],
-  }
-
   return (
     <>
       <script
@@ -115,15 +87,6 @@ export function StructuredData({
           }}
         />
       )}
-      {(url === baseUrl || url === `${baseUrl}/`) && (
-        <script
-          id={`software-json-ld-${structuredDataId}`}
-          type='application/ld+json'
-          dangerouslySetInnerHTML={{
-            __html: serializeJsonLd(softwareStructuredData),
-          }}
-        />
-      )}
     </>
   )
 }

apps/docs/lib/integration-landing.ts

@@ -0,0 +1,22 @@
+import { LANDING_SLUG_BY_DOCS_SLUG } from '@sim/deployment-config/integration-docs'
+import { INTEGRATION_METADATA } from '@sim/deployment-config/integration-metadata'
+import { SIM_SITE_URL } from '@/lib/urls'
+
+/** Lead-in for the docs page's link to its landing page, shared by the HTML and markdown forms. */
+export const INTEGRATION_LANDING_LABEL = 'Use cases and templates:'
+
+const INTEGRATION_NAME_BY_LANDING_SLUG = new Map(
+  INTEGRATION_METADATA.map((integration) => [integration.slug, integration.name])
+)
+
+/** The landing page a docs integration page (`/integrations/<docs slug>`) links to, if any. */
+export function getIntegrationLandingPage(slugs: readonly string[] | undefined) {
+  if (slugs?.length !== 2 || slugs[0] !== 'integrations') return undefined
+  const landingSlug = LANDING_SLUG_BY_DOCS_SLUG.get(slugs[1])
+  if (!landingSlug) return undefined
+  const name = INTEGRATION_NAME_BY_LANDING_SLUG.get(landingSlug) ?? landingSlug
+  return {
+    title: `${name} integration for AI agents`,
+    url: `${SIM_SITE_URL}/integrations/${landingSlug}`,
+  }
+}

apps/docs/lib/llms.ts

@@ -1,11 +1,66 @@
 import type { InferPageType } from 'fumadocs-core/source'
+import type { ApiPageProps } from 'fumadocs-openapi/ui'
+import { getIntegrationLandingPage, INTEGRATION_LANDING_LABEL } from '@/lib/integration-landing'
+import { getApiSpecContent } from '@/lib/openapi'
 import type { PageData, source } from '@/lib/source'
 
+const TABLE_ROW = /^\|.*\|$/
+const TABLE_DELIMITER_CELL = /^\s*(:?)-+(:?)\s*$/
+const UNESCAPED_PIPE = /(?<!\\)\|/
+
+/**
+ * The processed-markdown serializer pads every table cell to its column's widest cell, so one long
+ * enum (a parameter listing every timezone) pads every row of its table and roughly doubles the
+ * integration reference. Drop that trailing padding; a cell's own leading indentation (the `↳`
+ * nesting in output tables) and inline code are kept, and the tables stay valid GFM.
+ */
+function compactTables(markdown: string): string {
+  let inFence = false
+  return markdown
+    .split('\n')
+    .map((line) => {
+      if (line.startsWith('```')) inFence = !inFence
+      if (inFence || !TABLE_ROW.test(line)) return line
+      const cells = line.slice(1, -1).split(UNESCAPED_PIPE)
+      if (cells.every((cell) => TABLE_DELIMITER_CELL.test(cell))) {
+        return `|${cells.map((cell) => cell.replace(TABLE_DELIMITER_CELL, ' $1---$2 ')).join('|')}|`
+      }
+      return `|${cells.map((cell) => ` ${cell.replace(/^ /, '').trimEnd()} `).join('|')}|`
+    })
+    .join('\n')
+}
+
+type LLMPageData = PageData & { getAPIPageProps?: () => ApiPageProps }
+
+/**
+ * OpenAPI operation pages are virtual: no MDX body, and a fully resolved request/response schema
+ * (~40 KB each, mostly shared components) that is useful per page but would double `llms-full.txt`.
+ */
+export function isApiOperationPage(page: InferPageType<typeof source>): boolean {
+  return typeof (page.data as unknown as LLMPageData).getAPIPageProps === 'function'
+}
+
 export async function getLLMText(page: InferPageType<typeof source>) {
-  const data = page.data as unknown as PageData
+  const data = page.data as unknown as LLMPageData
+  const heading = `# ${data.title} (${page.url})`
+
+  // Operation pages get the same markdown the page's "Copy page" action builds.
+  if (typeof data.getAPIPageProps === 'function') {
+    const spec = getApiSpecContent(
+      data.title,
+      data.description,
+      data.getAPIPageProps().operations ?? []
+    )
+    return spec.replace(/^# .*$/m, heading)
+  }
+
   if (typeof data.getText !== 'function') return ''
   const processed = await data.getText('processed')
-  return `# ${data.title} (${page.url})
+  const landingPage = getIntegrationLandingPage(page.slugs)
+  const landingLink = landingPage
+    ? `${INTEGRATION_LANDING_LABEL} [${landingPage.title}](${landingPage.url})\n\n`
+    : ''
+  return `${heading}
 
-${processed}`
+${landingLink}${compactTables(processed)}`
 }

apps/docs/lib/redirects.ts

@@ -20,6 +20,72 @@ type DocsRedirect = Awaited<ReturnType<NonNullable<NextConfig['redirects']>>>[nu
  * modules would make match order an emergent property of two files.
  */
 const PATH_REDIRECTS: DocsRedirect[] = [
+  // The academy was removed; each lesson redirects to the reference page covering its topic.
+  { source: '/academy', destination: '/introduction', permanent: true },
+  { source: '/academy/agents/intro', destination: '/agents', permanent: true },
+  { source: '/academy/agents/block', destination: '/workflows/blocks/agent', permanent: true },
+  { source: '/academy/agents/memory', destination: '/workflows/blocks/agent', permanent: true },
+  { source: '/academy/agents/skills', destination: '/agents/skills', permanent: true },
+  {
+    source: '/academy/agents/tool-calling',
+    destination: '/workflows/blocks/agent',
+    permanent: true,
+  },
+  { source: '/academy/chat/intro', destination: '/chat', permanent: true },
+  { source: '/academy/chat/building', destination: '/chat/workflows', permanent: true },
+  { source: '/academy/files/intro', destination: '/files', permanent: true },
+  { source: '/academy/files/object', destination: '/files/passing-files', permanent: true },
+  { source: '/academy/knowledge-bases/intro', destination: '/knowledgebase', permanent: true },
+  {
+    source: '/academy/knowledge-bases/connectors',
+    destination: '/knowledgebase/connectors',
+    permanent: true,
+  },
+  { source: '/academy/tables/intro', destination: '/tables', permanent: true },
+  {
+    source: '/academy/tables/operations',
+    destination: '/tables/using-in-workflows',
+    permanent: true,
+  },
+  {
+    source: '/academy/tables/workflow-columns',
+    destination: '/tables/workflow-columns',
+    permanent: true,
+  },
+  { source: '/academy/workflows/intro', destination: '/workflows', permanent: true },
+  {
+    source: '/academy/workflows/branching',
+    destination: '/workflows/blocks/condition',
+    permanent: true,
+  },
+  { source: '/academy/workflows/loops', destination: '/workflows/blocks/loop', permanent: true },
+  {
+    source: '/academy/workflows/subworkflows',
+    destination: '/workflows/blocks/workflow',
+    permanent: true,
+  },
+  {
+    source: '/academy/workflows/deployment',
+    destination: '/workflows/deployment',
+    permanent: true,
+  },
+  { source: '/academy/workflows/logs', destination: '/logs-debugging', permanent: true },
+  {
+    source: '/academy/use-cases/document-extraction',
+    destination: '/files/using-in-workflows',
+    permanent: true,
+  },
+  {
+    source: '/academy/use-cases/monitoring-research',
+    destination: '/workflows/triggers/schedule',
+    permanent: true,
+  },
+  {
+    source: '/academy/use-cases/sales-data-enrichment',
+    destination: '/tables/workflow-columns',
+    permanent: true,
+  },
+  { source: '/academy/use-cases/slack-it-triage', destination: '/chat', permanent: true },
   { source: '/academy/:path*', destination: '/introduction', permanent: true },
   ...Object.entries(integrationNavigation.redirects).map(([from, to]) => ({
     source: `/integrations/${from}`,

apps/docs/next.config.ts

@@ -36,6 +36,10 @@ const config: NextConfig = {
         source: '/:path*.mdx',
         destination: '/llms.mdx/:path*',
       },
+      {
+        source: '/:path*.md',
+        destination: '/llms.mdx/:path*',
+      },
     ]
   },
 }

apps/docs/package.json

@@ -7,7 +7,7 @@
     "dev": "bun run dev:cache:cap && bun run llms-full:generate && bunx concurrently --kill-others --names Next,LLMs \"next dev --port 3001\" \"bun scripts/watch-llms-full.ts\"",
     "dev:cache:cap": "bun run ../../scripts/prune-turbopack-cache.ts",
     "dev:clean": "rm -rf .next/dev/cache",
-    "build": "bun run llms-full:generate && NODE_OPTIONS='--max-old-space-size=8192' next build",
+    "build": "bun scripts/fetch-git-history.ts && bun run llms-full:generate && NODE_OPTIONS='--max-old-space-size=8192' next build",
     "start": "next start",
     "postinstall": "fumadocs-mdx",
     "test": "vitest run",

apps/docs/proxy.ts

@@ -1,14 +1,24 @@
 import { isMarkdownPreferred, rewritePath } from 'fumadocs-core/negotiation'
 import { type NextRequest, NextResponse } from 'next/server'
 
-const { rewrite: rewriteLLM } = rewritePath('/docs/*path', '/llms.mdx/*path')
+/** Docs pages live at the root (`/introduction`), so every page path maps to its markdown route. */
+const { rewrite: rewriteLLM } = rewritePath('/*path', '/llms.mdx/*path')
+
+/**
+ * Docs page paths never carry an extension, so a final segment with one is a file or route
+ * (`/openapi.json`, `/introduction.md`) that must not be rewritten to a markdown page.
+ */
+const FILE_EXTENSION = /\.[^/]+$/
 
 export default function proxy(request: NextRequest) {
-  if (isMarkdownPreferred(request)) {
-    const result = rewriteLLM(request.nextUrl.pathname)
+  const { pathname } = request.nextUrl
+  if (isMarkdownPreferred(request) && !FILE_EXTENSION.test(pathname)) {
+    const result = rewriteLLM(pathname)
 
     if (result) {
-      return NextResponse.rewrite(new URL(result, request.nextUrl))
+      const response = NextResponse.rewrite(new URL(result, request.nextUrl))
+      response.headers.set('Vary', 'Accept')
+      return response
     }
   }
 
@@ -17,6 +27,6 @@ export default function proxy(request: NextRequest) {
 
 export const config = {
   matcher: [
-    '/((?!api|_next/static|_next/image|favicon|icon.svg|static|robots.txt|sitemap.xml|llms.txt|llms-full.txt).*)',
+    '/((?!api/|api$|llms.mdx|_next/static|_next/image|favicon|icon.svg|static|robots.txt|sitemap.xml|llms.txt|llms-full.txt).*)',
   ],
 }