Skip to content

Commit c1297d3

Browse files
waleedlatif1waleedlatif1
committed
improvement(seo): fix soft 404s, per-integration freshness, and docs markdown for AI crawlers
1 parent ac8e215 commit c1297d3

45 files changed

Lines changed: 885 additions & 466 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎apps/docs/app/[[...slug]]/page.tsx‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import type React from 'react'
22
import { Children, cloneElement, isValidElement } from 'react'
3+
import { INTEGRATION_METADATA } from '@sim/deployment-config/integration-metadata'
34
import { highlight } from 'fumadocs-core/highlight'
45
import { findNeighbour } from 'fumadocs-core/page-tree'
56
import type { ApiPageProps } from 'fumadocs-openapi/ui'
@@ -20,10 +21,21 @@ import { ResponseSection } from '@/components/ui/response-section'
2021
import { getApiSpecContent, getAuthenticatedCodeSamples, openapi } from '@/lib/openapi'
2122
import { simShikiOptions } from '@/lib/shiki-theme'
2223
import { type PageData, source } from '@/lib/source'
23-
import { DOCS_BASE_URL } from '@/lib/urls'
24+
import { DOCS_BASE_URL, SIM_SITE_URL } from '@/lib/urls'
2425

2526
const BASE_URL = DOCS_BASE_URL
2627

28+
/**
29+
* Landing page for each docs integration page. Docs pages are keyed by block type without its
30+
* version suffix (`gmail_v2` → `/integrations/gmail`); landing slugs are hyphenated catalog slugs.
31+
*/
32+
const INTEGRATION_LANDING_PAGES = new Map(
33+
INTEGRATION_METADATA.map((integration) => [
34+
integration.type.replace(/_v\d+$/, ''),
35+
{ name: integration.name, url: `${SIM_SITE_URL}/integrations/${integration.slug}` },
36+
])
37+
)
38+
2739
/**
2840
* Most pages close with a `## Next` / `## Next steps` grid of onward links.
2941
* That heading is navigation, not content, so it is kept out of the table of
@@ -122,6 +134,10 @@ export default async function Page(props: { params: Promise<{ slug?: string[] }>
122134
}
123135
const isOpenAPI = '_openapi' in data && data._openapi != null
124136
const isApiReference = slug?.some((s) => s === 'api-reference') ?? false
137+
const integrationLandingPage =
138+
slug?.length === 2 && slug[0] === 'integrations'
139+
? INTEGRATION_LANDING_PAGES.get(slug[1])
140+
: undefined
125141
const isCli = slug?.[0] === 'cli'
126142
const isMcp = slug?.[0] === 'mcp'
127143

@@ -266,6 +282,14 @@ export default async function Page(props: { params: Promise<{ slug?: string[] }>
266282
<PageNavigationArrows previous={neighbours?.previous} next={neighbours?.next} />
267283
</PageHeader>
268284
<DocsBody>
285+
{integrationLandingPage && (
286+
<p>
287+
Use cases and templates:{' '}
288+
<a href={integrationLandingPage.url}>
289+
{integrationLandingPage.name} integration for AI agents
290+
</a>
291+
</p>
292+
)}
269293
<MDX
270294
components={{
271295
...defaultMdxComponents,
@@ -367,6 +391,7 @@ export async function generateMetadata(props: { params: Promise<{ slug?: string[
367391
},
368392
alternates: {
369393
canonical: fullUrl,
394+
types: { 'text/markdown': `${fullUrl}.md` },
370395
},
371396
}
372397
}

‎apps/docs/app/layout.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ const structuredData = {
3232
url: DOCS_BASE_URL,
3333
publisher: {
3434
'@type': 'Organization',
35+
'@id': `${SIM_SITE_URL}#organization`,
3536
name: 'Sim',
3637
url: SIM_SITE_URL,
3738
logo: {

‎apps/docs/components/footer/footer.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ const PRODUCT_LINKS: FooterItem[] = [
3434

3535
const RESOURCES_LINKS: FooterItem[] = [
3636
{ label: 'Blog', href: `${SIM_SITE_URL}/blog`, external: true },
37-
{ label: 'Compare', href: `${SIM_SITE_URL}/comparison`, external: true },
37+
{ label: 'Compare', href: `${SIM_SITE_URL}/comparisons`, external: true },
3838
{ label: 'Careers', href: `${SIM_SITE_URL}/careers`, external: true },
3939
{ label: 'Changelog', href: `${SIM_SITE_URL}/changelog`, external: true },
4040
{ label: 'Contact', href: `${SIM_SITE_URL}/contact`, external: true },

‎apps/docs/components/structured-data.tsx‎

Lines changed: 6 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
import { INTEGRATION_COUNT_LABEL } from '@sim/deployment-config/integration-claims'
21
import { serializeJsonLd } from '@/lib/json-ld'
3-
import { DOCS_BASE_URL } from '@/lib/urls'
2+
import { DOCS_BASE_URL, SIM_SITE_URL } from '@/lib/urls'
43

54
interface StructuredDataProps {
65
title: string
@@ -29,13 +28,15 @@ export function StructuredData({
2928
...(dateModified && { dateModified }),
3029
author: {
3130
'@type': 'Organization',
32-
name: 'Sim Team',
33-
url: baseUrl,
31+
'@id': `${SIM_SITE_URL}#organization`,
32+
name: 'Sim',
33+
url: SIM_SITE_URL,
3434
},
3535
publisher: {
3636
'@type': 'Organization',
37+
'@id': `${SIM_SITE_URL}#organization`,
3738
name: 'Sim',
38-
url: baseUrl,
39+
url: SIM_SITE_URL,
3940
logo: {
4041
'@type': 'ImageObject',
4142
url: `${baseUrl}/static/logo.png`,
@@ -68,35 +69,6 @@ export function StructuredData({
6869
})),
6970
}
7071

71-
const softwareStructuredData = {
72-
'@context': 'https://schema.org',
73-
'@type': 'SoftwareApplication',
74-
name: 'Sim',
75-
applicationCategory: 'BusinessApplication',
76-
applicationSubCategory: 'AI Workspace',
77-
operatingSystem: 'Any',
78-
description: `Sim is the open-source AI workspace where teams build, deploy, and manage AI agents. Connect ${INTEGRATION_COUNT_LABEL} integrations and every major LLM to create agents that automate real work.`,
79-
url: baseUrl,
80-
author: {
81-
'@type': 'Organization',
82-
name: 'Sim Team',
83-
},
84-
offers: {
85-
'@type': 'Offer',
86-
category: 'Developer Tools',
87-
},
88-
featureList: [
89-
'AI workspace for teams',
90-
'Chat — natural language agent creation',
91-
'Visual workflow builder',
92-
`${INTEGRATION_COUNT_LABEL} integrations`,
93-
'LLM orchestration (OpenAI, Anthropic, Google, xAI, Mistral, Perplexity)',
94-
'Knowledge base creation',
95-
'Table creation',
96-
'Document creation',
97-
],
98-
}
99-
10072
return (
10173
<>
10274
<script
@@ -115,15 +87,6 @@ export function StructuredData({
11587
}}
11688
/>
11789
)}
118-
{(url === baseUrl || url === `${baseUrl}/`) && (
119-
<script
120-
id={`software-json-ld-${structuredDataId}`}
121-
type='application/ld+json'
122-
dangerouslySetInnerHTML={{
123-
__html: serializeJsonLd(softwareStructuredData),
124-
}}
125-
/>
126-
)}
12790
</>
12891
)
12992
}

‎apps/docs/lib/llms.ts‎

Lines changed: 51 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,59 @@
11
import type { InferPageType } from 'fumadocs-core/source'
2+
import type { ApiPageProps } from 'fumadocs-openapi/ui'
3+
import { getApiSpecContent } from '@/lib/openapi'
24
import type { PageData, source } from '@/lib/source'
35

6+
const TABLE_ROW = /^\|.*\|$/
7+
const TABLE_DELIMITER_CELL = /^\s*(:?)-+(:?)\s*$/
8+
9+
/**
10+
* The processed-markdown serializer pads every table cell to its column's widest cell, so one long
11+
* enum (a parameter listing every timezone) pads every row of its table and roughly doubles the
12+
* integration reference. Collapse the padding; the tables stay valid GFM.
13+
*/
14+
function compactTables(markdown: string): string {
15+
let inFence = false
16+
return markdown
17+
.split('\n')
18+
.map((line) => {
19+
if (line.startsWith('```')) inFence = !inFence
20+
if (inFence || !TABLE_ROW.test(line)) return line
21+
const cells = line.slice(1, -1).split('|')
22+
if (cells.every((cell) => TABLE_DELIMITER_CELL.test(cell))) {
23+
return `|${cells.map((cell) => cell.replace(TABLE_DELIMITER_CELL, ' $1---$2 ')).join('|')}|`
24+
}
25+
return line.replace(/ {2,}/g, ' ')
26+
})
27+
.join('\n')
28+
}
29+
30+
type LLMPageData = PageData & { getAPIPageProps?: () => ApiPageProps }
31+
32+
/**
33+
* OpenAPI operation pages are virtual: no MDX body, and a fully resolved request/response schema
34+
* (~40 KB each, mostly shared components) that is useful per page but would double `llms-full.txt`.
35+
*/
36+
export function isApiOperationPage(page: InferPageType<typeof source>): boolean {
37+
return typeof (page.data as unknown as LLMPageData).getAPIPageProps === 'function'
38+
}
39+
440
export async function getLLMText(page: InferPageType<typeof source>) {
5-
const data = page.data as unknown as PageData
41+
const data = page.data as unknown as LLMPageData
42+
const heading = `# ${data.title} (${page.url})`
43+
44+
// Operation pages get the same markdown the page's "Copy page" action builds.
45+
if (typeof data.getAPIPageProps === 'function') {
46+
const spec = getApiSpecContent(
47+
data.title,
48+
data.description,
49+
data.getAPIPageProps().operations ?? []
50+
)
51+
return spec.replace(/^# .*$/m, heading)
52+
}
53+
654
if (typeof data.getText !== 'function') return ''
755
const processed = await data.getText('processed')
8-
return `# ${data.title} (${page.url})
56+
return `${heading}
957
10-
${processed}`
58+
${compactTables(processed)}`
1159
}

‎apps/docs/lib/redirects.ts‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,72 @@ type DocsRedirect = Awaited<ReturnType<NonNullable<NextConfig['redirects']>>>[nu
2020
* modules would make match order an emergent property of two files.
2121
*/
2222
const PATH_REDIRECTS: DocsRedirect[] = [
23+
// The academy was removed; each lesson redirects to the reference page covering its topic.
24+
{ source: '/academy', destination: '/introduction', permanent: true },
25+
{ source: '/academy/agents/intro', destination: '/agents', permanent: true },
26+
{ source: '/academy/agents/block', destination: '/workflows/blocks/agent', permanent: true },
27+
{ source: '/academy/agents/memory', destination: '/workflows/blocks/agent', permanent: true },
28+
{ source: '/academy/agents/skills', destination: '/agents/skills', permanent: true },
29+
{
30+
source: '/academy/agents/tool-calling',
31+
destination: '/workflows/blocks/agent',
32+
permanent: true,
33+
},
34+
{ source: '/academy/chat/intro', destination: '/chat', permanent: true },
35+
{ source: '/academy/chat/building', destination: '/chat/workflows', permanent: true },
36+
{ source: '/academy/files/intro', destination: '/files', permanent: true },
37+
{ source: '/academy/files/object', destination: '/files/passing-files', permanent: true },
38+
{ source: '/academy/knowledge-bases/intro', destination: '/knowledgebase', permanent: true },
39+
{
40+
source: '/academy/knowledge-bases/connectors',
41+
destination: '/knowledgebase/connectors',
42+
permanent: true,
43+
},
44+
{ source: '/academy/tables/intro', destination: '/tables', permanent: true },
45+
{
46+
source: '/academy/tables/operations',
47+
destination: '/tables/using-in-workflows',
48+
permanent: true,
49+
},
50+
{
51+
source: '/academy/tables/workflow-columns',
52+
destination: '/tables/workflow-columns',
53+
permanent: true,
54+
},
55+
{ source: '/academy/workflows/intro', destination: '/workflows', permanent: true },
56+
{
57+
source: '/academy/workflows/branching',
58+
destination: '/workflows/blocks/condition',
59+
permanent: true,
60+
},
61+
{ source: '/academy/workflows/loops', destination: '/workflows/blocks/loop', permanent: true },
62+
{
63+
source: '/academy/workflows/subworkflows',
64+
destination: '/workflows/blocks/workflow',
65+
permanent: true,
66+
},
67+
{
68+
source: '/academy/workflows/deployment',
69+
destination: '/workflows/deployment',
70+
permanent: true,
71+
},
72+
{ source: '/academy/workflows/logs', destination: '/logs-debugging', permanent: true },
73+
{
74+
source: '/academy/use-cases/document-extraction',
75+
destination: '/files/using-in-workflows',
76+
permanent: true,
77+
},
78+
{
79+
source: '/academy/use-cases/monitoring-research',
80+
destination: '/workflows/triggers/schedule',
81+
permanent: true,
82+
},
83+
{
84+
source: '/academy/use-cases/sales-data-enrichment',
85+
destination: '/tables/workflow-columns',
86+
permanent: true,
87+
},
88+
{ source: '/academy/use-cases/slack-it-triage', destination: '/chat', permanent: true },
2389
{ source: '/academy/:path*', destination: '/introduction', permanent: true },
2490
...Object.entries(integrationNavigation.redirects).map(([from, to]) => ({
2591
source: `/integrations/${from}`,

‎apps/docs/next.config.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@ const config: NextConfig = {
3636
source: '/:path*.mdx',
3737
destination: '/llms.mdx/:path*',
3838
},
39+
{
40+
source: '/:path*.md',
41+
destination: '/llms.mdx/:path*',
42+
},
3943
]
4044
},
4145
}

‎apps/docs/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
"dev": "bun run dev:cache:cap && bun run llms-full:generate && bunx concurrently --kill-others --names Next,LLMs \"next dev --port 3001\" \"bun scripts/watch-llms-full.ts\"",
88
"dev:cache:cap": "bun run ../../scripts/prune-turbopack-cache.ts",
99
"dev:clean": "rm -rf .next/dev/cache",
10-
"build": "bun run llms-full:generate && NODE_OPTIONS='--max-old-space-size=8192' next build",
10+
"build": "bun scripts/fetch-git-history.ts && bun run llms-full:generate && NODE_OPTIONS='--max-old-space-size=8192' next build",
1111
"start": "next start",
1212
"postinstall": "fumadocs-mdx",
1313
"test": "vitest run",

‎apps/docs/proxy.ts‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,16 @@
11
import { isMarkdownPreferred, rewritePath } from 'fumadocs-core/negotiation'
22
import { type NextRequest, NextResponse } from 'next/server'
33

4-
const { rewrite: rewriteLLM } = rewritePath('/docs/*path', '/llms.mdx/*path')
4+
/** Docs pages live at the root (`/introduction`), so every page path maps to its markdown route. */
5+
const { rewrite: rewriteLLM } = rewritePath('/*path', '/llms.mdx/*path')
6+
7+
/** `.md`/`.mdx` URLs already resolve to markdown through the `next.config.ts` rewrites. */
8+
const MARKDOWN_SUFFIX = /\.mdx?$/
59

610
export default function proxy(request: NextRequest) {
7-
if (isMarkdownPreferred(request)) {
8-
const result = rewriteLLM(request.nextUrl.pathname)
11+
const { pathname } = request.nextUrl
12+
if (isMarkdownPreferred(request) && !MARKDOWN_SUFFIX.test(pathname)) {
13+
const result = rewriteLLM(pathname)
914

1015
if (result) {
1116
return NextResponse.rewrite(new URL(result, request.nextUrl))
@@ -17,6 +22,6 @@ export default function proxy(request: NextRequest) {
1722

1823
export const config = {
1924
matcher: [
20-
'/((?!api|_next/static|_next/image|favicon|icon.svg|static|robots.txt|sitemap.xml|llms.txt|llms-full.txt).*)',
25+
'/((?!api|llms.mdx|_next/static|_next/image|favicon|icon.svg|static|robots.txt|sitemap.xml|llms.txt|llms-full.txt).*)',
2126
],
2227
}

‎apps/docs/scripts/fetch-git-history.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/**
2+
* Vercel builds from a `--depth=10` clone, and `source.config.ts` refuses to date pages from a
3+
* shallow clone (every file untouched since the boundary commit would get that commit's date), so
4+
* the docs sitemap shipped with no `lastModified` at all. Fetch the commit history before the
5+
* build. `--filter=blob:none` skips file contents: `git log --name-only` needs commits and trees.
6+
*
7+
* A failed fetch leaves the clone shallow, so pages ship undated as before. That is logged rather
8+
* than failing the deploy, because dates are an enhancement and the docs must still ship.
9+
*/
10+
import { execFileSync } from 'node:child_process'
11+
import { createLogger } from '@sim/logger'
12+
import { getErrorMessage } from '@sim/utils/errors'
13+
14+
const logger = createLogger('DocsGitHistory')
15+
16+
function git(args: string[]): string {
17+
return execFileSync('git', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim()
18+
}
19+
20+
if (process.env.VERCEL) {
21+
try {
22+
if (git(['rev-parse', '--is-shallow-repository']) === 'true') {
23+
git(['fetch', '--unshallow', '--filter=blob:none', '--no-tags', 'origin'])
24+
logger.info('Fetched full commit history for docs last-modified dates')
25+
}
26+
} catch (error) {
27+
logger.warn(
28+
'Could not fetch commit history; docs pages will ship without last-modified dates',
29+
{
30+
error: getErrorMessage(error),
31+
}
32+
)
33+
}
34+
}

0 commit comments

Comments
 (0)