From d4094fb4952b3e20a482df791b4a24a8b882ebcd Mon Sep 17 00:00:00 2001 From: Alex Krawiec Date: Mon, 18 May 2026 11:10:56 -0700 Subject: [PATCH 1/3] Add user agent rules for ai crawlers in robots.txt --- app/robots.txt/route.ts | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/app/robots.txt/route.ts b/app/robots.txt/route.ts index bfdcd62e62d09..0876f6c4931d4 100644 --- a/app/robots.txt/route.ts +++ b/app/robots.txt/route.ts @@ -1,10 +1,43 @@ import {isDeveloperDocs} from 'sentry-docs/isDeveloperDocs'; +const sitemap = isDeveloperDocs + ? 'https://develop.sentry.dev/sitemap.xml' + : 'https://docs.sentry.io/sitemap.xml'; + export const GET = (_request: Request) => { return new Response( ` -Sitemap: ${isDeveloperDocs ? 'https://develop.sentry.dev/sitemap.xml' : 'https://docs.sentry.io/sitemap.xml'} +Sitemap: ${sitemap} + User-agent: * +Allow: / + +User-agent: GPTBot +Allow: / + +User-agent: OAI-SearchBot +Allow: / + +User-agent: Claude-Web +Allow: / + +User-agent: anthropic-ai +Allow: / + +User-agent: Google-Extended +Allow: / + +User-agent: Amazonbot +Allow: / + +User-agent: Bytespider +Allow: / + +User-agent: CCBot +Allow: / + +User-agent: Applebot-Extended +Allow: / `.trim(), {headers: {'content-type': 'text/plain'}} ); From 4bf623fedc669b42cc278cc349129f3ce4a8f9d5 Mon Sep 17 00:00:00 2001 From: Alex Krawiec Date: Mon, 18 May 2026 13:38:11 -0700 Subject: [PATCH 2/3] Add ai content signals to robots.txt --- app/robots.txt/route.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/robots.txt/route.ts b/app/robots.txt/route.ts index 0876f6c4931d4..3625cb7556f00 100644 --- a/app/robots.txt/route.ts +++ b/app/robots.txt/route.ts @@ -11,33 +11,43 @@ Sitemap: ${sitemap} User-agent: * Allow: / +Content-Signal: ai-train=yes, search=yes, ai-input=yes User-agent: GPTBot Allow: / +Content-Signal: ai-train=yes, search=yes, ai-input=yes User-agent: OAI-SearchBot Allow: / +Content-Signal: ai-train=yes, search=yes, ai-input=yes User-agent: Claude-Web Allow: / +Content-Signal: ai-train=yes, search=yes, ai-input=yes User-agent: anthropic-ai Allow: / +Content-Signal: ai-train=yes, search=yes, ai-input=yes User-agent: Google-Extended Allow: / +Content-Signal: ai-train=yes, search=yes, ai-input=yes User-agent: Amazonbot Allow: / +Content-Signal: ai-train=yes, search=yes, ai-input=yes User-agent: Bytespider Allow: / +Content-Signal: ai-train=yes, search=yes, ai-input=yes User-agent: CCBot Allow: / +Content-Signal: ai-train=yes, search=yes, ai-input=yes User-agent: Applebot-Extended Allow: / +Content-Signal: ai-train=yes, search=yes, ai-input=yes `.trim(), {headers: {'content-type': 'text/plain'}} ); From db96e70404f3075f123d1654bda50ff66ea29bd8 Mon Sep 17 00:00:00 2001 From: Alex Krawiec Date: Tue, 19 May 2026 13:19:59 -0700 Subject: [PATCH 3/3] Collapse individual bot rules to a wildcard --- app/robots.txt/route.ts | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/app/robots.txt/route.ts b/app/robots.txt/route.ts index 3625cb7556f00..d01ef2dfac6ad 100644 --- a/app/robots.txt/route.ts +++ b/app/robots.txt/route.ts @@ -12,42 +12,6 @@ Sitemap: ${sitemap} User-agent: * Allow: / Content-Signal: ai-train=yes, search=yes, ai-input=yes - -User-agent: GPTBot -Allow: / -Content-Signal: ai-train=yes, search=yes, ai-input=yes - -User-agent: OAI-SearchBot -Allow: / -Content-Signal: ai-train=yes, search=yes, ai-input=yes - -User-agent: Claude-Web -Allow: / -Content-Signal: ai-train=yes, search=yes, ai-input=yes - -User-agent: anthropic-ai -Allow: / -Content-Signal: ai-train=yes, search=yes, ai-input=yes - -User-agent: Google-Extended -Allow: / -Content-Signal: ai-train=yes, search=yes, ai-input=yes - -User-agent: Amazonbot -Allow: / -Content-Signal: ai-train=yes, search=yes, ai-input=yes - -User-agent: Bytespider -Allow: / -Content-Signal: ai-train=yes, search=yes, ai-input=yes - -User-agent: CCBot -Allow: / -Content-Signal: ai-train=yes, search=yes, ai-input=yes - -User-agent: Applebot-Extended -Allow: / -Content-Signal: ai-train=yes, search=yes, ai-input=yes `.trim(), {headers: {'content-type': 'text/plain'}} );