diff --git a/app/robots.txt/route.ts b/app/robots.txt/route.ts index bfdcd62e62d09..d01ef2dfac6ad 100644 --- a/app/robots.txt/route.ts +++ b/app/robots.txt/route.ts @@ -1,10 +1,17 @@ import {isDeveloperDocs} from 'sentry-docs/isDeveloperDocs'; +const sitemap = isDeveloperDocs + ? 'https://develop.sentry.dev/sitemap.xml' + : 'https://docs.sentry.io/sitemap.xml'; + export const GET = (_request: Request) => { return new Response( ` -Sitemap: ${isDeveloperDocs ? 'https://develop.sentry.dev/sitemap.xml' : 'https://docs.sentry.io/sitemap.xml'} +Sitemap: ${sitemap} + User-agent: * +Allow: / +Content-Signal: ai-train=yes, search=yes, ai-input=yes `.trim(), {headers: {'content-type': 'text/plain'}} );