fix(site): allow /api/og/ and social card crawlers in robots.txt (#776)

* fix(site): allow /api/og/ and social card crawlers in robots.txt

* fix(site): remove Disallow /api/ so all social, search, and AI bots can crawl OG cards and APIs

* test(site): update baseline robots.txt contract for open social & AI crawling

Author: codercatdev

apps/site/baseline/robots.txt

@@ -1,6 +1,7 @@
 User-Agent: *
-Content-Signal: ai-train=no, search=yes, ai-input=yes
+Content-Signal: ai-train=yes, search=yes, ai-input=yes
 Allow: /
+Allow: /api/og/
 Allow: /api/search
 Allow: /api/mcp
 Allow: /.well-known/
@@ -9,8 +10,26 @@ Allow: /llms-full.txt
 Allow: /auth.md
 Allow: /AGENTS.md
 Allow: /pricing.md
-Disallow: /api/
 Disallow: /dashboard/
+Disallow: /guest-portal/
+
+User-Agent: Twitterbot
+Allow: /
+
+User-Agent: Redditbot
+Allow: /
+
+User-Agent: Discordbot
+Allow: /
+
+User-Agent: Slackbot
+Allow: /
+
+User-Agent: facebookexternalhit
+Allow: /
+
+User-Agent: LinkedInBot
+Allow: /
 
 User-Agent: GPTBot
 Allow: /

apps/site/src/pages/robots.txt.ts

@@ -20,8 +20,9 @@ export const GET: APIRoute = async ({ locals }) => {
 
 	const body = isProduction
 		? `User-Agent: *
-Content-Signal: ai-train=no, search=yes, ai-input=yes
+Content-Signal: ai-train=yes, search=yes, ai-input=yes
 Allow: /
+Allow: /api/og/
 Allow: /api/search
 Allow: /api/mcp
 Allow: /.well-known/
@@ -30,8 +31,26 @@ Allow: /llms-full.txt
 Allow: /auth.md
 Allow: /AGENTS.md
 Allow: /pricing.md
-Disallow: /api/
 Disallow: /dashboard/
+Disallow: /guest-portal/
+
+User-Agent: Twitterbot
+Allow: /
+
+User-Agent: Redditbot
+Allow: /
+
+User-Agent: Discordbot
+Allow: /
+
+User-Agent: Slackbot
+Allow: /
+
+User-Agent: facebookexternalhit
+Allow: /
+
+User-Agent: LinkedInBot
+Allow: /
 
 User-Agent: GPTBot
 Allow: /

apps/site/tests/tier1-features.test.mjs

@@ -653,15 +653,16 @@ describe("Tier 1 - Feature Coverage", () => {
 			);
 		});
 
-		it("4.2 Production robots allows root and disallows /api/ and /dashboard/", async () => {
+		it("4.2 Production robots allows root and /api/og/ and disallows /dashboard/ and /guest-portal/", async () => {
 			const locals = { siteUrl: new URL("https://codingcat.dev") };
 			const res = await robotsModule.GET({ locals });
 			const body = await res.text();
 
 			assert.ok(body.includes("User-Agent: *"));
 			assert.ok(body.includes("Allow: /"));
-			assert.ok(body.includes("Disallow: /api/"));
+			assert.ok(body.includes("Allow: /api/og/"));
 			assert.ok(body.includes("Disallow: /dashboard/"));
+			assert.ok(body.includes("Disallow: /guest-portal/"));
 			assert.ok(body.includes("Host: https://codingcat.dev"));
 			assert.ok(body.includes("Sitemap: https://codingcat.dev/sitemap.xml"));
 		});

apps/site/tests/tier4-baseline-contract.test.mjs

@@ -320,10 +320,10 @@ describe("Tier 4 - Real-World Application & Baseline Contract", () => {
 		const baselineRobots = loadBaseline("robots.txt");
 		const robotsModule = loadModule("src/pages/robots.txt.ts");
 
-		it("4.1 Baseline robots contains User-Agent: * block with /api/ and /dashboard/ disallowed", () => {
+		it("4.1 Baseline robots contains User-Agent: * block with /api/og/ allowed and /dashboard/ disallowed", () => {
 			assert.ok(baselineRobots.includes("User-Agent: *"));
 			assert.ok(baselineRobots.includes("Allow: /"));
-			assert.ok(baselineRobots.includes("Disallow: /api/"));
+			assert.ok(baselineRobots.includes("Allow: /api/og/"));
 			assert.ok(baselineRobots.includes("Disallow: /dashboard/"));
 		});