From e24e8c9a44b621cb2b9cc13240f733ef5eb9c360 Mon Sep 17 00:00:00 2001
From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com>
Date: Fri, 28 Aug 2026 22:55:36 +0200
Subject: [PATCH 1/3] feat: LLM discoverability sync from kurrentschrift
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Carry the sister project's assistant-protocol findings (kurrentschrift
PRs #428, #435, #439, #440, 2026-08-28) over to anyplot. Assistants'
fetch tools often allow only URLs that already appeared verbatim in
fetched content, so the machine guide and one complete, callable
example per surface now stand wherever an agent actually reads:
- SPA shell: in the head and a
diff --git a/app/nginx.conf b/app/nginx.conf
index c95212b642..001e0352e4 100644
--- a/app/nginx.conf
+++ b/app/nginx.conf
@@ -84,6 +84,10 @@ map $http_user_agent $is_bot {
# "insufficient relevant content" (AI-access audit 2026-08-19). No common
# browser UA contains the substring.
~*grok 1;
+ # xAI's company token, for fetcher UAs that carry "xAI" but not "grok"
+ # (verified 2026-08-28: a UA "xAI-Bot" fell through ~*grok to the empty
+ # shell). No common browser UA contains the substring.
+ ~*xai 1;
~*youbot 1;
~*cohere-ai 1;
~*diffbot 1;
@@ -223,6 +227,16 @@ server {
# it generates is emitted as http://, downgrading the crawler's next
# hop. The other proxied locations already set it.
proxy_set_header X-Forwarded-Proto https;
+ # The CRAWLER's address must reach the API: it reports each read to
+ # the bot Plausible site with the visitor IP, and Plausible drops
+ # events whose forwarded IP is a hosting-provider address. On this
+ # hop cf-connecting-ip at the API is this container's Google egress
+ # IP, so the crawler is only ever visible in X-Forwarded-For —
+ # Cloudflare put it first at the edge; $proxy_add_x_forwarded_for
+ # appends our hop instead of replacing the list (kurrentschrift
+ # measured 1 counted read in 20 without this, 2026-08-28;
+ # api/request_context.py::visitor_ip reads the list leftmost-first).
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_ssl_server_name on;
proxy_ssl_verify on;
# api.anyplot.ai serves a 4-deep Let's Encrypt chain (leaf -> YE1 ->
@@ -244,6 +258,14 @@ server {
try_files $uri =404;
}
+ # llms.txt has no registered .well-known name, but agents guess this path
+ # (docs/reference/seo.md "Discoverability") — answer with the file's
+ # canonical location instead of the SPA shell, which soft-404'd it with
+ # 200 + the home page until 2026-08-28.
+ location = /.well-known/llms.txt {
+ return 302 /llms.txt;
+ }
+
# llms-full.txt is generated from the DB (one line per spec), so unlike
# llms.txt it cannot be a static file — proxy it to the API's
# /llms-full.txt endpoint for EVERY client, mapped crawler UA or not.
diff --git a/app/public/llms.txt b/app/public/llms.txt
index 3f720509e5..20dc7e623f 100644
--- a/app/public/llms.txt
+++ b/app/public/llms.txt
@@ -44,6 +44,7 @@ and the spec itself at `.../plots/{spec_id}/specification.md`.
- [Plot gallery](https://anyplot.ai/plots): every rendered implementation, filterable by library, language and tags
- [Specifications](https://anyplot.ai/specs): the library-agnostic plot specs the implementations are generated from
- [Libraries](https://anyplot.ai/libraries): the fifteen supported plotting libraries across four languages
+- [Network map](https://anyplot.ai/map): every spec as a node, grouped by visual similarity — the catalogue's neighbourhoods (interactive; the data behind it is `GET https://api.anyplot.ai/specs/map`)
- [Sitemap](https://anyplot.ai/sitemap.xml): every spec hub and implementation page
## Docs
diff --git a/app/public/robots.txt b/app/public/robots.txt
index 657a9163f6..9d45369602 100644
--- a/app/public/robots.txt
+++ b/app/public/robots.txt
@@ -1,3 +1,6 @@
+# llms.txt (machine guide): https://anyplot.ai/llms.txt
+# OpenAPI: https://api.anyplot.ai/openapi.json
+#
# Crawler policy — everything here is open, to humans and machines alike.
#
# The catalogue is MIT-licensed and published to be reused: every plot page
@@ -37,7 +40,10 @@
# first-match parser lets /debug and /interactive through (verified with
# Python's urllib.robotparser, which is exactly such a parser).
#
-# Content signals per contentsignals.org:
+# Content signals per contentsignals.org (these three tokens are the whole
+# vocabulary; nothing else is valid on the line — a `use=reference` token
+# used to ride along here, and a strict parser may discard the entire line
+# over an unknown token, taking the three real signals with it):
# search: indexing and returning links/excerpts — yes
# ai-input: retrieval for AI answers, grounding, citation — yes
# ai-train: training or fine-tuning models — yes
@@ -50,16 +56,22 @@
# Control does the actual enforcing. If its behaviour changes, this group can
# go — nothing in the licence or the policy above argues against it.
User-agent: Bytespider
-Content-Signal: search=yes,ai-input=yes,ai-train=yes,use=reference
+Content-Signal: search=yes,ai-input=yes,ai-train=yes
Disallow: /
# Everyone else: search engines, AI assistants and their training crawlers,
# social and link previews, feed readers. /debug and /interactive are app
# internals rather than catalogue content and are not worth crawling.
User-agent: *
-Content-Signal: search=yes,ai-input=yes,ai-train=yes,use=reference
+Content-Signal: search=yes,ai-input=yes,ai-train=yes
Disallow: /debug
Disallow: /interactive
Allow: /
Sitemap: https://anyplot.ai/sitemap.xml
+
+# Machine guide for AI assistants — every retrieval recipe with a full example
+# URL (runnable source via the JSON API, render PNGs, OpenAPI, the MCP
+# endpoint), and the whole catalogue as one line per spec:
+# https://anyplot.ai/llms.txt
+# https://anyplot.ai/llms-full.txt
diff --git a/app/src/routes/seoCoverage.test.ts b/app/src/routes/seoCoverage.test.ts
new file mode 100644
index 0000000000..77fc043b0e
--- /dev/null
+++ b/app/src/routes/seoCoverage.test.ts
@@ -0,0 +1,106 @@
+// Drift guard for the machine-facing surface: the route registry in paths.ts,
+// public/llms.txt, public/robots.txt and the SPA shell (index.html) must
+// describe the SAME site and point at each other. Each of these has drifted
+// silently before — a route missing from llms.txt, a guide findable only by
+// guessing the convention, a shell that told a JS-less agent nothing — because
+// nothing coupled them. This test is that coupling; the API side of the same
+// contract (the API host's robots.txt, /llms.txt redirect, the bot-page nav)
+// lives in tests/unit/api/test_routers.py.
+//
+// The files are read through Vite's `?raw` imports rather than node:fs so the
+// test needs no Node typings — vitest resolves them exactly as the build does.
+
+import { describe, expect, it } from 'vitest';
+
+import shell from '../../index.html?raw';
+import llmsTxt from '../../public/llms.txt?raw';
+import robotsTxt from '../../public/robots.txt?raw';
+import { paths } from './paths';
+
+const ORIGIN = 'https://anyplot.ai';
+const API = 'https://api.anyplot.ai';
+
+// The public content routes are the top-level string values of paths, minus
+// the ones that are deliberately absent from the machine guide: '/' is the
+// document's own subject, /debug is an app internal (robots.txt disallows it),
+// and the search deep link is a view of /plots, not a page.
+const publicPaths = Object.values(paths)
+ .flatMap(v => (typeof v === 'string' ? [v] : []))
+ .filter(p => p !== paths.home && p !== paths.debug && !p.includes('?'));
+
+// One complete, callable example per surface. Assistants' fetch tools often
+// allow only URLs that already appeared verbatim in fetched content, so the
+// same example pair must stand in the guide AND in the shell — a template with
+// placeholders satisfies neither (kurrentschrift finding 2026-08-28).
+const EXAMPLE_CODE_URL = `${API}/specs/bar-error/seaborn/code`;
+const EXAMPLE_RENDER_URL =
+ 'https://storage.googleapis.com/anyplot-images/plots/bar-error/python/seaborn/plot-light.png';
+
+const escapeRe = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+describe('seo coverage', () => {
+ it('llms.txt links every public content route', () => {
+ // Word-boundary match, not substring: a link to /plots must not be
+ // satisfied by /plots?focus=search, nor /specs by /specs/…. The literal
+ // parts are regex-escaped — the dots in the origin would otherwise match
+ // any character and quietly weaken the guard.
+ for (const p of publicPaths) {
+ expect(llmsTxt, `llms.txt lacks ${p}`).toMatch(
+ new RegExp(`${escapeRe(ORIGIN + p)}(?![\\w./?-])`)
+ );
+ }
+ });
+
+ it('llms.txt carries the complete example pair and the machine surfaces', () => {
+ expect(llmsTxt).toContain(EXAMPLE_CODE_URL);
+ expect(llmsTxt).toContain(EXAMPLE_RENDER_URL);
+ expect(llmsTxt).toContain(`${ORIGIN}/llms-full.txt`);
+ expect(llmsTxt).toContain(`${API}/openapi.json`);
+ expect(llmsTxt).toContain(`${API}/mcp/`);
+ });
+
+ it('robots.txt announces the sitemap and the machine guide, head and foot', () => {
+ // Two terse lines at the very top for readers that only take the file
+ // head seriously, the fuller block beside `Sitemap:` for everyone else.
+ const lines = robotsTxt.split('\n');
+ expect(lines[0]).toBe(`# llms.txt (machine guide): ${ORIGIN}/llms.txt`);
+ expect(lines[1]).toBe(`# OpenAPI: ${API}/openapi.json`);
+ expect(robotsTxt).toContain(`Sitemap: ${ORIGIN}/sitemap.xml`);
+ expect(robotsTxt).toContain(`${ORIGIN}/llms-full.txt`);
+ });
+
+ it('robots.txt uses only the three contentsignals.org tokens, in every group', () => {
+ // `use=reference` used to ride along; it is not a signal, and a strict
+ // parser may discard the whole line over an unknown token — taking the
+ // three real ones with it.
+ const signals = robotsTxt.split('\n').filter(l => l.startsWith('Content-Signal:'));
+ const groups = robotsTxt.split('\n').filter(l => l.startsWith('User-agent:'));
+ expect(signals).toHaveLength(groups.length);
+ for (const s of signals) expect(s).toBe('Content-Signal: search=yes,ai-input=yes,ai-train=yes');
+ });
+
+ it('the SPA shell itself carries the machine fallback', () => {
+ // Agents NOT in the nginx $is_bot map get the shell on every route, and
+ // most read raw HTML without executing JS. The shell must therefore name
+ // the machine guide and one complete, callable example per surface — in
+ // the head link and the