diff --git a/.github/workflows/bot-serving-check.yml b/.github/workflows/bot-serving-check.yml
index 2a0d5eca54..607023608a 100644
--- a/.github/workflows/bot-serving-check.yml
+++ b/.github/workflows/bot-serving-check.yml
@@ -28,13 +28,13 @@ permissions:
jobs:
bot-serving:
runs-on: ubuntu-latest
- # 22 check() calls x (--retry 2 -> up to 3 attempts x --max-time 30) can
- # reach ~33 min worst-case, plus two non-retried probes (llms.txt charset,
- # trailing slash — 30s each); 38 leaves room to report a clean failure
- # rather than dying to the job timeout, which reports nothing useful.
- # Recompute this when adding checks: the ceiling is check() calls x 90s,
- # plus margin.
- timeout-minutes: 38
+ # 27 check() calls x (--retry 2 -> up to 3 attempts x --max-time 30) can
+ # reach ~41 min worst-case, plus four non-retried probes (llms.txt charset,
+ # trailing slash, og-image, .well-known redirect — 30s each); 46 leaves
+ # room to report a clean failure rather than dying to the job timeout,
+ # which reports nothing useful. Recompute this when adding checks: the
+ # ceiling is check() calls x 90s, plus margin.
+ timeout-minutes: 46
steps:
- name: Crawler UAs must get 200 + per-route titles
run: |
@@ -50,16 +50,16 @@ jobs:
fail=0
check() {
- local ua="$1" url="$2" expect="$3"
+ local ua="$1" url="$2" expect="$3" want="${4:-200}"
local code
# On curl failure REPLACE the code — a failing curl can still have
# printed a partial -w code; appending would yield e.g. "200000".
code=$(curl -sS --retry 2 --max-time 30 -A "$ua" -o body.html -w '%{http_code}' "$url") || code="000"
- if [ "$code" != "200" ]; then
- echo "::error::$url with UA '$ua' returned HTTP $code (expected 200)"
+ if [ "$code" != "$want" ]; then
+ echo "::error::$url with UA '$ua' returned HTTP $code (expected $want)"
fail=1
elif ! grep -qF "$expect" body.html; then
- echo "::error::$url with UA '$ua' returned 200 but the body is missing: $expect"
+ echo "::error::$url with UA '$ua' returned $want but the body is missing: $expect"
fail=1
else
echo "OK: $url ($ua)"
@@ -101,11 +101,42 @@ jobs:
"Mozilla/5.0 (compatible; Amzn-User/1.0)" \
"Mozilla/5.0 (compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)" \
"meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)" \
- "Grok/1.0"
+ "Grok/1.0" \
+ "Mozilla/5.0 (compatible; xAI-Bot/1.0)"
do
check "$ua" "$ORIGIN/scatter-basic" "
Basic Scatter Plot | anyplot.ai"
done
+ # A crawler asking for a URL that is no page gets a real 404 from the
+ # seo-proxy — the SPA shell would answer 200 (soft-404), and did for
+ # 161 stale migration URLs before the proxy learned to say no.
+ check "$GOOGLEBOT" "$ORIGIN/this-spec-does-not-exist" '"status":404' 404
+
+ # The machine files must be served directly, never proxied to the
+ # seo backend — including for a mapped crawler UA, which is the whole
+ # point of the `location =` bypasses. robots.txt and sitemap.xml
+ # join llms.txt below: the sitemap is proxied to the API for every
+ # client, and a broken proxy would hand a crawler the SPA shell.
+ check "$GOOGLEBOT" "$ORIGIN/robots.txt" "User-agent: Bytespider"
+ check "$GOOGLEBOT" "$ORIGIN/sitemap.xml" " $wk_target" ;;
+ *) echo "::error::.well-known/llms.txt did not redirect to the guide: '$wk_target'"; fail=1 ;;
+ esac
+
# llms.txt must be served directly, never proxied to the seo backend —
# including for a mapped crawler UA, which is the whole point of the
# `location = /llms.txt` bypass.
@@ -145,7 +176,9 @@ jobs:
*) echo "OK: trailing slash -> $slash_target" ;;
esac
- # Control: humans must still get the SPA shell
- check "$HUMAN" "$ORIGIN/" '
'
+ # Control: humans must still get the SPA shell — on the home page
+ # and on a deep route.
+ check "$HUMAN" "$ORIGIN/" '
'
+ check "$HUMAN" "$ORIGIN/scatter-basic" '
'
exit $fail
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b6ec19a528..517a8c4267 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,26 @@ aggregate instead: an italic *Catalog* line at the end of the version section an
### Added
+- **LLM discoverability sync from kurrentschrift** — the sister project's
+ assistant-protocol findings (2026-08-28) carried over: assistants' fetch tools often allow
+ only URLs that already appeared verbatim in fetched content, so the machine guide and one
+ complete, callable example per surface now stand wherever an agent actually reads. The SPA
+ shell carries a `` and a `
diff --git a/app/nginx.conf b/app/nginx.conf
index c95212b642..001e0352e4 100644
--- a/app/nginx.conf
+++ b/app/nginx.conf
@@ -84,6 +84,10 @@ map $http_user_agent $is_bot {
# "insufficient relevant content" (AI-access audit 2026-08-19). No common
# browser UA contains the substring.
~*grok 1;
+ # xAI's company token, for fetcher UAs that carry "xAI" but not "grok"
+ # (verified 2026-08-28: a UA "xAI-Bot" fell through ~*grok to the empty
+ # shell). No common browser UA contains the substring.
+ ~*xai 1;
~*youbot 1;
~*cohere-ai 1;
~*diffbot 1;
@@ -223,6 +227,16 @@ server {
# it generates is emitted as http://, downgrading the crawler's next
# hop. The other proxied locations already set it.
proxy_set_header X-Forwarded-Proto https;
+ # The CRAWLER's address must reach the API: it reports each read to
+ # the bot Plausible site with the visitor IP, and Plausible drops
+ # events whose forwarded IP is a hosting-provider address. On this
+ # hop cf-connecting-ip at the API is this container's Google egress
+ # IP, so the crawler is only ever visible in X-Forwarded-For —
+ # Cloudflare put it first at the edge; $proxy_add_x_forwarded_for
+ # appends our hop instead of replacing the list (kurrentschrift
+ # measured 1 counted read in 20 without this, 2026-08-28;
+ # api/request_context.py::visitor_ip reads the list leftmost-first).
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_ssl_server_name on;
proxy_ssl_verify on;
# api.anyplot.ai serves a 4-deep Let's Encrypt chain (leaf -> YE1 ->
@@ -244,6 +258,14 @@ server {
try_files $uri =404;
}
+ # llms.txt has no registered .well-known name, but agents guess this path
+ # (docs/reference/seo.md "Discoverability") — answer with the file's
+ # canonical location instead of the SPA shell, which soft-404'd it with
+ # 200 + the home page until 2026-08-28.
+ location = /.well-known/llms.txt {
+ return 302 /llms.txt;
+ }
+
# llms-full.txt is generated from the DB (one line per spec), so unlike
# llms.txt it cannot be a static file — proxy it to the API's
# /llms-full.txt endpoint for EVERY client, mapped crawler UA or not.
diff --git a/app/public/llms.txt b/app/public/llms.txt
index 3f720509e5..20dc7e623f 100644
--- a/app/public/llms.txt
+++ b/app/public/llms.txt
@@ -44,6 +44,7 @@ and the spec itself at `.../plots/{spec_id}/specification.md`.
- [Plot gallery](https://anyplot.ai/plots): every rendered implementation, filterable by library, language and tags
- [Specifications](https://anyplot.ai/specs): the library-agnostic plot specs the implementations are generated from
- [Libraries](https://anyplot.ai/libraries): the fifteen supported plotting libraries across four languages
+- [Network map](https://anyplot.ai/map): every spec as a node, grouped by visual similarity — the catalogue's neighbourhoods (interactive; the data behind it is `GET https://api.anyplot.ai/specs/map`)
- [Sitemap](https://anyplot.ai/sitemap.xml): every spec hub and implementation page
## Docs
diff --git a/app/public/robots.txt b/app/public/robots.txt
index 657a9163f6..9d45369602 100644
--- a/app/public/robots.txt
+++ b/app/public/robots.txt
@@ -1,3 +1,6 @@
+# llms.txt (machine guide): https://anyplot.ai/llms.txt
+# OpenAPI: https://api.anyplot.ai/openapi.json
+#
# Crawler policy — everything here is open, to humans and machines alike.
#
# The catalogue is MIT-licensed and published to be reused: every plot page
@@ -37,7 +40,10 @@
# first-match parser lets /debug and /interactive through (verified with
# Python's urllib.robotparser, which is exactly such a parser).
#
-# Content signals per contentsignals.org:
+# Content signals per contentsignals.org (these three tokens are the whole
+# vocabulary; nothing else is valid on the line — a `use=reference` token
+# used to ride along here, and a strict parser may discard the entire line
+# over an unknown token, taking the three real signals with it):
# search: indexing and returning links/excerpts — yes
# ai-input: retrieval for AI answers, grounding, citation — yes
# ai-train: training or fine-tuning models — yes
@@ -50,16 +56,22 @@
# Control does the actual enforcing. If its behaviour changes, this group can
# go — nothing in the licence or the policy above argues against it.
User-agent: Bytespider
-Content-Signal: search=yes,ai-input=yes,ai-train=yes,use=reference
+Content-Signal: search=yes,ai-input=yes,ai-train=yes
Disallow: /
# Everyone else: search engines, AI assistants and their training crawlers,
# social and link previews, feed readers. /debug and /interactive are app
# internals rather than catalogue content and are not worth crawling.
User-agent: *
-Content-Signal: search=yes,ai-input=yes,ai-train=yes,use=reference
+Content-Signal: search=yes,ai-input=yes,ai-train=yes
Disallow: /debug
Disallow: /interactive
Allow: /
Sitemap: https://anyplot.ai/sitemap.xml
+
+# Machine guide for AI assistants — every retrieval recipe with a full example
+# URL (runnable source via the JSON API, render PNGs, OpenAPI, the MCP
+# endpoint), and the whole catalogue as one line per spec:
+# https://anyplot.ai/llms.txt
+# https://anyplot.ai/llms-full.txt
diff --git a/app/src/routes/seoCoverage.test.ts b/app/src/routes/seoCoverage.test.ts
new file mode 100644
index 0000000000..77fc043b0e
--- /dev/null
+++ b/app/src/routes/seoCoverage.test.ts
@@ -0,0 +1,106 @@
+// Drift guard for the machine-facing surface: the route registry in paths.ts,
+// public/llms.txt, public/robots.txt and the SPA shell (index.html) must
+// describe the SAME site and point at each other. Each of these has drifted
+// silently before — a route missing from llms.txt, a guide findable only by
+// guessing the convention, a shell that told a JS-less agent nothing — because
+// nothing coupled them. This test is that coupling; the API side of the same
+// contract (the API host's robots.txt, /llms.txt redirect, the bot-page nav)
+// lives in tests/unit/api/test_routers.py.
+//
+// The files are read through Vite's `?raw` imports rather than node:fs so the
+// test needs no Node typings — vitest resolves them exactly as the build does.
+
+import { describe, expect, it } from 'vitest';
+
+import shell from '../../index.html?raw';
+import llmsTxt from '../../public/llms.txt?raw';
+import robotsTxt from '../../public/robots.txt?raw';
+import { paths } from './paths';
+
+const ORIGIN = 'https://anyplot.ai';
+const API = 'https://api.anyplot.ai';
+
+// The public content routes are the top-level string values of paths, minus
+// the ones that are deliberately absent from the machine guide: '/' is the
+// document's own subject, /debug is an app internal (robots.txt disallows it),
+// and the search deep link is a view of /plots, not a page.
+const publicPaths = Object.values(paths)
+ .flatMap(v => (typeof v === 'string' ? [v] : []))
+ .filter(p => p !== paths.home && p !== paths.debug && !p.includes('?'));
+
+// One complete, callable example per surface. Assistants' fetch tools often
+// allow only URLs that already appeared verbatim in fetched content, so the
+// same example pair must stand in the guide AND in the shell — a template with
+// placeholders satisfies neither (kurrentschrift finding 2026-08-28).
+const EXAMPLE_CODE_URL = `${API}/specs/bar-error/seaborn/code`;
+const EXAMPLE_RENDER_URL =
+ 'https://storage.googleapis.com/anyplot-images/plots/bar-error/python/seaborn/plot-light.png';
+
+const escapeRe = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+describe('seo coverage', () => {
+ it('llms.txt links every public content route', () => {
+ // Word-boundary match, not substring: a link to /plots must not be
+ // satisfied by /plots?focus=search, nor /specs by /specs/…. The literal
+ // parts are regex-escaped — the dots in the origin would otherwise match
+ // any character and quietly weaken the guard.
+ for (const p of publicPaths) {
+ expect(llmsTxt, `llms.txt lacks ${p}`).toMatch(
+ new RegExp(`${escapeRe(ORIGIN + p)}(?![\\w./?-])`)
+ );
+ }
+ });
+
+ it('llms.txt carries the complete example pair and the machine surfaces', () => {
+ expect(llmsTxt).toContain(EXAMPLE_CODE_URL);
+ expect(llmsTxt).toContain(EXAMPLE_RENDER_URL);
+ expect(llmsTxt).toContain(`${ORIGIN}/llms-full.txt`);
+ expect(llmsTxt).toContain(`${API}/openapi.json`);
+ expect(llmsTxt).toContain(`${API}/mcp/`);
+ });
+
+ it('robots.txt announces the sitemap and the machine guide, head and foot', () => {
+ // Two terse lines at the very top for readers that only take the file
+ // head seriously, the fuller block beside `Sitemap:` for everyone else.
+ const lines = robotsTxt.split('\n');
+ expect(lines[0]).toBe(`# llms.txt (machine guide): ${ORIGIN}/llms.txt`);
+ expect(lines[1]).toBe(`# OpenAPI: ${API}/openapi.json`);
+ expect(robotsTxt).toContain(`Sitemap: ${ORIGIN}/sitemap.xml`);
+ expect(robotsTxt).toContain(`${ORIGIN}/llms-full.txt`);
+ });
+
+ it('robots.txt uses only the three contentsignals.org tokens, in every group', () => {
+ // `use=reference` used to ride along; it is not a signal, and a strict
+ // parser may discard the whole line over an unknown token — taking the
+ // three real ones with it.
+ const signals = robotsTxt.split('\n').filter(l => l.startsWith('Content-Signal:'));
+ const groups = robotsTxt.split('\n').filter(l => l.startsWith('User-agent:'));
+ expect(signals).toHaveLength(groups.length);
+ for (const s of signals) expect(s).toBe('Content-Signal: search=yes,ai-input=yes,ai-train=yes');
+ });
+
+ it('the SPA shell itself carries the machine fallback', () => {
+ // Agents NOT in the nginx $is_bot map get the shell on every route, and
+ // most read raw HTML without executing JS. The shell must therefore name
+ // the machine guide and one complete, callable example per surface — in
+ // the head link and the