diff --git a/.github/workflows/bot-serving-check.yml b/.github/workflows/bot-serving-check.yml index 2a0d5eca54..607023608a 100644 --- a/.github/workflows/bot-serving-check.yml +++ b/.github/workflows/bot-serving-check.yml @@ -28,13 +28,13 @@ permissions: jobs: bot-serving: runs-on: ubuntu-latest - # 22 check() calls x (--retry 2 -> up to 3 attempts x --max-time 30) can - # reach ~33 min worst-case, plus two non-retried probes (llms.txt charset, - # trailing slash — 30s each); 38 leaves room to report a clean failure - # rather than dying to the job timeout, which reports nothing useful. - # Recompute this when adding checks: the ceiling is check() calls x 90s, - # plus margin. - timeout-minutes: 38 + # 27 check() calls x (--retry 2 -> up to 3 attempts x --max-time 30) can + # reach ~41 min worst-case, plus four non-retried probes (llms.txt charset, + # trailing slash, og-image, .well-known redirect — 30s each); 46 leaves + # room to report a clean failure rather than dying to the job timeout, + # which reports nothing useful. Recompute this when adding checks: the + # ceiling is check() calls x 90s, plus margin. + timeout-minutes: 46 steps: - name: Crawler UAs must get 200 + per-route titles run: | @@ -50,16 +50,16 @@ jobs: fail=0 check() { - local ua="$1" url="$2" expect="$3" + local ua="$1" url="$2" expect="$3" want="${4:-200}" local code # On curl failure REPLACE the code — a failing curl can still have # printed a partial -w code; appending would yield e.g. "200000". code=$(curl -sS --retry 2 --max-time 30 -A "$ua" -o body.html -w '%{http_code}' "$url") || code="000" - if [ "$code" != "200" ]; then - echo "::error::$url with UA '$ua' returned HTTP $code (expected 200)" + if [ "$code" != "$want" ]; then + echo "::error::$url with UA '$ua' returned HTTP $code (expected $want)" fail=1 elif ! grep -qF "$expect" body.html; then - echo "::error::$url with UA '$ua' returned 200 but the body is missing: $expect" + echo "::error::$url with UA '$ua' returned $want but the body is missing: $expect" fail=1 else echo "OK: $url ($ua)" @@ -101,11 +101,42 @@ jobs: "Mozilla/5.0 (compatible; Amzn-User/1.0)" \ "Mozilla/5.0 (compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)" \ "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)" \ - "Grok/1.0" + "Grok/1.0" \ + "Mozilla/5.0 (compatible; xAI-Bot/1.0)" do check "$ua" "$ORIGIN/scatter-basic" "Basic Scatter Plot | anyplot.ai" done + # A crawler asking for a URL that is no page gets a real 404 from the + # seo-proxy — the SPA shell would answer 200 (soft-404), and did for + # 161 stale migration URLs before the proxy learned to say no. + check "$GOOGLEBOT" "$ORIGIN/this-spec-does-not-exist" '"status":404' 404 + + # The machine files must be served directly, never proxied to the + # seo backend — including for a mapped crawler UA, which is the whole + # point of the `location =` bypasses. robots.txt and sitemap.xml + # join llms.txt below: the sitemap is proxied to the API for every + # client, and a broken proxy would hand a crawler the SPA shell. + check "$GOOGLEBOT" "$ORIGIN/robots.txt" "User-agent: Bytespider" + check "$GOOGLEBOT" "$ORIGIN/sitemap.xml" " $wk_target" ;; + *) echo "::error::.well-known/llms.txt did not redirect to the guide: '$wk_target'"; fail=1 ;; + esac + # llms.txt must be served directly, never proxied to the seo backend — # including for a mapped crawler UA, which is the whole point of the # `location = /llms.txt` bypass. @@ -145,7 +176,9 @@ jobs: *) echo "OK: trailing slash -> $slash_target" ;; esac - # Control: humans must still get the SPA shell - check "$HUMAN" "$ORIGIN/" '
' + # Control: humans must still get the SPA shell — on the home page + # and on a deep route. + check "$HUMAN" "$ORIGIN/" '
' + check "$HUMAN" "$ORIGIN/scatter-basic" '
' exit $fail diff --git a/CHANGELOG.md b/CHANGELOG.md index b6ec19a528..517a8c4267 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,26 @@ aggregate instead: an italic *Catalog* line at the end of the version section an ### Added +- **LLM discoverability sync from kurrentschrift** — the sister project's + assistant-protocol findings (2026-08-28) carried over: assistants' fetch tools often allow + only URLs that already appeared verbatim in fetched content, so the machine guide and one + complete, callable example per surface now stand wherever an agent actually reads. The SPA + shell carries a `` and a ` diff --git a/app/nginx.conf b/app/nginx.conf index c95212b642..001e0352e4 100644 --- a/app/nginx.conf +++ b/app/nginx.conf @@ -84,6 +84,10 @@ map $http_user_agent $is_bot { # "insufficient relevant content" (AI-access audit 2026-08-19). No common # browser UA contains the substring. ~*grok 1; + # xAI's company token, for fetcher UAs that carry "xAI" but not "grok" + # (verified 2026-08-28: a UA "xAI-Bot" fell through ~*grok to the empty + # shell). No common browser UA contains the substring. + ~*xai 1; ~*youbot 1; ~*cohere-ai 1; ~*diffbot 1; @@ -223,6 +227,16 @@ server { # it generates is emitted as http://, downgrading the crawler's next # hop. The other proxied locations already set it. proxy_set_header X-Forwarded-Proto https; + # The CRAWLER's address must reach the API: it reports each read to + # the bot Plausible site with the visitor IP, and Plausible drops + # events whose forwarded IP is a hosting-provider address. On this + # hop cf-connecting-ip at the API is this container's Google egress + # IP, so the crawler is only ever visible in X-Forwarded-For — + # Cloudflare put it first at the edge; $proxy_add_x_forwarded_for + # appends our hop instead of replacing the list (kurrentschrift + # measured 1 counted read in 20 without this, 2026-08-28; + # api/request_context.py::visitor_ip reads the list leftmost-first). + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_ssl_server_name on; proxy_ssl_verify on; # api.anyplot.ai serves a 4-deep Let's Encrypt chain (leaf -> YE1 -> @@ -244,6 +258,14 @@ server { try_files $uri =404; } + # llms.txt has no registered .well-known name, but agents guess this path + # (docs/reference/seo.md "Discoverability") — answer with the file's + # canonical location instead of the SPA shell, which soft-404'd it with + # 200 + the home page until 2026-08-28. + location = /.well-known/llms.txt { + return 302 /llms.txt; + } + # llms-full.txt is generated from the DB (one line per spec), so unlike # llms.txt it cannot be a static file — proxy it to the API's # /llms-full.txt endpoint for EVERY client, mapped crawler UA or not. diff --git a/app/public/llms.txt b/app/public/llms.txt index 3f720509e5..20dc7e623f 100644 --- a/app/public/llms.txt +++ b/app/public/llms.txt @@ -44,6 +44,7 @@ and the spec itself at `.../plots/{spec_id}/specification.md`. - [Plot gallery](https://anyplot.ai/plots): every rendered implementation, filterable by library, language and tags - [Specifications](https://anyplot.ai/specs): the library-agnostic plot specs the implementations are generated from - [Libraries](https://anyplot.ai/libraries): the fifteen supported plotting libraries across four languages +- [Network map](https://anyplot.ai/map): every spec as a node, grouped by visual similarity — the catalogue's neighbourhoods (interactive; the data behind it is `GET https://api.anyplot.ai/specs/map`) - [Sitemap](https://anyplot.ai/sitemap.xml): every spec hub and implementation page ## Docs diff --git a/app/public/robots.txt b/app/public/robots.txt index 657a9163f6..9d45369602 100644 --- a/app/public/robots.txt +++ b/app/public/robots.txt @@ -1,3 +1,6 @@ +# llms.txt (machine guide): https://anyplot.ai/llms.txt +# OpenAPI: https://api.anyplot.ai/openapi.json +# # Crawler policy — everything here is open, to humans and machines alike. # # The catalogue is MIT-licensed and published to be reused: every plot page @@ -37,7 +40,10 @@ # first-match parser lets /debug and /interactive through (verified with # Python's urllib.robotparser, which is exactly such a parser). # -# Content signals per contentsignals.org: +# Content signals per contentsignals.org (these three tokens are the whole +# vocabulary; nothing else is valid on the line — a `use=reference` token +# used to ride along here, and a strict parser may discard the entire line +# over an unknown token, taking the three real signals with it): # search: indexing and returning links/excerpts — yes # ai-input: retrieval for AI answers, grounding, citation — yes # ai-train: training or fine-tuning models — yes @@ -50,16 +56,22 @@ # Control does the actual enforcing. If its behaviour changes, this group can # go — nothing in the licence or the policy above argues against it. User-agent: Bytespider -Content-Signal: search=yes,ai-input=yes,ai-train=yes,use=reference +Content-Signal: search=yes,ai-input=yes,ai-train=yes Disallow: / # Everyone else: search engines, AI assistants and their training crawlers, # social and link previews, feed readers. /debug and /interactive are app # internals rather than catalogue content and are not worth crawling. User-agent: * -Content-Signal: search=yes,ai-input=yes,ai-train=yes,use=reference +Content-Signal: search=yes,ai-input=yes,ai-train=yes Disallow: /debug Disallow: /interactive Allow: / Sitemap: https://anyplot.ai/sitemap.xml + +# Machine guide for AI assistants — every retrieval recipe with a full example +# URL (runnable source via the JSON API, render PNGs, OpenAPI, the MCP +# endpoint), and the whole catalogue as one line per spec: +# https://anyplot.ai/llms.txt +# https://anyplot.ai/llms-full.txt diff --git a/app/src/routes/seoCoverage.test.ts b/app/src/routes/seoCoverage.test.ts new file mode 100644 index 0000000000..77fc043b0e --- /dev/null +++ b/app/src/routes/seoCoverage.test.ts @@ -0,0 +1,106 @@ +// Drift guard for the machine-facing surface: the route registry in paths.ts, +// public/llms.txt, public/robots.txt and the SPA shell (index.html) must +// describe the SAME site and point at each other. Each of these has drifted +// silently before — a route missing from llms.txt, a guide findable only by +// guessing the convention, a shell that told a JS-less agent nothing — because +// nothing coupled them. This test is that coupling; the API side of the same +// contract (the API host's robots.txt, /llms.txt redirect, the bot-page nav) +// lives in tests/unit/api/test_routers.py. +// +// The files are read through Vite's `?raw` imports rather than node:fs so the +// test needs no Node typings — vitest resolves them exactly as the build does. + +import { describe, expect, it } from 'vitest'; + +import shell from '../../index.html?raw'; +import llmsTxt from '../../public/llms.txt?raw'; +import robotsTxt from '../../public/robots.txt?raw'; +import { paths } from './paths'; + +const ORIGIN = 'https://anyplot.ai'; +const API = 'https://api.anyplot.ai'; + +// The public content routes are the top-level string values of paths, minus +// the ones that are deliberately absent from the machine guide: '/' is the +// document's own subject, /debug is an app internal (robots.txt disallows it), +// and the search deep link is a view of /plots, not a page. +const publicPaths = Object.values(paths) + .flatMap(v => (typeof v === 'string' ? [v] : [])) + .filter(p => p !== paths.home && p !== paths.debug && !p.includes('?')); + +// One complete, callable example per surface. Assistants' fetch tools often +// allow only URLs that already appeared verbatim in fetched content, so the +// same example pair must stand in the guide AND in the shell — a template with +// placeholders satisfies neither (kurrentschrift finding 2026-08-28). +const EXAMPLE_CODE_URL = `${API}/specs/bar-error/seaborn/code`; +const EXAMPLE_RENDER_URL = + 'https://storage.googleapis.com/anyplot-images/plots/bar-error/python/seaborn/plot-light.png'; + +const escapeRe = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +describe('seo coverage', () => { + it('llms.txt links every public content route', () => { + // Word-boundary match, not substring: a link to /plots must not be + // satisfied by /plots?focus=search, nor /specs by /specs/…. The literal + // parts are regex-escaped — the dots in the origin would otherwise match + // any character and quietly weaken the guard. + for (const p of publicPaths) { + expect(llmsTxt, `llms.txt lacks ${p}`).toMatch( + new RegExp(`${escapeRe(ORIGIN + p)}(?![\\w./?-])`) + ); + } + }); + + it('llms.txt carries the complete example pair and the machine surfaces', () => { + expect(llmsTxt).toContain(EXAMPLE_CODE_URL); + expect(llmsTxt).toContain(EXAMPLE_RENDER_URL); + expect(llmsTxt).toContain(`${ORIGIN}/llms-full.txt`); + expect(llmsTxt).toContain(`${API}/openapi.json`); + expect(llmsTxt).toContain(`${API}/mcp/`); + }); + + it('robots.txt announces the sitemap and the machine guide, head and foot', () => { + // Two terse lines at the very top for readers that only take the file + // head seriously, the fuller block beside `Sitemap:` for everyone else. + const lines = robotsTxt.split('\n'); + expect(lines[0]).toBe(`# llms.txt (machine guide): ${ORIGIN}/llms.txt`); + expect(lines[1]).toBe(`# OpenAPI: ${API}/openapi.json`); + expect(robotsTxt).toContain(`Sitemap: ${ORIGIN}/sitemap.xml`); + expect(robotsTxt).toContain(`${ORIGIN}/llms-full.txt`); + }); + + it('robots.txt uses only the three contentsignals.org tokens, in every group', () => { + // `use=reference` used to ride along; it is not a signal, and a strict + // parser may discard the whole line over an unknown token — taking the + // three real ones with it. + const signals = robotsTxt.split('\n').filter(l => l.startsWith('Content-Signal:')); + const groups = robotsTxt.split('\n').filter(l => l.startsWith('User-agent:')); + expect(signals).toHaveLength(groups.length); + for (const s of signals) expect(s).toBe('Content-Signal: search=yes,ai-input=yes,ai-train=yes'); + }); + + it('the SPA shell itself carries the machine fallback', () => { + // Agents NOT in the nginx $is_bot map get the shell on every route, and + // most read raw HTML without executing JS. The shell must therefore name + // the machine guide and one complete, callable example per surface — in + // the head link and the