feat: LLM discoverability sync from kurrentschrift

.github/workflows/bot-serving-check.yml

@@ -28,13 +28,13 @@ permissions:
 jobs:
   bot-serving:
     runs-on: ubuntu-latest
-    # 22 check() calls x (--retry 2 -> up to 3 attempts x --max-time 30) can
-    # reach ~33 min worst-case, plus two non-retried probes (llms.txt charset,
-    # trailing slash — 30s each); 38 leaves room to report a clean failure
-    # rather than dying to the job timeout, which reports nothing useful.
-    # Recompute this when adding checks: the ceiling is check() calls x 90s,
-    # plus margin.
-    timeout-minutes: 38
+    # 27 check() calls x (--retry 2 -> up to 3 attempts x --max-time 30) can
+    # reach ~41 min worst-case, plus four non-retried probes (llms.txt charset,
+    # trailing slash, og-image, .well-known redirect — 30s each); 46 leaves
+    # room to report a clean failure rather than dying to the job timeout,
+    # which reports nothing useful. Recompute this when adding checks: the
+    # ceiling is check() calls x 90s, plus margin.
+    timeout-minutes: 46
     steps:
       - name: Crawler UAs must get 200 + per-route titles
         run: |
@@ -50,16 +50,16 @@ jobs:
           fail=0
 
           check() {
-            local ua="$1" url="$2" expect="$3"
+            local ua="$1" url="$2" expect="$3" want="${4:-200}"
             local code
             # On curl failure REPLACE the code — a failing curl can still have
             # printed a partial -w code; appending would yield e.g. "200000".
             code=$(curl -sS --retry 2 --max-time 30 -A "$ua" -o body.html -w '%{http_code}' "$url") || code="000"
-            if [ "$code" != "200" ]; then
-              echo "::error::$url with UA '$ua' returned HTTP $code (expected 200)"
+            if [ "$code" != "$want" ]; then
+              echo "::error::$url with UA '$ua' returned HTTP $code (expected $want)"
               fail=1
             elif ! grep -qF "$expect" body.html; then
-              echo "::error::$url with UA '$ua' returned 200 but the body is missing: $expect"
+              echo "::error::$url with UA '$ua' returned $want but the body is missing: $expect"
               fail=1
             else
               echo "OK: $url ($ua)"
@@ -101,11 +101,42 @@ jobs:
             "Mozilla/5.0 (compatible; Amzn-User/1.0)" \
             "Mozilla/5.0 (compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)" \
             "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)" \
-            "Grok/1.0"
+            "Grok/1.0" \
+            "Mozilla/5.0 (compatible; xAI-Bot/1.0)"
           do
             check "$ua" "$ORIGIN/scatter-basic" "<title>Basic Scatter Plot | anyplot.ai</title>"
           done
 
+          # A crawler asking for a URL that is no page gets a real 404 from the
+          # seo-proxy — the SPA shell would answer 200 (soft-404), and did for
+          # 161 stale migration URLs before the proxy learned to say no.
+          check "$GOOGLEBOT" "$ORIGIN/this-spec-does-not-exist" '"status":404' 404
+
+          # The machine files must be served directly, never proxied to the
+          # seo backend — including for a mapped crawler UA, which is the whole
+          # point of the `location =` bypasses. robots.txt and sitemap.xml
+          # join llms.txt below: the sitemap is proxied to the API for every
+          # client, and a broken proxy would hand a crawler the SPA shell.
+          check "$GOOGLEBOT" "$ORIGIN/robots.txt"  "User-agent: Bytespider"
+          check "$GOOGLEBOT" "$ORIGIN/sitemap.xml" "<urlset"
+
+          # The site card must be the FILE for a preview bot, not the proxy —
+          # a preview bot that lands on /seo-proxy/og-image.png shows nothing.
+          code=$(curl -sS --max-time 30 -A "$TWITTERBOT" -o /dev/null -w '%{http_code} %{content_type}' "$ORIGIN/og-image.png") || code="000"
+          case "$code" in
+            "200 image/png"*) echo "OK: og-image.png served as image to a preview bot" ;;
+            *) echo "::error::og-image.png for a preview bot: $code (expected 200 image/png)"; fail=1 ;;
+          esac
+
+          # A guessed /.well-known/llms.txt must land on the file, not on the
+          # SPA shell (which soft-404'd it with 200 until 2026-08-28).
+          wk_target=$(curl -sS --max-time 30 -o /dev/null -A "$CHATGPTUSER" \
+            -w '%{redirect_url}' "$ORIGIN/.well-known/llms.txt")
+          case "$wk_target" in
+            */llms.txt) echo "OK: .well-known/llms.txt -> $wk_target" ;;
+            *) echo "::error::.well-known/llms.txt did not redirect to the guide: '$wk_target'"; fail=1 ;;
+          esac
+
           # llms.txt must be served directly, never proxied to the seo backend —
           # including for a mapped crawler UA, which is the whole point of the
           # `location = /llms.txt` bypass.
@@ -145,7 +176,9 @@ jobs:
             *) echo "OK: trailing slash -> $slash_target" ;;
           esac
 
-          # Control: humans must still get the SPA shell
-          check "$HUMAN" "$ORIGIN/" '<div id="root">'
+          # Control: humans must still get the SPA shell — on the home page
+          # and on a deep route.
+          check "$HUMAN" "$ORIGIN/"              '<div id="root">'
+          check "$HUMAN" "$ORIGIN/scatter-basic" '<div id="root">'
 
           exit $fail

CHANGELOG.md

@@ -18,6 +18,26 @@ aggregate instead: an italic *Catalog* line at the end of the version section an
 
 ### Added
 
+- **LLM discoverability sync from kurrentschrift** — the sister project's
+  assistant-protocol findings (2026-08-28) carried over: assistants' fetch tools often allow
+  only URLs that already appeared verbatim in fetched content, so the machine guide and one
+  complete, callable example per surface now stand wherever an agent actually reads. The SPA
+  shell carries a `<link rel="alternate" href="/llms.txt">` and a `<noscript>` block with
+  absolute URLs (guide, `llms-full.txt`, one code endpoint, one render PNG, OpenAPI, MCP);
+  `/.well-known/llms.txt` redirects to the guide instead of soft-404ing with the shell, and
+  `api.anyplot.ai/llms.txt` redirects to the site file instead of 404ing; both `robots.txt`
+  files name `llms.txt` and the OpenAPI spec (two terse lines at the very top of the site
+  file, a fuller block beside `Sitemap:`, the API host's comment head); every bot page links
+  the guide from its footer nav, and every implementation page spells out its own code URL
+  and carries a visible JSON retrieval record beside the JSON-LD (HTML-to-Markdown converters
+  drop `<script>`, keep `<pre>`). The nginx map and `AI_AGENTS` gain `xai` (a `xAI-Bot` UA
+  fell through to the empty shell) and the bare `grok` token that was prerendered but never
+  counted. A new `asset_fetch` event on `bots.anyplot.ai` records which implementation's
+  code and which spec's detail assistants fetch through the API (cache misses — see
+  `docs/reference/plausible.md`). Guards: `app/src/routes/seoCoverage.test.ts` pins llms.txt
+  ↔ route registry ↔ robots.txt ↔ shell; `bot-serving-check` adds the `.well-known`
+  redirect, the robots/sitemap bypasses, the site card, a real 404 and a deep-route SPA
+  control. Doctrine: `docs/reference/seo.md` "Discoverability for assistants" (#10808).
 - **`llms-full.txt` — the whole catalogue in one fetch** — a new
   `GET /llms-full.txt` lists every spec on one line (id, title, hub URL, implemented
   libraries) under a header that documents the retrieval recipes that work for *every*
@@ -32,6 +52,17 @@ aggregate instead: an italic *Catalog* line at the end of the version section an
 
 ### Fixed
 
+- **Crawler reads on the bot site were mostly dropped by Plausible** — on the prerender path
+  (Cloud Run app → Cloudflare → API) `cf-connecting-ip` is the app container's Google egress
+  address, exactly the "hosting provider IP" Plausible discards, and `visitor_ip` preferred
+  it over the forwarded list where the crawler actually stands. kurrentschrift measured one
+  counted read in twenty with probe events (2026-08-28). `visitor_ip` now reads the leftmost
+  valid `x-forwarded-for` entry first and falls back to `cf-connecting-ip`; nginx's
+  `@seo_proxy` forwards the crawler in `X-Forwarded-For` explicitly (#10808).
+- **`robots.txt` carried an invalid `use=reference` Content-Signal token** — the
+  contentsignals.org vocabulary is exactly `search`, `ai-input` and `ai-train`; a strict
+  parser may discard the whole line over an unknown token, taking the three real signals with
+  it. Dropped from both groups; `seoCoverage.test.ts` pins the vocabulary (#10808).
 - **The generation retry cap counted a library's entire failure history, not the current
   run** — `impl-generate.yml` derives its "3 attempts" budget from hidden marker comments
   on the spec issue, and nothing ever deletes those markers. A `(spec, library)` pair that

api/analytics.py

@@ -70,6 +70,13 @@
     ("grok-deepsearch", "grok", "user_directed"),
     ("grokbot", "grok", "user_directed"),
     ("xai-grok", "grok", "user_directed"),
+    # The bare tokens the nginx map already serves (`~*grok`, `~*xai`): xAI's
+    # fetcher was seen sending plain "Grok" (AI-access audit 2026-08-19) and
+    # "xAI-Bot" without the grok substring (kurrentschrift, 2026-08-28). Both
+    # were prerendered but never counted, because nothing above matched them.
+    # Last among the xAI patterns so the specific ones keep winning.
+    ("grok", "grok", "user_directed"),
+    ("xai", "grok", "user_directed"),
     ("youbot", "you", "index"),
     ("cohere-ai", "cohere", "user_directed"),
     # Classic search crawlers. Worth recording for the same reason the AI ones
@@ -307,6 +314,68 @@ def track_og_image(
     task.add_done_callback(_handle_task_exception)
 
 
+# The single-asset API reads an assistant makes after landing on a page: the
+# runnable source of one implementation and the detail of one spec. Nothing
+# else on the API is counted — the list endpoints, /plots/filter and the
+# machine files are catalogue navigation, not a request for one thing.
+# `/specs/map` shares the shape of a spec id and is excluded by name.
+_ASSET_ROUTES: tuple[tuple[re.Pattern[str], str], ...] = (
+    (re.compile(r"^/specs/([a-z0-9]+(?:-[a-z0-9]+)*)/([a-z0-9]+(?:-[a-z0-9]+)*)/code$"), "code"),
+    (re.compile(r"^/specs/([a-z0-9]+(?:-[a-z0-9]+)*)$"), "spec"),
+)
+_NOT_A_SPEC = frozenset({"map"})
+
+
+def classify_asset(path: str) -> tuple[str, str, str | None] | None:
+    """(asset, spec_id, library_id) for a single-asset API path, else None.
+
+    `asset` is `code` (one implementation's runnable source) or `spec` (one
+    spec's detail); `library_id` is None for a spec read.
+    """
+    for pattern, asset in _ASSET_ROUTES:
+        m = pattern.match(path)
+        if m is None:
+            continue
+        spec_id = m.group(1)
+        if spec_id in _NOT_A_SPEC:
+            return None
+        return asset, spec_id, (m.group(2) if m.lastindex and m.lastindex >= 2 else None)
+    return None
+
+
+def track_asset_fetch(request: Request, *, asset: str, spec: str, library: str | None, status: int) -> None:
+    """Record an AI or search agent fetching one asset through the API (fire-and-forget).
+
+    Same site, same gate and same audience rule as `track_bot_fetch`: humans
+    and unclassified clients cost nothing beyond a substring scan, and the
+    event goes to BOT_DOMAIN. The props answer "which plots do assistants
+    actually pull the code of": `asset` (code · spec), `spec`, `library`.
+
+    One caveat the dashboard must know: these routes answer `public,
+    max-age=300`, so a read that Cloudflare serves from its edge never reaches
+    this process. The count is therefore the cache MISSES — the first fetch
+    per URL per edge TTL — which for sporadic assistant traffic is nearly all
+    of them, but not the same thing as every request.
+    """
+    detected = detect_ai_agent(request.headers.get("user-agent", ""))
+    if detected is None:
+        return
+    assistant, kind = detected
+    props: dict[str, str] = {"asset": asset, "spec": spec, "assistant": assistant, "kind": kind, "status": str(status)}
+    if library is not None:
+        props["library"] = library
+    url = f"https://api.anyplot.ai{request.url.path}"
+
+    task = asyncio.create_task(
+        _send_plausible_event(
+            request.headers.get("user-agent", ""), visitor_ip(request), "asset_fetch", url, props, domain=BOT_DOMAIN
+        )
+    )
+    _BACKGROUND_TASKS.add(task)
+    task.add_done_callback(_BACKGROUND_TASKS.discard)
+    task.add_done_callback(_handle_task_exception)
+
+
 def track_bot_fetch(request: Request, path: str, status: int) -> None:
     """Record an AI or search agent requesting a page (fire-and-forget).
 

api/main.py

@@ -16,7 +16,7 @@
 from fastapi.middleware.cors import CORSMiddleware  # noqa: E402
 from starlette.middleware.gzip import GZipMiddleware  # noqa: E402
 
-from api.analytics import track_bot_fetch  # noqa: E402
+from api.analytics import classify_asset, track_asset_fetch, track_bot_fetch  # noqa: E402
 from api.cache import cache_key, set_cache  # noqa: E402
 from api.exceptions import (  # noqa: E402
     AnyplotException,
@@ -181,16 +181,27 @@ async def lifespan(app: FastAPI):
 # longer exists is a signal worth keeping, it just is not a page view.
 @app.middleware("http")
 async def record_bot_fetch(request: Request, call_next):
-    """Report AI/search agent page requests to the bot analytics site.
+    """Report AI/search agent page requests and single-asset API reads to the bot site.
 
     Requests, not reads: the status is recorded rather than filtered on, so a
-    404 is visible as a miss instead of counted as a page view.
+    404 is visible as a miss instead of counted as a page view. Page reads
+    (/seo-proxy) become `bot_fetch`; one implementation's code or one spec's
+    detail fetched through the API becomes `asset_fetch` — the list endpoints
+    and the machine files are not recorded.
     """
     response: Response = await call_next(request)
     path = request.url.path
     if path.startswith("/seo-proxy"):
         # The public URL, never this router's internal prefix
         track_bot_fetch(request, path.removeprefix("/seo-proxy") or "/", response.status_code)
+        return response
+    asset = classify_asset(path)
+    if asset is not None:
+        # `asset_type` is what was fetched (code · spec) — not the agent's
+        # `kind` (user_directed · index · …), which track_asset_fetch derives
+        # from the user agent itself.
+        asset_type, spec_id, library_id = asset
+        track_asset_fetch(request, asset=asset_type, spec=spec_id, library=library_id, status=response.status_code)
     return response
 
 

api/request_context.py

@@ -51,17 +51,26 @@ def visitor_ip(request: Request) -> str:
     Spoofing is not a concern in this direction: a forged value skews a
     geolocation bucket, where forging the rate-limit key locked people out.
 
-    Order: `cf-connecting-ip`, which Cloudflare overwrites on proxied traffic
-    and is therefore both real and unforgeable; then the leftmost non-empty
-    forwarded entry; then the socket peer.
+    Order: the leftmost valid `x-forwarded-for` entry FIRST, then
+    `cf-connecting-ip`, then the socket peer. It used to be the other way
+    round, and that lost nearly every crawler read: the prerendered pages
+    reach this host through the site's nginx (Cloud Run app → Cloudflare →
+    API), so on that hop `cf-connecting-ip` is the app container's Google
+    egress address — exactly the "hosting provider IP" Plausible drops — while
+    the crawler itself is only visible as the first forwarded entry, where the
+    site-side Cloudflare put it. kurrentschrift measured the effect with probe
+    events on 2026-08-28 (34.90.x / 35.204.x dropped, a home IP kept): one
+    counted read in twenty. For a direct client behind Cloudflare the leftmost
+    entry and `cf-connecting-ip` are the same address, so nothing changes
+    there.
     """
-    cf_ip = request.headers.get("cf-connecting-ip", "").strip()
-    if _is_ip(cf_ip):
-        return cf_ip
     for entry in request.headers.get("x-forwarded-for", "").split(","):
         candidate = entry.strip()
         if _is_ip(candidate):
             return candidate
+    cf_ip = request.headers.get("cf-connecting-ip", "").strip()
+    if _is_ip(cf_ip):
+        return cf_ip
     return request.client.host if request.client else ""