doc/user: publish agent-readable Markdown and llms.txt for every page

bin/docs-markdown

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+#
+# docs-markdown -- renders a Markdown copy of every page in a built docs site
+
+exec "$(dirname "$0")"/pyactivate -m materialize.cli.docs_markdown "$@"

ci/deploy_website/docs-content-negotiation.js

@@ -0,0 +1,31 @@
+// Copyright Materialize, Inc. and contributors. All rights reserved.
+//
+// Use of this software is governed by the Business Source License
+// included in the LICENSE file at the root of this repository.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0.
+
+// CloudFront Function (cloudfront-js-2.0, viewer request) for the /docs/*
+// behavior of materialize.com. It serves a page's Markdown rendition, the
+// index.md that bin/docs-markdown writes beside each index.html, to clients
+// that send `Accept: text/markdown`.
+//
+// It must run on viewer request, not origin request: the rewritten URI is
+// then the cache key, so the HTML and Markdown renditions of a page are cached
+// separately. Responses on the behavior must carry `Vary: Accept` (set by its
+// response headers policy) so that caches downstream of CloudFront do the
+// same.
+//
+// This file is not deployed by CI. Publish changes to the function attached
+// to the distribution by hand.
+
+function handler(event) {
+  var request = event.request;
+  var accept = request.headers.accept ? request.headers.accept.value : "";
+  if (request.uri.endsWith("/") && accept.indexOf("text/markdown") !== -1) {
+    request.uri += "index.md";
+  }
+  return request;
+}

ci/deploy_website/website.sh

@@ -40,10 +40,10 @@ cd doc/user
 if [[ "$BUILDKITE_ORGANIZATION_SLUG" == "materialize" ]] && [[ "$BUILDKITE_BRANCH" == self-managed-docs/* ]]; then
     VERSION=${BUILDKITE_BRANCH#self-managed-docs/}
     hugo --gc --baseURL "/docs/self-managed/$VERSION" --destination "public/docs/self-managed/$VERSION"
+    ../../bin/docs-markdown "public/docs/self-managed/$VERSION" --base-url "https://materialize.com/docs/self-managed/$VERSION/"
 else
     hugo --gc --baseURL /docs --destination public/docs
-    # Build skill docs to public/docs/markdown-docs/.
-    hugo --gc --baseURL /docs --config config.toml,config.skill.toml --disableKinds 404,sitemap,robotsTXT,taxonomy --destination public/docs/markdown-docs
+    ../../bin/docs-markdown public/docs --base-url https://materialize.com/docs/
 fi
 hugo deploy --maxDeletes -1
 

ci/test/lint-docs.sh

@@ -17,7 +17,8 @@ set -euo pipefail
 
 git clean -ffdX ci/www/public
 try hugo --gc --baseURL https://ci.materialize.com/docs --source doc/user --destination ../../ci/www/public/docs
-try hugo --gc --baseURL https://ci.materialize.com/docs --source doc/user --config config.toml,config.skill.toml --disableKinds 404,sitemap,robotsTXT,taxonomy --destination ../../ci/www/public/docs/markdown-docs
+try bin/docs-markdown ci/www/public/docs --base-url https://ci.materialize.com/docs/
+try bin/pytest -qq misc/python/materialize/cli/docs_markdown_test.py
 echo "<!doctype html>" > ci/www/public/index.html
 try htmltest -s ci/www/public -c doc/user/.htmltest.yml
 try ci/test/lint-docs-catalog.sh

ci/test/preview-docs.sh

@@ -22,16 +22,13 @@ cd doc/user
 
 # Build main docs to public/
 hugo --gc --environment preview --baseURL "/materialize/$BUILDKITE_PULL_REQUEST"
-
-# Build skill docs to public/markdown-docs/
-hugo --config config.toml,config.skill.toml --gc --baseURL "/materialize/$BUILDKITE_PULL_REQUEST" --disableKinds 404,sitemap,robotsTXT,taxonomy
+../../bin/docs-markdown public --base-url "https://preview.materialize.com/materialize/$BUILDKITE_PULL_REQUEST/"
 
 cat > config.deployment.toml <<EOF
 [[deployment.targets]]
 name = "preview"
 url = "s3://materialize-website-previews?region=us-east-1&prefix=materialize/$BUILDKITE_PULL_REQUEST/"
 EOF
-# Single deploy: public/ contains both main site and markdown-docs/
 hugo deploy --config config.toml,config.deployment.toml --force
 
 curl -fsSL \

doc/user/AGENTS.md

@@ -18,18 +18,21 @@ This directory is the root of Materialize's Hugo user documentation site.
 - `resources/` and `public/` are generated build output. Avoid hand-editing
   them.
 
-### Output formats
+### Markdown rendition
 
-Every page renders as both the HTML site and a plain-Markdown `skill` output.
-The `config.skill.toml` overlay writes the Markdown output to
-`public/markdown-docs/` for use as agent-readable documentation. Templates
-therefore come in pairs, such as `layouts/_default/single.html` and
-`layouts/_default/single.skill.md`.
+Every page also publishes as Markdown at `<page URL>index.md`, for agents and
+other non-browser readers. `bin/docs-markdown` generates it after the Hugo
+build by converting each page's `<article>` element, so shortcodes need no
+Markdown variant. Chrome inside the `<article>` that should stay out of the
+Markdown, such as a copy button, takes the `data-markdown-ignore` attribute.
 
-When adding or changing a shortcode that emits HTML, add or update its
-`.skill.md` variant too. Without one, Hugo falls back to the `.html` template
-and can leak raw HTML into the Markdown output. Existing shortcodes without a
-variant are a known gap, not a pattern to copy.
+To inspect the Markdown for a template or shortcode change, run from the
+repository root:
+
+```shell
+hugo --source doc/user --destination /tmp/docs/docs --baseURL /docs
+bin/docs-markdown /tmp/docs/docs --base-url https://materialize.com/docs/
+```
 
 ### Reusing content
 
@@ -71,12 +74,10 @@ Run commands from `doc/user` unless noted otherwise:
 - `../../bin/format-docs` trims trailing whitespace and ensures Markdown files
   end with a newline.
 - `sql-grammar/generate.sh` regenerates railroad diagrams after BNF changes.
-- `hugo --config config.toml,config.skill.toml --disableKinds
-  sitemap,robotsTXT,taxonomy` builds the agent-readable Markdown output.
 - `./docs-pre-pr-review-claude.sh --worktree`, `--staged`, `--range A..B`, or a
   list of files runs the pre-PR content review.
 
-The full docs lint builds both output formats before running `htmltest` and the
+The full docs lint builds the site before running `htmltest` and the
 catalog and metrics checks. Install `htmltest` with `brew install htmltest` if
 it is unavailable locally.
 
@@ -200,10 +201,6 @@ run `../../ci/test/lint-docs.sh` to catch broken links, HTML errors, and
 catalog inconsistencies. Review generated diagrams and rendered examples when
 changing shortcodes, layouts, or SQL grammar.
 
-When changing templates or shortcodes, inspect both the HTML and Markdown
-outputs. A successful HTML render does not prove that the `skill` output is
-valid.
-
 ## Test syntax as you write the documentation
 
 When documentation adds or changes SQL syntax, test each example using the

doc/user/assets/sass/_base.scss

@@ -221,6 +221,18 @@ input[type="submit"] {
 
 .osano-cm-widget { display: none; }
 
+.llms-txt-directive {
+    position: absolute;
+    width: 1px;
+    height: 1px;
+    margin: -1px;
+    padding: 0;
+    overflow: hidden;
+    clip: rect(0, 0, 0, 0);
+    white-space: nowrap;
+    border: 0;
+}
+
 /**
     The following section handles these cases:
     1. Buttons with links

doc/user/config.toml

@@ -101,15 +101,6 @@ parent = "about"
 url = "https://materialize.com/securitydisclosure"
 weight = 55
 
-#
-# Custom output format for Claude skill generation
-#
-[outputFormats.skill]
-  mediaType = "text/markdown"
-  baseName = "index"
-  isPlainText = true
-  notAlternative = true
-
 #
 # Custom output format for the llms.txt docs index
 #
@@ -119,8 +110,25 @@ weight = 55
   isPlainText = true
   notAlternative = true
 
+#
+# The full sidebar tree, fetched by the sidebar script (baseof.html).
+#
+[outputFormats.sidebar]
+  mediaType = "text/html"
+  baseName = "sidebar"
+  isHTML = true
+  notAlternative = true
+
 [outputs]
-  home = ["HTML", "RSS", "llmstxt"]
+  home = ["HTML", "RSS", "llmstxt", "sidebar"]
+
+# Each top-level section publishes its own llms.txt, which the root llms.txt
+# links to.
+[[cascade]]
+  outputs = ["HTML", "RSS", "llmstxt"]
+  [cascade.target]
+    kind = "section"
+    path = "/*"
 
 [markup.goldmark.renderer]
 # allow <a name="link-target">, the old syntax no longer works
@@ -136,6 +144,16 @@ unsafe = true
     startLevel = 2
     endLevel = 4
 
+[[deployment.matchers]]
+# RFC 7763 makes charset a required parameter of text/markdown.
+pattern = "^.+\\.md$"
+contentType = "text/markdown; charset=utf-8"
+
+[[deployment.matchers]]
+# Hugo deploy would otherwise serve the sitemap as application/rss+xml.
+pattern = "^(.+/)?sitemap\\.xml$"
+contentType = "application/xml"
+
 [[deployment.targets]]
 name = "production"
 url = "s3://materialize-website?region=us-east-1"