From e075fc3f922e12812a473ec12692ce5eb1c521b4 Mon Sep 17 00:00:00 2001 From: Seth Wiesman Date: Thu, 1 Oct 2026 10:48:58 -0500 Subject: [PATCH 01/10] bin: remove gen-claude-skill The docs Markdown output is moving to serve agents on the web, with absolute links and per-page index files, which no longer fits a local skill bundle. Co-Authored-By: Claude Opus 5.5 --- bin/gen-claude-skill | 72 -------------------------------------------- 1 file changed, 72 deletions(-) delete mode 100755 bin/gen-claude-skill diff --git a/bin/gen-claude-skill b/bin/gen-claude-skill deleted file mode 100755 index 4625d4e745ccb..0000000000000 --- a/bin/gen-claude-skill +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License included in -# the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with the Business -# Source License, use of this software will be governed by the Apache License, -# Version 2.0. -# -# gen-claude-skill -- Generate a shared agent skill bundle from the -# documentation. -# -# Usage: bin/gen-claude-skill [OUTPUT_DIR] -# -# Arguments: OUTPUT_DIR Directory to output the skill files. The OUTPUT_DIR -# directory is cleaned before generation. -# Default: .agents/skills/mz-docs. -# For publishing to the website, -# we will output to doc/user/public/markdown-docs directory. -# -# Examples: -# bin/gen-claude-skill # Generate to default location -# bin/gen-claude-skill /path/to/output # Generate to custom location - -set -euo pipefail - -cd "$(dirname "$0")/.." - -# For publishing to the website, we will output to doc/user/public/markdown-docs -# directory. The OUTPUT_DIR directory is cleaned before generation. -OUTPUT_DIR="${1:-.agents/skills/mz-docs}" - -# Ensure output directory exists -mkdir -p "$OUTPUT_DIR" - -echo "Generating agent skill from documentation..." -echo "Output directory: $OUTPUT_DIR" - -# Build documentation with skill output format -cd doc/user -hugo --config config.toml,config.skill.toml \ - --disableKinds 404,sitemap,robotsTXT,taxonomy \ - --destination "../../$OUTPUT_DIR" \ - --cleanDestinationDir \ - --gc \ - --quiet - -cd ../.. - -# Rename the home page to SKILL.md (shared skill convention) -if [[ -f "$OUTPUT_DIR/index.md" ]]; then - mv "$OUTPUT_DIR/index.md" "$OUTPUT_DIR/SKILL.md" -fi - -# Collapse runs of consecutive blank lines down to a single blank line. -# Hugo emits a blank line in place of each stripped shortcode/template -# directive, which produces long runs around //
blocks -# in pages like ingest-data/mysql/* and self-managed-deployments/*. -find "$OUTPUT_DIR" -name '*.md' -type f -print0 | while IFS= read -r -d '' f; do - awk ' - /^[[:space:]]*$/ { if (blank) next; blank = 1; print ""; next } - { blank = 0; print } - ' "$f" > "$f.tmp" && mv "$f.tmp" "$f" -done - -# Count generated files -FILE_COUNT=$(find "$OUTPUT_DIR" -name "*.md" | wc -l | tr -d ' ') - -echo "Done! Generated $FILE_COUNT markdown files." -echo "Skill entry point: $OUTPUT_DIR/SKILL.md" From 02c84f8cb333ac32e186cf37636ddb2e18fa71b7 Mon Sep 17 00:00:00 2001 From: Seth Wiesman Date: Thu, 1 Oct 2026 11:55:12 -0500 Subject: [PATCH 02/10] doc/user: remove the Markdown output format The Markdown rendition is regenerated from the built HTML in a following commit, which removes the need for a Markdown twin of every shortcode. Co-Authored-By: Claude Opus 5.5 --- ci/deploy_website/website.sh | 2 - ci/test/lint-docs.sh | 1 - ci/test/preview-docs.sh | 4 - doc/user/AGENTS.md | 21 +--- doc/user/config.skill.toml | 39 -------- doc/user/config.toml | 9 -- doc/user/layouts/_default/list.skill.md | 20 ---- doc/user/layouts/_default/single.skill.md | 10 -- doc/user/layouts/index.skill.md | 56 ----------- .../yaml-tables/generic-table.skill.md | 21 ---- .../layouts/shortcodes/annotation.skill.md | 3 - doc/user/layouts/shortcodes/callout.skill.md | 9 -- doc/user/layouts/shortcodes/diagram.skill.md | 3 - .../explain-plans/operator-table.skill.md | 55 ----------- doc/user/layouts/shortcodes/fnlist.skill.md | 41 -------- .../layouts/shortcodes/if-released.skill.md | 9 -- .../layouts/shortcodes/important.skill.md | 2 - .../shortcodes/include-example.skill.md | 29 ------ .../shortcodes/include-from-yaml.skill.md | 27 ----- .../shortcodes/include-headless-with.skill.md | 17 ---- .../shortcodes/include-headless.skill.md | 14 --- .../layouts/shortcodes/include-md.skill.md | 5 - .../shortcodes/include-syntax.skill.md | 39 -------- .../layouts/shortcodes/json-parser.skill.md | 2 - doc/user/layouts/shortcodes/kwlist.skill.md | 29 ------ doc/user/layouts/shortcodes/note.skill.md | 2 - .../shortcodes/private-preview.skill.md | 1 - .../shortcodes/public-preview.skill.md | 2 - .../source-versioning-disambiguation.skill.md | 5 - .../sql-commands-table-by-label.skill.md | 98 ------------------- doc/user/layouts/shortcodes/tab.skill.md | 5 - doc/user/layouts/shortcodes/tabs.skill.md | 2 - doc/user/layouts/shortcodes/tip.skill.md | 2 - doc/user/layouts/shortcodes/warning.skill.md | 2 - .../layouts/shortcodes/yaml-list.skill.md | 23 ----- .../layouts/shortcodes/yaml-sections.skill.md | 19 ---- .../layouts/shortcodes/yaml-table.skill.md | 47 --------- 37 files changed, 1 insertion(+), 674 deletions(-) delete mode 100644 doc/user/config.skill.toml delete mode 100644 doc/user/layouts/_default/list.skill.md delete mode 100644 doc/user/layouts/_default/single.skill.md delete mode 100644 doc/user/layouts/index.skill.md delete mode 100644 doc/user/layouts/partials/yaml-tables/generic-table.skill.md delete mode 100644 doc/user/layouts/shortcodes/annotation.skill.md delete mode 100644 doc/user/layouts/shortcodes/callout.skill.md delete mode 100644 doc/user/layouts/shortcodes/diagram.skill.md delete mode 100644 doc/user/layouts/shortcodes/explain-plans/operator-table.skill.md delete mode 100644 doc/user/layouts/shortcodes/fnlist.skill.md delete mode 100644 doc/user/layouts/shortcodes/if-released.skill.md delete mode 100644 doc/user/layouts/shortcodes/important.skill.md delete mode 100644 doc/user/layouts/shortcodes/include-example.skill.md delete mode 100644 doc/user/layouts/shortcodes/include-from-yaml.skill.md delete mode 100644 doc/user/layouts/shortcodes/include-headless-with.skill.md delete mode 100644 doc/user/layouts/shortcodes/include-headless.skill.md delete mode 100644 doc/user/layouts/shortcodes/include-md.skill.md delete mode 100644 doc/user/layouts/shortcodes/include-syntax.skill.md delete mode 100644 doc/user/layouts/shortcodes/json-parser.skill.md delete mode 100644 doc/user/layouts/shortcodes/kwlist.skill.md delete mode 100644 doc/user/layouts/shortcodes/note.skill.md delete mode 100644 doc/user/layouts/shortcodes/private-preview.skill.md delete mode 100644 doc/user/layouts/shortcodes/public-preview.skill.md delete mode 100644 doc/user/layouts/shortcodes/source-versioning-disambiguation.skill.md delete mode 100644 doc/user/layouts/shortcodes/sql-commands-table-by-label.skill.md delete mode 100644 doc/user/layouts/shortcodes/tab.skill.md delete mode 100644 doc/user/layouts/shortcodes/tabs.skill.md delete mode 100644 doc/user/layouts/shortcodes/tip.skill.md delete mode 100644 doc/user/layouts/shortcodes/warning.skill.md delete mode 100644 doc/user/layouts/shortcodes/yaml-list.skill.md delete mode 100644 doc/user/layouts/shortcodes/yaml-sections.skill.md delete mode 100644 doc/user/layouts/shortcodes/yaml-table.skill.md diff --git a/ci/deploy_website/website.sh b/ci/deploy_website/website.sh index 686a70171b5b2..d27e3a1e73676 100755 --- a/ci/deploy_website/website.sh +++ b/ci/deploy_website/website.sh @@ -42,8 +42,6 @@ if [[ "$BUILDKITE_ORGANIZATION_SLUG" == "materialize" ]] && [[ "$BUILDKITE_BRANC hugo --gc --baseURL "/docs/self-managed/$VERSION" --destination "public/docs/self-managed/$VERSION" else hugo --gc --baseURL /docs --destination public/docs - # Build skill docs to public/docs/markdown-docs/. - hugo --gc --baseURL /docs --config config.toml,config.skill.toml --disableKinds 404,sitemap,robotsTXT,taxonomy --destination public/docs/markdown-docs fi hugo deploy --maxDeletes -1 diff --git a/ci/test/lint-docs.sh b/ci/test/lint-docs.sh index 250f03ad4c05c..77d65ef0ffd25 100755 --- a/ci/test/lint-docs.sh +++ b/ci/test/lint-docs.sh @@ -17,7 +17,6 @@ set -euo pipefail git clean -ffdX ci/www/public try hugo --gc --baseURL https://ci.materialize.com/docs --source doc/user --destination ../../ci/www/public/docs -try hugo --gc --baseURL https://ci.materialize.com/docs --source doc/user --config config.toml,config.skill.toml --disableKinds 404,sitemap,robotsTXT,taxonomy --destination ../../ci/www/public/docs/markdown-docs echo "" > ci/www/public/index.html try htmltest -s ci/www/public -c doc/user/.htmltest.yml try ci/test/lint-docs-catalog.sh diff --git a/ci/test/preview-docs.sh b/ci/test/preview-docs.sh index 8a122faf50a6f..d57efec9163f8 100755 --- a/ci/test/preview-docs.sh +++ b/ci/test/preview-docs.sh @@ -23,15 +23,11 @@ cd doc/user # Build main docs to public/ hugo --gc --environment preview --baseURL "/materialize/$BUILDKITE_PULL_REQUEST" -# Build skill docs to public/markdown-docs/ -hugo --config config.toml,config.skill.toml --gc --baseURL "/materialize/$BUILDKITE_PULL_REQUEST" --disableKinds 404,sitemap,robotsTXT,taxonomy - cat > config.deployment.toml < **{{ $type }}:** {{ .Inner | replaceRE "^\\s+" "" | replaceRE "\\n\\s*" " " }} diff --git a/doc/user/layouts/shortcodes/callout.skill.md b/doc/user/layouts/shortcodes/callout.skill.md deleted file mode 100644 index d7acd184fe089..0000000000000 --- a/doc/user/layouts/shortcodes/callout.skill.md +++ /dev/null @@ -1,9 +0,0 @@ -{{- /* Skill output: render callout as markdown blockquote */ -}} -> {{ .Inner | replaceRE "^\\s+" "" | replaceRE "\\s+$" "" | replaceRE "\\n\\s*\\n+" "\n" | replaceRE "\\n" "\n> " | replaceRE "\\n> \\s*$" "" }} -{{ if and ($.Params) (isset $.Params "primary_text") }} -> -> **{{ .Get "primary_text" }}**: {{ .Get "primary_url" }} -{{ end }} -{{ if and ($.Params) (isset $.Params "secondary_text") }} -> **{{ .Get "secondary_text" }}**: {{ .Get "secondary_url" }} -{{ end }} diff --git a/doc/user/layouts/shortcodes/diagram.skill.md b/doc/user/layouts/shortcodes/diagram.skill.md deleted file mode 100644 index 33ce07ed4dc28..0000000000000 --- a/doc/user/layouts/shortcodes/diagram.skill.md +++ /dev/null @@ -1,3 +0,0 @@ -{{- /* Skill output: diagrams are visual, describe or omit */ -}} -{{- $diagramName := .Get 0 | replaceRE "\\.svg$" "" -}} -_See syntax diagram: {{ $diagramName }}_ diff --git a/doc/user/layouts/shortcodes/explain-plans/operator-table.skill.md b/doc/user/layouts/shortcodes/explain-plans/operator-table.skill.md deleted file mode 100644 index 258f531059b1f..0000000000000 --- a/doc/user/layouts/shortcodes/explain-plans/operator-table.skill.md +++ /dev/null @@ -1,55 +0,0 @@ -{{- /* Skill output: operator-table renders as markdown table */ -}} -{{- $dataFile := .Get "data" -}} -{{- $planType := .Get "planType" -}} - -{{- if not $dataFile -}} - {{- errorf "operator-table shortcode requires a 'data' parameter" -}} -{{- end -}} -{{- if not $planType -}} - {{- errorf "operator-table shortcode requires a 'planType' parameter" -}} -{{- end -}} - -{{- $data := index $.Site.Data $dataFile -}} - -{{- $filteredOperators := slice -}} -{{- range $data.operators -}} - {{- if in .plan_types $planType -}} - {{- $filteredOperators = $filteredOperators | append . -}} - {{- end -}} -{{- end -}} - -{{- if gt (len $filteredOperators) 0 -}} -The following table lists the operators that are available in the {{ $planType }} plan. - -- For those operators that require memory to maintain intermediate state, **Uses memory** is marked with **Yes**. -- For those operators that expand the data size (either rows or columns), **Can increase data size** is marked with **Yes**. - -{{- $rows := slice -}} -{{- range $filteredOperators -}} - {{- $description := .description | markdownify -}} - {{- $expansiveText := "No" -}} - {{- if .expansive -}} - {{- $expansiveText = .expansive_details | markdownify -}} - {{- end -}} - {{- $memoryText := "No" -}} - {{- if .uses_memory -}} - {{- $memoryText = printf "✅ %s" (.memory_details | markdownify) -}} - {{- end -}} - {{- $fullDescription := printf "%s\n\n**Can increase data size:** %s\n**Uses memory:** %s" $description $expansiveText $memoryText -}} - {{- $row := dict "Operator" (printf "**%s**" .operator) "Description" $fullDescription "Example" (.example | markdownify) -}} - {{- $rows = $rows | append $row -}} -{{- end -}} - -{{- $columns := slice -}} -{{- $columns = $columns | append (dict "column" "Operator") -}} -{{- $columns = $columns | append (dict "column" "Description") -}} -{{- $columns = $columns | append (dict "column" "Example") -}} - -{{- partial "yaml-tables/generic-table.skill.md" (dict "rows" $rows "columns" $columns) -}} - -**Notes:** -- **Can increase data size:** Specifies whether the operator can increase the data size (can be the number of rows or the number of columns). -- **Uses memory:** Specifies whether the operator use memory to maintain state for its inputs. -{{- else -}} -*No operators found for plan type "{{ $planType }}".* -{{- end -}} diff --git a/doc/user/layouts/shortcodes/fnlist.skill.md b/doc/user/layouts/shortcodes/fnlist.skill.md deleted file mode 100644 index 3bb0b610c55a9..0000000000000 --- a/doc/user/layouts/shortcodes/fnlist.skill.md +++ /dev/null @@ -1,41 +0,0 @@ -{{- /* Skill output: converts data/sql_funcs.yml into markdown list */ -}} -{{- $parentPath := partial "relative-link.html" . -}} -{{- $releasedVersions := dict -}} -{{- range (where $.Site.RegularPages "Section" "releases") -}} - {{- $releasedVersions = merge $releasedVersions (dict .File.ContentBaseName .) -}} -{{- end -}} - -{{- range $.Site.Data.sql_funcs -}} - -{{- if not (isset $.Params 0) -}} - -### {{ .type }} functions - -{{- if .description -}} -{{ .description | $.Page.RenderString }} -{{- end -}} - -{{- end -}} - -{{- if or (eq ($.Get 0) .type) (not (isset $.Params 0)) -}} - -{{- range .functions -}} -#### `{{ .signature }}` - -{{ .description | $.Page.RenderString }}{{ if .url }} [(docs)]({{ $parentPath }}{{ .url | relURL }}){{ end }}{{ if .unmaterializable }} - -**Note:** This function is [unmaterializable](#unmaterializable-functions).{{ end }}{{ if .unmaterializable_unless_temporal_filter }} - -**Note:** This function is [unmaterializable](#unmaterializable-functions), but can be used in limited contexts in materialized views as a [temporal filter]({{ $parentPath }}{{ "/transform-data/patterns/temporal-filters/" | relURL }}).{{ end }}{{ if .known_time_zone_limitation_cast }} - -**Known limitation:** You must explicitly cast the type for the time zone.{{ end }}{{ if .side_effecting }} - -**Note:** This function is [side-effecting](#side-effecting-functions).{{ end }}{{ $versionAdded := index . "version-added" }}{{ if $versionAdded }}{{ $releasePage := index $releasedVersions $versionAdded }}{{ if not $releasePage.Params.released }} - -**Unreleased:** This function will be released in [**{{ $versionAdded }}**]({{ printf "%s/releases#release-notes" $parentPath | relURL }}). It may not be available in your region yet. The release is scheduled to complete by **{{ dateFormat "January 2, 2006" $releasePage.Params.date }}**.{{ end }}{{ end }} - -{{- end -}} - -{{- end -}} - -{{- end -}} diff --git a/doc/user/layouts/shortcodes/if-released.skill.md b/doc/user/layouts/shortcodes/if-released.skill.md deleted file mode 100644 index 2d4123f27c562..0000000000000 --- a/doc/user/layouts/shortcodes/if-released.skill.md +++ /dev/null @@ -1,9 +0,0 @@ -{{$version := .Get 0}} -{{$releasePages := where $.Site.RegularPages "Section" "releases"}} -{{$releasePage := (index (where $releasePages ".File.ContentBaseName" $version) 0)}} -{{$isReleased := $releasePage.Params.released}} -{{$releaseDate := $releasePage.Params.date}} - -{{if $isReleased}} -{{ .Inner }} -{{end}} diff --git a/doc/user/layouts/shortcodes/important.skill.md b/doc/user/layouts/shortcodes/important.skill.md deleted file mode 100644 index 4c086066ae782..0000000000000 --- a/doc/user/layouts/shortcodes/important.skill.md +++ /dev/null @@ -1,2 +0,0 @@ -{{- /* Skill output: render important as markdown blockquote */ -}} -> **Important:** {{ .Inner | replaceRE "^\\s+" "" | replaceRE "\\s+$" "" | replaceRE "\\n\\s*\\n+" "\n" | replaceRE "\\n" "\n> " | replaceRE "\\n> \\s*$" "" }} diff --git a/doc/user/layouts/shortcodes/include-example.skill.md b/doc/user/layouts/shortcodes/include-example.skill.md deleted file mode 100644 index 9227d8dcbb3c9..0000000000000 --- a/doc/user/layouts/shortcodes/include-example.skill.md +++ /dev/null @@ -1,29 +0,0 @@ -{{- /* Skill output: include-example renders code examples from YAML */ -}} -{{- $pathArray := split (lower (.Get "file")) "/" -}} -{{- $data := .Site.Data -}} -{{- range $pathArray }} - {{- $data = index $data . -}} -{{- end }} -{{- $example := .Get "example" -}} -{{- $indent := .Get "indent" -}} -{{- range $data }} -{{- if eq .name $example -}} -{{- .description -}} -{{- if .code -}} -{{- $code := .code -}} -{{- if $indent }} -{{- $code = replaceRE "(?m)^" " " $code -}} -{{- end }} -{{- /* Trim trailing whitespace from each line */ -}} -{{- $code = replaceRE "(?m)[ \t]+$" "" $code -}} - -{{ if $indent }} {{ end }}```mzsql -{{ $code | safeHTML }} -{{ if $indent }} {{ end }}``` -{{- end -}} -{{- if .results }} - -{{- if $indent }} {{ end }}{{ .results -}} -{{- end -}} -{{- end -}} -{{- end -}} diff --git a/doc/user/layouts/shortcodes/include-from-yaml.skill.md b/doc/user/layouts/shortcodes/include-from-yaml.skill.md deleted file mode 100644 index 871212b87cce9..0000000000000 --- a/doc/user/layouts/shortcodes/include-from-yaml.skill.md +++ /dev/null @@ -1,27 +0,0 @@ -{{- /* Skill output: include-from-yaml renders content from YAML data */ -}} -{{- $pathArray := split (lower (.Get "data")) "/" -}} -{{- $data := $.Site.Data -}} -{{- range $pathArray }} - {{- $data = index $data . -}} -{{- end }} -{{- $name := .Get "name" -}} -{{- $field := .Get "field" | default "content" -}} - -{{- $rows := $data -}} -{{- if reflect.IsMap $data -}} - {{- $rows = $data.rows -}} -{{- end -}} - -{{- range $rows -}} -{{- if eq .name $name -}} -{{- $content := index . $field -}} -{{- /* If the snippet contains nested shortcodes, render it (skill shortcode - variants resolve, but output is HTML). Otherwise emit the raw markdown so - skill output stays clean markdown. */ -}} -{{- if findRE "{{[<%]" $content -}} -{{- $content | $.Page.RenderString -}} -{{- else -}} -{{- $content -}} -{{- end -}} -{{- end -}} -{{- end -}} diff --git a/doc/user/layouts/shortcodes/include-headless-with.skill.md b/doc/user/layouts/shortcodes/include-headless-with.skill.md deleted file mode 100644 index 8473e43e7973d..0000000000000 --- a/doc/user/layouts/shortcodes/include-headless-with.skill.md +++ /dev/null @@ -1,17 +0,0 @@ -{{- /* Skill output: inline the included markdown content, substituting named params */ -}} -{{- $file := .Get "file" -}} -{{- with $.Page.GetPage $file -}} - {{- $out := .RenderShortcodes -}} - {{- range $k, $v := $.Params -}} - {{- if ne $k "file" -}} - {{- $out = replace $out (printf "__%s__" (upper $k)) $v -}} - {{- end -}} - {{- end -}} - {{- $out = replaceRE `{{__hugo_ctx[^}]*}}` "" $out -}} - {{- $out = replace $out "{{__hugo_ctx/}}" "" -}} - {{- $out = replaceRE "^\\s+" "" $out -}} - {{- $out = replaceRE "\\s+$" "" $out -}} - {{- $out -}} -{{- else -}} - {{- errorf "The %q shortcode was unable to find %q. See %s" $.Name $file $.Position -}} -{{- end -}} diff --git a/doc/user/layouts/shortcodes/include-headless.skill.md b/doc/user/layouts/shortcodes/include-headless.skill.md deleted file mode 100644 index b20b6e25e1522..0000000000000 --- a/doc/user/layouts/shortcodes/include-headless.skill.md +++ /dev/null @@ -1,14 +0,0 @@ -{{- with .Get 0 -}} - {{- with $.Page.GetPage . -}} - {{- $out := .RenderShortcodes -}} - {{- $out = replaceRE `{{__hugo_ctx[^}]*}}` "" $out -}} - {{- $out = replace $out "{{__hugo_ctx/}}" "" -}} - {{- $out = replaceRE "^\\s+" "" $out -}} - {{- $out = replaceRE "\\s+$" "" $out -}} - {{- $out -}} - {{- else -}} - {{- errorf "The %q shortcode was unable to find %q. See %s" $.Name . $.Position -}} - {{- end -}} -{{- else -}} - {{- errorf "The %q shortcode requires a positional parameter indicating the logical path of the file to include. See %s" .Name .Position -}} -{{- end -}} diff --git a/doc/user/layouts/shortcodes/include-md.skill.md b/doc/user/layouts/shortcodes/include-md.skill.md deleted file mode 100644 index c6dac9a15ee8d..0000000000000 --- a/doc/user/layouts/shortcodes/include-md.skill.md +++ /dev/null @@ -1,5 +0,0 @@ -{{- /* Skill output: inline the included markdown content with shortcode processing */ -}} -{{- $path := .Get "file" -}} -{{- $content := readFile $path -}} -{{- $content = replaceRE "(?s)^---\n.*?\n---\n" "" $content -}} -{{- $content | $.Page.RenderString -}} diff --git a/doc/user/layouts/shortcodes/include-syntax.skill.md b/doc/user/layouts/shortcodes/include-syntax.skill.md deleted file mode 100644 index 296b20a14facf..0000000000000 --- a/doc/user/layouts/shortcodes/include-syntax.skill.md +++ /dev/null @@ -1,39 +0,0 @@ -{{- /* Skill output: include-syntax renders code block and markdown table */ -}} -{{- $pathArray := split (lower (.Get "file")) "/" -}} -{{- $data := .Site.Data -}} -{{- range $pathArray }} - {{- $data = index $data . -}} -{{- end }} -{{- $example := .Get "example" -}} -{{- $indent := .Get "indent" -}} -{{- range $data }} -{{- if eq .name $example -}} -{{- if .description }} -{{- .description -}} -{{ end -}} -{{- if .code -}} -{{- $code := .code -}} -{{- if $indent }} -{{- $code = replaceRE "(?m)^" " " $code -}} -{{- end }} - -{{ if $indent }} {{ end }}```mzsql -{{ $code | safeHTML }} -{{ if $indent }} {{ end }}``` -{{- end -}} -{{- if .syntax_elements }} - -{{ partial "yaml-tables/generic-table.skill.md" (dict - "columns" (slice - (dict "column" "name" "header" "Syntax element") - (dict "column" "description" "header" "Description") - ) - "rows" .syntax_elements -) }} -{{- end -}} -{{- if .addenda }} - -{{- .addenda -}} -{{- end -}} -{{- end -}} -{{- end -}} diff --git a/doc/user/layouts/shortcodes/json-parser.skill.md b/doc/user/layouts/shortcodes/json-parser.skill.md deleted file mode 100644 index dd10b2fce07a5..0000000000000 --- a/doc/user/layouts/shortcodes/json-parser.skill.md +++ /dev/null @@ -1,2 +0,0 @@ -{{- /* Skill output: link to JSON parsing widget */ -}} -Manually parsing JSON-formatted data in SQL can be tedious. You can use the [interactive JSON parser widget](https://materialize.com/docs/sql/types/jsonb/#parsing) to automatically turn a sample JSON payload into a parsing view with the individual fields mapped to columns. diff --git a/doc/user/layouts/shortcodes/kwlist.skill.md b/doc/user/layouts/shortcodes/kwlist.skill.md deleted file mode 100644 index fe513a9c88e1c..0000000000000 --- a/doc/user/layouts/shortcodes/kwlist.skill.md +++ /dev/null @@ -1,29 +0,0 @@ -{{- /* Skill output: list SQL keywords from keywords.txt */ -}} -{{- $keywords := slice -}} -{{- range $line := split (readFile "sql-grammar/keywords.txt") "\n" -}} - {{- if and (not (hasPrefix $line "#")) $line -}} - {{- $keywords = $keywords | append ($line | upper) -}} - {{- end -}} -{{- end -}} - -{{- $columnCount := 4 -}} -{{- $totalKeywords := len $keywords -}} -{{- $numRows := div $totalKeywords $columnCount -}} -{{- if gt (mod $totalKeywords $columnCount) 0 -}} - {{- $numRows = add $numRows 1 -}} -{{- end -}} - -| | | | | -|--|--|--|--| -{{- range $row := seq $numRows -}} -{{- $rowIndex := sub $row 1 -}} -| {{- range $col := seq $columnCount -}} - {{- $index := add (mul $rowIndex $columnCount) (sub $col 1) -}} - {{- if and (ge $index 0) (lt $index $totalKeywords) -}} -`{{ index $keywords $index }}` - {{- else -}} -  - {{- end -}} - {{- if lt $col $columnCount }} |{{ end -}} -{{- end -}} | -{{- end -}} diff --git a/doc/user/layouts/shortcodes/note.skill.md b/doc/user/layouts/shortcodes/note.skill.md deleted file mode 100644 index c5523fc240cd7..0000000000000 --- a/doc/user/layouts/shortcodes/note.skill.md +++ /dev/null @@ -1,2 +0,0 @@ -{{- /* Skill output: render note as markdown blockquote */ -}} -> **Note:** {{ .Inner | replaceRE "^\\s+" "" | replaceRE "\\s+$" "" | replaceRE "\\n\\s*\\n+" "\n" | replaceRE "\\n" "\n> " | replaceRE "\\n> \\s*$" "" }} diff --git a/doc/user/layouts/shortcodes/private-preview.skill.md b/doc/user/layouts/shortcodes/private-preview.skill.md deleted file mode 100644 index 850f9b3590cf6..0000000000000 --- a/doc/user/layouts/shortcodes/private-preview.skill.md +++ /dev/null @@ -1 +0,0 @@ -{{- /* Skill output: omit private preview marker */ -}} diff --git a/doc/user/layouts/shortcodes/public-preview.skill.md b/doc/user/layouts/shortcodes/public-preview.skill.md deleted file mode 100644 index 9298735aa0f64..0000000000000 --- a/doc/user/layouts/shortcodes/public-preview.skill.md +++ /dev/null @@ -1,2 +0,0 @@ -{{- /* Skill output: show public preview note */ -}} -> **Public Preview:** This feature is in public preview. diff --git a/doc/user/layouts/shortcodes/source-versioning-disambiguation.skill.md b/doc/user/layouts/shortcodes/source-versioning-disambiguation.skill.md deleted file mode 100644 index a3ca15c8787a6..0000000000000 --- a/doc/user/layouts/shortcodes/source-versioning-disambiguation.skill.md +++ /dev/null @@ -1,5 +0,0 @@ -{{- /* Skill output: render source versioning disambiguation as blockquote */ -}} -{{- $is_new := .Get "is_new" -}} -{{- $include_blurb := .Get "include_blurb" -}} -{{- $other_ref := .Get "other_ref" -}} -> **Disambiguation:** {{ if $include_blurb }}{{ if $is_new }}This page reflects the new syntax which allows Materialize to handle upstream DDL changes, specifically adding or dropping columns, without downtime. For the deprecated syntax, see the {{ $other_ref }}.{{ else }}This page reflects the legacy syntax, which requires downtime to handle upstream DDL changes. For the new syntax which can handle adding or dropping columns to the upstream tables without downtime, see the {{ $other_ref }}.{{ end }}{{ else }}{{ if $is_new }}This page reflects the new syntax. For the legacy syntax, see the {{ $other_ref }}.{{ else }}This page reflects the legacy syntax. For the new syntax, see {{ $other_ref }}.{{ end }}{{ end }} diff --git a/doc/user/layouts/shortcodes/sql-commands-table-by-label.skill.md b/doc/user/layouts/shortcodes/sql-commands-table-by-label.skill.md deleted file mode 100644 index 0be8a3f77345d..0000000000000 --- a/doc/user/layouts/shortcodes/sql-commands-table-by-label.skill.md +++ /dev/null @@ -1,98 +0,0 @@ -{{- /* Skill output: sql-commands-table-by-label renders as markdown table */ -}} -{{- $dataPath := .Get "data" | default "sql_commands_all" -}} -{{- $label := .Get "label" -}} -{{- $columns := .Get "columns" | default "command" -}} -{{- $groupBy := .Get "group_by" -}} - -{{- if not $label -}} - {{- errorf "sql-commands-by-label shortcode requires a 'label' parameter" -}} -{{- end -}} - -{{- $parts := split $dataPath "/" -}} -{{- $data := index site.Data (index $parts 0) -}} -{{- range $i, $key := after 1 $parts -}} - {{- $data = index $data $key -}} -{{- end -}} - -{{- $columnNames := split $columns "," -}} -{{- $columnList := slice -}} -{{- range $columnNames -}} - {{- $colName := trim . " " -}} - {{- $capitalized := $colName | title -}} - {{- $columnList = $columnList | append (dict "column" $capitalized) -}} -{{- end -}} - -{{- $filteredRows := slice -}} -{{- range $data.rows -}} - {{- if in .labels $label -}} - {{- $filteredRows = $filteredRows | append . -}} - {{- end -}} -{{- end -}} - -{{- if gt (len $filteredRows) 0 -}} - {{- if $groupBy -}} - {{- $scratch := newScratch -}} - {{- $groupKeys := slice -}} - {{- $verbs := slice -}} - {{- range $filteredRows -}} - {{- $groupKey := index . $groupBy -}} - {{- if $groupKey -}} - {{- if not (in $groupKeys $groupKey) -}} - {{- $groupKeys = $groupKeys | append $groupKey -}} - {{- end -}} - {{- $commandText := index . "command" -}} - {{- $commandMatch := findRE "`([A-Z]+(?:\\s+[A-Z]+)*)`" $commandText -}} - {{- if $commandMatch -}} - {{- $fullCommand := index $commandMatch 0 -}} - {{- $commandWords := split (trim (replaceRE "`" "" $fullCommand) " ") " " -}} - {{- $verb := index $commandWords 0 -}} - {{- if not (in $verbs $verb) -}} - {{- $verbs = $verbs | append $verb -}} - {{- end -}} - {{- $existingCommands := $scratch.Get (printf "%s_%s" $groupKey $verb) -}} - {{- if not $existingCommands -}} - {{- $scratch.Set (printf "%s_%s" $groupKey $verb) $commandText -}} - {{- end -}} - {{- end -}} - {{- end -}} - {{- end -}} - {{- $groupKeys = sort $groupKeys -}} - {{- $sortedVerbs := slice -}} - {{- $priorityVerbs := slice "CREATE" "ALTER" "DROP" -}} - {{- range $priorityVerb := $priorityVerbs -}} - {{- if in $verbs $priorityVerb -}} - {{- $sortedVerbs = $sortedVerbs | append $priorityVerb -}} - {{- end -}} - {{- end -}} - {{- $remainingVerbs := slice -}} - {{- range $verb := $verbs -}} - {{- if not (in $priorityVerbs $verb) -}} - {{- $remainingVerbs = $remainingVerbs | append $verb -}} - {{- end -}} - {{- end -}} - {{- $remainingVerbs = sort $remainingVerbs -}} - {{- $verbs = $sortedVerbs -}} - {{- range $verb := $remainingVerbs -}} - {{- $verbs = $verbs | append $verb -}} - {{- end -}} - {{- $pivotRows := slice -}} - {{- range $groupKey := $groupKeys -}} - {{- $row := dict "Object" $groupKey -}} - {{- range $verb := $verbs -}} - {{- $command := $scratch.Get (printf "%s_%s" $groupKey $verb) -}} - {{- $row = merge $row (dict $verb ($command | default "")) -}} - {{- end -}} - {{- $pivotRows = $pivotRows | append $row -}} - {{- end -}} - {{- $pivotColumns := slice -}} - {{- range $verb := $verbs -}} - {{- $capitalized := $verb | title -}} - {{- $pivotColumns = $pivotColumns | append (dict "column" $capitalized) -}} - {{- end -}} - {{- partial "yaml-tables/generic-table.skill.md" (dict "rows" $pivotRows "columns" $pivotColumns) -}} - {{- else -}} - {{- partial "yaml-tables/generic-table.skill.md" (dict "rows" $filteredRows "columns" $columnList) -}} - {{- end -}} -{{- else -}} -*No commands found with label "{{ $label }}".* -{{- end -}} diff --git a/doc/user/layouts/shortcodes/tab.skill.md b/doc/user/layouts/shortcodes/tab.skill.md deleted file mode 100644 index c85fefb8c6195..0000000000000 --- a/doc/user/layouts/shortcodes/tab.skill.md +++ /dev/null @@ -1,5 +0,0 @@ -{{- /* Skill output: render tab with heading */ -}} -{{- $title := .Get 0 -}} -**{{ $title }}:** - -{{- .Inner -}} diff --git a/doc/user/layouts/shortcodes/tabs.skill.md b/doc/user/layouts/shortcodes/tabs.skill.md deleted file mode 100644 index 11d6d747be258..0000000000000 --- a/doc/user/layouts/shortcodes/tabs.skill.md +++ /dev/null @@ -1,2 +0,0 @@ -{{- /* Skill output: render tabs content sequentially */ -}} -{{- .Inner -}} diff --git a/doc/user/layouts/shortcodes/tip.skill.md b/doc/user/layouts/shortcodes/tip.skill.md deleted file mode 100644 index d22769a0d8b90..0000000000000 --- a/doc/user/layouts/shortcodes/tip.skill.md +++ /dev/null @@ -1,2 +0,0 @@ -{{- /* Skill output: render tip as markdown blockquote */ -}} -> **Tip:** {{ .Inner | replaceRE "^\\s+" "" | replaceRE "\\s+$" "" | replaceRE "\\n\\s*\\n+" "\n" | replaceRE "\\n" "\n> " | replaceRE "\\n> \\s*$" "" }} diff --git a/doc/user/layouts/shortcodes/warning.skill.md b/doc/user/layouts/shortcodes/warning.skill.md deleted file mode 100644 index 3c3648ad5d6d9..0000000000000 --- a/doc/user/layouts/shortcodes/warning.skill.md +++ /dev/null @@ -1,2 +0,0 @@ -{{- /* Skill output: render warning as markdown blockquote */ -}} -> **Warning:** {{ .Inner | replaceRE "^\\s+" "" | replaceRE "\\s+$" "" | replaceRE "\\n\\s*\\n+" "\n" | replaceRE "\\n" "\n> " | replaceRE "\\n> \\s*$" "" }} diff --git a/doc/user/layouts/shortcodes/yaml-list.skill.md b/doc/user/layouts/shortcodes/yaml-list.skill.md deleted file mode 100644 index b875647a9b406..0000000000000 --- a/doc/user/layouts/shortcodes/yaml-list.skill.md +++ /dev/null @@ -1,23 +0,0 @@ -{{- /* Skill output: yaml-list renders as markdown list */ -}} -{{- $column := .Get "column" -}} -{{- $label := .Get "label" -}} -{{- $pathArray := split (lower (.Get "data")) "/" -}} -{{- $data := $.Site.Data -}} -{{- range $pathArray -}} - {{- $data = index $data . -}} -{{- end -}} -{{ if $label }} -{{- $filteredRows := slice -}} -{{- range $data.rows -}} - {{- if in .labels $label -}} - {{- $filteredRows = $filteredRows | append . -}} - {{- end -}} -{{- end -}} -{{ range $filteredRows }} -- {{ index . "command" }} -{{ end -}} -{{ else }} -{{ range $data.rows -}} -- {{ index . $column }} -{{ end -}} -{{ end -}} diff --git a/doc/user/layouts/shortcodes/yaml-sections.skill.md b/doc/user/layouts/shortcodes/yaml-sections.skill.md deleted file mode 100644 index f646022442ef8..0000000000000 --- a/doc/user/layouts/shortcodes/yaml-sections.skill.md +++ /dev/null @@ -1,19 +0,0 @@ -{{- /* Skill output: yaml-sections renders sections with markdown headings */ -}} -{{- $dataPath := .Get "data" -}} -{{- $headingField := .Get "heading-field" | default "title" -}} -{{- $headingLevel := .Get "heading-level" | default "3" | int -}} -{{- $headingPrefix := "" -}} -{{- range seq $headingLevel -}} - {{- $headingPrefix = printf "%s#" $headingPrefix -}} -{{- end -}} -{{- $parts := split $dataPath "/" -}} -{{- $data := index site.Data (index $parts 0) -}} -{{- range $i, $key := after 1 $parts -}} - {{- $data = index $data $key -}} -{{- end -}} -{{ range $section := $data }} -{{ $heading := index $section $headingField }} -{{ $headingPrefix }} {{ $heading }} - -{{ $section.description }} -{{ end -}} diff --git a/doc/user/layouts/shortcodes/yaml-table.skill.md b/doc/user/layouts/shortcodes/yaml-table.skill.md deleted file mode 100644 index ec5a0c76d1097..0000000000000 --- a/doc/user/layouts/shortcodes/yaml-table.skill.md +++ /dev/null @@ -1,47 +0,0 @@ -{{- /* Skill output: yaml-table renders as markdown table with shortcode processing */ -}} -{{- $pathArray := split (lower (.Get "data")) "/" -}} -{{- $noHeader := .Get "noHeader" -}} -{{- $columnsParam := .Get "columns" -}} -{{- $data := $.Site.Data -}} -{{- range $pathArray }} - {{- $data = index $data . -}} -{{- end }} - -{{- $columns := $data.columns -}} -{{- if $columnsParam -}} - {{- $wanted := split $columnsParam "," -}} - {{- $filtered := slice -}} - {{- range $wanted -}} - {{- $name := strings.TrimSpace . -}} - {{- range $data.columns -}} - {{- if eq .column $name -}} - {{- $filtered = $filtered | append . -}} - {{- end -}} - {{- end -}} - {{- end -}} - {{- $columns = $filtered -}} -{{- end -}} - -{{- $fields := slice -}} -{{- $headers := slice -}} -{{- $separators := slice -}} -{{- range $columns -}} - {{- $headers = $headers | append (.header | default .column) -}} - {{- $fields = $fields | append (dict "field" .column) -}} - {{- $separators = $separators | append "---" -}} -{{- end -}} -{{- if not $noHeader }} -| {{ delimit $headers " | " }} | -| {{ delimit $separators " | " }} | -{{- end }} -{{- range $data.rows }} -{{- $row := . -}} -{{- $cells := slice -}} -{{- range $fields -}} - {{- $field := .field -}} - {{- $value := index $row $field | default "" -}} - {{- $rendered := $value | $.Page.RenderString -}} - {{- $cells = $cells | append ($rendered | replaceRE "\\|" "\\|" | replaceRE "\n" " ") -}} -{{- end }} -| {{ delimit $cells " | " }} | -{{- end }} From 7671f3db85fbb169d44fcd488abbdde38b228e41 Mon Sep 17 00:00:00 2001 From: Seth Wiesman Date: Thu, 1 Oct 2026 11:59:06 -0500 Subject: [PATCH 03/10] doc/user: wrap the page body in
The
gives the Markdown rendition and agent tooling one element to read the page body from. Section pages now wrap their child-page links in a
    ; they were bare
  • elements. Co-Authored-By: Claude Opus 5.5 --- doc/user/layouts/_default/baseof.html | 7 ++++++- doc/user/layouts/_default/list.html | 4 +++- doc/user/layouts/_default/single.html | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/doc/user/layouts/_default/baseof.html b/doc/user/layouts/_default/baseof.html index 167df07159dfe..b357ad8c315ea 100644 --- a/doc/user/layouts/_default/baseof.html +++ b/doc/user/layouts/_default/baseof.html @@ -59,7 +59,12 @@
    {{ partial "breadcrumbs.html" . }} - {{ block "main" . }}{{ end }} + {{- /* + The
    is the page body that the Markdown rendition is + generated from (bin/docs-markdown). Mark chrome inside it with + data-markdown-ignore to keep it out of the Markdown. + */}} +
    {{ block "main" . }}{{ end }}
    Back to top ↑ diff --git a/doc/user/layouts/_default/list.html b/doc/user/layouts/_default/list.html index 997e34ff42049..1624010b61da8 100644 --- a/doc/user/layouts/_default/list.html +++ b/doc/user/layouts/_default/list.html @@ -3,18 +3,20 @@ {{ if not (.Params.disable_h1) }}

    {{.Title}}

    - View as Markdown + View as Markdown
    {{ end }} {{ .Content }} {{ if not (.Params.disable_list) }} +
      {{ range .Pages.ByWeight }}
    • {{.Title}}
      • {{ end }}{{/* {{ range .Pages.ByWeight }} */}} +
    {{ end }}{{/* {{ if not (.Params.disable_list) }} */}} {{ end }}{{/* {{ define "main"}} */}} diff --git a/doc/user/layouts/_default/single.html b/doc/user/layouts/_default/single.html index b54667f3f6dca..f8125dc75a8e7 100644 --- a/doc/user/layouts/_default/single.html +++ b/doc/user/layouts/_default/single.html @@ -3,7 +3,7 @@ {{ if not (.Params.disable_h1) }}

    {{.Title | markdownify}}

    - View as Markdown + View as Markdown
    {{ end }} From b694a77aa9c4d7095af26888fd0a7e4edd0f0c0b Mon Sep 17 00:00:00 2001 From: Seth Wiesman Date: Thu, 1 Oct 2026 12:03:39 -0500 Subject: [PATCH 04/10] doc/user: generate the Markdown rendition from the built HTML bin/docs-markdown converts each page's
    into an index.md beside its index.html, so https://materialize.com/docs//index.md serves Markdown for every page. Converting the rendered HTML keeps the two renditions in parity without a Markdown twin of every shortcode. Links and images are absolute, since an agent holding the Markdown has usually lost the URL it fetched. Co-Authored-By: Claude Opus 5.5 --- bin/docs-markdown | 14 + ci/deploy_website/website.sh | 2 + ci/test/lint-docs.sh | 2 + ci/test/preview-docs.sh | 1 + doc/user/AGENTS.md | 16 + doc/user/config.toml | 5 + doc/user/layouts/index.llmstxt.txt | 19 +- doc/user/layouts/partials/markdown-url.html | 10 +- misc/python/materialize/cli/docs_markdown.py | 504 ++++++++++++++++++ .../materialize/cli/docs_markdown_test.py | 176 ++++++ 10 files changed, 726 insertions(+), 23 deletions(-) create mode 100755 bin/docs-markdown create mode 100644 misc/python/materialize/cli/docs_markdown.py create mode 100644 misc/python/materialize/cli/docs_markdown_test.py diff --git a/bin/docs-markdown b/bin/docs-markdown new file mode 100755 index 0000000000000..a61bf653fe201 --- /dev/null +++ b/bin/docs-markdown @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. +# +# docs-markdown -- renders a Markdown copy of every page in a built docs site + +exec "$(dirname "$0")"/pyactivate -m materialize.cli.docs_markdown "$@" diff --git a/ci/deploy_website/website.sh b/ci/deploy_website/website.sh index d27e3a1e73676..fa3ba50893203 100755 --- a/ci/deploy_website/website.sh +++ b/ci/deploy_website/website.sh @@ -40,8 +40,10 @@ cd doc/user if [[ "$BUILDKITE_ORGANIZATION_SLUG" == "materialize" ]] && [[ "$BUILDKITE_BRANCH" == self-managed-docs/* ]]; then VERSION=${BUILDKITE_BRANCH#self-managed-docs/} hugo --gc --baseURL "/docs/self-managed/$VERSION" --destination "public/docs/self-managed/$VERSION" + ../../bin/docs-markdown "public/docs/self-managed/$VERSION" --base-url "https://materialize.com/docs/self-managed/$VERSION/" else hugo --gc --baseURL /docs --destination public/docs + ../../bin/docs-markdown public/docs --base-url https://materialize.com/docs/ fi hugo deploy --maxDeletes -1 diff --git a/ci/test/lint-docs.sh b/ci/test/lint-docs.sh index 77d65ef0ffd25..860709ee28293 100755 --- a/ci/test/lint-docs.sh +++ b/ci/test/lint-docs.sh @@ -17,6 +17,8 @@ set -euo pipefail git clean -ffdX ci/www/public try hugo --gc --baseURL https://ci.materialize.com/docs --source doc/user --destination ../../ci/www/public/docs +try bin/docs-markdown ci/www/public/docs --base-url https://ci.materialize.com/docs/ +try bin/pytest -qq misc/python/materialize/cli/docs_markdown_test.py echo "" > ci/www/public/index.html try htmltest -s ci/www/public -c doc/user/.htmltest.yml try ci/test/lint-docs-catalog.sh diff --git a/ci/test/preview-docs.sh b/ci/test/preview-docs.sh index d57efec9163f8..22a9fbeb0402d 100755 --- a/ci/test/preview-docs.sh +++ b/ci/test/preview-docs.sh @@ -22,6 +22,7 @@ cd doc/user # Build main docs to public/ hugo --gc --environment preview --baseURL "/materialize/$BUILDKITE_PULL_REQUEST" +../../bin/docs-markdown public --base-url "https://preview.materialize.com/materialize/$BUILDKITE_PULL_REQUEST/" cat > config.deployment.toml <index.md`, for agents and +other non-browser readers. `bin/docs-markdown` generates it after the Hugo +build by converting each page's `
    ` element, so shortcodes need no +Markdown variant. Chrome inside the `
    ` that should stay out of the +Markdown, such as a copy button, takes the `data-markdown-ignore` attribute. + +To inspect the Markdown for a template or shortcode change, run from the +repository root: + +```shell +hugo --source doc/user --destination /tmp/docs/docs --baseURL /docs +bin/docs-markdown /tmp/docs/docs --base-url https://materialize.com/docs/ +``` + ### Reusing content Use the existing single-sourcing mechanisms before duplicating prose: diff --git a/doc/user/config.toml b/doc/user/config.toml index 0bd3b4854e42a..b4539dfff7129 100644 --- a/doc/user/config.toml +++ b/doc/user/config.toml @@ -127,6 +127,11 @@ unsafe = true startLevel = 2 endLevel = 4 +[[deployment.matchers]] +# RFC 7763 makes charset a required parameter of text/markdown. +pattern = "^.+\\.md$" +contentType = "text/markdown; charset=utf-8" + [[deployment.targets]] name = "production" url = "s3://materialize-website?region=us-east-1" diff --git a/doc/user/layouts/index.llmstxt.txt b/doc/user/layouts/index.llmstxt.txt index f1c320c6ed30a..b7ff46810b77e 100644 --- a/doc/user/layouts/index.llmstxt.txt +++ b/doc/user/layouts/index.llmstxt.txt @@ -17,20 +17,9 @@ Links point at each page's Markdown rendition rather than its HTML page, so a consumer fetches prose instead of site chrome. - Only the HTML pass renders this file. config.skill.toml resets outputs.home to - ["skill"], and the rightmost --config file wins, so the Markdown pass emits - markdown-docs/index.md instead of a second copy of this index. - The output must be byte-stable: no dates, no build metadata, and no ranging over maps. Every collection below is explicitly ordered. */ -}} -{{- $basePath := (urls.Parse site.BaseURL).Path -}} -{{- /* - Only the main docs build produces the markdown-docs tree these links resolve - against; self-managed-docs/* branches build HTML alone. Emit nothing there - rather than publish an index of dead links. -*/ -}} -{{- if not (in $basePath "self-managed") -}} {{- $eligible := slice -}} {{- range site.Pages -}} {{- $keep := true -}} @@ -119,13 +108,11 @@ > {{ $intro }} -Every link below points at the Markdown source of a documentation page. For the -HTML page instead, drop `markdown-docs/` from the URL and drop the trailing -`index.md`. Per-version release notes are not listed individually; see the -Releases section. +Every link below points at the Markdown rendition of a documentation page. For +the HTML page instead, drop the trailing `index.md`. Per-version release notes +are not listed individually; see the Releases section. {{ range $groups }}{{ $items := sort (where $eligible "Section" .key) "RelPermalink" }}{{ if $items }} ## {{ .title }} {{ range $items }}- [{{ .Title }}]({{ partial "markdown-url.html" (dict "page" . "absolute" true) }}){{ with .Description }}{{ $d := trim (replaceRE `\s+` " " .) " " }}{{ if $d }}: {{ $d }}{{ end }}{{ end }} {{ end }}{{ end }}{{ end -}} -{{- end -}} diff --git a/doc/user/layouts/partials/markdown-url.html b/doc/user/layouts/partials/markdown-url.html index 27890bbd1ed2a..da551737ee041 100644 --- a/doc/user/layouts/partials/markdown-url.html +++ b/doc/user/layouts/partials/markdown-url.html @@ -1,7 +1,6 @@ {{- /* - Returns the URL of a page's Markdown rendition, the file the `skill` output - format writes into /markdown-docs/ on the second Hugo pass (see - ci/deploy_website/website.sh). + Returns the URL of a page's Markdown rendition, the index.md that + bin/docs-markdown writes beside the page's index.html. Arguments (dict): page (required) the Page to link to @@ -17,10 +16,7 @@ {{ partial "markdown-url.html" (dict "page" .) }} {{ partial "markdown-url.html" (dict "page" $p "absolute" true) }} */ -}} -{{- $basePath := (urls.Parse site.BaseURL).Path -}} -{{- if not (strings.HasSuffix $basePath "/") -}}{{- $basePath = printf "%s/" $basePath -}}{{- end -}} -{{- $pagePath := .page.RelPermalink | strings.TrimPrefix $basePath -}} -{{- $url := printf "%smarkdown-docs/%sindex.md" $basePath $pagePath -}} +{{- $url := printf "%sindex.md" .page.RelPermalink -}} {{- if .absolute -}} {{- $parsed := urls.Parse site.BaseURL -}} {{- $origin := site.Params.canonicalHost -}} diff --git a/misc/python/materialize/cli/docs_markdown.py b/misc/python/materialize/cli/docs_markdown.py new file mode 100644 index 0000000000000..ee1bdd8d6c1f7 --- /dev/null +++ b/misc/python/materialize/cli/docs_markdown.py @@ -0,0 +1,504 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. +# +# docs_markdown.py - Render a Markdown copy of every page in a built Hugo site. + +"""Render a Markdown copy of every page in a built Hugo docs site. + +For each `/index.html` that has a `
    `, writes `/index.md` from +the page's `
    `. Generating from the built HTML keeps the Markdown in +parity with what readers see: every shortcode, include, and data-driven table +has already been expanded by Hugo. + +Elements carrying `data-markdown-ignore` are left out, which is the same +marker agent-docs checkers honor when comparing the two renditions. + +Links and images are rewritten to absolute URLs against `--base-url`, because +an agent that receives the Markdown has usually lost the URL it came from. +""" + +import argparse +import json +import re +import sys +from collections.abc import Iterator +from dataclasses import dataclass, field +from html.parser import HTMLParser +from pathlib import Path +from urllib.parse import urljoin + +VOID_TAGS = frozenset( + "area base br col embed hr img input link meta source track wbr".split() +) + +# Interactive or decorative elements with no Markdown meaning. +DROPPED_TAGS = frozenset( + "button form input noscript script select style svg template textarea".split() +) + +BLOCK_TAGS = frozenset( + """address article aside blockquote dd details div dl dt fieldset figcaption + figure footer h1 h2 h3 h4 h5 h6 header hr legend li main nav ol p pre section + summary table tbody td tfoot th thead tr ul""".split() +) + +# Callout shortcodes render as a
    with one of these classes. +CALLOUT_CLASSES = frozenset( + """annotation callout error important note private-preview public-preview + tip warning""".split() +) + +# Chroma language names that consumers do not recognize, mapped to ones they do. +CODE_LANGUAGE_ALIASES = {"mzsql": "sql", "nofmt": "text", "none": ""} + + +@dataclass +class Element: + tag: str + attrs: dict[str, str] + children: list["Element | str"] = field(default_factory=list) + + @property + def classes(self) -> set[str]: + return set(self.attrs.get("class", "").split()) + + def iter(self) -> Iterator["Element"]: + yield self + for child in self.children: + if isinstance(child, Element): + yield from child.iter() + + def find(self, tag: str) -> "Element | None": + return next((e for e in self.iter() if e.tag == tag), None) + + def text(self) -> str: + return "".join( + c if isinstance(c, str) else c.text() + for c in self.children + if isinstance(c, str) or c.tag not in DROPPED_TAGS + ) + + +class TreeBuilder(HTMLParser): + """Builds an `Element` tree, tolerating the unclosed tags HTML allows.""" + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.root = Element("#root", {}) + self.stack = [self.root] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + element = Element(tag, {k: v or "" for k, v in attrs}) + self.stack[-1].children.append(element) + if tag not in VOID_TAGS: + self.stack.append(element) + + def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + self.stack[-1].children.append(Element(tag, {k: v or "" for k, v in attrs})) + + def handle_endtag(self, tag: str) -> None: + # Close back to the matching open element. An end tag with no open + # match is stray and ignored. + for i in range(len(self.stack) - 1, 0, -1): + if self.stack[i].tag == tag: + del self.stack[i:] + return + + def handle_data(self, data: str) -> None: + self.stack[-1].children.append(data) + + +def parse_html(html: str) -> Element: + builder = TreeBuilder() + builder.feed(html) + builder.close() + return builder.root + + +@dataclass +class Page: + title: str + description: str + body: str + + +class PageError(Exception): + pass + + +def collapse_whitespace(text: str) -> str: + return re.sub(r"\s+", " ", text) + + +def escape_text(text: str) -> str: + """Escapes characters that would otherwise start Markdown syntax.""" + text = re.sub(r"([\\`*\[\]])", r"\\\1", text) + # An underscore between word characters cannot open emphasis. + text = re.sub(r"(? str: + """Escapes a leading character that would turn a paragraph into a block.""" + if re.match(r"(#{1,6}(\s|$)|>|[-+*](\s|$)|=+\s*$)", line): + return "\\" + line + return re.sub(r"^(\d+)([.)])(\s|$)", r"\1\\\2\3", line) + + +def code_span(text: str) -> str: + text = collapse_whitespace(text) + if not text.strip(): + return "" + fence = "`" * (max((len(m) for m in re.findall(r"`+", text)), default=0) + 1) + if text.startswith("`") or text.endswith("`"): + text = f" {text} " + return f"{fence}{text}{fence}" + + +def wrap_inline(marker: str, text: str) -> str: + """Wraps `text` in an emphasis marker, keeping edge whitespace outside it.""" + stripped = text.strip() + if not stripped: + return text + lead = text[: len(text) - len(text.lstrip())] + trail = text[len(text.rstrip()) :] + return f"{lead}{marker}{stripped}{marker}{trail}" + + +class Converter: + def __init__(self, page_url: str) -> None: + self.page_url = page_url + + def url(self, href: str) -> str: + if href.startswith("#"): + return href + return urljoin(self.page_url, href) + + def convert(self, document: Element) -> Page: + main = document.find("main") + if main is None: + raise PageError("page has no
    ") + article = main.find("article") + if article is None: + raise PageError("
    has no
    ") + h1 = article.find("h1") + title = collapse_whitespace(h1.text()).strip() if h1 is not None else "" + if not title: + title_element = document.find("title") + if title_element is not None: + title = collapse_whitespace(title_element.text()).strip() + description = "" + for meta in document.iter(): + if meta.tag == "meta" and meta.attrs.get("name") == "description": + description = collapse_whitespace(meta.attrs.get("content", "")).strip() + break + return Page(title, description, "\n\n".join(self.blocks(article))) + + def blocks(self, element: Element) -> list[str]: + """Renders an element's children as a list of Markdown blocks.""" + out: list[str] = [] + run: list[Element | str] = [] + + def flush() -> None: + paragraph = self.paragraph(run) + if paragraph: + out.append(paragraph) + run.clear() + + for child in element.children: + if isinstance(child, Element) and self.is_block(child): + flush() + out.extend(self.block(child)) + else: + run.append(child) + flush() + return out + + def is_block(self, element: Element) -> bool: + return element.tag in BLOCK_TAGS + + def paragraph(self, nodes: list[Element | str]) -> str: + text = "".join(self.inline(node) for node in nodes) + lines = [line.strip() for line in text.split("\n")] + lines = [escape_block_start(line) for line in lines if line] + return " \n".join(lines) + + def block(self, element: Element) -> list[str]: + if element.tag in DROPPED_TAGS or "data-markdown-ignore" in element.attrs: + return [] + tag = element.tag + classes = element.classes + if tag in ("h1", "h2", "h3", "h4", "h5", "h6"): + text = self.inline_text(element) + return [f"{'#' * int(tag[1])} {text}"] if text else [] + if tag == "p" and "heading" in classes: + text = self.inline_text(element) + return [f"**{text}**"] if text else [] + if tag == "pre": + return [self.code_block(element)] + if tag in ("ul", "ol"): + return [self.list_block(element)] if self.list_items(element) else [] + if tag == "li": + return [self.list_item(element, "- ")] + if tag == "table": + return self.table(element) + if tag == "hr": + return ["---"] + if tag == "blockquote" or (tag == "div" and classes & CALLOUT_CLASSES): + inner = self.blocks(element) + return [quote(inner)] if inner else [] + if tag == "div" and "annotation-title" in classes: + text = self.inline_text(element) + return [f"**{text}**"] if text else [] + if tag == "div" and "tab-pane" in classes: + label = collapse_whitespace(element.attrs.get("title", "")).strip() + heading = [f"**{escape_text(label)}**"] if label else [] + return heading + self.blocks(element) + if tag in ("summary", "dt"): + text = self.inline_text(element) + return [f"**{text}**"] if text else [] + return self.blocks(element) + + def inline_text(self, element: Element) -> str: + return collapse_whitespace(self.inline(element)).strip() + + def inline(self, node: Element | str) -> str: + if isinstance(node, str): + return escape_text(collapse_whitespace(node)) + if node.tag in DROPPED_TAGS or "data-markdown-ignore" in node.attrs: + return "" + tag = node.tag + if tag == "br": + return "\n" + if tag == "img": + src = node.attrs.get("src", "") + if not src: + return "" + alt = escape_text(collapse_whitespace(node.attrs.get("alt", "")).strip()) + return f"![{alt}]({self.url(src)})" + if tag in ("code", "kbd", "samp"): + return code_span(node.text()) + inner = "".join(self.inline(child) for child in node.children) + if tag == "a": + href = node.attrs.get("href", "") + text = collapse_whitespace(inner).strip() + if not href or not text: + return inner + title = node.attrs.get("title", "").replace('"', '\\"') + suffix = f' "{title}"' if title else "" + lead = " " if inner[:1].isspace() else "" + trail = " " if inner[-1:].isspace() else "" + return f"{lead}[{text}]({self.url(href)}{suffix}){trail}" + if tag in ("strong", "b"): + return wrap_inline("**", inner) + if tag in ("em", "i"): + return wrap_inline("*", inner) + if tag in ("del", "s"): + return wrap_inline("~~", inner) + if tag == "li": + return f"\n- {inner.strip()}\n" + if tag in BLOCK_TAGS: + # A block element in inline context, such as a

    in a table + # cell, is separated from its neighbors by a line break. + return f"\n{inner.strip()}\n" + return inner + + def code_block(self, pre: Element) -> str: + code = pre.find("code") + source = code if code is not None else pre + text = source.text().rstrip("\n") + if "mermaid" in pre.classes: + language = "mermaid" + else: + language = source.attrs.get("data-lang", "") + if not language: + language = next( + ( + c.removeprefix("language-") + for c in source.classes + if c.startswith("language-") + ), + "", + ) + language = CODE_LANGUAGE_ALIASES.get(language, language) + longest = max((len(m) for m in re.findall(r"`+", text)), default=0) + fence = "`" * max(3, longest + 1) + return f"{fence}{language}\n{text}\n{fence}" + + def list_items(self, element: Element) -> list[Element]: + return [c for c in element.children if isinstance(c, Element) and c.tag == "li"] + + def list_block(self, element: Element) -> str: + ordered = element.tag == "ol" + start = int(element.attrs.get("start", "1") or "1") if ordered else 1 + items = [] + for i, item in enumerate(self.list_items(element)): + marker = f"{start + i}. " if ordered else "- " + items.append(self.list_item(item, marker)) + return "\n".join(items) + + def list_item(self, item: Element, marker: str) -> str: + blocks = self.blocks(item) + if not blocks: + return marker.rstrip() + # Paragraph-only items stay tight; any other block needs a blank line + # to stay part of the item. + tight = all(re.match(r"(- |\d+\. )", b) for b in blocks[1:]) + separator = "\n" if tight else "\n\n" + body = separator.join(blocks) + indent = " " * len(marker) + lines = body.split("\n") + return "\n".join( + [marker + lines[0]] + [indent + line if line else "" for line in lines[1:]] + ) + + def table(self, table: Element) -> list[str]: + rows = [ + [ + c + for c in row.children + if isinstance(c, Element) and c.tag in ("td", "th") + ] + for row in table_rows(table) + ] + rows = [row for row in rows if row] + if not rows: + return [] + if any( + e.tag in ("pre", "table") + for row in rows + for cell in row + for e in cell.iter() + if e is not cell + ): + return self.table_as_blocks(rows) + header, body = rows[0], rows[1:] + width = max(len(row) for row in rows) + + def line(cells: list[str]) -> str: + cells = cells + [""] * (width - len(cells)) + return "| " + " | ".join(cells) + " |" + + out = [line([self.cell(c) for c in header]), line(["---"] * width)] + out.extend(line([self.cell(c) for c in row]) for row in body) + return ["\n".join(out)] + + def cell(self, cell: Element) -> str: + text = "".join(self.inline(child) for child in cell.children) + lines = [line.strip() for line in text.split("\n")] + return "
    ".join(line for line in lines if line).replace("|", "\\|") + + def table_as_blocks(self, rows: list[list[Element]]) -> list[str]: + """Renders a table whose cells hold code blocks or nested tables. + + A pipe table cannot hold either, so each row is written as a sequence + of blocks, labeled by the header cells when the table has several + columns. + """ + header: list[str] = [] + if all(c.tag == "th" for c in rows[0]): + header = [self.inline_text(c) for c in rows[0]] + rows = rows[1:] + out: list[str] = [] + for row in rows: + for i, cell in enumerate(row): + blocks = self.blocks(cell) + if not blocks: + continue + if len(row) > 1 and i < len(header) and header[i]: + out.append(f"**{header[i]}**") + out.extend(blocks) + return out + + +def table_rows(table: Element) -> list[Element]: + """Returns the table's own rows, excluding rows of nested tables.""" + rows = [] + for child in table.children: + if not isinstance(child, Element): + continue + if child.tag == "tr": + rows.append(child) + elif child.tag in ("thead", "tbody", "tfoot"): + rows.extend(table_rows(child)) + return rows + + +def quote(blocks: list[str]) -> str: + lines = "\n\n".join(blocks).split("\n") + return "\n".join(f"> {line}" if line else ">" for line in lines) + + +def render(page: Page) -> str: + front_matter = [f"title: {json.dumps(page.title, ensure_ascii=False)}"] + if page.description: + front_matter.append( + f"description: {json.dumps(page.description, ensure_ascii=False)}" + ) + return "---\n" + "\n".join(front_matter) + "\n---\n\n" + page.body.strip() + "\n" + + +def page_url(base_url: str, site_root: Path, html_path: Path) -> str: + relative = html_path.parent.relative_to(site_root).as_posix() + base = base_url.rstrip("/") + "/" + return base if relative == "." else f"{base}{relative}/" + + +def convert_site(site_root: Path, base_url: str) -> tuple[int, list[str]]: + """Writes index.md beside every page's index.html. + + Returns the number of pages written and a list of per-page errors. Files + without a

    , such as alias redirect stubs, are skipped. + """ + written = 0 + errors: list[str] = [] + for html_path in sorted(site_root.rglob("index.html")): + document = parse_html(html_path.read_text()) + if document.find("main") is None: + continue + converter = Converter(page_url(base_url, site_root, html_path)) + try: + page = converter.convert(document) + except PageError as e: + errors.append(f"{html_path.relative_to(site_root)}: {e}") + continue + html_path.with_name("index.md").write_text(render(page)) + written += 1 + return written, errors + + +def main() -> int: + parser = argparse.ArgumentParser( + prog="docs-markdown", + description="Render a Markdown copy of every page in a built Hugo site.", + ) + parser.add_argument( + "site_root", + type=Path, + help="the directory Hugo built the site into, e.g. public/docs", + ) + parser.add_argument( + "--base-url", + required=True, + help="the absolute URL site_root is served at, e.g. https://materialize.com/docs/", + ) + args = parser.parse_args() + if not re.match(r"https?://", args.base_url): + parser.error("--base-url must be an absolute http(s) URL") + written, errors = convert_site(args.site_root, args.base_url) + for error in errors: + print(f"docs-markdown: {error}", file=sys.stderr) + print(f"docs-markdown: wrote {written} Markdown pages") + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/misc/python/materialize/cli/docs_markdown_test.py b/misc/python/materialize/cli/docs_markdown_test.py new file mode 100644 index 0000000000000..39851cb77f6ab --- /dev/null +++ b/misc/python/materialize/cli/docs_markdown_test.py @@ -0,0 +1,176 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +from pathlib import Path + +import pytest + +from materialize.cli.docs_markdown import ( + Converter, + PageError, + convert_site, + parse_html, + render, +) + +PAGE_URL = "https://materialize.com/docs/sql/create-index/" + + +def body(article: str) -> str: + html = f"
    {article}
    " + return Converter(PAGE_URL).convert(parse_html(html)).body + + +def test_links_and_images_are_absolute() -> None: + assert body('

    See SQL.

    ') == ( + "See [SQL](https://materialize.com/docs/sql/)." + ) + assert body('

    views

    ') == ( + "[views](https://materialize.com/docs/sql/views/)" + ) + assert body('

    A

    ') == ( + "![A](https://materialize.com/docs/images/a.svg)" + ) + + +def test_fragment_links_stay_relative() -> None: + assert body('

    Syntax

    ') == "[Syntax](#syntax)" + + +def test_chroma_code_block() -> None: + html = ( + '
    '
+        'SELECT 1;\n'
+        "SELECT 2;\n
    " + ) + assert body(html) == "```sql\nSELECT 1;\nSELECT 2;\n```" + + +def test_code_fence_outgrows_backticks_in_code() -> None: + assert body("
    a ``` b
    ") == "````\na ``` b\n````" + + +def test_inline_code_with_backtick() -> None: + assert body("

    a`b

    ") == "``a`b``" + + +def test_text_that_looks_like_markdown_is_escaped() -> None: + assert body("

    *not emphasis* and [not a link]

    ") == ( + "\\*not emphasis\\* and \\[not a link\\]" + ) + assert body("

    # not a heading

    ") == "\\# not a heading" + assert body("

    1. not a list

    ") == "1\\. not a list" + assert body("

    <cluster_name>

    ") == "\\" + + +def test_intraword_underscores_are_not_escaped() -> None: + assert body("

    mz_internal and _x_

    ") == "mz_internal and \\_x\\_" + + +def test_nested_lists() -> None: + html = "
    • a
      • b
    • c
    " + assert body(html) == "- a\n - b\n- c" + + +def test_ordered_list_start() -> None: + assert body('
    1. x
    2. y
    ') == "3. x\n4. y" + + +def test_list_item_with_code_block() -> None: + html = "

    1. Run:

      ls\n
    " + assert body(html) == "1. Run:\n\n ```\n ls\n ```" + + +def test_pipe_table() -> None: + html = ( + "" + "" + "
    FieldUse
    a|b

    One.

    Two.

    " + ) + assert body(html) == ("| Field | Use |\n| --- | --- |\n| `a\\|b` | One.
    Two. |") + + +def test_table_with_code_blocks_renders_as_blocks() -> None: + html = ( + "
    Function
    f(x)
    " + "

    Does f.

    " + ) + assert body(html) == "```\nf(x)\n```\n\nDoes f." + + +def test_nested_table_rows_stay_in_the_nested_table() -> None: + html = ( + "
    Outer
    " + "
    Inner
    x
    " + ) + assert body(html) == "| Inner |\n| --- |\n| x |" + + +def test_callout_and_tabs() -> None: + html = ( + '
    NOTE: Careful.
    ' + '
      ' + '

      Cloud text.

      ' + '

      SM text.

      ' + "
      " + ) + assert body(html) == ( + "> **NOTE:** Careful.\n\n**Cloud**\n\nCloud text.\n\n" + "**Self-Managed**\n\nSM text." + ) + + +def test_ignored_and_interactive_elements_are_dropped() -> None: + html = ( + '

      Title

      ' + 'View as Markdown
      ' + "

      Texticon.

      " + "" + ) + assert body(html) == "# Title\n\nText." + + +def test_hard_line_break() -> None: + assert body("

      a
      b

      ") == "a \nb" + + +def test_front_matter() -> None: + html = ( + 'T | Docs

      Title

      ' + "
      " + ) + page = Converter(PAGE_URL).convert(parse_html(html)) + assert render(page) == ( + '---\ntitle: "Title"\ndescription: "Says \\"hi\\"."\n---\n\n# Title\n' + ) + + +def test_page_without_article_is_an_error() -> None: + with pytest.raises(PageError): + Converter(PAGE_URL).convert(parse_html("

      x

      ")) + + +def test_convert_site(tmp_path: Path) -> None: + page = tmp_path / "sql" / "index.html" + page.parent.mkdir() + page.write_text('
      ') + # Alias redirect stubs have no
      and get no Markdown. + alias = tmp_path / "old" / "index.html" + alias.parent.mkdir() + alias.write_text('') + + written, errors = convert_site(tmp_path, "https://materialize.com/docs") + + assert (written, errors) == (1, []) + assert ( + "[x](https://materialize.com/docs/x/)" in (page.parent / "index.md").read_text() + ) + assert not (alias.parent / "index.md").exists() From 730d80fe1d41f29145ba97fd293b948e30b3b921 Mon Sep 17 00:00:00 2001 From: Seth Wiesman Date: Thu, 1 Oct 2026 12:06:28 -0500 Subject: [PATCH 05/10] doc/user: point agents at llms.txt from every page Every HTML page now opens with a visually hidden directive naming the docs index and the Markdown rendition, and advertises its Markdown with . bin/docs-markdown copies the directive to the top of each Markdown page. Co-Authored-By: Claude Opus 5.5 --- doc/user/assets/sass/_base.scss | 12 ++++++ doc/user/layouts/_default/baseof.html | 12 ++++++ doc/user/layouts/partials/head.html | 3 ++ doc/user/layouts/partials/llms-txt-url.html | 2 + doc/user/layouts/partials/markdown-url.html | 15 +------ doc/user/layouts/partials/site-origin.html | 17 ++++++++ misc/python/materialize/cli/docs_markdown.py | 8 +++- .../materialize/cli/docs_markdown_test.py | 39 +++++++++++++------ 8 files changed, 82 insertions(+), 26 deletions(-) create mode 100644 doc/user/layouts/partials/llms-txt-url.html create mode 100644 doc/user/layouts/partials/site-origin.html diff --git a/doc/user/assets/sass/_base.scss b/doc/user/assets/sass/_base.scss index d1c87ae66585e..1a2109f4d44e9 100644 --- a/doc/user/assets/sass/_base.scss +++ b/doc/user/assets/sass/_base.scss @@ -221,6 +221,18 @@ input[type="submit"] { .osano-cm-widget { display: none; } +.llms-txt-directive { + position: absolute; + width: 1px; + height: 1px; + margin: -1px; + padding: 0; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; + border: 0; +} + /** The following section handles these cases: 1. Buttons with links diff --git a/doc/user/layouts/_default/baseof.html b/doc/user/layouts/_default/baseof.html index b357ad8c315ea..1d023ad13f50f 100644 --- a/doc/user/layouts/_default/baseof.html +++ b/doc/user/layouts/_default/baseof.html @@ -10,6 +10,18 @@ + {{- /* + Points agents that fetch this page at the docs index. It must be the + first element in , ahead of the navigation, so that it survives a + truncated fetch. It is hidden visually rather than with display:none, + which HTML-to-Markdown converters drop. bin/docs-markdown copies its + text to the top of the Markdown rendition. + */}} +