diff --git a/bin/docs-markdown b/bin/docs-markdown new file mode 100755 index 0000000000000..a61bf653fe201 --- /dev/null +++ b/bin/docs-markdown @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. +# +# docs-markdown -- renders a Markdown copy of every page in a built docs site + +exec "$(dirname "$0")"/pyactivate -m materialize.cli.docs_markdown "$@" diff --git a/bin/gen-claude-skill b/bin/gen-claude-skill deleted file mode 100755 index 4625d4e745ccb..0000000000000 --- a/bin/gen-claude-skill +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License included in -# the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with the Business -# Source License, use of this software will be governed by the Apache License, -# Version 2.0. -# -# gen-claude-skill -- Generate a shared agent skill bundle from the -# documentation. -# -# Usage: bin/gen-claude-skill [OUTPUT_DIR] -# -# Arguments: OUTPUT_DIR Directory to output the skill files. The OUTPUT_DIR -# directory is cleaned before generation. -# Default: .agents/skills/mz-docs. -# For publishing to the website, -# we will output to doc/user/public/markdown-docs directory. -# -# Examples: -# bin/gen-claude-skill # Generate to default location -# bin/gen-claude-skill /path/to/output # Generate to custom location - -set -euo pipefail - -cd "$(dirname "$0")/.." - -# For publishing to the website, we will output to doc/user/public/markdown-docs -# directory. The OUTPUT_DIR directory is cleaned before generation. -OUTPUT_DIR="${1:-.agents/skills/mz-docs}" - -# Ensure output directory exists -mkdir -p "$OUTPUT_DIR" - -echo "Generating agent skill from documentation..." -echo "Output directory: $OUTPUT_DIR" - -# Build documentation with skill output format -cd doc/user -hugo --config config.toml,config.skill.toml \ - --disableKinds 404,sitemap,robotsTXT,taxonomy \ - --destination "../../$OUTPUT_DIR" \ - --cleanDestinationDir \ - --gc \ - --quiet - -cd ../.. - -# Rename the home page to SKILL.md (shared skill convention) -if [[ -f "$OUTPUT_DIR/index.md" ]]; then - mv "$OUTPUT_DIR/index.md" "$OUTPUT_DIR/SKILL.md" -fi - -# Collapse runs of consecutive blank lines down to a single blank line. -# Hugo emits a blank line in place of each stripped shortcode/template -# directive, which produces long runs around //
blocks -# in pages like ingest-data/mysql/* and self-managed-deployments/*. -find "$OUTPUT_DIR" -name '*.md' -type f -print0 | while IFS= read -r -d '' f; do - awk ' - /^[[:space:]]*$/ { if (blank) next; blank = 1; print ""; next } - { blank = 0; print } - ' "$f" > "$f.tmp" && mv "$f.tmp" "$f" -done - -# Count generated files -FILE_COUNT=$(find "$OUTPUT_DIR" -name "*.md" | wc -l | tr -d ' ') - -echo "Done! Generated $FILE_COUNT markdown files." -echo "Skill entry point: $OUTPUT_DIR/SKILL.md" diff --git a/ci/deploy_website/docs-content-negotiation.js b/ci/deploy_website/docs-content-negotiation.js new file mode 100644 index 0000000000000..54c616f268f18 --- /dev/null +++ b/ci/deploy_website/docs-content-negotiation.js @@ -0,0 +1,31 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file at the root of this repository. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +// CloudFront Function (cloudfront-js-2.0, viewer request) for the /docs/* +// behavior of materialize.com. It serves a page's Markdown rendition, the +// index.md that bin/docs-markdown writes beside each index.html, to clients +// that send `Accept: text/markdown`. +// +// It must run on viewer request, not origin request: the rewritten URI is +// then the cache key, so the HTML and Markdown renditions of a page are cached +// separately. Responses on the behavior must carry `Vary: Accept` (set by its +// response headers policy) so that caches downstream of CloudFront do the +// same. +// +// This file is not deployed by CI. Publish changes to the function attached +// to the distribution by hand. + +function handler(event) { + var request = event.request; + var accept = request.headers.accept ? request.headers.accept.value : ""; + if (request.uri.endsWith("/") && accept.indexOf("text/markdown") !== -1) { + request.uri += "index.md"; + } + return request; +} diff --git a/ci/deploy_website/website.sh b/ci/deploy_website/website.sh index 686a70171b5b2..fa3ba50893203 100755 --- a/ci/deploy_website/website.sh +++ b/ci/deploy_website/website.sh @@ -40,10 +40,10 @@ cd doc/user if [[ "$BUILDKITE_ORGANIZATION_SLUG" == "materialize" ]] && [[ "$BUILDKITE_BRANCH" == self-managed-docs/* ]]; then VERSION=${BUILDKITE_BRANCH#self-managed-docs/} hugo --gc --baseURL "/docs/self-managed/$VERSION" --destination "public/docs/self-managed/$VERSION" + ../../bin/docs-markdown "public/docs/self-managed/$VERSION" --base-url "https://materialize.com/docs/self-managed/$VERSION/" else hugo --gc --baseURL /docs --destination public/docs - # Build skill docs to public/docs/markdown-docs/. - hugo --gc --baseURL /docs --config config.toml,config.skill.toml --disableKinds 404,sitemap,robotsTXT,taxonomy --destination public/docs/markdown-docs + ../../bin/docs-markdown public/docs --base-url https://materialize.com/docs/ fi hugo deploy --maxDeletes -1 diff --git a/ci/test/lint-docs.sh b/ci/test/lint-docs.sh index 250f03ad4c05c..860709ee28293 100755 --- a/ci/test/lint-docs.sh +++ b/ci/test/lint-docs.sh @@ -17,7 +17,8 @@ set -euo pipefail git clean -ffdX ci/www/public try hugo --gc --baseURL https://ci.materialize.com/docs --source doc/user --destination ../../ci/www/public/docs -try hugo --gc --baseURL https://ci.materialize.com/docs --source doc/user --config config.toml,config.skill.toml --disableKinds 404,sitemap,robotsTXT,taxonomy --destination ../../ci/www/public/docs/markdown-docs +try bin/docs-markdown ci/www/public/docs --base-url https://ci.materialize.com/docs/ +try bin/pytest -qq misc/python/materialize/cli/docs_markdown_test.py echo "" > ci/www/public/index.html try htmltest -s ci/www/public -c doc/user/.htmltest.yml try ci/test/lint-docs-catalog.sh diff --git a/ci/test/preview-docs.sh b/ci/test/preview-docs.sh index 8a122faf50a6f..22a9fbeb0402d 100755 --- a/ci/test/preview-docs.sh +++ b/ci/test/preview-docs.sh @@ -22,16 +22,13 @@ cd doc/user # Build main docs to public/ hugo --gc --environment preview --baseURL "/materialize/$BUILDKITE_PULL_REQUEST" - -# Build skill docs to public/markdown-docs/ -hugo --config config.toml,config.skill.toml --gc --baseURL "/materialize/$BUILDKITE_PULL_REQUEST" --disableKinds 404,sitemap,robotsTXT,taxonomy +../../bin/docs-markdown public --base-url "https://preview.materialize.com/materialize/$BUILDKITE_PULL_REQUEST/" cat > config.deployment.toml <index.md`, for agents and +other non-browser readers. `bin/docs-markdown` generates it after the Hugo +build by converting each page's `
` element, so shortcodes need no +Markdown variant. Chrome inside the `
` that should stay out of the +Markdown, such as a copy button, takes the `data-markdown-ignore` attribute. -When adding or changing a shortcode that emits HTML, add or update its -`.skill.md` variant too. Without one, Hugo falls back to the `.html` template -and can leak raw HTML into the Markdown output. Existing shortcodes without a -variant are a known gap, not a pattern to copy. +To inspect the Markdown for a template or shortcode change, run from the +repository root: + +```shell +hugo --source doc/user --destination /tmp/docs/docs --baseURL /docs +bin/docs-markdown /tmp/docs/docs --base-url https://materialize.com/docs/ +``` ### Reusing content @@ -71,12 +74,10 @@ Run commands from `doc/user` unless noted otherwise: - `../../bin/format-docs` trims trailing whitespace and ensures Markdown files end with a newline. - `sql-grammar/generate.sh` regenerates railroad diagrams after BNF changes. -- `hugo --config config.toml,config.skill.toml --disableKinds - sitemap,robotsTXT,taxonomy` builds the agent-readable Markdown output. - `./docs-pre-pr-review-claude.sh --worktree`, `--staged`, `--range A..B`, or a list of files runs the pre-PR content review. -The full docs lint builds both output formats before running `htmltest` and the +The full docs lint builds the site before running `htmltest` and the catalog and metrics checks. Install `htmltest` with `brew install htmltest` if it is unavailable locally. @@ -200,10 +201,6 @@ run `../../ci/test/lint-docs.sh` to catch broken links, HTML errors, and catalog inconsistencies. Review generated diagrams and rendered examples when changing shortcodes, layouts, or SQL grammar. -When changing templates or shortcodes, inspect both the HTML and Markdown -outputs. A successful HTML render does not prove that the `skill` output is -valid. - ## Test syntax as you write the documentation When documentation adds or changes SQL syntax, test each example using the diff --git a/doc/user/assets/sass/_base.scss b/doc/user/assets/sass/_base.scss index d1c87ae66585e..1a2109f4d44e9 100644 --- a/doc/user/assets/sass/_base.scss +++ b/doc/user/assets/sass/_base.scss @@ -221,6 +221,18 @@ input[type="submit"] { .osano-cm-widget { display: none; } +.llms-txt-directive { + position: absolute; + width: 1px; + height: 1px; + margin: -1px; + padding: 0; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; + border: 0; +} + /** The following section handles these cases: 1. Buttons with links diff --git a/doc/user/config.skill.toml b/doc/user/config.skill.toml deleted file mode 100644 index 334d03451427b..0000000000000 --- a/doc/user/config.skill.toml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. -# -# config.skill.toml - Configuration overlay for generating Claude skill output. -# Use with: hugo --cleanDestinationDir --config config.toml,config.skill.toml --disableKinds 404,sitemap,robotsTXT,taxonomy - -publishDir = "public/markdown-docs" - -# Output only the skill format (markdown), not HTML -[outputs] - home = ["skill"] - section = ["skill"] - page = ["skill"] - -[outputFormats.HTML] -disable = true - -[outputFormats.RSS] -disable = true - -# Exclude static files by excluding everything in the static mount -[[module.mounts]] - source = "static" - target = "static" - excludeFiles = ["**", "*"] - -# Skill-specific parameters -[params] - # Sections to exclude from skill generation - excludeFromSkill = [] - -[cascade] - aliases = [] diff --git a/doc/user/config.toml b/doc/user/config.toml index 213219ed88f7b..73a1028eb2288 100644 --- a/doc/user/config.toml +++ b/doc/user/config.toml @@ -101,15 +101,6 @@ parent = "about" url = "https://materialize.com/securitydisclosure" weight = 55 -# -# Custom output format for Claude skill generation -# -[outputFormats.skill] - mediaType = "text/markdown" - baseName = "index" - isPlainText = true - notAlternative = true - # # Custom output format for the llms.txt docs index # @@ -119,8 +110,25 @@ weight = 55 isPlainText = true notAlternative = true +# +# The full sidebar tree, fetched by the sidebar script (baseof.html). +# +[outputFormats.sidebar] + mediaType = "text/html" + baseName = "sidebar" + isHTML = true + notAlternative = true + [outputs] - home = ["HTML", "RSS", "llmstxt"] + home = ["HTML", "RSS", "llmstxt", "sidebar"] + +# Each top-level section publishes its own llms.txt, which the root llms.txt +# links to. +[[cascade]] + outputs = ["HTML", "RSS", "llmstxt"] + [cascade.target] + kind = "section" + path = "/*" [markup.goldmark.renderer] # allow , the old syntax no longer works @@ -136,6 +144,16 @@ unsafe = true startLevel = 2 endLevel = 4 +[[deployment.matchers]] +# RFC 7763 makes charset a required parameter of text/markdown. +pattern = "^.+\\.md$" +contentType = "text/markdown; charset=utf-8" + +[[deployment.matchers]] +# Hugo deploy would otherwise serve the sitemap as application/rss+xml. +pattern = "^(.+/)?sitemap\\.xml$" +contentType = "application/xml" + [[deployment.targets]] name = "production" url = "s3://materialize-website?region=us-east-1" diff --git a/doc/user/layouts/_default/baseof.html b/doc/user/layouts/_default/baseof.html index 167df07159dfe..788e41630a836 100644 --- a/doc/user/layouts/_default/baseof.html +++ b/doc/user/layouts/_default/baseof.html @@ -10,6 +10,18 @@ + {{- /* + Points agents that fetch this page at the docs index. It must be the + first element in , ahead of the navigation, so that it survives a + truncated fetch. It is hidden visually rather than with display:none, + which HTML-to-Markdown converters drop. bin/docs-markdown copies its + text to the top of the Markdown rendition. + */}} + " + ) + assert body(html) == "# Title\n\nText." + + +def test_hard_line_break() -> None: + assert body("

a
b

") == "a \nb" + + +def test_front_matter_and_directive_lead_the_page() -> None: + html = document( + "

Title

", + head='T | Docs', + ) + page = Converter(PAGE_URL).convert(parse_html(html)) + assert render(page) == ( + '---\ntitle: "Title"\ndescription: "Says \\"hi\\"."\n---\n\n' + "> See /llms.txt.\n\n# Title\n" + ) + + +def test_page_without_article_is_an_error() -> None: + with pytest.raises(PageError, match="article"): + Converter(PAGE_URL).convert(parse_html(f"{DIRECTIVE}

x

")) + + +def test_page_without_directive_is_an_error() -> None: + with pytest.raises(PageError, match="directive"): + Converter(PAGE_URL).convert(parse_html("
")) + + +def test_convert_site(tmp_path: Path) -> None: + page = tmp_path / "sql" / "index.html" + page.parent.mkdir() + page.write_text(document('x')) + # Alias redirect stubs have no
and get no Markdown. + alias = tmp_path / "old" / "index.html" + alias.parent.mkdir() + alias.write_text('') + + written, errors = convert_site(tmp_path, "https://materialize.com/docs") + + assert (written, errors) == (1, []) + assert ( + "[x](https://materialize.com/docs/x/)" in (page.parent / "index.md").read_text() + ) + assert not (alias.parent / "index.md").exists()