docs: add versioned documentation site built with Zensical

.github/workflows/docs-deploy.yml

@@ -0,0 +1,77 @@
+name: Deploy documentation
+
+# Publishes versioned docs to the `gh-pages` branch using the Zensical-compatible
+# fork of `mike` (https://github.com/squidfunk/mike): each version is committed
+# as a subdirectory of `site_url` (see zensical.toml), with a `latest` alias and
+# a root redirect pointing at the newest release.
+#
+# Runs automatically on release tags (semantic-release tags the bare version,
+# e.g. 0.95.1) and can be triggered manually. Release tags publish the minor
+# series (e.g. 0.95) so that patch releases update the same series page.
+
+on:
+  push:
+    tags: ["[0-9]+.[0-9]+.[0-9]+"]
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Version label to publish (e.g. 0.95 or dev)"
+        required: true
+        default: "dev"
+      alias:
+        description: "Alias to update"
+        required: false
+        default: "latest"
+      set_default:
+        description: "Point the site root at this alias"
+        type: boolean
+        default: true
+
+permissions:
+  contents: write # mike pushes the built docs to the gh-pages branch
+
+concurrency:
+  group: docs-deploy
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    name: Deploy docs with mike
+    runs-on: ubuntu-latest
+    # Auto-deploy only on the canonical repo; manual runs are allowed anywhere
+    # (e.g. to bootstrap gh-pages on a fork while testing).
+    if: ${{ github.event_name == 'workflow_dispatch' || github.repository == 'substrait-io/substrait-java' }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 0 # mike reads/writes the full gh-pages history
+      - name: Set up pixi
+        uses: prefix-dev/setup-pixi@v0.10.0
+        with:
+          environments: docs
+      - name: Configure git identity
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+      - name: Resolve version and alias
+        id: v
+        run: |
+          if [ "${{ github.event_name }}" = "push" ]; then
+            full="${GITHUB_REF_NAME}"                        # 0.95.1
+            echo "version=${full%.*}" >> "$GITHUB_OUTPUT"    # 0.95 (minor series)
+            echo "alias=latest" >> "$GITHUB_OUTPUT"
+            echo "set_default=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "version=${{ inputs.version }}" >> "$GITHUB_OUTPUT"
+            echo "alias=${{ inputs.alias }}" >> "$GITHUB_OUTPUT"
+            echo "set_default=${{ inputs.set_default }}" >> "$GITHUB_OUTPUT"
+          fi
+      - name: Deploy version
+        run: |
+          pixi run -e docs mike deploy --push --update-aliases \
+            "${{ steps.v.outputs.version }}" "${{ steps.v.outputs.alias }}"
+      - name: Set default version
+        if: ${{ steps.v.outputs.set_default == 'true' }}
+        run: |
+          pixi run -e docs mike set-default --push "${{ steps.v.outputs.alias }}"

.github/workflows/docs.yml

@@ -0,0 +1,28 @@
+name: Build documentation
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+# Build-check only: this verifies the docs compile on every PR so broken pages
+# or a broken link are caught early. It does NOT publish. Versioned publishing
+# to GitHub Pages is handled separately by .github/workflows/docs-deploy.yml
+# (mike -> gh-pages branch).
+
+jobs:
+  build:
+    name: Build docs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+      - name: Set up pixi
+        uses: prefix-dev/setup-pixi@v0.10.0
+        with:
+          environments: docs
+      - name: Build the documentation site
+        run: pixi run docs-build

.gitignore

@@ -20,3 +20,7 @@ bin
 .classpath
 .settings
 bin/
+
+# Documentation site (Zensical + pixi)
+/site
+.pixi

AGENTS.md

@@ -192,6 +192,40 @@ compile — they have their own visitor implementors:
   fidelity. See `core/src/test/java/io/substrait/type/proto/DynamicParameterRoundtripTest.java`
   for the canonical pattern.
 
+## Documentation
+
+User-facing documentation lives in `docs/` (Markdown, one file per page) and is built with
+**Zensical** (config in `zensical.toml`). Python and the doc tooling are managed with **pixi**,
+independent of the Gradle build — the only prerequisite is a `pixi` install.
+
+- Preview locally with `pixi run docs-serve` (live reload); build the static site with
+  `pixi run docs-build` (output to `site/`, gitignored). `docs-build` validates internal links,
+  so run it before pushing doc changes.
+- Pages are grouped under `docs/{core,isthmus,isthmus-cli,spark}/` plus a landing page and
+  getting-started. Navigation is **explicit** in `zensical.toml` (`nav`) — when you add a page,
+  add it to `nav`.
+- **Keep the guide in sync with the code as substrait-java evolves.** When you add or change
+  user-facing behavior — a new expression/relation type, a `SubstraitBuilder`/`ExpressionCreator`
+  factory, a newly supported function, an isthmus/spark capability, or a CLI flag — update the
+  matching `docs/` page in the same PR. (As in source) keep GitHub issue/PR numbers out of the docs.
+- **Runnable code samples are pulled from compiled, CI-run tests via `pymdownx.snippets`** — they
+  are not hand-written in the Markdown, so they can't silently drift from the API. Backing tests
+  live in an `io.substrait…docs` package under each module's test sources: `core`
+  (`core/src/test/java/io/substrait/docs/`) and `isthmus`
+  (`isthmus/src/test/java/io/substrait/isthmus/docs/`) as JUnit tests, and `spark`
+  (`spark/src/test/scala/io/substrait/spark/docs/DocExamplesSuite.scala`, a full round-trip run on
+  every variant). A test marks a region with `// --8<-- [start:name]` / `[end:name]`; the Markdown
+  references it inside a fenced block, e.g.
+  `--8<-- "core/src/test/java/io/substrait/docs/TypesDocTest.java:aliases"`. Edit the **test**, not
+  the Markdown, to change a sample; `pixi run docs-build` fails on a missing file or region
+  (`check_paths`), and the backing test failing to compile/run fails the Gradle build. `import`
+  lines stay literal alongside the include; pure API-signature and resource-dependent snippets
+  remain inline.
+- CI: `.github/workflows/docs.yml` build-checks docs on every PR and push to `main`;
+  `.github/workflows/docs-deploy.yml` publishes versioned docs to the `gh-pages` branch via the
+  Zensical-compatible `mike` fork on release tags (the bare `X.Y.Z` tag publishes the minor
+  series `X.Y` and updates the `latest` alias). Site: <https://substrait-io.github.io/substrait-java/>.
+
 ## Conventions & workflow
 
 - **Conventional commits** are required (CI lints them, and PR title + body must form a
@@ -204,6 +238,7 @@ compile — they have their own visitor implementors:
 - Many features track upstream Substrait spec releases (see epic-style issues); a new
   proto message usually needs: POJO + visitor wiring + both proto converters + a
   round-trip test, and often `ExpressionCreator` factories and `dsl/SubstraitBuilder`
-  helpers for ergonomics.
+  helpers for ergonomics — plus a matching update to the user guide under `docs/`
+  (see [Documentation](#documentation)).
 - The macOS native image is not built on PRs (only Linux is), so macOS-specific native
   regressions surface on `main` or the weekly `native-image-macos.yml` run, not on the PR.

CONTRIBUTING.md

@@ -4,6 +4,7 @@ This page provides some orientation and recommendations on how to get the best r
 
 1. [Commit conventions](#commit-conventions)
 2. [Style Guide](#style-guide)
+3. [Documentation](#documentation)
 
 ## Commit Conventions
 
@@ -45,3 +46,39 @@ regardless of which JDK launches Gradle, as long as a JDK 17 is installed and di
 Spotless is the exception: the `google-java-format` version it uses only runs on JDK 17 and
 fails with `NoSuchMethodError` / `NoClassDefFoundError` when the Gradle daemon runs on a newer
 JDK, so `./gradlew spotlessApply` (and `spotlessCheck`) require the daemon itself to be on JDK 17.
+
+## Documentation
+
+The user-facing documentation site lives under [`docs/`](docs) and is built with
+[Zensical](https://zensical.org). Python and the documentation dependencies are managed with
+[pixi](https://pixi.sh), so the only prerequisite is a pixi installation.
+
+Preview your changes locally with a live-reloading dev server, or produce the static site:
+
+```bash
+pixi run docs-serve   # live-reloading preview at http://localhost:8000
+pixi run docs-build   # build the static site into ./site
+```
+
+Guidelines:
+
+* Every page is a Markdown file under `docs/`; the navigation is defined explicitly in
+  [`zensical.toml`](zensical.toml). When you add a page, add it to the `nav`.
+* **Code samples are verified, not hand-written.** Runnable code samples are pulled from compiled,
+  CI-executed tests with
+  [`pymdownx.snippets`](https://facelessuser.github.io/pymdown-extensions/extensions/snippets/), so
+  they cannot drift from the API. The backing test marks a region with `// --8<-- [start:name]` /
+  `// --8<-- [end:name]`, and the Markdown references it inside a fenced code block, e.g.
+  `--8<-- "core/src/test/java/io/substrait/docs/BuildingPlansDocTest.java:create-builder"`. Backing
+  tests live in an `io.substrait…docs` package under each module's test sources (`core` and
+  `isthmus` as JUnit tests, `spark` as a scalatest suite that runs a full round trip). To add or
+  change a sample, edit the test — not the Markdown — then run that module's tests; `pixi run
+  docs-build` fails if a referenced file or region is missing. `import` lines and similar
+  illustrative context may be kept as literal text alongside the include. A few pure
+  API-signature snippets (and examples that need external resources) remain inline.
+* As elsewhere in the codebase, do not reference GitHub issue or PR numbers in the docs.
+
+Documentation is built on every pull request by the `Build documentation` workflow. On each
+release, the `Deploy documentation` workflow publishes a versioned copy to
+<https://substrait-io.github.io/substrait-java/> (via the `mike` version manager, on the
+`gh-pages` branch).

core/src/test/java/io/substrait/docs/BuildingPlansDocTest.java

@@ -0,0 +1,89 @@
+package io.substrait.docs;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import io.substrait.TestBase;
+import io.substrait.dsl.SubstraitBuilder;
+import io.substrait.extension.SimpleExtension;
+import io.substrait.plan.Plan;
+import io.substrait.relation.Filter;
+import io.substrait.relation.NamedScan;
+import io.substrait.relation.Project;
+import io.substrait.type.TypeCreator;
+import java.util.List;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Backs the code samples in {@code docs/core/building-plans.md}. Each region marked with {@code //
+ * --8<-- [start:name]} / {@code [end:name]} is pulled into the docs via a {@code --8<--} snippet
+ * include, so the samples shown there are exactly this compiled and executed code.
+ */
+class BuildingPlansDocTest extends TestBase {
+
+  @Test
+  void creatingABuilder() {
+    // --8<-- [start:create-builder]
+    SubstraitBuilder b = new SubstraitBuilder();
+    // --8<-- [end:create-builder]
+    assertNotNull(b);
+  }
+
+  @Test
+  void creatingABuilderWithACustomCollection() {
+    SimpleExtension.ExtensionCollection myExtensionCollection = extensions;
+    // --8<-- [start:create-builder-custom]
+    SubstraitBuilder b = new SubstraitBuilder(myExtensionCollection);
+    // --8<-- [end:create-builder-custom]
+    assertNotNull(b);
+  }
+
+  @Test
+  void typeShortcuts() {
+    // --8<-- [start:type-shortcuts]
+    TypeCreator R = TypeCreator.REQUIRED; // non-nullable types
+    TypeCreator N = TypeCreator.NULLABLE; // nullable types
+    // --8<-- [end:type-shortcuts]
+    assertNotNull(R.I32);
+    assertNotNull(N.STRING);
+  }
+
+  @Test
+  void relationHelpersTakeLambdas() {
+    SubstraitBuilder b = new SubstraitBuilder();
+    // --8<-- [start:filter-lambda]
+    NamedScan scan = b.namedScan(List.of("t"), List.of("a", "b"), List.of(R.I32, R.STRING));
+
+    Filter filter = b.filter(rel -> b.equal(b.fieldReference(rel, 0), b.i32(10)), scan);
+    // --8<-- [end:filter-lambda]
+    assertNotNull(filter);
+  }
+
+  @Test
+  void endToEndExample() {
+    // --8<-- [start:end-to-end]
+    TypeCreator R = TypeCreator.REQUIRED;
+    SubstraitBuilder b = new SubstraitBuilder();
+
+    // 1. scan: table "t" with columns a (i32) and b (string)
+    NamedScan scan = b.namedScan(List.of("t"), List.of("a", "b"), List.of(R.I32, R.STRING));
+
+    // 2. filter: keep rows where a == 10
+    Filter filter = b.filter(rel -> b.equal(b.fieldReference(rel, 0), b.i32(10)), scan);
+
+    // 3. project: output columns a and b. project appends its expressions after
+    //    the input fields, so remap keeps only the two projected columns.
+    Project project =
+        b.project(
+            rel -> List.of(b.fieldReference(rel, 0), b.fieldReference(rel, 1)),
+            b.remap(2, 3),
+            filter);
+
+    // 4. root: name the plan's output columns
+    Plan.Root root = b.root(project, List.of("a", "b"));
+
+    // 5. plan: wrap the root
+    Plan plan = b.plan(root);
+    // --8<-- [end:end-to-end]
+    assertNotNull(plan);
+  }
+}

core/src/test/java/io/substrait/docs/CoreIndexDocTest.java

@@ -0,0 +1,36 @@
+package io.substrait.docs;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import io.substrait.TestBase;
+import io.substrait.plan.Plan;
+import io.substrait.relation.NamedScan;
+import io.substrait.type.NamedStruct;
+import java.util.List;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Backs the code sample in {@code docs/core/index.md}. The region is pulled into the docs via a
+ * {@code --8<--} snippet include.
+ */
+class CoreIndexDocTest extends TestBase {
+
+  @Test
+  void quickExample() {
+    NamedStruct schema = NamedStruct.of(List.of("id"), R.struct(R.I32));
+    // --8<-- [start:quick-example]
+    NamedScan scan = NamedScan.builder().addNames("my_table").initialSchema(schema).build();
+
+    Plan.Root root = Plan.Root.builder().input(scan).build();
+    Plan plan =
+        Plan.builder()
+            .addRoots(root)
+            .executionBehavior(
+                Plan.ExecutionBehavior.builder()
+                    .variableEvaluationMode(Plan.ExecutionBehavior.VariableEvaluationMode.PER_PLAN)
+                    .build())
+            .build();
+    // --8<-- [end:quick-example]
+    assertNotNull(plan);
+  }
+}