diff --git a/ci/plugins/mzcompose/hooks/command b/ci/plugins/mzcompose/hooks/command index 9fd0938b3c561..b0e74df8d7a7e 100644 --- a/ci/plugins/mzcompose/hooks/command +++ b/ci/plugins/mzcompose/hooks/command @@ -444,6 +444,7 @@ cleanup() { && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Canary Environment" ] \ && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Benchmarking Staging Environment" ] \ && [ "$BUILDKITE_LABEL" != ":rust: Miri test (full)" ] \ + && [ "$BUILDKITE_LABEL" != "Lean models" ] \ && [[ ! "$BUILDKITE_LABEL" =~ Terraform\ .* ]] \ && [[ ! "$BUILDKITE_LABEL" =~ Orchestratord\ .* ]] \ && [[ ! "$BUILDKITE_LABEL" =~ Cluster\ spec\ sheet.* ]] \ diff --git a/ci/test/lint-main/checks/check-copyright.sh b/ci/test/lint-main/checks/check-copyright.sh index 7ed9a509609a5..d415a11ddfb6a 100755 --- a/ci/test/lint-main/checks/check-copyright.sh +++ b/ci/test/lint-main/checks/check-copyright.sh @@ -32,6 +32,7 @@ copyright_files=$(grep -vE \ -e '(^|/)Cargo\.lock$' \ -e '(^|/)types\.lock$' \ -e '(^|/)\.mzprofile$' \ + -e '(^|/)lean-toolchain$' \ -e '^src/mz-deploy/src/cli/scaffold/gitignore$' \ -e '^test/mz-deploy/projects/multi-profile/v1/models/app/core/ambiguous\.sql$' \ -e '^about\.toml$' \ diff --git a/ci/test/pipeline.template.yml b/ci/test/pipeline.template.yml index e64cff9510b85..fc98e0a2be2d8 100644 --- a/ci/test/pipeline.template.yml +++ b/ci/test/pipeline.template.yml @@ -668,6 +668,21 @@ steps: agents: queue: hetzner-aarch64-4cpu-8gb + - id: lean + label: Lean models + depends_on: build-aarch64 + timeout_in_minutes: 30 + # Design doc models live outside the composition directory. A git + # pathspec `*` also matches `/`, so this covers every subdirectory. + inputs: ["doc/developer/design/*.lean"] + plugins: + - ./ci/plugins/mzcompose: + composition: lean + agents: + queue: hetzner-aarch64-4cpu-8gb + coverage: skip + sanitizer: skip + - id: mcp label: MCP e2e depends_on: build-aarch64 diff --git a/doc/developer/design/README.md b/doc/developer/design/README.md index 95feb3d0d4bf9..415bdfb6e67da 100644 --- a/doc/developer/design/README.md +++ b/doc/developer/design/README.md @@ -126,6 +126,11 @@ your thinking and inform the writing process. of stakeholders. Note that we have a bot that automatically posts notifications about new design documents in #rnd-design-docs. +A design document can carry a Lean 4 model that CI checks. Put the document +in its own date-prefixed directory and the model's `.lean` files next to it. +See [`misc/lean/README.md`](../../../misc/lean/README.md) for how the models +are built. + ### Iteration 6. As you begin to get feedback on your document, address the comments diff --git a/misc/images/lean/.gitignore b/misc/images/lean/.gitignore new file mode 100644 index 0000000000000..4fc9b82b8ab13 --- /dev/null +++ b/misc/images/lean/.gitignore @@ -0,0 +1,3 @@ +/lean-toolchain +/lakefile.lean +/lake-manifest.json diff --git a/misc/images/lean/Dockerfile b/misc/images/lean/Dockerfile new file mode 100644 index 0000000000000..6d2f98150bf3b --- /dev/null +++ b/misc/images/lean/Dockerfile @@ -0,0 +1,78 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +# Lean 4 toolchain for the Lake project in `misc/lean`. The toolchain comes +# from that project's `lean-toolchain`, so a local elan install and this image +# use the same Lean version. The project's dependencies are built into +# `/mz/misc/lean/.lake` at image build time. The project sits at its +# repository path because `lakefile.lean` finds design doc models relative to +# itself, in `/mz/doc/developer/design`. + +MZFROM debian-base + +ARG ELAN_VERSION=v4.2.4 + +ENV ELAN_HOME=/opt/elan +ENV PATH=$ELAN_HOME/bin:$PATH + +RUN apt-get update \ + && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + git \ + jq \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +COPY lean-toolchain /mz/misc/lean/ + +# NOTE: the toolchain is kept whole although checking proofs needs only part of +# it. Batteries, a dependency of Aesop and Mathlib, has the `runLinter` +# executable among its default targets. Linking it needs the toolchain's static +# libraries and LLVM, so removing them breaks the dependency build below. +RUN case "$(uname -m)" in \ + x86_64) triple=x86_64-unknown-linux-gnu; sha256=42b94d4244e8353142c456ec0e4ca6528fd898a6c604d4059f494e706e431f63 ;; \ + aarch64) triple=aarch64-unknown-linux-gnu; sha256=05febd124d84ebf994b2e7479922a5650b1e950c17ae3bd1ddd776b65bb72bf9 ;; \ + *) echo "unsupported architecture $(uname -m)" >&2; exit 1 ;; \ + esac \ + && curl -fsSL -o /tmp/elan.tar.gz \ + "https://github.com/leanprover/elan/releases/download/$ELAN_VERSION/elan-$triple.tar.gz" \ + && echo "$sha256 /tmp/elan.tar.gz" | sha256sum -c - \ + && tar -C /tmp -xzf /tmp/elan.tar.gz \ + && /tmp/elan-init -y --no-modify-path --default-toolchain "$(cat /mz/misc/lean/lean-toolchain)" \ + && rm /tmp/elan.tar.gz /tmp/elan-init \ + && elan toolchain install "$(cat /mz/misc/lean/lean-toolchain)" \ + && chmod -R a+rX "$ELAN_HOME" \ + && lean --version + +# Git refuses repositories owned by a different uid than the one running it, +# and Lake then re-clones the baked dependencies. +RUN git config --system --add safe.directory '*' + +WORKDIR /mz/misc/lean + +COPY lakefile.lean lake-manifest.json /mz/misc/lean/ + +# Builds the dependency closure. Building the placeholder library alone only +# clones the dependencies, because Lake compiles just the modules a target +# imports, so each package's default targets are built by name. NOTE: a +# package that declares no default targets builds nothing here and compiles +# on every run instead. One `lake build` for all packages fails if any of them +# fails. With no dependencies it builds the placeholder. +RUN mkdir Mz \ + && touch Mz.lean \ + && targets="$(jq -r '.packages[] | "@" + .name' lake-manifest.json)" \ + && lake build $targets \ + && rm -rf Mz Mz.lean .lake/build \ + && chmod -R a+rwX /mz/misc/lean + +COPY mz-lake /usr/local/bin/ + +ENTRYPOINT ["mz-lake"] +CMD ["build"] diff --git a/misc/images/lean/mz-lake b/misc/images/lean/mz-lake new file mode 100755 index 0000000000000..38d19de493d0f --- /dev/null +++ b/misc/images/lean/mz-lake @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. +# +# mz-lake: copy the Lake project mounted at /src into /mz/misc/lean and run +# `lake` there with the given arguments. Design doc models are read from +# /mz/doc/developer/design, which mirrors the repository layout that +# `lakefile.lean` expects. + +set -euo pipefail + +# Mounting the project over /mz/misc/lean would hide its .lake, which holds the +# dependency closure built into the image. The host's own .lake is skipped for +# the same reason. The entries are copied one by one because copying `.` itself +# makes a non-root user try to change the mode and mtime of the root-owned +# project directory, which fails. +cd /src +find . -mindepth 1 -maxdepth 1 ! -name .lake -exec cp -R -t /mz/misc/lean {} + + +cd /mz/misc/lean +# The image build cached a configuration elaborated without any design doc +# models. `-R` elaborates `lakefile.lean` again so it finds them. +exec lake -R "$@" diff --git a/misc/images/lean/mzbuild.yml b/misc/images/lean/mzbuild.yml new file mode 100644 index 0000000000000..e4700f1b39a05 --- /dev/null +++ b/misc/images/lean/mzbuild.yml @@ -0,0 +1,26 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +name: lean +description: Lean 4 toolchain and dependency closure for the models in misc/lean. +# Only the files that pin the toolchain and the dependencies are copied, so +# editing a model does not change the image fingerprint. +pre-image: + - type: copy + source: misc/lean + destination: . + matching: lean-toolchain + - type: copy + source: misc/lean + destination: . + matching: lakefile.lean + - type: copy + source: misc/lean + destination: . + matching: lake-manifest.json diff --git a/misc/lean/.gitignore b/misc/lean/.gitignore new file mode 100644 index 0000000000000..bfb30ec8c762c --- /dev/null +++ b/misc/lean/.gitignore @@ -0,0 +1 @@ +/.lake diff --git a/misc/lean/Mz.lean b/misc/lean/Mz.lean new file mode 100644 index 0000000000000..d0ffff89084fe --- /dev/null +++ b/misc/lean/Mz.lean @@ -0,0 +1,10 @@ +-- Copyright Materialize, Inc. and contributors. All rights reserved. +-- +-- Use of this software is governed by the Business Source License +-- included in the LICENSE file at the root of this repository. +-- +-- As of the Change Date specified in that file, in accordance with +-- the Business Source License, use of this software will be governed +-- by the Apache License, Version 2.0. + +import Mz.Collection diff --git a/misc/lean/Mz/Collection.lean b/misc/lean/Mz/Collection.lean new file mode 100644 index 0000000000000..4511f4cd8b96e --- /dev/null +++ b/misc/lean/Mz/Collection.lean @@ -0,0 +1,200 @@ +-- Copyright Materialize, Inc. and contributors. All rights reserved. +-- +-- Use of this software is governed by the Business Source License +-- included in the LICENSE file at the root of this repository. +-- +-- As of the Change Date specified in that file, in accordance with +-- the Business Source License, use of this software will be governed +-- by the Apache License, Version 2.0. + +/-! +# Collections of updates + +A collection is a list of `(record, diff)` updates. Its meaning is the +multiplicity of each record, the sum of that record's diffs. Two collections +are equivalent when they agree on every multiplicity, regardless of how the +updates are ordered or split. + +`consolidate` merges updates to the same record and drops the ones that cancel. +It is correct when it preserves every multiplicity, leaves no zero diff, and +leaves at most one update per record. +-/ + +namespace Mz + +abbrev Collection (α : Type) := List (α × Int) + +namespace Collection + +variable {α : Type} [DecidableEq α] + +/-- The multiplicity of `x` in `c`. -/ +def count : Collection α → α → Int + | [], _ => 0 + | (a, d) :: rest, x => (if a = x then d else 0) + count rest x + +/-- Flips the sign of every diff. -/ +def negate (c : Collection α) : Collection α := + c.map fun (a, d) => (a, -d) + +/-- Adds `d` to the update for `a`, dropping that update if it cancels. -/ +def insert (a : α) (d : Int) : Collection α → Collection α + | [] => if d = 0 then [] else [(a, d)] + | (b, e) :: rest => + if b = a then + if d + e = 0 then rest else (a, d + e) :: rest + else + (b, e) :: insert a d rest + +/-- Merges updates to the same record and drops the ones that cancel. -/ +def consolidate : Collection α → Collection α + | [] => [] + | (a, d) :: rest => insert a d (consolidate rest) + +theorem count_append (c₁ c₂ : Collection α) (x : α) : + count (c₁ ++ c₂) x = count c₁ x + count c₂ x := by + induction c₁ with + | nil => simp [count] + | cons u rest ih => + obtain ⟨a, d⟩ := u + simp [count, ih, Int.add_assoc] + +theorem count_negate (c : Collection α) (x : α) : + count (negate c) x = -count c x := by + induction c with + | nil => simp [negate, count] + | cons u rest ih => + obtain ⟨a, d⟩ := u + simp only [negate, List.map_cons, count] at ih ⊢ + rw [ih] + split <;> omega + +/-- Appending a collection's negation cancels it. -/ +theorem count_append_negate (c : Collection α) (x : α) : + count (c ++ negate c) x = 0 := by + rw [count_append, count_negate] + omega + +theorem count_insert (a : α) (d : Int) (c : Collection α) (x : α) : + count (insert a d c) x = (if a = x then d else 0) + count c x := by + induction c with + | nil => + simp only [insert, count] + split <;> simp [count, *] + | cons u rest ih => + obtain ⟨b, e⟩ := u + by_cases hb : b = a + · subst hb + by_cases hz : d + e = 0 + · rw [insert, ite_eq_left rfl, ite_eq_left hz, count] + by_cases hx : b = x <;> simp [hx] <;> omega + · rw [insert, ite_eq_left rfl, ite_eq_right hz, count, count] + by_cases hx : b = x <;> simp [hx] <;> omega + · rw [insert, ite_eq_right hb, count, count, ih] + omega + +/-- Consolidation preserves the multiplicity of every record. -/ +theorem count_consolidate (c : Collection α) (x : α) : + count (consolidate c) x = count c x := by + induction c with + | nil => rfl + | cons u rest ih => + obtain ⟨a, d⟩ := u + simp only [consolidate, count_insert, ih, count] + +/-- Every update has a nonzero diff. -/ +def NoZeros (c : Collection α) : Prop := + ∀ u ∈ c, u.2 ≠ 0 + +theorem noZeros_insert (a : α) (d : Int) (c : Collection α) (h : NoZeros c) : + NoZeros (insert a d c) := by + induction c with + | nil => + simp only [insert] + split + · simp [NoZeros] + · simp [NoZeros, *] + | cons u rest ih => + obtain ⟨b, e⟩ := u + have hrest : NoZeros rest := fun v hv => h v (List.mem_cons_of_mem _ hv) + have he : e ≠ 0 := h (b, e) List.mem_cons_self + simp only [insert] + split + · split + · exact hrest + · intro v hv + cases List.mem_cons.mp hv with + | inl hv => subst hv; assumption + | inr hv => exact hrest v hv + · intro v hv + cases List.mem_cons.mp hv with + | inl hv => subst hv; exact he + | inr hv => exact ih hrest v hv + +/-- Consolidation leaves no update with a zero diff. -/ +theorem noZeros_consolidate (c : Collection α) : NoZeros (consolidate c) := by + induction c with + | nil => simp [consolidate, NoZeros] + | cons u rest ih => + obtain ⟨a, d⟩ := u + exact noZeros_insert a d _ ih + +/-- The records that have an update. -/ +def keys (c : Collection α) : List α := + c.map Prod.fst + +theorem mem_keys_insert {a x : α} {d : Int} {c : Collection α} + (h : x ∈ keys (insert a d c)) : x = a ∨ x ∈ keys c := by + induction c with + | nil => + simp only [insert] at h + split at h <;> simp_all [keys] + | cons u rest ih => + obtain ⟨b, e⟩ := u + simp only [insert] at h + split at h + · split at h <;> simp_all [keys] + · simp only [keys, List.map_cons, List.mem_cons] at h ⊢ + rcases h with h | h + · exact Or.inr (Or.inl h) + · rcases ih h with h | h + · exact Or.inl h + · exact Or.inr (Or.inr h) + +theorem nodup_insert (a : α) (d : Int) (c : Collection α) (h : (keys c).Nodup) : + (keys (insert a d c)).Nodup := by + induction c with + | nil => + simp only [insert] + split <;> simp [keys] + | cons u rest ih => + obtain ⟨b, e⟩ := u + simp only [keys, List.map_cons, List.nodup_cons] at h + obtain ⟨hb, hrest⟩ := h + simp only [insert] + split + · subst_vars + split + · exact hrest + · simp only [keys, List.map_cons, List.nodup_cons] + exact ⟨hb, hrest⟩ + · simp only [keys, List.map_cons, List.nodup_cons] + refine ⟨fun hmem => ?_, ih hrest⟩ + rcases mem_keys_insert hmem with h | h + · contradiction + · exact hb h + +/-- Consolidation leaves at most one update per record. -/ +theorem nodup_consolidate (c : Collection α) : (keys (consolidate c)).Nodup := by + induction c with + | nil => simp [consolidate, keys] + | cons u rest ih => + obtain ⟨a, d⟩ := u + exact nodup_insert a d _ ih + +example : consolidate ([(1, 2), (2, 1), (1, -2)] : Collection Nat) = [(2, 1)] := by + decide + +end Collection + +end Mz diff --git a/misc/lean/README.md b/misc/lean/README.md new file mode 100644 index 0000000000000..3cf2d898f8a35 --- /dev/null +++ b/misc/lean/README.md @@ -0,0 +1,42 @@ +# Lean models + +This directory is a Lake project for Lean 4 models of Materialize. +Models live in two places: under `Mz/` in this directory, and next to the design doc they belong to, as `doc/developer/design/