From 39d3fb4c76326004dd0ff2165a0d25534d2104e5 Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Wed, 9 Sep 2026 06:59:14 +0000 Subject: [PATCH] docs: a walkthrough that runs, and a front page whose output is not typed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured before writing any of this: `docs/src` had 32 fenced ```julia blocks whose output was typed by hand, against one `@example` and one `jldoctest`. Two of those 32 had already been caught producing output the code cannot produce — a report shown in the wrong order, and a frame count written from a simplified trace. Worse, the single most important output in the documentation — the exit summary, the thing the front page exists to sell — was in a ```console block reading `… your output …`. It could not be an `@example`: the summary fires at exit and a documentation build does not exit between blocks. That is exactly why it was typed. `examples/walkthrough.jl` is a Literate source with three consumers and one copy of the truth: * Literate generates `docs/src/walkthrough.md` at build time — gitignored, so no second drifting copy is committed * `julia --project=examples examples/walkthrough.jl` runs it standalone * `test/test_examples.jl` runs it and asserts what the page claims It walks one small package through the whole arc — mark, `entered`, `@entered`, `record`, `reach`, `audit`, `compare`/`isbreaking` — and every step carries the control, because a claim without one is worth little here: a definition the run never touched is *absent* rather than zero, a caller with nothing unvalidated behind it gives an empty record, the same removal is breaking or not depending on whether the notice was in the source. The exit summary is shown by running `examples/sweep.jl` as its own process and printing what came back. That also made a property visible that was nowhere in the documentation: the summary goes to `stderr` and the result to `stdout`, so `julia sweep.jl > result.dat` keeps the data clean. The test asserts it, and would fail if the summary ever moved streams. Two things the page says because they were measured, not assumed: * `:clean` is version-sensitive. A `lengths` that called `tanh` was `:clean` on 1.12.2 and `:unknown` on 1.14.0-DEV, because resolving `tanh` reaches Base internals that moved. The walkthrough uses a formulation that is `:clean` on both AND states the sensitivity, since it is one of the reasons `reach` is itself declared `@experimental`. * Pinax is not the subject. It has zero marks today, so it could demonstrate `audit` and not the headline, and making the docs depend on it would put a `[sources]` rev in the docs environment. Its real audited numbers stay in the prose of `adopting.md`. Co-Authored-By: Claude Opus 5 --- .gitignore | 4 + docs/Project.toml | 1 + docs/make.jl | 11 ++ docs/src/index.md | 24 ++-- examples/Project.toml | 7 ++ examples/sweep.jl | 46 ++++++++ examples/walkthrough.jl | 238 ++++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 1 + test/test_examples.jl | 76 +++++++++++++ 9 files changed, 401 insertions(+), 7 deletions(-) create mode 100644 examples/Project.toml create mode 100644 examples/sweep.jl create mode 100644 examples/walkthrough.jl create mode 100644 test/test_examples.jl diff --git a/.gitignore b/.gitignore index 6646a61..f5e53c6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,8 @@ docs/build/ +# Generated from `examples/walkthrough.jl` by Literate at docs-build time. Committing it would +# put a second, drifting copy of the outputs in the repository — the exact failure the page is +# built to avoid. +docs/src/walkthrough.md Manifest.toml # `flush_coverage` writes these next to the source it measured, so a coverage-enabled run # of the suite leaves them in `src/`. diff --git a/docs/Project.toml b/docs/Project.toml index c78fb39..9aad5f8 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,6 +1,7 @@ [deps] Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" ExperimentalAPI = "fd2d14cb-3a46-42a9-afd8-e8499236f05e" +Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" [sources] ExperimentalAPI = {path = ".."} diff --git a/docs/make.jl b/docs/make.jl index b947016..33ec791 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,5 +1,15 @@ using ExperimentalAPI using Documenter +using Literate + +# The walkthrough page is GENERATED from a script that runs, rather than written as prose with +# its outputs typed underneath. Twice in this package's first week a documented sample output was +# something the code could not produce; the source of that page is `examples/walkthrough.jl`, it +# is executed by `test/test_examples.jl`, and the outputs below it are whatever it printed. +const EXAMPLES = joinpath(@__DIR__, "..", "examples") +Literate.markdown( + joinpath(EXAMPLES, "walkthrough.jl"), joinpath(@__DIR__, "src"); documenter=true +) makedocs(; sitename="ExperimentalAPI.jl", @@ -24,6 +34,7 @@ makedocs(; checkdocs=:public, pages=[ "Home" => "index.md", + "Walkthrough" => "walkthrough.md", "Declaring" => "declaring.md", "Observing" => "observing.md", "Analysing" => "analysing.md", diff --git a/docs/src/index.md b/docs/src/index.md index 42dc7f8..5366afb 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -19,17 +19,27 @@ using ExperimentalAPI energy(0.5) ``` -```console -$ julia sweep.jl -… your output … -┌ ExperimentalAPI: this run entered 1 experimental definition -│ MyModel.energy — convergence not established below β ≈ 0.1 -└ set ENV["EXPERIMENTALAPI_SUMMARY"] = "0" before `using` to silence this +Now run a program that uses it — `examples/sweep.jl` in this repository — and read what it says on +its way out. The block below is not a transcript: it runs that script as its own process during the +documentation build and prints what came back on `stderr`. + +```@example frontpage +script = joinpath(pkgdir(ExperimentalAPI), "examples", "sweep.jl") +err = IOBuffer() +run( + pipeline( + `$(Base.julia_cmd()) --startup-file=no --project=$(Base.active_project()) $script`; + stdout=devnull, stderr=err, + ), +) +print(String(take!(err))) ``` Nobody asked for that summary. It is on by default, it carries the **reason** rather than just the symbol, and a marked definition the run never entered is *absent* — not reported with a count of -zero. The same answer is available programmatically through [`entered`](@ref). +zero. It goes to `stderr`, so `julia sweep.jl > result.dat` keeps the data clean and still puts the +notice in front of whoever ran it. The same answer is available programmatically through +[`entered`](@ref), and the [walkthrough](@ref "A walkthrough, run rather than typed") asks all five questions against one running package. ## Five questions, and the first is the reason to have this diff --git a/examples/Project.toml b/examples/Project.toml new file mode 100644 index 0000000..31a46e6 --- /dev/null +++ b/examples/Project.toml @@ -0,0 +1,7 @@ +# Hand-written, never `Pkg.add`ed: a resolve here would rewrite the `[sources]` path and re-sort +# what follows. `docs/Project.toml` is the same shape for the same reason. +[deps] +ExperimentalAPI = "fd2d14cb-3a46-42a9-afd8-e8499236f05e" + +[sources] +ExperimentalAPI = {path = ".."} diff --git a/examples/sweep.jl b/examples/sweep.jl new file mode 100644 index 0000000..99436de --- /dev/null +++ b/examples/sweep.jl @@ -0,0 +1,46 @@ +# A sweep that returns a number, and nothing in the number says which code path behind it was +# never validated. Run it as its own process — `julia --project=examples examples/sweep.jl` — and +# read what comes out AFTER the result: the summary fires at exit, so it cannot be demonstrated +# from inside a docs build that never exits. + +module Ising + +using ExperimentalAPI + +public energy, correlator, susceptibility, sweep + +""" + energy(β) + +Free energy density of the 1D Ising chain at inverse temperature `β`. +""" +@experimental "convergence not established below β ≈ 0.1" energy(β) = -log(2cosh(β)) / β + +""" + correlator(β, r) + +Two-point function at separation `r`. Settled: the closed form is exact for the 1D chain. +""" +correlator(β, r) = tanh(β)^r + +""" + susceptibility(β) + +Magnetic susceptibility. Marked, and never called by this script — so the summary has to leave it +out rather than report it with a count of zero. +""" +@experimental "extrapolated; no reference value" susceptibility(β) = exp(2β) / β + +""" + sweep(βs) + +Mean energy density over `βs`. The caller never names `energy`. +""" +sweep(βs) = sum(energy, βs) / length(βs) + +end # module Ising + +result = Ising.sweep(0.05:0.05:2.0) + +println("mean energy density = ", result) +println("correlation length at β = 1: ", -1 / log(tanh(1.0))) diff --git a/examples/walkthrough.jl b/examples/walkthrough.jl new file mode 100644 index 0000000..e5ce85c --- /dev/null +++ b/examples/walkthrough.jl @@ -0,0 +1,238 @@ +#md # ```@meta +#md # CurrentModule = ExperimentalAPI +#md # ``` +#md # +# # A walkthrough, run rather than typed +# +# Every output on this page was produced by executing the code above it. The script is +# `examples/walkthrough.jl` in the repository and it runs on its own: +# +# ```console +# $ julia --project=examples examples/walkthrough.jl +# ``` +# +# That is not a stylistic choice. Twice in this package's first week a documented sample output +# turned out to be something the code could not produce — a report shown in the wrong order, and +# a frame count written from a simplified trace. Both were prose describing behaviour rather than +# behaviour producing prose. This page is the other way round. + +# ## The situation the package is for +# +# A run finishes and hands back a number. Nothing in the number says which of the code paths +# behind it had never been validated. + +module Ising + +using ExperimentalAPI + +public energy, correlator, susceptibility, sweep, report, scale, normalise, apply + +""" + energy(β) + +Free energy density of the 1D Ising chain at inverse temperature `β`. +""" +@experimental "convergence not established below β ≈ 0.1" energy(β::Float64) = + -log(2cosh(β)) / β + +""" + correlator(β, r) + +Two-point function at separation `r`. Exact in closed form for the 1D chain. +""" +correlator(β::Float64, r::Int) = tanh(β)^r + +""" + susceptibility(β) + +Magnetic susceptibility. Marked, and deliberately not on the path this page walks — it is the +control for every "the run entered nothing" answer below. +""" +@experimental "extrapolated; no reference value" susceptibility(β::Float64) = exp(2β) / β + +"Mean energy density over `βs`." +sweep(βs::Vector{Float64}) = sum(energy, βs) / length(βs) + +"The number a caller actually asks for. It never names `energy`." +report(βs::Vector{Float64}) = round(sweep(βs); digits=4) + +"Rescale one already-computed value." +scale(x::Float64, c::Float64) = c * x + +"Post-processing. Nothing unvalidated is behind this one." +function normalise(xs::Vector{Float64}, c::Float64) + total = 0.0 + for x in xs + total += scale(x, c) + end + return total +end + +apply(fs::Vector{Function}, β::Float64) = fs[1](β) + +end # module Ising + +βs = collect(0.05:0.05:2.0) +Ising.report(βs) + +# One `Float64`. It is correct, it is the number that goes in the plot, and it went through a +# definition whose own author wrote down that they had not established convergence below +# `β ≈ 0.1` — which is where this sweep starts. +# +# The mark that says so is one line at the definition site: +# +# ```julia +# @experimental "convergence not established below β ≈ 0.1" energy(β::Float64) = … +# ``` +# +# `public` already decides *who may call* `energy`. Whether it is *finished* is the orthogonal +# question, and the usual place it gets answered is a sentence in a docstring that no tool reads. + +# ## What the run went through +# +# [`entered`](@ref) answers it after the fact, about the run, for somebody who is not the author. + +# One macro is exported, because it is written at a definition site. Everything else is `public` +# and asked for by name. + +using ExperimentalAPI: + ExperimentalAPI, audit, compare, entered, isbreaking, reach, record, snapshot, verdict + +entered(Ising) + +# The reason travels with the answer, so a reader a year later needs no source — and +# `susceptibility` is **absent** rather than reported with a count of zero, which is what makes a +# clean answer worth anything. Two definitions are marked; one is in this run. +# +# A fresh accounting comes from opening a [`record`](@ref) block, which clears every flag on entry. +# That is also the control for the paragraph above: `normalise` is arithmetic on numbers already in +# hand, so a caller with nothing unvalidated behind it produces an **empty** answer, not a small +# one: + +record(() -> Ising.normalise(βs, 0.5); paths=false, timing=false) + +# ## One call, rather than the whole process +# +# [`@entered`](@ref) asks the same question about a single expression, and returns its value, so +# it drops into existing code the way `@time` does. + +value = ExperimentalAPI.@entered Ising.report(βs) + +# ## How often, and how much of the run +# +# [`record`](@ref) is the opt-in layer. It counts exactly — the count survives inlining, because +# opening a block clears every flag and the *write* side does the counting — and it reports its +# own overhead so the number is never quietly load-bearing. + +rec = record(() -> Ising.report(βs); paths=false, timing=false) +[(h.mod, h.name, h.count) for h in rec] + +# Forty calls for a forty-point sweep. Nothing in `Ising` changed to get that number. + +# ## Without running anything +# +# The previous answers are all about a run that happened. [`reach`](@ref) asks about code: +# *could* this caller get there? The answer is three-valued, and the third value is the point. + +verdict(reach(Ising.report, Tuple{Vector{Float64}})) + +# `report` never writes the word `energy`. It calls `sweep`, which does. A grep for the name +# finds nothing; the call graph finds it. + +verdict(reach(Ising.normalise, Tuple{Vector{Float64},Float64})) + +# `:clean` is the expensive claim — it means the *whole* call graph was resolved and nothing marked +# is in it, Base included. `:depends` needs one witness; `:clean` needs all of them. +# +# That asymmetry has a consequence worth knowing before you rely on it. Measured 2026-09-09: a +# version of this function that called `tanh` came back `:clean` on Julia 1.12.2 and `:unknown` on +# 1.14.0-DEV, because resolving `tanh` reaches into Base internals that moved between them. The +# verdict did not get worse; the compiler did. A `:clean` is a statement about the analysis on the +# Julia you ran it on, which is one of the reasons [`reach`](@ref) is itself declared +# `@experimental`. +# +# And the third value is the point of having three: + +verdict(reach(Ising.apply, Tuple{Vector{Function},Float64})) + +# `fs[1](β)` really can reach anything. Reporting that as `:clean` would not be a weaker claim, +# it would be a false one. + +# ## The surface, and what neither account covers +# +# [`audit`](@ref) compares `names(M)` against two independent accounts — a docstring, and a mark. +# They are not alternatives; the docstring is owed either way. + +audit(Ising) + +# `apply` is public, undocumented and unmarked. It is not a bug and the audit does not call it +# one — it says that the module has published a name and said nothing about it, which is a +# decision somebody should make on purpose. + +# ## Is dropping it breaking? +# +# This is the payoff for marking anything at all. Take a snapshot at each release; hand the old +# one and the new module to [`compare`](@ref). Here the "old" release is the current surface plus +# two names that this release drops — one settled, one marked. + +old = snapshot(Ising) +push!(old["stable"], "old_verb") +old["experimental"]["draft_energy"] = Dict( + "reason" => "never validated; superseded by energy" +) + +d = compare(old, Ising) +(removed_stable=d.removed_stable, removed_experimental=d.removed_experimental) + +# Two removals, and the gate answers differently for them: + +isbreaking(d) + +# `true`, because `old_verb` was settled. Drop only the marked one and the same function says: + +old2 = snapshot(Ising) +old2["experimental"]["draft_energy"] = Dict( + "reason" => "never validated; superseded by energy" +) +isbreaking(compare(old2, Ising)) + +# `false`. "Changing an experimental name is not breaking" stops being an argument in a review +# thread and becomes a function call — because the notice was given in the source, at the +# definition, before the removal. +# +# Read it as a floor on breakage and never as a clearance: `compare` reads name sets, so a name +# present in both whose signature changed is a break it cannot see. +# [`compare_methods`](@ref) is the finer instrument. + +# ## The one answer nobody asked for +# +# Everything above was asked for. This is not: a process that loaded a marked package and entered +# marked code says so on its way out. +# +# It cannot be shown from inside this page, because the summary fires at exit and a documentation +# build does not exit between blocks — which is exactly why this output used to be typed by hand. +# So the block below runs `examples/sweep.jl` as its own process and prints what came back. + +script = joinpath(pkgdir(ExperimentalAPI), "examples", "sweep.jl") +out, err = IOBuffer(), IOBuffer() +run( + pipeline( + `$(Base.julia_cmd()) --startup-file=no --project=$(Base.active_project()) $script`; + stdout=out, + stderr=err, + ), +) +print(String(take!(out))) + +# That is the program's own output, on `stdout`, exactly as a pipe or a redirect would receive +# it. The summary is not in it, because it goes to `stderr`: + +print(String(take!(err))) + +# Nobody wrote a line to produce that. It carries the **reason**, not just the symbol; a marked +# definition the run never entered is absent rather than reported as zero; and being on `stderr` +# means `julia sweep.jl > result.dat` keeps the data clean and still puts the notice in front of +# whoever ran it. +# +# `ENV["EXPERIMENTALAPI_SUMMARY"] = "0"` before `using` turns it off. The default is on because +# the person who needs this answer is usually not the person who would think to ask for it. diff --git a/test/runtests.jl b/test/runtests.jl index bee91a3..49957e8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -25,6 +25,7 @@ using Test include("spec/test_spec_dispatch.jl") include("spec/test_spec_lifecycle.jl") include("test_spec_table.jl") + include("test_examples.jl") include("test_readme.jl") include("test_aqua.jl") end diff --git a/test/test_examples.jl b/test/test_examples.jl new file mode 100644 index 0000000..45db80a --- /dev/null +++ b/test/test_examples.jl @@ -0,0 +1,76 @@ +# The examples are documentation that runs. +# +# `docs/src/walkthrough.md` is generated from `examples/walkthrough.jl` by Literate at build time, +# so every output on that page is whatever the script printed. This file is the other half of that +# arrangement: without it, a change in `src/` could quietly rewrite what the documentation claims +# and nothing would go red until somebody read the rendered page. +# +# Twice in this package's first week a documented sample output turned out to be something the +# code could not produce. Both would have failed here. + +using ExperimentalAPI: ExperimentalAPI, entered, reach, verdict +using Test + +const EXAMPLES = joinpath(@__DIR__, "..", "examples") + +"Everything `script` wrote, as its own process, with the two streams kept apart." +function run_example(script) + out, err = IOBuffer(), IOBuffer() + cmd = `$(Base.julia_cmd()) --startup-file=no --project=$(Base.active_project()) $script` + run(pipeline(cmd; stdout=out, stderr=err)) + return String(take!(out)), String(take!(err)) +end + +@testset "examples/sweep.jl keeps the result and the notice on different streams" begin + out, err = run_example(joinpath(EXAMPLES, "sweep.jl")) + @test occursin("mean energy density", out) + # The claim the front page makes: `julia sweep.jl > result.dat` keeps the data clean. If the + # summary ever moves to `stdout` it corrupts every machine-readable output in the wild. + @test !occursin("ExperimentalAPI:", out) + @test occursin("this run entered 1 experimental definition", err) + # The REASON travels, not just the symbol — that is the whole difference from a linter. + @test occursin("convergence not established below", err) + # …and the control, which is what makes a clean answer worth anything: `susceptibility` is + # marked and was never called, so it is ABSENT rather than reported with a count of zero. + @test !occursin("susceptibility", err) +end + +# Run once; assert against it below. Loading it into a module of its own keeps its `Ising` out of +# `Main`, where the rest of the suite lives. +const WALK = Module(:WalkthroughExample) +Base.include(WALK, joinpath(EXAMPLES, "walkthrough.jl")) + +@testset "examples/walkthrough.jl still shows what the page says it shows" begin + Ising = WALK.Ising + βs = collect(0.05:0.05:2.0) + + # The three-valued answer, in the order the page presents it. A verdict that flips is a page + # that lies, and these are the three sentences the page spends the most words on. + @test verdict(reach(Ising.report, Tuple{Vector{Float64}})) === :depends + @test verdict(reach(Ising.normalise, Tuple{Vector{Float64},Float64})) === :clean + @test verdict(reach(Ising.apply, Tuple{Vector{Function},Float64})) === :unknown + + # "Two definitions are marked; one is in this run." Both halves, because the second is the + # control for the first. + marked = Set(mk.name for mk in ExperimentalAPI.experimental(Ising)) + @test marked == Set([:energy, :susceptibility]) + rec = ExperimentalAPI.record(() -> Ising.report(βs); paths=false, timing=false) + @test Set(h.name for h in rec) == Set([:energy]) + # Forty calls for a forty-point sweep — the page prints this count, and it is exact rather + # than sampled, which is the property that makes printing it defensible. + @test only(rec).count == length(βs) + + # The audit's finding. `apply` is public, undocumented and unmarked, and the page says so. + a = ExperimentalAPI.audit(Ising) + @test a.unaccounted == [:apply] + + # The release gate, both ways round: the same removal is breaking or not depending on whether + # the notice was given at the definition site. + old = ExperimentalAPI.snapshot(Ising) + push!(old["stable"], "old_verb") + old["experimental"]["draft_energy"] = Dict("reason" => "never validated") + @test ExperimentalAPI.isbreaking(ExperimentalAPI.compare(old, Ising)) + only_experimental = ExperimentalAPI.snapshot(Ising) + only_experimental["experimental"]["draft_energy"] = Dict("reason" => "never validated") + @test !ExperimentalAPI.isbreaking(ExperimentalAPI.compare(only_experimental, Ising)) +end