diff --git a/ext/ExperimentalAPITestExt.jl b/ext/ExperimentalAPITestExt.jl index 043e02e..5c45cf9 100644 --- a/ext/ExperimentalAPITestExt.jl +++ b/ext/ExperimentalAPITestExt.jl @@ -14,10 +14,8 @@ using ExperimentalAPI: partition_holds using Test: Test, @test, @testset -# The knobs below are the newest part of this package and the least settled: which of them a -# project should turn on is a question no measurement has answered yet, and the answer will change -# what they mean. The mark is written here, in the extension that defines them, which is also the -# case `experimental(m; extensions = true)` exists for. +# The newest part of the package: which of these a project should turn on is unmeasured. Marked +# here, in the extension that defines them — the case `experimental(m; extensions = true)` is for. ExperimentalAPI.@experimental( "which gates a project should run, and therefore what these keywords should default to, is " * "undecided; `require_tracking` and `max_marks` may be replaced by one policy argument", @@ -44,10 +42,8 @@ function ExperimentalAPI.test_surface( a = audit(m; methods) outputlevel ≥ 1 && show(stdout, MIME"text/plain"(), a) @testset "public surface of $(nameof(m))" begin - # Assertions that run whatever the audit found. Everything below iterates over a set of - # findings, so on a clean module all of it collapses to nothing and the testset would - # report `0 tests passed` — a green indistinguishable from the extension having failed to - # load, or from `m` having no public names at all. These make the pass mean something. + # Everything below iterates over findings, so a clean module reports `0 tests passed` — a green + # indistinguishable from the extension failing to load. These make the pass mean something. @testset "every public name has a docstring" begin @test isempty(setdiff(a.undocumented, skip)) @test isempty(a.dangling) diff --git a/src/audit.jl b/src/audit.jl index 321d3ff..0b5652d 100644 --- a/src/audit.jl +++ b/src/audit.jl @@ -1,14 +1,10 @@ -# The check. Everything above this file is material; this is the part that turns a marker into -# something that can fail. +# The check: the part that turns a marker into something that can fail. # -# The name half of the audit is one set difference: public surface, minus the names with a -# docstring, minus the names with a mark. What is left is the set of names a caller can reach and -# nobody has said anything about — and it is exactly the set that a package cannot leave non-empty -# once this runs in CI. +# The name half is one set difference — public surface, minus documented, minus marked. # -# The method half exists because `names(m)` cannot see a method a package contributed to somebody -# else's generic, and for a package whose surface IS such methods — `fetch(model, quantity)` with -# 570 of them — a clean name audit reports nothing while covering nothing. +# The method half exists because `names(m)` cannot see a method contributed to somebody else's +# generic. For a package whose surface IS such methods (`fetch` with 570 of them) a clean name +# audit reports nothing while covering nothing. """ surface(m::Module) -> Vector{Symbol} @@ -93,11 +89,8 @@ This answers *whether prose exists*, never whether it is any good. A docstring r documented as far as this package is concerned. """ function isdocumented(m::Module, name::Symbol) - # `Docs.hasdoc` is public API — `public`, and exported from `Base.Docs`. Its set-valued - # sibling `Docs.undocumented_names` answers the same question for a whole module at once and - # is what `Aqua.test_undocumented_names` is built on; the per-name form is used here because - # `audit` has to tell a module's own gap apart from a dependency's, and the set form reports - # a re-exported name's missing docstring as though it were this module's to fix. + # `Docs.hasdoc` is public API. The per-name form rather than `Docs.undocumented_names` because the + # set form reports a re-exported name's missing docstring as this module's to fix. return Base.Docs.hasdoc(m, name) end @@ -105,10 +98,8 @@ function isdocumented(m::Method) id = _ftype_identity(_sig_ftype(m.sig)) owner = id === nothing ? m.module : id.mod name = id === nothing ? m.name : id.name - # Only the module that WROTE the method is asked. `Docs` files a docstring under the - # binding's module but in the *writing* module's table, so this is the right table — and - # looking in the owner's as well would let the generic's own docstring, whose key is - # `Tuple{Any, Any}`, account for all 570 methods anybody ever contributed to it. + # Only the module that WROTE the method. Looking in the owner's table too would let the generic's + # own docstring, keyed `Tuple{Any, Any}`, account for every method contributed to it. d = try Base.Docs.meta(m.module; autoinit=false) catch @@ -138,10 +129,9 @@ function _argument_tuple(@nospecialize(sig)) end end -# Whether a mark says anything about `m`'s own public surface. A mark that attached to a -# signature is asked about the generic it extends, not about whether the name happens to be bound -# here: `using ..Upstream` leaves no binding for `fetch_value`, and reading that absence as "ours" -# would report every contributed method as a dangling promise. +# Whether a mark says anything about `m`'s own surface. A signature mark is asked about the +# generic it extends: `using ..Upstream` leaves no binding, and reading that absence as "ours" +# would report every contributed method as dangling. function _is_surface_claim(m::Module, mk::Mark) if mk.sig !== nothing id = _ftype_identity(_sig_ftype(mk.sig)) @@ -166,15 +156,12 @@ function _is_own(m::Module, name::Symbol) end end -# `own_methods` is a scan over every public callable of every loaded module — 1916 candidates and -# 11026 methods behind them for this package — and a suite that audits several modules pays it once -# per audit. Its ANSWER, though, is "the methods whose defining module is `m`", and that set can -# only change when a method is defined or deleted. Both bump the world counter: measured on 1.11.9, -# 1.12.2 and 1.14.0-DEV, a method definition bumps it in all three. +# A scan over every public callable of every loaded module — 1916 candidates, 11026 methods here. +# The answer changes only when a method is defined or deleted, and a method definition bumps the +# world counter on 1.11.9, 1.12.2 and 1.14.0-DEV alike. # -# A `const` binding does NOT bump it on 1.11 (it does on 1.12 and later), which is why the key is -# argued rather than assumed. A new `const` cannot change this answer: either it aliases something -# whose methods belong to another module, or creating it defined a method and bumped the counter. +# A `const` does NOT bump it on 1.11, but cannot change this answer either: it aliases something +# whose methods belong elsewhere, or creating it defined a method. const _OWN_METHODS = Ref{Tuple{UInt64,Dict{Module,Vector{Method}}}}(( typemax(UInt64), Dict{Module,Vector{Method}}() )) @@ -223,10 +210,8 @@ function _own_methods(m::Module) mm.module === m && !(mm in seen) && (push!(seen, mm); push!(out, mm)) end end - # The key is built ONCE per method, not once per comparison. `sort!(…; by = f)` calls `f` on - # both sides of every comparison, and `string(mm.sig)` is not cheap: measured on this - # package's own 301 methods, the sort was 0.601s while building all 301 keys was 0.039s. That - # one line was 80% of `audit`, which is called once per module in every surface check. + # Keys built once per method: `sort!(…; by = f)` calls `f` on both sides of every comparison. On + # 301 methods the sort was 0.601s and building all 301 keys 0.039s. return out[sortperm([(string(mm.name), string(mm.sig)) for mm in out])] end @@ -459,10 +444,8 @@ function audit(m::Module; methods::Bool=true) n in marked || push!(unaccounted, n) end end - # A mark on a generic another module owns is the foreign-method form — `Base.show(io, ::T)` - # — and it promises nothing about THIS module's surface, so it cannot dangle here; - # `contributed_methods` is where it is accounted for. A mark on something of our own that is - # not public does dangle, whether or not it carries a signature. + # A mark on another module's generic promises nothing about this surface, so it cannot dangle — + # `contributed_methods` accounts for it. A mark on something of ours that is not public does. dangling = sort!( unique( mk.name for mk in all_marks if _is_surface_claim(m, mk) && !(mk.name in surf) diff --git a/src/detect.jl b/src/detect.jl index 5f8a1fc..8f5d3a5 100644 --- a/src/detect.jl +++ b/src/detect.jl @@ -1,27 +1,19 @@ -# The default layer: which marked definitions a run actually entered. +# The default layer: which marked definitions a run entered. Presence, not counts, and only for +# definitions with a body. # -# Scope: presence, not counts, and only for definitions with a body. A mark written as a name list, -# or attached to a struct, const, module or macro, is a declaration only — nothing observes it. +# The statement the macro puts in a marked body reads one field and writes it once: 1.03x on one +# thread, 0.985x on eight, over 10M calls of a numeric body. A shared counter is 3.76x at eight +# threads and loses 40% of its increments to races unless atomic. # -# The one statement the macro puts in a marked body reads a single field and writes it once: -# measured at 1.03x on one thread and 0.985x on eight, over 10M calls of a numeric body. A flag -# written once and only read afterwards stops dirtying the cache line, which a counter (3.76x at -# eight threads, and losing 40% of its increments to races unless atomic) does not. -# -# `record` reaches the same statement without changing it: opening a recording clears every -# probe's flag, so the short-circuit fails and the write side runs on every call. The cost of -# counting is paid only inside `record`, and the fast path is one field load either way. +# `record` reaches the same statement without changing it: opening a recording clears every flag, +# so the short-circuit fails and the write side runs on every call. # Padding, in Int64 slots, between one thread's counter and the next. A cache line is 64 bytes on # every platform this runs on; two threads sharing one would serialise on the store. const _COUNTER_STRIDE = 8 -# How many distinct backtraces one probe keeps while recording, and how many times it will look. -# A backtrace costs microseconds, so capturing one per call would dominate any run long enough to -# be worth recording. The paths a marked definition is reached by are few and repeat, so the -# attempt budget is what bounds the cost: without it, a definition reached by three paths would -# keep paying for a backtrace on every one of ten million calls, having found its third path in -# the first microsecond. +# How many distinct backtraces one probe keeps, and how many times it will look. A backtrace costs +# microseconds and the paths repeat, so the attempt budget is what bounds the cost. const _TRACE_CAP = 64 const _TRACE_ATTEMPTS = 256 @@ -62,13 +54,10 @@ end # The fast path, and the only thing a marked body does when nothing is recording. Base.getindex(p::Probe) = p.entered -# The write side. Reached once per process when nothing is recording, and on every call while a -# recording is open — which is what makes counting cost nothing outside `record`. +# The write side: once per process when nothing is recording, every call while one is open. # -# `@noinline` for two reasons, and neither is speed on this path. It keeps the marked body small, -# so the fast path is a load and a branch over a call; and it makes the call a real frame, so the -# backtrace taken underneath it resolves to the marked definition rather than to whatever the -# optimiser left at that address. +# `@noinline` keeps the marked body small, and makes the call a real frame so a backtrace taken +# underneath resolves to the marked definition. @noinline function Base.setindex!(p::Probe, v::Bool) if _RECORDING[] _hit!(p) @@ -116,10 +105,9 @@ function _resize_hits!(p::Probe, tid::Int) return nothing end -# The address list only. Resolving it to names here would mean walking the debug info while the -# sampling profiler may be in its signal handler doing the same thing, and the two take the same -# lock: `record`'s own paths would deadlock against its own timing. `_trace_names` is called -# once, at the end of the block, with the sampler stopped. +# Addresses only. Resolving names here walks the debug info under the same lock the sampler takes +# in its signal handler — paths would deadlock against timing. `_trace_names` runs at the end of +# the block, sampler stopped. @noinline function _capture_trace!(p::Probe) bt = backtrace() @lock p.lock begin @@ -197,26 +185,19 @@ probes() = reduce(vcat, (probes(m) for m in marked_modules()); init=Probe[]) # What the macro puts in the body: one statement, a read that writes only on the first call. _probe(flag) = :($flag[] || ($flag[] = true)) -# Returns the definition with the probe spliced in, or `nothing` if this form has no body to -# instrument. +# The definition with the probe spliced in, or `nothing` if the form has no body. # -# The `LineNumberNode` is the declaration's own, and it is load bearing rather than cosmetic. The -# write side is a cold branch, so the optimiser is free to sink it to the end of the function; -# without a location of its own it inherits whichever statement happens to be next, and a -# backtrace taken inside it then resolves to that statement's inlining context instead of to the -# marked definition. `record`'s call paths are built out of exactly that. +# The `LineNumberNode` is load bearing: the write side is a cold branch the optimiser may sink, and +# without its own location it inherits the next statement's — a backtrace taken inside it then +# resolves to that statement's inlining context, which is what `record`'s paths are built from. function _instrument(def, flag, src::LineNumberNode) def isa Expr || return nothing if def.head === :macrocall - # An annotating macro — `@inline` and its neighbours — leaves the body alone, so the probe - # rides inside the definition it wraps and the wrapper is rebuilt around the result. Only - # those reach here: `_subject` marks a macrocall instrumentable exactly when the macro is - # in `_ANNOTATING_MACROS`, and refuses or opts out of every other one. + # An annotating macro leaves the body alone, so the probe rides inside and the wrapper is rebuilt + # around it. Only `_ANNOTATING_MACROS` reach here. # - # Returning `nothing` here instead — which is what this did — did not merely lose the - # observation. The flag is registered either way, so `@experimental "…" @inline f(x) = x` - # counted as an observable definition that no call could ever set: `entered` reported it - # as not entered no matter what ran, and `unverified` reported it forever. + # Returning `nothing` registered the flag anyway, so `@experimental "…" @inline f(x) = x` counted + # as observable and no call could ever set it. inner = _instrument(def.args[end], flag, src) inner === nothing && return nothing return Expr(:macrocall, def.args[1:(end - 1)]..., inner) @@ -314,11 +295,9 @@ function marked_modules() return out end -# Cached per world age. The walk is over every binding of every loaded module, and `record` asks -# for it twice per block — with a large dependency tree loaded that is the most expensive thing -# in a recording that counts a hundred calls. Keying on the world counter is exact rather than -# approximate: a module gains a registry only by defining a `const`, and defining one advances -# the counter. +# Cached per world age: the walk is over every binding of every loaded module and `record` asks +# twice per block. Exact, not approximate — a module gains a registry only by defining a `const`, +# which advances the counter. const _MARKED_MODULES = Ref{Tuple{UInt64,Vector{Module}}}((typemax(UInt64), Module[])) function _walk_modules!(out::Vector{Module}, seen::Set{Module}, m::Module) diff --git a/src/macros.jl b/src/macros.jl index 413d618..4a8b263 100644 --- a/src/macros.jl +++ b/src/macros.jl @@ -1,9 +1,6 @@ -# The expression-level spelling of the observing layer. -# -# `record(() -> f(x))` is the function form and it is what everything here is built on. The macro -# earns its place by knowing two things a closure cannot: the source text of the expression, and -# the line it was written on. A report that says which call went through unvalidated code, and -# where that call is, is a different thing from a list of names. +# The expression-level spelling of the observing layer, built on `record(() -> f(x))`. The macro +# earns its place by knowing two things a closure cannot: the source text of the expression and +# the line it was written on. @experimental """ the report is a text format with no schema, and it has already changed twice in its first week — \ diff --git a/src/mark.jl b/src/mark.jl index de65b76..c67a4d9 100644 --- a/src/mark.jl +++ b/src/mark.jl @@ -1,9 +1,8 @@ # What one `@experimental` declaration records, where it is stored, and the macro that writes it. # -# Storage is a `const` vector inside the MARKED module: a registry here would be populated during -# the marked package's precompilation, and nothing written into a third module then survives into -# its cache image. `Docs.META` is per-module for the same reason. Pinned by -# `test/test_precompile.jl`. +# Storage is a `const` vector inside the MARKED module: a registry here would be written during +# that package's precompilation and would not survive into its cache image. `Docs.META` is +# per-module for the same reason. """ Mark @@ -150,10 +149,8 @@ end const MARKS_BINDING = :__EXPERIMENTAL_API_MARKS__ const SUPERSEDED_BINDING = :__EXPERIMENTAL_API_SUPERSEDED__ -# Created by code the macro emits into the marked module, not by `Core.eval` from here: Julia -# 1.12 rejects reading a binding created earlier in the same top-level statement, which is what a -# `Core.eval`-then-`getglobal` helper does. The emitted form puts the `const` and the `push!` in -# separate statements, so the world age has advanced in between. +# Emitted into the marked module, not `Core.eval`ed from here: 1.12 rejects reading a binding +# created in the same top-level statement. The `const` and the `push!` are separate statements. function _registry_of(m::Module) v = getglobal(m, MARKS_BINDING) v isa Vector{Mark} || throw( @@ -167,10 +164,9 @@ end _has_registry(m::Module) = isdefined(m, MARKS_BINDING) -# The superseded log is created on the first replacement rather than emitted by the macro: most -# modules never supersede a mark, and an empty binding in every marked module would be noise in -# `names(m; all=true)`. `Core.eval` is safe here because the value is used through the return -# value, never by reading the binding back in the same world age. +# Created on the first replacement, not emitted by the macro: most modules never supersede a mark +# and an empty binding would be noise in `names(m; all=true)`. `Core.eval` is safe because the +# value is used through the return, never read back in the same world age. function _superseded_registry!(m::Module) isdefined(m, SUPERSEDED_BINDING) && return getglobal(m, SUPERSEDED_BINDING) return Core.eval(m, Expr(:const, Expr(:(=), SUPERSEDED_BINDING, Mark[]))) @@ -358,12 +354,10 @@ macro experimental(args...) src = __source__ marks = esc(MARKS_BINDING) - # The `const` must land in its own top-level statement: the `_mark!` calls below read that - # binding, and Julia 1.12 forbids reading one created in the same world age. - # - # Built with `Expr` so no `LineNumberNode` from this file reaches the expansion. `const` in - # local scope is a lowering error this macro cannot catch, so the only lever is where the - # error points, and it must point at the caller. Pinned in `test_spec_forms.jl`. + # The `const` lands in its own top-level statement: the `_mark!` calls below read it, and 1.12 + # forbids reading a binding created in the same world age. Built with `Expr` so no + # `LineNumberNode` from this file reaches the expansion and the lowering error points at the + # caller. init = Expr( :if, :(!$(isdefined)($__module__, $(QuoteNode(MARKS_BINDING)))), @@ -517,14 +511,12 @@ struct _Subject includes_constructors::Bool end -# Macros known to hand their definition through unchanged. An allowlist rather than "unwrap the -# last argument of any macrocall": `@deprecate old new` also ends in something name-shaped, and -# marking the wrong symbol silently is the failure this package exists to remove. +# An allowlist, not "unwrap the last argument of any macrocall": `@deprecate old new` also ends in +# something name-shaped. # -# Split by whether the body underneath is still a body. `@inline` and its neighbours annotate a -# definition and leave the body alone, so the probe can ride inside and the definition is -# observed. `@generated`'s body returns an expression — a probe there would be generated rather -# than run — and `Base.@kwdef` wraps a struct, which has no body at all. +# Split by whether the body underneath is still a body. `@inline` and its neighbours leave it +# alone, so the probe rides inside. `@generated`'s body returns an expression and `Base.@kwdef` +# wraps a struct. const _ANNOTATING_MACROS = ( Symbol("@inline"), Symbol("@noinline"), diff --git a/src/query.jl b/src/query.jl index 907b15f..e25b805 100644 --- a/src/query.jl +++ b/src/query.jl @@ -1,11 +1,9 @@ -# Reading a module's marks back out. Everything here is read-only and allocation-cheap: these -# are the functions a release script, a docs build or a test calls, and none of them should be -# able to create a registry as a side effect of asking a question. +# Reading a module's marks back out. Read-only: none of these may create a registry as a side +# effect of asking a question. # -# Two units live side by side. A NAME is what `names(m)` reports and what a release promises; a -# METHOD is what a call site actually reaches. A mark attached to a definition is both — it names -# something unsettled and knows which signature it attached to — so the queries below differ in -# which of the two they answer, never in which marks they can see. +# Two units side by side. A NAME is what `names(m)` reports and what a release promises; a METHOD +# is what a call site reaches. A mark on a definition is both, so the queries differ in which of +# the two they answer, never in which marks they can see. """ experimental(m::Module; extensions = false) -> Vector{Mark} @@ -190,13 +188,9 @@ function _marks_covering(m::Method) return sort!(out; by=mk -> (mk.sig !== nothing, _mark_order(mk))) end -# Where a mark covering `m` can live, and nowhere else. The macro writes into the module the -# definition is in, and `mark_method!` writes into `m.module` for the same reason — so a -# signature-level mark is always in `m.module`. A whole-name mark is in the module that owns the -# name, which is the only module in which that name means this generic. -# -# Two modules rather than a walk over every marked module in the process: this runs once per -# resolved call site inside `reach`, and a world walk there would dominate the analysis. +# Where a mark covering `m` can live, and nowhere else: a signature mark is always in `m.module`, a +# whole-name mark in the module that owns the name. Two modules rather than a walk over every +# marked module — this runs once per resolved call site inside `reach`. function _search_modules(m::Method) id = _ftype_identity(_sig_ftype(m.sig)) (id === nothing || id.mod === m.module) && return (m.module,) diff --git a/src/reach.jl b/src/reach.jl index eb6b789..83f3dad 100644 --- a/src/reach.jl +++ b/src/reach.jl @@ -1,21 +1,15 @@ -# A caller that never names a marked thing still depends on it. -# -# Modelled on Lean's `sorry`, but Julia's call graph is not closed, so the answer is three-valued: +# A caller that never names a marked thing still depends on it. Julia's call graph is not closed, +# so the answer is three-valued: # # :depends a marked definition is reachable # :clean the whole call graph was resolved and nothing marked is in it # :unknown some call site could not be resolved — the honest non-answer # -# Collapsing `:unknown` into `:clean` is the one failure this file exists to prevent. It is not a -# weaker claim, it is a false one: `Holder.f::Function` and `TABLE[i](x)` really can reach a -# marked function while being statically invisible. +# Reporting `:unknown` as `:clean` is the one failure this file guards. # -# The walk is over INFERRED, UNOPTIMISED IR — `code_typed_by_type(sig; optimize=false)`. Inference -# runs before inlining, so every call is still a call and every argument still has a type; -# `optimize=true` would show `mul_float` and find nothing. That also makes the two hard cases fall -# out rather than needing special handling: a callee inference typed as `Function` is exactly a -# call site with no unique method, and a function passed as a value is specialised on `typeof(f)` -# and resolves. +# The walk is over inferred, UNOPTIMISED IR — `code_typed_by_type(sig; optimize=false)`. Inference +# runs before inlining, so every call is still a call and every argument has a type; +# `optimize=true` shows `mul_float` and finds nothing. @experimental """ the line between `:clean` and `:unknown` is drawn by Julia's own IR accessors, which are internal \ @@ -360,12 +354,9 @@ function reach_script( push!(body.args, st) elseif _is_toplevel_only(st) Core.eval(scratch, st) - # `const RESULT = simulate(model)` is a script's WORK wearing a declaration's syntax, - # and it is how a researcher writes the line that produces the figure. Evaluating it - # and stopping there analysed nothing: measured on a two-line script whose only call - # was a `const`, and the answer came back `:clean`. The binding still has to be made — - # a later `struct` may use it — so the value is computed at top level and the - # right-hand side is analysed as well. + # `const RESULT = simulate(model)` is a script's work wearing a declaration's syntax. + # Evaluating it and stopping there reported `:clean` for a two-line script whose only + # call was the `const`. The binding is still made and the right-hand side analysed too. rhs = _const_rhs(st) rhs === nothing || push!(body.args, rhs) else @@ -373,11 +364,9 @@ function reach_script( end end thunk = Core.eval(scratch, Expr(:function, Expr(:call, gensym(:script)), body)) - # `invokelatest`, because the walk reads the bindings the script's own `const` lines were just - # evaluated into and this call's world age was fixed before they existed. Julia 1.12 warns — - # "Detected access to binding … in a world prior to its definition world" — and says it will - # be an error in a future version. The analysis reads globals out of the IR, which is what - # makes this the one place in the package that reaches a binding younger than its caller. + # `invokelatest`: the walk reads bindings the script's `const` lines were just evaluated + # into, and this call's world age was fixed before they existed. 1.12 warns that it will + # become an error. r = Base.invokelatest(reach, thunk, Tuple{}; maxdepth, maxcandidates, maxwork, ignore) return Reach( path, @@ -428,12 +417,9 @@ function _enter!(st::_Walk, match, depth::Int, path::Vector{Symbol}) ) return nothing end - # Depth bounds how FAR the walk goes, not how much of it there is: `maxdepth` levels each - # branching by `maxcandidates` is not a finite amount of work in any useful sense, and - # `visited` only prunes signatures that repeat. Measured on 1.14.0-DEV, `[f(x) for x in xs]` - # and `sum(map(f, xs))` produced new signatures faster than the depth limit could stop them - # and the call did not return; the same two answer in milliseconds on 1.12. So the walk also - # has a budget, and spends `:unknown` when it runs out — which is what `:unknown` is for. + # Depth bounds how far the walk goes, not how much of it there is, and `visited` only + # prunes signatures that repeat. Measured on 1.14.0-DEV, `[f(x) for x in xs]` and + # `sum(map(f, xs))` never returned; both answer in milliseconds on 1.12. if st.budget[] <= 0 st.truncated = true push!( @@ -445,10 +431,8 @@ function _enter!(st::_Walk, match, depth::Int, path::Vector{Symbol}) st.budget[] -= 1 sig in st.visited && return nothing push!(st.visited, sig) - # The flag `@experimental` emits is this package's own code, and under `ignore` the walk goes - # through it rather than stopping at the mark. Following it would report the recorder's - # internals — `backtrace`, and everything Base does to format one — as the caller's - # dependencies. Nothing in here is ever marked, so there is nothing to lose by stopping. + # Under `ignore` the walk goes through the mark into this package's own flag and would + # report `backtrace` and its formatting as the caller's. Nothing here is ever marked. Base.moduleroot(mm.module) === ExperimentalAPI && return nothing mm.module in st.modules || push!(st.modules, mm.module) @@ -533,10 +517,9 @@ function _resolve_call!(st::_Walk, ci, stmt::Expr, pc::Int, mm::Method, depth::I pinned = _const_value(ci, args[3]) if target !== nothing && pinned isa Type sig = Base.signature_type(target, pinned) - # `invoke` semantics, not dispatch semantics: the method chosen for arguments of the - # DECLARED type. Resolving `Tuple{typeof(more_specific), Integer}` by dispatch finds - # both `::Int` and `::Integer` and reports the site unresolved, which is exactly the - # over-caution an analysis that ignores `invoke` would show. + # `invoke` semantics, not dispatch: the method for the DECLARED type. By dispatch + # `Tuple{typeof(f), Integer}` matches both `::Int` and `::Integer` and the site would + # be reported unresolved. pin = try which(sig) catch @@ -610,15 +593,9 @@ function _resolve_sig!( push!(st.unresolved, Unresolved(name, sig, :nomethod, mm.file, line, mm, Mark[])) return nothing end - # Several methods match and nothing in the IR says which. Reporting `:depends` because one of - # them is marked would over-claim; reporting `:clean` because none is *proved* reached is the - # false answer this whole file guards. - # - # But "which method runs" is only worth knowing if the answer could differ. Every candidate is - # walked in its own right, and when none of them reaches anything marked the site is resolved - # after all — that is not a guess, it is having checked all of them. Without this, - # `convert(::Type, ::UInt32)` — dozens of matching methods, none of them anybody's research - # code — makes every caller that formats a string `:unknown`. + # Several methods match and nothing in the IR says which. Every candidate is walked, and + # if none reaches a mark the site resolves — checked, not guessed. Without this, + # `convert(::Type, ::UInt32)` makes every caller that formats a string `:unknown`. if length(matches) > st.maxcandidates push!(st.unresolved, Unresolved(name, sig, :ambiguous, mm.file, line, mm, Mark[])) return nothing @@ -800,11 +777,9 @@ function _callee_name(ci, @nospecialize(x)) end w = _widen(t === nothing ? Any : t) if w isa DataType && isdefined(w, :instance) - # `nameof` accepts a `Function`, a `Type` or a `Module` and nothing else. A struct whose - # fields are all singletons is itself a singleton, so `w.instance` exists for callables - # that are none of the three — `Base.MappingRF{…}`, which is what `sum(f(x) for x in xs)` - # lowers to. Asking that for a name threw a `MethodError` out of an analysis whose entire - # contract is to come back with one of three verdicts. + # `nameof` accepts only a `Function`, `Type` or `Module`. A struct whose fields are all + # singletons is itself one, so `w.instance` exists for callables that are none of the + # three — `Base.MappingRF{…}`, what `sum(f(x) for x in xs)` lowers to. inst = w.instance inst isa Union{Function,Type,Module} && return nameof(inst) end @@ -828,11 +803,9 @@ function _tuple_type(@nospecialize(ft), args::Vector{Any}) end end -# Whether a type can be the first parameter of a signature that dispatch could pin. `Function` -# and `Any` cannot: they are the shapes a field read or a table lookup produces. -# -# `Type{Float64}` is the exception the abstractness flag alone gets wrong. Julia marks it abstract, -# but a constant type in call position is a constructor call and dispatch pins it exactly. +# Whether a type can head a signature dispatch could pin. `Function` and `Any` cannot — they +# are what a field read or a table lookup produces. `Type{Float64}` is marked abstract, but a +# constant type in call position is a constructor call and pins exactly. function _is_callable_type(@nospecialize(ft)) ft === Any && return false ft === Function && return false @@ -843,13 +816,9 @@ function _is_callable_type(@nospecialize(ft)) return true end -# The `X` in `Type{X}`, or `nothing` if `t` is not a constant type. -# -# Spelled as a question about `t` rather than as `t isa DataType && t <: Type`, because both halves -# of that moved: on 1.14-DEV `Type{X}` is no longer a `DataType`, and `Core.Typeof(Float64)` -# returns the new `Core.TypeEgal{Float64}` rather than `Type{Float64}`. Measured on -# 1.14.0-DEV.3115; the old spelling made every constructor call in the graph `:unknown`, which -# reported four otherwise-clean fixtures as unresolved. +# The `X` in `Type{X}`, or `nothing`. Not `t isa DataType && t <: Type`: on 1.14.0-DEV.3115 +# `Type{X}` is no longer a `DataType` and `Core.Typeof(Float64)` returns +# `Core.TypeEgal{Float64}`. The old spelling made every constructor call `:unknown`. function _type_parameter(@nospecialize(t)) (t isa Type && t <: Type && t !== Type) || return nothing ps = try diff --git a/src/record.jl b/src/record.jl index 29b95f7..705c5c5 100644 --- a/src/record.jl +++ b/src/record.jl @@ -1,12 +1,10 @@ # The opt-in layer: how often a run entered marked code, by which paths, and how much of the run # was spent inside it. # -# The boundary against the default layer was measured rather than chosen (`test/spec/README.md`). # A counter in the body costs 3.76x on eight threads and loses 40% of its increments to races -# unless it is atomic; a set-once flag is free. So the flag stays, and `record` reaches the same -# statement from the other side: opening a recording clears every probe, the short-circuit fails, -# and the write side — which is a function call, not an inlined store — does the counting. Nothing -# in the body changes, and nothing outside `record` pays for any of it. +# unless atomic; a set-once flag is free. So `record` reaches the same statement from the other +# side: opening a recording clears every probe, the short-circuit fails, and the write side — a +# call, not an inlined store — counts. @experimental """ the record's shape and its collection knobs are both still moving: `Record` gained a field after \ @@ -127,9 +125,8 @@ end # ── the timing backend ─────────────────────────────────────────────────────────────────────── # # Sampling is the only way to say how much of a run was spent inside a definition without wrapping -# the call, and wrapping is exactly what the emitted statement may not do. The sampler is Julia's -# own, reached through an extension so that `using ExperimentalAPI` — which every marked package -# does at run time — never loads `Profile`. +# the call. Julia's own sampler, behind an extension so that `using ExperimentalAPI` never loads +# `Profile`. """ TimingBackend @@ -257,17 +254,12 @@ function record( closed[] && return nothing closed[] = true sampled && stop_timing!(timing_backend()) - # The probe set is re-derived here rather than reused from before the call. A mark can - # come into existence WHILE the block runs — a package extension loaded by `f` is the - # ordinary way — and a probe that was not in the snapshot is entered by code that ran, - # counted by nobody, and left with its flag `false` for the rest of the process. That - # loses the entry from `entered()` and from the exit summary too, which is the one thing - # the default layer promises never to do. + # Re-derived, not reused from before the call: a mark can come into existence WHILE the block + # runs — a package extension loaded by `f` is the ordinary way — and a probe missing from the + # snapshot is left `false` for the rest of the process, losing it from `entered()` too. # - # `invokelatest`, because reading those probes is the whole point and their bindings are - # younger than this frame: `probes()` reaches `M.__EXPERIMENTAL_API_ENTERED_newborn__`, - # created while `f` ran. Julia 1.12 warns that reading a binding in a world prior to its - # definition world will be an error. + # `invokelatest` because those bindings are younger than this frame; 1.12 warns that reading one + # in a world prior to its definition will become an error. append!(measured, Base.invokelatest(probes)) for p in measured counts[p] = _probe_count(p) - get(before, p, 0) @@ -293,11 +285,8 @@ function record( catch e err = e if rethrow - # Closed here, and re-raised from inside the `catch`, because that is the only place - # the exception's own backtrace survives. Closing first and calling `throw(err)` - # afterwards — which is what this did — manufactures a fresh backtrace rooted in this - # function, so the caller debugging a failed run sees `record.jl` where their own call - # chain should be. + # Re-raised from inside the `catch`, the only place the exception's own backtrace survives. + # Closing first and `throw(err)` afterwards manufactures one rooted in `record.jl`. close!() Base.rethrow() end diff --git a/src/release.jl b/src/release.jl index 8f41b94..fed30b2 100644 --- a/src/release.jl +++ b/src/release.jl @@ -1,10 +1,8 @@ -# The release-decision layer: write the covenant down at release time, and read a diff of two of -# them at review time. This is the payoff for marking anything at all — "changing an experimental -# name is not breaking" stops being an argument and becomes a function call. +# Write the covenant down at release time, diff two of them at review time — "changing an +# experimental name is not breaking" stops being an argument and becomes a function call. # -# Two units, one file, one schema. Names are what `names(m)` promises; methods are what a call -# site reaches, and a package whose surface is methods on somebody else's generic has no name-level -# covenant at all. The two live in one snapshot so that a repository has one file to commit. +# Names and methods in one snapshot, so a repository has one file to commit: a package whose +# surface is methods on somebody else's generic has no name-level covenant at all. @experimental """ the snapshot schema is young: nothing has been released against it, and the method half was added \ after the name half, so a file written by one version may not be readable by the next diff --git a/src/verify.jl b/src/verify.jl index 49e68f1..a5bf59c 100644 --- a/src/verify.jl +++ b/src/verify.jl @@ -1,13 +1,9 @@ -# How well a marked definition is exercised by the tests. +# How well a marked definition is exercised by the tests. No new machinery: the mark carries the +# file and line, `--code-coverage` writes a count per line, and joining them answers the worst case +# — unvalidated code its own suite never runs. # -# No new machinery: the mark already carries the file and line its definition starts at, and -# `--code-coverage` already writes a count per line. Joining the two answers the worst case a -# marked definition can be in — unvalidated code that its own suite never runs — and it answers it -# without anyone writing another list. -# -# Coverage counts are flushed from the running process rather than read from the `.cov` files -# Julia writes at exit, because a test that has to wait for the process to end cannot assert -# anything. +# Counts are flushed from the running process, not read from the `.cov` files Julia writes at exit: +# a test that has to wait for the process to end cannot assert anything. @experimental """ the counts come from two interfaces Julia does not document — `ccall(:jl_write_coverage_data, …)` \ @@ -163,13 +159,10 @@ function _entered_flag(mk::Mark) return p === nothing ? nothing : p[] end -# The line range one declaration occupies: from the `@experimental` line to just before the next -# statement BESIDE it. Read from the source rather than from the method, because a mark may cover -# a `struct` or a name list, which have no method to ask. -# -# "Beside it" is the whole difficulty. The next `LineNumberNode` above the declaration is usually -# the first line of its own body, so a span computed that way is one line long and reports every -# multi-line definition as fully covered by its own signature. +# The line range one declaration occupies, from the `@experimental` line to just before the next +# statement BESIDE it. Read from the source, not the method: a mark may cover a `struct` or a name +# list. The next `LineNumberNode` is usually the first line of its own body, and a span computed +# that way is one line long and reports every multi-line definition as fully covered. function _definition_span(mk::Mark) lines = _source_lines(String(mk.file)) lines === nothing && return nothing diff --git a/test/spec/summary.jl b/test/spec/summary.jl index b654d73..8a39cc2 100644 --- a/test/spec/summary.jl +++ b/test/spec/summary.jl @@ -1,7 +1,5 @@ -# Generates the coverage table for `test/spec/README.md` and the pull request that ships it. -# -# The measure is DISTINCT BEHAVIOURS — one per leaf `@testset` — not assertions, which move -# without any implementation progress when they sit inside a loop over the fixture's marks. +# Generates the coverage table for `test/spec/README.md`. The measure is DISTINCT BEHAVIOURS — one +# per leaf `@testset` — not assertions, which move without progress when they sit inside a loop. # # Run it: julia --project=test test/spec/summary.jl diff --git a/test/spec/test_spec_declare.jl b/test/spec/test_spec_declare.jl index 2d9fa1e..558203a 100644 --- a/test/spec/test_spec_declare.jl +++ b/test/spec/test_spec_declare.jl @@ -1,8 +1,6 @@ -# What can carry a mark: function, method, struct, const, module, macro, extension. -# -# Scope: both units. A mark names something, and — when it attached to a definition — also records -# the signature it attached to, so the same declaration answers `audit`'s question about the name -# and `reach`'s question about the method. +# What can carry a mark: function, method, struct, const, module, macro, extension. A mark names +# something and, when attached to a definition, records the signature too — so one declaration +# answers `audit`'s question about the name and `reach`'s about the method. using ExperimentalAPI: ExperimentalAPI, @experimental, Mark, experimental, isexperimental using Test diff --git a/test/spec/test_spec_dispatch.jl b/test/spec/test_spec_dispatch.jl index e670800..c557b84 100644 --- a/test/spec/test_spec_dispatch.jl +++ b/test/spec/test_spec_dispatch.jl @@ -1,9 +1,7 @@ -# One call site, several methods, only some of them marked. +# One call site, several methods, only some marked. # -# Scope: the call site the analysis cannot pin to one method. For a `Union`-typed or abstract -# argument `which(f, T)` throws, and an implementation that catches that and moves on reports -# `:clean` about a call that reaches a marked method half the time. That is the failure guarded -# here. +# For a `Union`-typed or abstract argument `which(f, T)` throws, and catching that and moving on +# reports `:clean` about a call that reaches a marked method half the time. using ExperimentalAPI: ExperimentalAPI, @experimental, experimental, isexperimental, mark using Test diff --git a/test/spec/test_spec_docstring.jl b/test/spec/test_spec_docstring.jl index db3c02e..8d053de 100644 --- a/test/spec/test_spec_docstring.jl +++ b/test/spec/test_spec_docstring.jl @@ -1,8 +1,5 @@ -# A mark and a docstring are different accounts, and must coexist. -# -# Scope: a mark is never a substitute for prose. `Base.Experimental` is the precedent — Base -# marks an experimental surface AND documents it. Pinned by named entries rather than by a count, -# which moves with the Julia version and the counting rule. +# A mark and a docstring are different accounts and must coexist — `Base.Experimental` is the +# precedent. Pinned by named entries, not by a count, which moves with the Julia version. using ExperimentalAPI: ExperimentalAPI, @experimental, audit, experimental, isdocumented, isexperimental, mark diff --git a/test/spec/test_spec_foreign.jl b/test/spec/test_spec_foreign.jl index 36dc568..6be6dc6 100644 --- a/test/spec/test_spec_foreign.jl +++ b/test/spec/test_spec_foreign.jl @@ -1,8 +1,7 @@ -# Marking a method on somebody else's generic — the `QAtlas.fetch` case, refused outright today. +# Marking a method on somebody else's generic — the `QAtlas.fetch` case. # -# Scope: `audit` files a name bound elsewhere under `foreign` and says nothing about the methods -# we contributed to it. Extending another package's generic is the normal Julia idiom, so a mark -# that cannot attach there cannot describe the surface that matters. +# `audit` files a name bound elsewhere under `foreign`. Extending another package's generic is the +# normal idiom, so a mark that cannot attach there cannot describe the surface that matters. using ExperimentalAPI: ExperimentalAPI, @experimental, audit, experimental, isexperimental using Test diff --git a/test/spec/test_spec_forms.jl b/test/spec/test_spec_forms.jl index 4dad829..15c0d74 100644 --- a/test/spec/test_spec_forms.jl +++ b/test/spec/test_spec_forms.jl @@ -1,8 +1,6 @@ -# The definition forms a real package hits on its second afternoon: kwargs, parametric -# signatures, callable structs, constructors, operators, stacked macros. -# -# Scope: each form either works, or the refusal names the alternative. Silently marking the wrong -# symbol is the outcome this file exists to prevent. +# The definition forms a real package hits on its second afternoon: kwargs, parametric signatures, +# callable structs, constructors, operators, stacked macros. Each either works or is refused by +# name; silently marking the wrong symbol is what this file prevents. using ExperimentalAPI: ExperimentalAPI, @experimental, experimental, isexperimental, mark using Test @@ -159,12 +157,9 @@ end end @testset "@inline and the mark compose in both orders" begin - # "Compose" was asserted as `Set([:f, :g])` — the names are marked — and that is satisfied by - # a mark that can never fire. Measured: the two orders did NOT compose the same way. With the - # mark outside, `_instrument` saw a `:macrocall` and returned `nothing`, so the flag was - # registered and nothing ever set it; with the mark inside, the flag worked. `entered` said - # `[:g]` after calling both. An `@inline` kernel is exactly what this package is for, so the - # claim has to be about the observation, not about the name. + # `Set([:f, :g])` — the names are marked — is satisfied by a mark that can never fire. Measured: + # with the mark outside the flag was registered and nothing set it, and `entered` said `[:g]` + # after calling both. The claim has to be about the observation. @eval module InlineMarked using ExperimentalAPI @experimental "kernel unverified" @inline f(x) = x @@ -223,13 +218,9 @@ end end @testset "a mark inside a function body is refused" begin - # Refused by Julia, not by this package: `const` in local scope fails during lowering, before - # any emitted code runs, so no check of ours can intercept it. The one lever is where the - # error points, and the expansion carries the caller's `LineNumberNode`. - # - # The misuse must arrive from a FILE: written through `@eval` the message carries no location - # at all, so a location assertion made that way is vacuous. Built line by line so the - # formatter cannot shift line 4. + # Refused by Julia during lowering, before any emitted code runs. The one lever is where the error + # points. The misuse must arrive from a FILE — through `@eval` the message carries no location at + # all, so the assertion would be vacuous. Built line by line so the formatter cannot shift line 4. dir = mktempdir() path = joinpath(dir, "caller_side.jl") write( @@ -264,21 +255,14 @@ end end @testset "the refusal cannot name @experimental, and that is now a decision" begin - # WITHDRAWN, with the measurement that withdrew it. The requirement was that the message name - # `@experimental`. It cannot, and the three routes are exhausted: + # WITHDRAWN: the message cannot name `@experimental`. Measured on 1.12.2 — # - # * `const` in local scope fails during LOWERING, before any emitted code runs, so no check - # of ours can intercept it — and Julia's message does not name the variable either, so - # naming the binding `var"@experimental ..."` does not smuggle the word in. Measured on - # 1.12.2: the message is byte-identical for `:__EXPERIMENTAL_API_MARKS__` and for a - # binding whose name is the whole sentence. - # * `global`, the one expansion that avoids `const`, fails SILENTLY in local scope — a - # worse outcome than a loud message pointing at the wrong vocabulary. - # * creating the registry through `Core.eval` removes the error altogether, which turns a - # refusal into a mark registered when the enclosing function is first called. + # * `const` fails during lowering and Julia's message names no variable, so a binding called + # `var"@experimental ..."` changes nothing: byte-identical either way. + # * `global`, the one expansion avoiding `const`, fails SILENTLY in local scope. + # * `Core.eval` removes the error and registers the mark on first call instead. # - # What is kept is the part that is in this package's hands and is asserted above: the blame - # lands on the line the author wrote, and never inside this package. + # What is kept is asserted above: the blame lands on the author's line, never inside this package. e = try @eval module ClosureMarked2 using ExperimentalAPI diff --git a/test/spec/test_spec_integration.jl b/test/spec/test_spec_integration.jl index acf8ba5..68ff74e 100644 --- a/test/spec/test_spec_integration.jl +++ b/test/spec/test_spec_integration.jl @@ -1,8 +1,5 @@ -# Where the mark has to surface outside this package: docs, Aqua, releases, provenance, CI. -# -# Scope: a mark only `ExperimentalAPI` can read is a private note. Everything here is pure or -# touches a temporary file except the Documenter block, which needs a test dependency this -# package does not have — so nothing in this group is blocked on infrastructure. +# Where the mark has to surface outside this package: docs, Aqua, releases, provenance, CI. A mark +# only `ExperimentalAPI` can read is a private note. using ExperimentalAPI: ExperimentalAPI, @experimental, experimental using Documenter: Documenter diff --git a/test/spec/test_spec_lifecycle.jl b/test/spec/test_spec_lifecycle.jl index 72cbabc..6f1f1dd 100644 --- a/test/spec/test_spec_lifecycle.jl +++ b/test/spec/test_spec_lifecycle.jl @@ -1,8 +1,5 @@ -# The mark's exit — when it may be removed — and entry points that are not a single function. -# -# Scope: a mark that can only ever be added is a decoration. The exit is what makes it a work -# item. The entry point has to be a module or a script, as `#print axioms` answers for any -# declaration and not only for one it is handed. +# The mark's exit — when it may be removed — and entry points that are not a single function. A +# mark that can only ever be added is a decoration; the exit makes it a work item. using ExperimentalAPI: ExperimentalAPI, @experimental, audit, experimental, mark using Test @@ -91,17 +88,12 @@ end @test ExperimentalAPI.verdict(ExperimentalAPI.reach(Main.CleanModule)) === :clean end -# Run in a child process and read its stderr, because the failure it guards is a WARNING today and -# an error later: reading a binding in a world prior to its definition world. `reach_script` -# evaluates a script's `const` lines into a scratch module and then analyses a thunk that names -# them, and the walk reads globals out of the IR — so it is the one place in this package that -# reaches a binding younger than its own caller. +# A child process, because what it guards is a warning today and an error later: reading a binding +# in a world prior to its definition world. # -# Two things about the child are load bearing, and the first cost a test that could not fail. -# `reach_script` is called from inside a FUNCTION: a caller's world age is fixed when it is -# entered, and at top level it moves with every statement, so nothing is caught there. And the -# child runs with the DEFAULT `depwarn`: measured on 1.12.2, `--depwarn=error` *suppresses* this -# warning rather than promoting it, which is the opposite of what Julia's own hint says. +# Two things about the child are load bearing. `reach_script` is called from inside a FUNCTION — +# at top level the world age moves with every statement and nothing is caught. And the child runs +# with the DEFAULT `depwarn`: measured on 1.12.2, `--depwarn=error` SUPPRESSES this warning. const _WORLD_AGE_SCRIPT = """ using ExperimentalAPI module L @@ -139,12 +131,9 @@ print("OK") end @testset "a script can be the entry point" begin - # The shape a researcher has: a file that produces a figure, not a package. The file must be - # written — `tempname()` alone throws regardless of the implementation. - # - # `hasproperty(…, :reached)` was the whole of this claim for a while, and it is satisfied by - # an implementation that returns an empty `Reach` for every input. So the script that reaches - # a mark and the one that does not are both run, and the verdicts have to differ. + # A file that produces a figure, not a package. `hasproperty(…, :reached)` was the whole of this + # claim once, and an implementation returning an empty `Reach` for every input satisfies it — so + # both scripts are run and the verdicts have to differ. dir = mktempdir() plain = joinpath(dir, "plain.jl") write(plain, "1 + 1\n") diff --git a/test/spec/test_spec_profile.jl b/test/spec/test_spec_profile.jl index a49f3dd..ccd35f7 100644 --- a/test/spec/test_spec_profile.jl +++ b/test/spec/test_spec_profile.jl @@ -1,8 +1,6 @@ # What a real run went through: which marked definitions it entered, how often, and how much of -# the run was spent inside them. -# -# Scope: two layers. Presence is detected by default and must cost nothing; counts, call sites -# and paths are opt-in. The measurements that put the boundary there are in `README.md`. +# the run was spent inside them. Presence is on by default and must cost nothing; counts, call +# sites and paths are opt-in. The measurements behind that boundary are in `README.md`. using ExperimentalAPI: ExperimentalAPI, @experimental using Profile: Profile @@ -197,12 +195,9 @@ end # ── proportion, not just presence ──────────────────────────────────────────────────────────── @testset "the record says what fraction of the run was inside experimental code" begin - # Long enough to be sampled: the timing backend is Julia's sampling profiler, and a run that - # finishes inside one sampling interval has no fraction to report. Two million iterations of - # a recorded body is tens of milliseconds — hundreds of samples, not a handful. - # `paths = false` because the two instruments cannot be read at once: `backtrace()` and the - # sampler both unwind the same threads' stacks, and asking for both is refused — see the - # measurement in `record`'s docstring, and the testset that pins the refusal below. + # Long enough to be sampled: a run finishing inside one sampling interval has no fraction to + # report. Two million iterations is tens of milliseconds — hundreds of samples. `paths = false` + # because the pair is refused; the measurement is below. r = ExperimentalAPI.record(() -> Sim.driver(M, 2_000_000); paths=false, timing=true) @test r.sampled # …the backend really was loaded f = ExperimentalAPI.experimental_fraction(r) @@ -315,16 +310,12 @@ end end @testset "paths and time cannot be collected in one block" begin - # A conjunction, and each half was measured alone before the pair was refused. `backtrace()` - # unwinds the calling thread; the sampler unwinds the same threads from outside. 1.12.7, 150 - # threaded records per run, four runs of each combination: + # A conjunction, each half measured alone first. `backtrace()` unwinds the calling thread; the + # sampler unwinds the same threads from outside. 1.12.7, 150 threaded records per run: # # paths alone 0/4 crashed # timing alone 0/4 # both 2/4 segmentation fault, no Julia backtrace - # - # So the pair is refused rather than risked, and the refusal names the measurement rather - # than saying "unsupported". e = try ExperimentalAPI.record(() -> Sim.driver(M, 1); paths=true, timing=true) nothing @@ -371,14 +362,9 @@ end end @testset "a mark born while the block runs is measured, not lost" begin - # The probe set was snapshotted BEFORE the call and never re-derived, so a mark that came - # into existence while `f` ran was entered by code that ran, counted by nobody, and left with - # its flag `false` for the rest of the process. That loses the entry from the OPT-IN layer and - # from the always-on one — `entered()` and the exit summary — which is the one thing the - # default layer promises never to do. - # - # A package extension loaded inside the block is the ordinary way this happens, and this - # package ships three of them; `Core.eval` is the same event without the loading machinery. + # The probe set was snapshotted before the call and never re-derived, so a mark born while `f` ran + # was left `false` for the rest of the process — lost from `entered()` and the exit summary too. + # A package extension loaded inside the block is the ordinary way this happens. @eval module Newborn using ExperimentalAPI public settled_mark @@ -545,14 +531,11 @@ end end # module Hot @testset "recording does not disturb Profile" begin - # `with_profile = true` means the caller is already using the buffer: whatever is in it stays. + # `with_profile = true`: whatever is in the buffer stays. # - # Two things this measures rather than assumes. The buffer has to be filled by a run that is - # long compared with the sampling interval — `Sim.driver(M, 200_000)` is about one interval at - # the default rate, and came back with **zero** samples on macOS, which made the whole - # assertion a coin flip. And the size is read with `Profile.len_data`, not by fetching: - # `fetch(; include_meta = false)` strips metadata behind an `@assert` that fires on - # 1.14.0-DEV.3115 for a buffer this test did not fill. + # The filling run must be long compared with the sampling interval — `Sim.driver(M, 200_000)` is + # about one interval and came back with zero samples on macOS. Size is read with + # `Profile.len_data`: `fetch(; include_meta = false)` hits an `@assert` on 1.14.0-DEV.3115. Hot.grind(10) Profile.clear() Profile.init(; delay=1e-5) diff --git a/test/spec/test_spec_propagate.jl b/test/spec/test_spec_propagate.jl index 33d4b1c..34febb6 100644 --- a/test/spec/test_spec_propagate.jl +++ b/test/spec/test_spec_propagate.jl @@ -1,17 +1,11 @@ -# A caller that never names a marked thing still depends on it. -# -# Modelled on Lean's `sorry`, but Julia's call graph is not closed, so the answer is three-valued: +# A caller that never names a marked thing still depends on it. Julia's call graph is not closed, +# so the answer is three-valued: # # :depends a marked definition is reachable # :clean the whole call graph was resolved and nothing marked is in it # :unknown some call site could not be resolved — the honest non-answer # -# Scope: collapsing `:unknown` into `:clean` is the one failure this file exists to prevent. It is -# not a weaker claim, it is a false one. -# -# The mechanism is a `Core.Compiler.AbstractInterpreter` hooking `abstract_call_method`, because -# inference runs before inlining; `code_typed(...; optimize=true)` sees only `mul_float` and finds -# nothing. +# Reporting `:unknown` as `:clean` is the one failure this file prevents. using ExperimentalAPI: ExperimentalAPI, @experimental, experimental using Test @@ -219,15 +213,11 @@ end # ── termination ────────────────────────────────────────────────────────────────────────────── @testset "a higher-order argument is answered, not thrown out of and not hung on" begin - # `sum(f(x) for x in xs)` lowers to a `Base.MappingRF` whose two fields are both singletons, - # which makes the STRUCT a singleton — so `w.instance` exists for a callable that is neither a - # `Function` nor a `Type`. `nameof` has no method for that, and the analysis died with a - # `MethodError` instead of returning one of its three verdicts. Measured on the shape this - # package's own `@entered` docstring uses as its worked example. + # `sum(f(x) for x in xs)` lowers to a `Base.MappingRF` whose fields are both singletons, so the + # struct is one too and `w.instance` exists for a callable `nameof` has no method for. # - # Removing the throw then exposed the second half: on 1.14.0-DEV `[f(x) for x in xs]` and - # `sum(map(f, xs))` generated new signatures faster than `maxdepth` could stop them and the - # call never returned, while both answer in milliseconds on 1.12. `maxwork` bounds the total. + # Removing the throw exposed the second half: on 1.14.0-DEV `[f(x) for x in xs]` and + # `sum(map(f, xs))` never returned, while both answer in milliseconds on 1.12. # # A throw is not a fourth verdict and neither is a hang. for f in (Chain.gen_bad, Chain.gen_good, Chain.comp_bad, Chain.map_bad, Chain.loop_good) @@ -308,14 +298,10 @@ end # ── across packages ────────────────────────────────────────────────────────────────────────── @testset "a mark in a dependency propagates into the dependent" begin - # `reach isa Function` was the whole of this claim for a while, and it is satisfied by an - # implementation that answers `:clean` for everything. The real question is whether a mark - # written while ANOTHER package was precompiled — in a process that has since exited — is - # visible to a caller here, and that needs a package rather than a module. - # - # `test/test_precompile.jl` is where it is asked, because it is the file that owns the scratch - # depot and the two subprocess runs. This asserts it is asked there rather than restating it: - # a claim checked in one place and mentioned in another is one that goes stale in the second. + # `reach isa Function` was the whole of this claim once, and an implementation answering `:clean` + # for everything satisfies it. The real question needs a package: is a mark written during ANOTHER + # package's precompilation visible here? `test/test_precompile.jl` owns the scratch depot, so this + # asserts the claim is asked there rather than restating it. src = read(joinpath(@__DIR__, "..", "test_precompile.jl"), String) @test occursin("REACH=", src) @test occursin("\"REACH\"] == \"depends\"", src) diff --git a/test/spec/test_spec_verify.jl b/test/spec/test_spec_verify.jl index 3258c12..7edf341 100644 --- a/test/spec/test_spec_verify.jl +++ b/test/spec/test_spec_verify.jl @@ -67,11 +67,9 @@ end end end -# Which half of each claim below can run depends on whether this process has coverage counters at -# all. Both halves are assertions: without `--code-coverage` the contract under test is that the -# answer is `missing` rather than a number, and that contract is exactly what stops every marked -# definition being reported unverified on an ordinary run. CI runs the suite with coverage on, so -# the measured half is what gates a pull request. +# Which half runs depends on whether this process has coverage counters. Both are assertions: +# without `--code-coverage` the contract is that the answer is `missing` rather than a number, +# which is what stops every marked definition being reported unverified on an ordinary run. const COVERED = ExperimentalAPI.coverage_enabled() @testset "the run knows whether it has coverage counters at all" begin diff --git a/test/test_dogfood.jl b/test/test_dogfood.jl index 0dab4b7..6a1a177 100644 --- a/test/test_dogfood.jl +++ b/test/test_dogfood.jl @@ -85,13 +85,9 @@ const YOUNG_LAYERS = Dict( end @testset "the settled core is not declared experimental" begin - # The control the testset above cannot be: an equality against a hand-written set is satisfied - # by marking every name and updating the set to match. These are the names the front page - # promises answers from, and a promise is exactly what a mark withdraws. - # - # Hoisted, and not for tidiness: `audit(ExperimentalAPI)` costs 0.75s — it reads a docstring - # for every public name and walks the method tables for the contributed ones — so calling it - # per iteration cost 17.8s, which was 16% of the whole suite. + # The control the testset above cannot be: an equality against a hand-written set is satisfied by + # marking every name and updating the set. `audit` is hoisted because it costs 0.75s — per + # iteration that was 17.8s, 16% of the suite. surface = audit(ExperimentalAPI).surface for n in [ Symbol("@experimental"), diff --git a/test/test_examples.jl b/test/test_examples.jl index 45db80a..5060cf7 100644 --- a/test/test_examples.jl +++ b/test/test_examples.jl @@ -1,12 +1,7 @@ -# The examples are documentation that runs. -# -# `docs/src/walkthrough.md` is generated from `examples/walkthrough.jl` by Literate at build time, -# so every output on that page is whatever the script printed. This file is the other half of that -# arrangement: without it, a change in `src/` could quietly rewrite what the documentation claims -# and nothing would go red until somebody read the rendered page. -# -# Twice in this package's first week a documented sample output turned out to be something the -# code could not produce. Both would have failed here. +# The examples are documentation that runs. `docs/src/walkthrough.md` is generated from +# `examples/walkthrough.jl` by Literate, so without this a change in `src/` could rewrite what the +# documentation claims and nothing would go red. Two shipped sample outputs turned out to be +# something the code could not produce; both would have failed here. using ExperimentalAPI: ExperimentalAPI, entered, reach, verdict using Test diff --git a/test/test_ext.jl b/test/test_ext.jl index 5fc0035..f119c41 100644 --- a/test/test_ext.jl +++ b/test/test_ext.jl @@ -1,9 +1,6 @@ -# `test_surface` is the reason the rest of the package exists, so the thing to establish is not -# that it passes on a clean module — it is that it FAILS on a dirty one, and fails naming the -# right symbol. A check that cannot be shown to fail has not been shown to check anything. -# -# `Recorder` collects results instead of throwing, which is what lets a passing suite contain a -# deliberate failure. +# The thing to establish is not that `test_surface` passes on a clean module — it is that it FAILS +# on a dirty one, naming the right symbol. `Recorder` collects results instead of throwing, which +# is what lets a passing suite contain a deliberate failure. using ExperimentalAPI: audit, test_surface using Test: Test diff --git a/test/test_macros.jl b/test/test_macros.jl index 4b05ee8..7d32287 100644 --- a/test/test_macros.jl +++ b/test/test_macros.jl @@ -1,9 +1,6 @@ -# `@entered expr` — the expression-level spelling of the observing layer. -# -# Scope: what the macro adds over `record(() -> expr)`. Two of those are things only a macro can -# get wrong — evaluating its argument twice, and reporting a location that is not the caller's — -# and one is the distinction the whole report exists for: a call that entered nothing is not the -# same state as a package that has nothing marked. +# `@entered expr` — what the macro adds over `record(() -> expr)`. Two are things only a macro can +# get wrong (evaluating its argument twice, reporting somebody else's location); one is the +# distinction the report exists for — entering nothing is not the same as nothing being marked. using ExperimentalAPI: ExperimentalAPI, @experimental using Test diff --git a/test/test_mark.jl b/test/test_mark.jl index c46bd53..a03eced 100644 --- a/test/test_mark.jl +++ b/test/test_mark.jl @@ -1,8 +1,5 @@ -# What `@experimental` accepts, what it refuses, and what the refusal says. -# -# Each accepted form gets its own module, because the thing under test is a top-level effect on -# the enclosing module — a `@testset` body is a function, and a mark written there would land -# somewhere no query looks. +# What `@experimental` accepts, what it refuses, and what the refusal says. Each form gets its own +# module: the effect under test is top-level, and a `@testset` body is a function. using ExperimentalAPI: ExperimentalAPI, @experimental, Mark, experimental, isexperimental, mark diff --git a/test/test_precompile.jl b/test/test_precompile.jl index f9813b1..fce0868 100644 --- a/test/test_precompile.jl +++ b/test/test_precompile.jl @@ -1,20 +1,15 @@ # The measurement that could kill the design. # -# Every other test in this suite marks a module defined in the running session, where any storage -# scheme works. The state a consumer is actually in is different: the marks are written while the -# consumer's package is being PRECOMPILED, in a process that then exits, and the query happens in -# a later process that only ever sees the cache image. A registry living in ExperimentalAPI's own -# state passes every other file here and returns an empty vector in that setting. +# Every other test marks a module defined in the running session, where any storage scheme works. +# A consumer's marks are written during PRECOMPILATION, in a process that then exits, and queried +# later from the cache image — where a registry living in ExperimentalAPI's own state returns an +# empty vector. # -# So this runs a real package through a real precompile, in a scratch depot, in a subprocess, and -# asks the loaded module what it is carrying. Twice: once compiling from source, once reading the -# cache the first run wrote. Only the second run is evidence. +# So: a real package, a real precompile, a scratch depot, a subprocess. Twice — compiling from +# source, then reading the cache the first run wrote. Only the second run is evidence. # -# It asks about all three KINDS of mark, because they are stored differently: a whole-name -# declaration, a mark attached to a definition (which also carries the `Type` it created), and a -# mark on a method of `Base.show`, whose signature names a type defined in the cached package. -# Only the first of those had ever been through a cache. It also runs `reach` across the package -# boundary, which is the query that reads the other two. +# All three KINDS of mark, because they are stored differently: a name declaration, a mark on a +# definition, and a mark on a `Base.show` method whose signature names a cached type. using Test: @test, @testset @@ -73,13 +68,10 @@ function run_probe(env, depots) return parse_probe(read(subprocess_env(cmd, depots), String)) end -# The child's environment is BUILT, not patched. `Pkg.test` exports a JULIA_LOAD_PATH pointing at -# its own temporary environment and omitting `@stdlib`; inherited, that overrides `--project` and -# the child cannot even load Pkg. Deleting the variable outright — rather than overwriting it with -# a hand-built list — leaves Julia's own default (`@:@v#.#:@stdlib`) in charge, which is both what -# is wanted and one fewer platform-specific string to get right. -# -# `setenv` REPLACES the environment rather than adding to it, so the base has to be `copy(ENV)`. +# The child's environment is BUILT, not patched: `Pkg.test` exports a JULIA_LOAD_PATH that omits +# `@stdlib`, and inherited it overrides `--project` so the child cannot load Pkg. Deleting the +# variable leaves Julia's own default in charge. `setenv` REPLACES the environment, so the base +# has to be `copy(ENV)`. function subprocess_env(cmd, depots) env = copy(ENV) delete!(env, "JULIA_LOAD_PATH") diff --git a/test/test_readme.jl b/test/test_readme.jl index a7dbde4..5feb760 100644 --- a/test/test_readme.jl +++ b/test/test_readme.jl @@ -1,8 +1,5 @@ -# The README's primary example, executed. -# -# Scope: the first ```julia block only — the rest of the README uses `MyPackage` as illustration. -# An example that does not run is the first thing a reader tries and the first thing that makes -# them close the tab. +# The README's primary example, executed — the first ```julia block only; the rest uses +# `MyPackage` as illustration. using ExperimentalAPI using Test @@ -59,11 +56,9 @@ end # ── every julia code block in the docs ─────────────────────────────────────────────────────── # -# Scope: a lint, not an execution. Most blocks reference a `MyPackage` that does not exist, so -# they cannot be run — but the defect that shipped here was not a runtime one. A trailing `\` -# used as a line continuation PARSES (Julia reads it as left-division) and fails only when the -# macro is expanded, so neither a parse check nor a `jldoctest` would have caught it. The check -# has to be for the character. +# A lint, not an execution: most blocks reference a `MyPackage` that does not exist. The defect +# that shipped was a trailing `\` line continuation, which PARSES as left-division and fails only +# at macro expansion — so the check has to be for the character. "Every ```julia fence in `path`, as (line number, text) pairs." function julia_blocks(path) @@ -134,9 +129,8 @@ end # ── the documentation pages ────────────────────────────────────────────────────────────────── # -# Scope: `docs/src` gets the same treatment the README does. Twenty julia blocks were shipped -# unexecuted, and two of them did not run — including the front page's, which is the defect the -# registry review named for the README and which was fixed there and not here. +# `docs/src` gets the same treatment. Twenty blocks shipped unexecuted and two did not run, +# including the front page's. const _DOCS = joinpath(@__DIR__, "..", "docs", "src") @@ -222,11 +216,9 @@ end # ── URLs inside examples ───────────────────────────────────────────────────────────────────── # -# A placeholder URL must not look like a real address that fails. `github.com/org/Pkg.jl/issues/12` -# resolved to GitHub and returned **404**, so a reviewer running a link checker saw a dead link — -# which is the finding that opened the review of another package in this organisation. -# `example.invalid` cannot resolve at all (RFC 2606 reserves it), which is what a placeholder -# should look like. Checked without the network: the property is the host, not the response. +# `github.com/org/Pkg.jl/issues/12` resolved to GitHub and returned 404, so a link checker saw a +# dead link. `example.invalid` cannot resolve at all (RFC 2606). Checked without the network: the +# property is the host, not the response. const _RESERVED_HOSTS = ["example.com", "example.net", "example.org", "example.invalid"] const _OWN_HOSTS = ["github.com/QAtlasHub/", "qatlashub.github.io/"] diff --git a/test/test_release.jl b/test/test_release.jl index 88c1fb3..530322d 100644 --- a/test/test_release.jl +++ b/test/test_release.jl @@ -1,9 +1,6 @@ -# The release-decision layer: a snapshot survives a round trip through TOML, and a diff of two -# snapshots says which moves break callers. -# -# The fixture is a pair of hand-written snapshots rather than two loaded versions of a package, -# because that is how a real caller uses it: the old side comes off disk, from a file committed -# at the last release. +# The release-decision layer: a snapshot survives a TOML round trip, and a diff of two says which +# moves break callers. Hand-written snapshots rather than two loaded versions, because that is how +# a caller uses it — the old side comes off disk. using ExperimentalAPI: Diff, compare, isbreaking, read_snapshot, snapshot, stable, write_snapshot, experimental