diff --git a/src/record.jl b/src/record.jl index 49bd5d1..29b95f7 100644 --- a/src/record.jl +++ b/src/record.jl @@ -523,6 +523,18 @@ make. """ attribute(data) = attribute(timing_backend(), data) +# Without this the call failed inside `Profile` with `no method matching getdict(::Record)`. +function attribute(rec::Record) + return throw( + ArgumentError( + "attribute: expected a profile buffer, got a `Record`. A `Record` already carries " * + "the attribution — read `inclusive`/`exclusive` off its hits, or call " * + "`record(f; timing = true)` to collect them. `attribute` is for a buffer somebody " * + "else profiled: `attribute(Profile.fetch())`.", + ), + ) +end + function attribute(::NoTiming, data) return throw( ArgumentError( @@ -608,6 +620,11 @@ Read back a record written by [`write_record`](@ref). The `method` field of every [`Hit`](@ref) comes back `nothing`, and so does the record's `value`: neither a `Method` nor a run's result is a thing a file can carry, and reconstructing one would mean claiming the code in this process is the code that produced the record. + +!!! warning "`mod` falls back to `Main` when the module is not loaded here" + The file carries a module's **name** and `Hit.mod` is a `Module`. Absent from this process — + the ordinary case when shards are merged elsewhere — `mod` is `Main`, and the report reads + `Main.energy` for a name that is not in `Main`. """ function read_record(path::AbstractString) d = TOML.parsefile(path) diff --git a/src/verify.jl b/src/verify.jl index 09242dd..49e68f1 100644 --- a/src/verify.jl +++ b/src/verify.jl @@ -198,10 +198,20 @@ function _sibling_spans(x, outer_end::Int) return out end +# `"docstring"` above a definition parses into `Core.@doc "…" `, one statement whose +# start line is the DOCSTRING's. Nothing then starts on the definition's own line, which is what +# the mark recorded. +function _is_doc_call(st) + return st isa Expr && + st.head === :macrocall && + (st.args[1] === GlobalRef(Core, Symbol("@doc")) || st.args[1] === Symbol("@doc")) +end + function _find_span(x, line::Int, outer_end::Int) (x isa Expr) || return nothing for (a, b, st) in _sibling_spans(x, outer_end) a == line && return (a, b) + (_is_doc_call(st) && a < line <= b) && return (line, b) (a <= line <= b) || continue for body in _containers(st) r = _find_span(body, line, b) diff --git a/test/spec/README.md b/test/spec/README.md index 34786e5..26f427d 100644 --- a/test/spec/README.md +++ b/test/spec/README.md @@ -52,10 +52,10 @@ that is entirely `@test_broken` is a claim written down, not a check being run. | `test_spec_forms.jl` | 28 | 28 | 0 | the definition forms a real package hits on its second afternoon | | `test_spec_integration.jl` | 19 | 19 | 0 | where the mark has to surface: docs, Aqua, releases, provenance, CI | | `test_spec_lifecycle.jl` | 16 | 16 | 0 | the mark's EXIT, and an entry point that is a module rather than a function | -| `test_spec_profile.jl` | 45 | 45 | 0 | what a real run went through, how often, and how much of it | +| `test_spec_profile.jl` | 47 | 47 | 0 | what a real run went through, how often, and how much of it | | `test_spec_propagate.jl` | 24 | 24 | 0 | a caller that never names a marked thing still depends on it | -| `test_spec_verify.jl` | 9 | 9 | 0 | how well is a marked thing exercised by the tests | -| **10 files** | **190** | **190** | **0** | | +| `test_spec_verify.jl` | 10 | 10 | 0 | how well is a marked thing exercised by the tests | +| **10 files** | **193** | **193** | **0** | | The table is generated and pinned by `test/test_spec_table.jl`, which fails if it goes stale — diff --git a/test/spec/test_spec_profile.jl b/test/spec/test_spec_profile.jl index 6cfd773..a49f3dd 100644 --- a/test/spec/test_spec_profile.jl +++ b/test/spec/test_spec_profile.jl @@ -632,6 +632,55 @@ end @test ExperimentalAPI.assert_clean(() -> 1 + 1) end +@testset "attribute refuses a Record by name instead of failing inside Profile" begin + # `attribute` takes a profile buffer. Given the `Record` next door it failed inside `Profile` + # with `no method matching getdict(::Record)`. + rec = ExperimentalAPI.record(() -> Sim.driver(M, 3); paths=false, timing=false) + e = try + ExperimentalAPI.attribute(rec) + nothing + catch err + err + end + @test e isa ArgumentError + msg = sprint(showerror, e) + @test occursin("attribute", msg) # the verb they wrote + @test occursin("Record", msg) # what they passed + @test occursin("Profile.fetch()", msg) # …and what to pass instead + @test !occursin("getdict", msg) # not an internal of somebody else's package +end + +@testset "a record read where its module is not loaded says Main, and that is stated" begin + # A file carries a module's name; `Hit.mod` is a `Module`. Present here it resolves, absent it + # falls back to `Main` — so a merged report reads `Main.energy` for a name not in `Main`. + dir = mktempdir() + p = joinpath(dir, "rec.toml") + rec = ExperimentalAPI.record(() -> Sim.driver(M, 3); paths=false, timing=false) + ExperimentalAPI.write_record(p, rec) + + here = ExperimentalAPI.read_record(p) + @test only(here).mod === Sim # present: resolved + + # Absent: read it in a process that never defined `Sim`. + probe = joinpath(dir, "probe.jl") + write( + probe, + """ + using ExperimentalAPI + r = ExperimentalAPI.read_record(ARGS[1]) + println(only(r).mod, " ", only(r).name, " ", only(r).count) + """, + ) + out = read( + `$(Base.julia_cmd()) --startup-file=no --project=$(Base.active_project()) $probe $p`, + String, + ) + parts = split(strip(out)) + @test parts[1] == "Main" + @test parts[2] == String(only(rec).name) + @test parts[3] == string(only(rec).count) +end + @testset "the assertion fails, naming the mark, when the run is not clean" begin # Control: a gate that cannot be shown to fire is not a gate. @test !ExperimentalAPI.assert_clean(() -> Sim.driver(M, 10); throw=false) diff --git a/test/spec/test_spec_verify.jl b/test/spec/test_spec_verify.jl index a191a47..3258c12 100644 --- a/test/spec/test_spec_verify.jl +++ b/test/spec/test_spec_verify.jl @@ -26,6 +26,9 @@ end @experimental "shipped without ever being called" never_exercised(x) = x * 0 +"Documented and marked — the shape `audit` asks for, and the one whose coverage came back `missing`." +@experimental "documented, and still unfinished" documented_and_marked(x) = x + 2 + end # module Covered # Deliberately partial: a fully exercised fixture cannot tell a working join from one that @@ -33,10 +36,28 @@ end # module Covered @testset "the fixture is exercised only partly, on purpose" begin @test Covered.exercised(1) == 2 @test Covered.half_exercised(1) == 1 + @test Covered.documented_and_marked(1) == 3 # `half_exercised(-1)` is NOT called # `never_exercised` is NOT called end +@testset "a docstring above the mark does not hide the coverage" begin + # Measured: entered once each in one process, bare reported 1/1 and documented 0/0 `missing`, + # because the docstring makes the pair one statement starting on the docstring's line. The + # documented-and-marked shape is the one `audit` asks for. + vs = Dict(v.mark.name => v for v in ExperimentalAPI.verification(Covered)) + doc = vs[:documented_and_marked] + bare = vs[:exercised] + if bare.fraction === missing + # No coverage in this run at all — then BOTH must say so, which is the claim either way. + @test doc.fraction === missing + else + @test doc.fraction !== missing + @test doc.fraction == bare.fraction == 1.0 + @test doc.total > 0 + end +end + @testset "marks carry the location a coverage file is keyed by" begin for mk in experimental(Covered) @test isfile(String(mk.file)) @@ -130,5 +151,5 @@ end @test length(vs) == length(ExperimentalAPI.experimental(Covered)) @test all(v -> v isa ExperimentalAPI.Verification, vs) @test Set(v.mark.name for v in vs) == - Set([:exercised, :half_exercised, :never_exercised]) + Set([:exercised, :half_exercised, :never_exercised, :documented_and_marked]) end