Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions src/record.jl
Original file line number Diff line number Diff line change
Expand Up @@ -523,6 +523,18 @@ make.
"""
attribute(data) = attribute(timing_backend(), data)

# Without this the call failed inside `Profile` with `no method matching getdict(::Record)`.
function attribute(rec::Record)
return throw(
ArgumentError(
"attribute: expected a profile buffer, got a `Record`. A `Record` already carries " *
"the attribution — read `inclusive`/`exclusive` off its hits, or call " *
"`record(f; timing = true)` to collect them. `attribute` is for a buffer somebody " *
"else profiled: `attribute(Profile.fetch())`.",
),
)
end

function attribute(::NoTiming, data)
return throw(
ArgumentError(
Expand Down Expand Up @@ -608,6 +620,11 @@ Read back a record written by [`write_record`](@ref).
The `method` field of every [`Hit`](@ref) comes back `nothing`, and so does the record's `value`:
neither a `Method` nor a run's result is a thing a file can carry, and reconstructing one would
mean claiming the code in this process is the code that produced the record.

!!! warning "`mod` falls back to `Main` when the module is not loaded here"
The file carries a module's **name** and `Hit.mod` is a `Module`. Absent from this process —
the ordinary case when shards are merged elsewhere — `mod` is `Main`, and the report reads
`Main.energy` for a name that is not in `Main`.
"""
function read_record(path::AbstractString)
d = TOML.parsefile(path)
Expand Down
10 changes: 10 additions & 0 deletions src/verify.jl
Original file line number Diff line number Diff line change
Expand Up @@ -198,10 +198,20 @@ function _sibling_spans(x, outer_end::Int)
return out
end

# `"docstring"` above a definition parses into `Core.@doc "…" <definition>`, one statement whose
# start line is the DOCSTRING's. Nothing then starts on the definition's own line, which is what
# the mark recorded.
function _is_doc_call(st)
return st isa Expr &&
st.head === :macrocall &&
(st.args[1] === GlobalRef(Core, Symbol("@doc")) || st.args[1] === Symbol("@doc"))
end

function _find_span(x, line::Int, outer_end::Int)
(x isa Expr) || return nothing
for (a, b, st) in _sibling_spans(x, outer_end)
a == line && return (a, b)
(_is_doc_call(st) && a < line <= b) && return (line, b)
(a <= line <= b) || continue
for body in _containers(st)
r = _find_span(body, line, b)
Expand Down
6 changes: 3 additions & 3 deletions test/spec/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,10 @@ that is entirely `@test_broken` is a claim written down, not a check being run.
| `test_spec_forms.jl` | 28 | 28 | 0 | the definition forms a real package hits on its second afternoon |
| `test_spec_integration.jl` | 19 | 19 | 0 | where the mark has to surface: docs, Aqua, releases, provenance, CI |
| `test_spec_lifecycle.jl` | 16 | 16 | 0 | the mark's EXIT, and an entry point that is a module rather than a function |
| `test_spec_profile.jl` | 45 | 45 | 0 | what a real run went through, how often, and how much of it |
| `test_spec_profile.jl` | 47 | 47 | 0 | what a real run went through, how often, and how much of it |
| `test_spec_propagate.jl` | 24 | 24 | 0 | a caller that never names a marked thing still depends on it |
| `test_spec_verify.jl` | 9 | 9 | 0 | how well is a marked thing exercised by the tests |
| **10 files** | **190** | **190** | **0** | |
| `test_spec_verify.jl` | 10 | 10 | 0 | how well is a marked thing exercised by the tests |
| **10 files** | **193** | **193** | **0** | |
<!-- END GENERATED -->

The table is generated and pinned by `test/test_spec_table.jl`, which fails if it goes stale —
Expand Down
49 changes: 49 additions & 0 deletions test/spec/test_spec_profile.jl
Original file line number Diff line number Diff line change
Expand Up @@ -632,6 +632,55 @@ end
@test ExperimentalAPI.assert_clean(() -> 1 + 1)
end

@testset "attribute refuses a Record by name instead of failing inside Profile" begin
# `attribute` takes a profile buffer. Given the `Record` next door it failed inside `Profile`
# with `no method matching getdict(::Record)`.
rec = ExperimentalAPI.record(() -> Sim.driver(M, 3); paths=false, timing=false)
e = try
ExperimentalAPI.attribute(rec)
nothing
catch err
err
end
@test e isa ArgumentError
msg = sprint(showerror, e)
@test occursin("attribute", msg) # the verb they wrote
@test occursin("Record", msg) # what they passed
@test occursin("Profile.fetch()", msg) # …and what to pass instead
@test !occursin("getdict", msg) # not an internal of somebody else's package
end

@testset "a record read where its module is not loaded says Main, and that is stated" begin
# A file carries a module's name; `Hit.mod` is a `Module`. Present here it resolves, absent it
# falls back to `Main` — so a merged report reads `Main.energy` for a name not in `Main`.
dir = mktempdir()
p = joinpath(dir, "rec.toml")
rec = ExperimentalAPI.record(() -> Sim.driver(M, 3); paths=false, timing=false)
ExperimentalAPI.write_record(p, rec)

here = ExperimentalAPI.read_record(p)
@test only(here).mod === Sim # present: resolved

# Absent: read it in a process that never defined `Sim`.
probe = joinpath(dir, "probe.jl")
write(
probe,
"""
using ExperimentalAPI
r = ExperimentalAPI.read_record(ARGS[1])
println(only(r).mod, " ", only(r).name, " ", only(r).count)
""",
)
out = read(
`$(Base.julia_cmd()) --startup-file=no --project=$(Base.active_project()) $probe $p`,
String,
)
parts = split(strip(out))
@test parts[1] == "Main"
@test parts[2] == String(only(rec).name)
@test parts[3] == string(only(rec).count)
end

@testset "the assertion fails, naming the mark, when the run is not clean" begin
# Control: a gate that cannot be shown to fire is not a gate.
@test !ExperimentalAPI.assert_clean(() -> Sim.driver(M, 10); throw=false)
Expand Down
23 changes: 22 additions & 1 deletion test/spec/test_spec_verify.jl
Original file line number Diff line number Diff line change
Expand Up @@ -26,17 +26,38 @@ end

@experimental "shipped without ever being called" never_exercised(x) = x * 0

"Documented and marked — the shape `audit` asks for, and the one whose coverage came back `missing`."
@experimental "documented, and still unfinished" documented_and_marked(x) = x + 2

end # module Covered

# Deliberately partial: a fully exercised fixture cannot tell a working join from one that
# always reports 100%.
@testset "the fixture is exercised only partly, on purpose" begin
@test Covered.exercised(1) == 2
@test Covered.half_exercised(1) == 1
@test Covered.documented_and_marked(1) == 3
# `half_exercised(-1)` is NOT called
# `never_exercised` is NOT called
end

@testset "a docstring above the mark does not hide the coverage" begin
# Measured: entered once each in one process, bare reported 1/1 and documented 0/0 `missing`,
# because the docstring makes the pair one statement starting on the docstring's line. The
# documented-and-marked shape is the one `audit` asks for.
vs = Dict(v.mark.name => v for v in ExperimentalAPI.verification(Covered))
doc = vs[:documented_and_marked]
bare = vs[:exercised]
if bare.fraction === missing
# No coverage in this run at all — then BOTH must say so, which is the claim either way.
@test doc.fraction === missing
else
@test doc.fraction !== missing
@test doc.fraction == bare.fraction == 1.0
@test doc.total > 0
end
end

@testset "marks carry the location a coverage file is keyed by" begin
for mk in experimental(Covered)
@test isfile(String(mk.file))
Expand Down Expand Up @@ -130,5 +151,5 @@ end
@test length(vs) == length(ExperimentalAPI.experimental(Covered))
@test all(v -> v isa ExperimentalAPI.Verification, vs)
@test Set(v.mark.name for v in vs) ==
Set([:exercised, :half_exercised, :never_exercised])
Set([:exercised, :half_exercised, :never_exercised, :documented_and_marked])
end
Loading