diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index b457130..7043575 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -49,6 +49,12 @@ jobs: - uses: julia-actions/cache@v3 - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 + env: + # `test/spec/test_spec_profile.jl` asserts `Threads.nthreads() > 1`. A + # `Threads.@threads` loop runs its body the same number of times whatever the thread + # count is, so a concurrency test on a single-threaded runner cannot fail for a + # recorder that is not thread safe — it would be untestable rather than passing. + JULIA_NUM_THREADS: '4' - uses: julia-actions/julia-processcoverage@v1 if: matrix.julia == '1.12' && matrix.os == 'ubuntu-latest' - uses: codecov/codecov-action@v7 diff --git a/Project.toml b/Project.toml index 14f2427..5c3fdcd 100644 --- a/Project.toml +++ b/Project.toml @@ -14,13 +14,15 @@ ExperimentalAPITestExt = "Test" [compat] Aqua = "0.8" +Profile = "1" TOML = "1" Test = "1" julia = "1.11" [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test", "Aqua"] +test = ["Test", "Aqua", "Profile"] diff --git a/README.md b/README.md index 358f2d9..77e9959 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,14 @@ reads as if it had none: - **Whether a name appears in your guide, README or docs site.** Docstring presence is not documentation-page presence, and those two gaps are usually different sets. +## Where this is going + +`test/spec/` is the specification for the rest: the propagation, profiling and lifecycle work is +written there as tests before it is implemented, so it cannot drift from the code. Most of it is +`@test_broken` today, and [`test/spec/README.md`](test/spec/README.md) explains why that register +was chosen, which of the negative controls are actually running, and the two defects the exercise +already found in the shipped code. + ## License MIT diff --git a/src/mark.jl b/src/mark.jl index ca6f149..4c421fb 100644 --- a/src/mark.jl +++ b/src/mark.jl @@ -1,11 +1,9 @@ -# The mark itself: what one `@experimental` declaration records, where it is stored, and the -# macro that writes it. +# What one `@experimental` declaration records, where it is stored, and the macro that writes it. # -# Storage is a `const` vector inside the MARKED module, not a table inside this one. That is not -# a style choice — a registry living here would be populated while the marked package is being -# precompiled, and nothing written into a third module at that moment is part of the cache image -# that gets loaded later. Base's own `Docs.META` is a per-module binding for exactly this reason, -# and `test/test_precompile.jl` is the test that would catch it if this stopped being true. +# Storage is a `const` vector inside the MARKED module: a registry here would be populated during +# the marked package's precompilation, and nothing written into a third module then survives into +# its cache image. `Docs.META` is per-module for the same reason. Pinned by +# `test/test_precompile.jl`. """ Mark @@ -35,6 +33,15 @@ struct Mark tracking::Union{String,Nothing} file::Symbol line::Int + + # The reason is the payload, so the invariant belongs in the type: `Mark` is `public`, and + # every construction route that is not the macro gets it for free here. + function Mark(mod, name, reason, since, tracking, file, line) + r = String(strip(reason)) + isempty(r) && + throw(ArgumentError("a Mark's reason may not be empty — it is the payload")) + return new(mod, name, r, since, tracking, file, line) + end end function Base.show(io::IO, m::Mark) @@ -52,18 +59,13 @@ function Base.show(io::IO, ::MIME"text/plain", m::Mark) return print(io, " declared: ", m.file, ":", m.line) end -# The binding every marked module gets. Named, not gensym'd, so `M.__EXPERIMENTAL_API_MARKS__` -# is greppable and inspectable when a query result surprises someone. +# Named rather than gensym'd, so it is greppable when a query result surprises someone. const MARKS_BINDING = :__EXPERIMENTAL_API_MARKS__ -# The registry is created by code the macro emits INTO the marked module — not by `Core.eval` -# from here. The difference is not stylistic. Julia 1.12 rejects reading a binding that was -# created earlier in the same top-level statement ("define the const at top-level before running -# the function that uses it"), and a `Core.eval`-then-`getglobal` helper does exactly that. The -# emitted form puts the `const` in one top-level statement and the `push!` in the next, so the -# world age has advanced in between and the read is legal. -# -# `@macroexpand` therefore shows the whole mechanism, which is the second reason to prefer it. +# Created by code the macro emits into the marked module, not by `Core.eval` from here: Julia +# 1.12 rejects reading a binding created earlier in the same top-level statement, which is what a +# `Core.eval`-then-`getglobal` helper does. The emitted form puts the `const` and the `push!` in +# separate statements, so the world age has advanced in between. function _registry_of(m::Module) v = getglobal(m, MARKS_BINDING) v isa Vector{Mark} || throw( @@ -75,8 +77,7 @@ function _registry_of(m::Module) return v end -# Re-marking a name replaces its entry instead of appending, so re-including a file (Revise, an -# `include` reached twice) cannot make one name appear in `experimental(M)` several times. +# Replaces rather than appends, so re-including a file cannot duplicate a name. function _mark!(reg::Vector{Mark}, mk::Mark) i = findfirst(x -> x.name === mk.name, reg) if i === nothing @@ -155,9 +156,8 @@ macro experimental(args...) isempty(rest) && throw(ArgumentError("@experimental: nothing to mark — give a definition or a name")) - # `k = v` pairs bind tighter than the subject only when they are NOT the last argument: the - # last argument is always the thing being marked, which keeps `@experimental "…" x = 3` - # unambiguous against `@experimental "…" since=v"1" x = 3`. + # The last argument is always the subject, which keeps `@experimental "…" x = 3` unambiguous + # against `@experimental "…" since=v"1" x = 3`. since, tracking, i = nothing, nothing, 1 while i < length(rest) a = rest[i] @@ -198,16 +198,18 @@ macro experimental(args...) names = [_defname(def)] end - # Statement order carries a constraint: the `const` must land in its own top-level statement, - # because the `_mark!` calls below READ that binding and Julia 1.12 forbids reading a binding - # created in the same world age. + # The `const` must land in its own top-level statement: the `_mark!` calls below read that + # binding, and Julia 1.12 forbids reading one created in the same world age. marks = esc(MARKS_BINDING) - init = :( - if !$(isdefined)($__module__, $(QuoteNode(MARKS_BINDING))) - const $marks = $(Mark)[] - end - ) src = __source__ + # Built with `Expr` so no `LineNumberNode` from this file reaches the expansion. `const` in + # local scope is a lowering error this macro cannot catch, so the only lever is where the + # error points, and it must point at the caller. Pinned in `test_spec_forms.jl`. + init = Expr( + :if, + :(!$(isdefined)($__module__, $(QuoteNode(MARKS_BINDING)))), + Expr(:block, src, Expr(:const, Expr(:(=), marks, :($(Mark)[])))), + ) records = [ :($(_mark!)( $marks, @@ -222,16 +224,14 @@ macro experimental(args...) ), )) for n in names ] - # `Expr(:meta, :doc)` is how a macro tells the documentation system which expression inside - # its expansion a preceding docstring belongs to — the mechanism `Base.@kwdef` uses. Without - # it, `"""docs""" @experimental "why" f(x) = x` fails with "cannot document the following - # expression", which would make the two accounts this package asks for mutually exclusive. + # Tells the documentation system which expression a preceding docstring belongs to, as + # `Base.@kwdef` does. Without it a docstring on a marked definition fails to attach at all. body = def === nothing ? nothing : Expr(:block, Expr(:meta, :doc), esc(def)) return Expr(:block, init, body, records..., nothing) end -# Whitespace is normalised so a reason written as a wrapped triple-quoted string does not carry -# its indentation into every message that prints it. Nothing else about the text is touched. +# Normalises whitespace so a wrapped triple-quoted reason does not carry its indentation into +# every message. Nothing else about the text is touched. function _reason(s) r = String(strip(s)) isempty(r) && throw( @@ -251,9 +251,8 @@ function _subject_names(subject) a.args[1] isa Symbol && length(a.args) == 2 && a.args[2] isa LineNumberNode - # A BARE `@foo` — the name a macro is public under. A macrocall carrying arguments - # is a definition this macro cannot read, not a name, and must fall through to be - # refused rather than silently recorded as `Symbol("@doc")`. + # A bare `@foo` is a name; a macrocall carrying arguments is a definition this macro + # cannot read, and must fall through to be refused rather than recorded. push!(names, a.args[1]) elseif a isa QuoteNode && a.value isa Symbol push!(names, a.value) # `:foo`, for a name a reader prefers to quote @@ -276,8 +275,8 @@ function _defname(ex::Expr) h === :abstract && return _typename(ex.args[1]) h === :primitive && return _typename(ex.args[1]) h === :const && return _defname(ex.args[1]) - # A `module` cannot be flattened out of the block this macro emits — Julia requires it as a - # direct top-level statement — so it takes the name-list form rather than being wrapped. + # Julia requires `module` as a direct top-level statement, so it cannot be wrapped and takes + # the name-list form. h === :module && throw( ArgumentError( "@experimental cannot attach to a `module`, which Julia requires at top level. " * diff --git a/test/runtests.jl b/test/runtests.jl index 8226297..a4e9c83 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -11,5 +11,18 @@ using Test include("test_ext.jl") include("test_precompile.jl") include("test_dogfood.jl") + # The case matrix. Written before the implementation, so most of it is @test_broken; + # see test/spec/README.md for why that is the right register. + include("spec/test_spec_declare.jl") + include("spec/test_spec_propagate.jl") + include("spec/test_spec_docstring.jl") + include("spec/test_spec_verify.jl") + include("spec/test_spec_profile.jl") + include("spec/test_spec_foreign.jl") + include("spec/test_spec_forms.jl") + include("spec/test_spec_integration.jl") + include("spec/test_spec_dispatch.jl") + include("spec/test_spec_lifecycle.jl") + include("test_spec_table.jl") include("test_aqua.jl") end diff --git a/test/spec/README.md b/test/spec/README.md new file mode 100644 index 0000000..549f6ff --- /dev/null +++ b/test/spec/README.md @@ -0,0 +1,141 @@ +# The case matrix + +These files are the specification. They are written before the implementation, so most of them +are `@test_broken`. + +`@test_broken` was chosen deliberately over comments or a TODO list: + + * an expression that throws (because the function does not exist yet) registers as **Broken**, + not as an error, so the suite stays green while the spec is incomplete; + * an expression that starts **passing** registers as `Error: Unexpected Pass`, which fails the + suite until someone promotes it to `@test`. + +So the spec cannot silently rot in either direction: it neither blocks work that has not been +done, nor lets finished work go unnoticed. + +One caveat, measured rather than assumed: an expression whose value is not a `Bool` — an +`@eval module … end`, whose value is a `Module` — reports `Error: Expression evaluated to +non-Boolean` instead of `Unexpected Pass`. Both fail the suite loudly, so nothing rots either +way, but every `@eval module` block in this directory now ends in a `Bool` and checks *what* got +marked, so the message is the expected one and the assertion has content beyond "it parsed". + +## Overlap with `test/test_*.jl` + +`test_spec_declare.jl` re-covers definition forms that `test/test_mark.jl` already pins, and +`test_spec_docstring.jl` re-covers part of `test/test_audit.jl`'s bucket contract, through +separate fixtures. That is deliberate while the spec is the design document — but two fixtures +pinning one contract have to be kept in sync by hand, so the older files should be folded in or +retired once the spec stops moving. + +Anything already implemented is a plain `@test`. + +## What is covered + +The measure is **distinct behaviours** — one per leaf `@testset` — not assertions. An assertion +count moves without any implementation progress: `test_spec_declare.jl` has 36 assertion lines +but 91 runtime assertions, because several run inside `for mk in experimental(Declared)`, so a +thirteenth fixture mark would buy four more passing assertions and cover nothing new. A leaf +testset is one claim, and adding one means writing one. + +*operating today* is the column that matters when reading a claim about this directory: a leaf +that is entirely `@test_broken` is a claim written down, not a check being run. + + +| file | behaviours | operating today | specified only | concern | +|---|---|---|---|---| +| `test_spec_declare.jl` | 11 | 7 | 4 | what can carry a mark: function, method, struct, const, module, macro, extension | +| `test_spec_dispatch.jl` | 14 | 4 | 10 | one call site, several methods, only some marked — the branch | +| `test_spec_docstring.jl` | 9 | 6 | 3 | a mark and a docstring are different accounts and must coexist | +| `test_spec_foreign.jl` | 14 | 5 | 9 | marking a method on somebody else's generic — the `QAtlas.fetch` case | +| `test_spec_forms.jl` | 26 | 15 | 11 | the definition forms a real package hits on its second afternoon | +| `test_spec_integration.jl` | 17 | 1 | 16 | where the mark has to surface: docs, Aqua, releases, provenance, CI | +| `test_spec_lifecycle.jl` | 15 | 7 | 8 | the mark's EXIT, and an entry point that is a module rather than a function | +| `test_spec_profile.jl` | 40 | 5 | 35 | what a real run went through, how often, and how much of it | +| `test_spec_propagate.jl` | 20 | 2 | 18 | a caller that never names a marked thing still depends on it | +| `test_spec_verify.jl` | 8 | 2 | 6 | how well is a marked thing exercised by the tests | +| **10 files** | **174** | **54** | **120** | | + + +The table is generated and pinned by `test/test_spec_table.jl`, which fails if it goes stale — +the hand-written version drifted inside the change that introduced it. + +## Two layers, and why the split is where it is + +The goal is a tool that says **where** experimental code was used, and a user who learns they +used it **without opting in**. Those are different jobs with different budgets, and the boundary +between them was measured rather than chosen. 10M calls of a realistic numeric body, Julia +1.12.2, minimum of 7–9 trials: + +| emitted into the body | 1 thread | 8 threads | counts correctly? | +|---|---|---|---| +| nothing | 1.00× | 1.00× | — | +| set-once flag, read-mostly | 1.03× | **0.985×** | yes | +| counter, plain shared `Ref` | 1.03× | 3.76× | **no** | +| counter, global atomic | 1.17× | 4.87× | yes | +| counter, per-thread atomic | 1.12× | 2.79× | yes | +| `@warn`, guarded so it fires once | 5.65× | — | yes | +| `@warn maxlog=1` | 59.57× | — | yes | + +Two results decided it. The plain counter is not merely slow in parallel — it recorded +95,406,048 of 160,000,000 calls, **losing 40% to races**, so it is wrong as well as expensive. +And a flag written once and only read afterwards never dirties the cache line again, which is +why it is free at eight threads while every counting scheme is not. + +So **presence is detected by default and costs nothing; counts, call sites and paths are +opt-in.** The `@warn` rows are why the default notice is a summary at process exit rather than a +warning at first entry: the cost is the logging call sitting in the body, not the warning being +printed, and guarding it so that it fires once does not recover it. + +### One requirement was withdrawn + +This directory used to require that `@experimental` **never wrap the call**, and +`test_spec_profile.jl` pinned it structurally. That is gone: presence cannot be detected without +emitting something into the body. What replaced it is narrower and measured — the emitted +statement must be read-mostly, must add exactly one statement, and must not bring the logging +machinery with it. The last of those is checked today, with a macro that *does* log as the +positive control. + +## Negative controls + +Each group has a negative control **specified**; most are not yet operating, because the control +is `@test_broken` alongside the claim it controls. They are listed here as design, not as +evidence: + +| group | the control | operating? | +|---|---|---| +| propagate | `top_good` is *exactly as deep* as `top_bad` and must come back `:clean` | no | +| profile | `cold` is marked and never called, and must be **absent**, not reported with count zero | no | +| verify | the fixture is exercised only *partly*, so a checker that always reports 100% cannot pass | no | +| integration | a settled name must get **no** docs note | no | +| dispatch | `which()` really does throw for the branching signatures the file rests on | **yes** | +| forms | the misuse refusals name the missing half, rather than one generic message | **yes** | +| lifecycle | deleting a settled name is still breaking, so `isbreaking` cannot answer `false` always | **yes** | + +"no" means the assertion about the *implementation* does not run, because the implementation is +not there. It does not mean the row is unchecked: the fixture premise each control rests on is +pinned live where it could be got wrong — `test_spec_verify.jl:38` asserts the fixture really is +only partly exercised, and `test_spec_dispatch.jl:93` asserts the specificity relation the whole +file assumes. A control resting on a false premise is the failure mode those guard. + +The one that matters most is not in the table because it is a rule rather than a fixture: +`:unknown` must never be reported as `:clean`. Two fixtures (`Holder.f::Function`, `TABLE[i](x)`) +really can reach the marked function while being statically invisible. Answering "no experimental +dependency" there is not a weaker claim, it is a false one. + +## What the spec already found + +Two defects, both live in the shipped code, both of the kind the spec was written to catch — +a mark that silently records the wrong thing rather than refusing: + +1. `@experimental "…" (c::C)(x) = c.k * x` marks **`:c`**, the argument name. Not the type, not + a function — a local that is not a binding anywhere. It produces **two** wrong signals, not + one: the audit reports `:c` as *dangling* (declared, no such binding) **and** `:C` as + *unaccounted* (public, never declared), so it tells the author to go declare the very thing + that line declares. Both halves have to move together; `test_spec_forms.jl` pins each. +2. A mark inside a function body is refused by *Julia*, not by this package: + `syntax: unsupported const declaration on local variable`. Half fixed — the expansion now + carries the caller's `LineNumberNode`, so the message names the line the author wrote instead + of `ExperimentalAPI/src/mark.jl`, which read as a bug in the package. The message still never + says `@experimental`, and may not be able to: `const` in local scope fails during lowering, + before any emitted code runs, and the one alternative that avoids `const` (`global`) fails + *silently* in local scope, which is worse. diff --git a/test/spec/summary.jl b/test/spec/summary.jl new file mode 100644 index 0000000..a220400 --- /dev/null +++ b/test/spec/summary.jl @@ -0,0 +1,156 @@ +# Generates the coverage table for `test/spec/README.md` and the pull request that ships it. +# +# The measure is DISTINCT BEHAVIOURS — one per leaf `@testset` — not assertions, which move +# without any implementation progress when they sit inside a loop over the fixture's marks. +# +# Run it: julia --project=test test/spec/summary.jl + +module SpecSummary + +const SPEC_DIR = @__DIR__ + +# One line per spec file. A file with no entry here is an error, not a missing row. +const CONCERNS = Dict( + "test_spec_declare.jl" => "what can carry a mark: function, method, struct, const, module, macro, extension", + "test_spec_forms.jl" => "the definition forms a real package hits on its second afternoon", + "test_spec_foreign.jl" => "marking a method on somebody else's generic — the `QAtlas.fetch` case", + "test_spec_propagate.jl" => "a caller that never names a marked thing still depends on it", + "test_spec_docstring.jl" => "a mark and a docstring are different accounts and must coexist", + "test_spec_verify.jl" => "how well is a marked thing exercised by the tests", + "test_spec_profile.jl" => "what a real run went through, how often, and how much of it", + "test_spec_dispatch.jl" => "one call site, several methods, only some marked — the branch", + "test_spec_lifecycle.jl" => "the mark's EXIT, and an entry point that is a module rather than a function", + "test_spec_integration.jl" => "where the mark has to surface: docs, Aqua, releases, provenance, CI", +) + +""" + spec_files() -> Vector{String} + +Every spec file on disk, read from the directory rather than from a list, so a file cannot be +added and left out of the table. +""" +function spec_files() + return sort( + filter(f -> startswith(f, "test_spec_") && endswith(f, ".jl"), readdir(SPEC_DIR)) + ) +end + +function _ismacrocall(x, name::Symbol) + return x isa Expr && x.head === :macrocall && !isempty(x.args) && x.args[1] === name +end + +function _walk(f, x) + f(x) + x isa Expr && for a in x.args + _walk(f, a) + end + return nothing +end + +_count(pred, x) = (n=0; _walk(y -> (pred(y) && (n += 1)), x); n) + +""" + Counts + +What one spec file contains. `operating` and `specified` partition `behaviours`: a leaf testset +either runs at least one assertion against the package today, or it is entirely `@test_broken` — +a claim written down and not yet checked against anything. + +The distinction is the point: "every group has a negative control" is true of this directory as a +specification and false of it as a running suite. +""" +struct Counts + behaviours::Int + operating::Int + specified::Int + broken_assertions::Int +end + +function counts(path::AbstractString) + ex = Meta.parseall(read(path, String); filename=path) + sets = Expr[] + _walk(x -> (_ismacrocall(x, Symbol("@testset")) && push!(sets, x)), ex) + behaviours = operating = 0 + for ts in sets + body = ts.args[2:end] + # A leaf has no testset inside it. A loop-generated `@testset "$T"` is one leaf, not one + # per iteration: a behaviour is something someone wrote. + sum(b -> _count(y -> _ismacrocall(y, Symbol("@testset")), b), body) == 0 || continue + behaviours += 1 + live = sum( + b -> _count( + y -> + _ismacrocall(y, Symbol("@test")) || + _ismacrocall(y, Symbol("@test_throws")), + b, + ), + body, + ) + live > 0 && (operating += 1) + end + broken = _count(y -> _ismacrocall(y, Symbol("@test_broken")), ex) + return Counts(behaviours, operating, behaviours - operating, broken) +end + +const _HEAD = "| file | behaviours | operating today | specified only | concern |" +const _RULE = "|---|---|---|---|---|" + +""" + table() -> String + +The markdown table, generated. `test/test_spec_table.jl` pins `README.md` against it, so the two +cannot come apart the way the hand-written one did. +""" +function table() + rows = String[_HEAD, _RULE] + tot = Counts(0, 0, 0, 0) + for f in spec_files() + haskey(CONCERNS, f) || error( + "test/spec/$f has no entry in SpecSummary.CONCERNS — add one line describing what " * + "it covers. Reading the directory rather than a list is what makes this an error " * + "instead of a silently missing row.", + ) + c = counts(joinpath(SPEC_DIR, f)) + push!( + rows, + "| `$f` | $(c.behaviours) | $(c.operating) | $(c.specified) | $(CONCERNS[f]) |", + ) + tot = Counts( + tot.behaviours + c.behaviours, + tot.operating + c.operating, + tot.specified + c.specified, + tot.broken_assertions + c.broken_assertions, + ) + end + push!( + rows, + "| **$(length(spec_files())) files** | **$(tot.behaviours)** | " * + "**$(tot.operating)** | **$(tot.specified)** | |", + ) + return join(rows, "\n") +end + +function total() + return foldl( + (a, b) -> Counts( + a.behaviours + b.behaviours, + a.operating + b.operating, + a.specified + b.specified, + a.broken_assertions + b.broken_assertions, + ), + (counts(joinpath(SPEC_DIR, f)) for f in spec_files()); + init=Counts(0, 0, 0, 0), + ) +end + +end # module SpecSummary + +if abspath(PROGRAM_FILE) == @__FILE__ + println(SpecSummary.table()) + t = SpecSummary.total() + println() + println( + "$(t.behaviours) behaviours: $(t.operating) operating today, $(t.specified) " * + "specified only ($(t.broken_assertions) `@test_broken` assertions).", + ) +end diff --git a/test/spec/test_spec_declare.jl b/test/spec/test_spec_declare.jl new file mode 100644 index 0000000..e57fe04 --- /dev/null +++ b/test/spec/test_spec_declare.jl @@ -0,0 +1,176 @@ +# What can carry a mark: function, method, struct, const, module, macro, extension. +# +# Scope: both what the name-keyed implementation does today (plain `@test`) and the method-level +# unit it has to become (`@test_broken`). + +using ExperimentalAPI: ExperimentalAPI, @experimental, Mark, experimental, isexperimental +using Test + +module Declared + +using ExperimentalAPI + +export documented_fn +public plain_fn, + short_fn, + Struct, + MutStruct, + AbstractKind, + PrimKind, + CONSTANT, + ASSIGNED, + Sub, + @marked_macro, + listed_a, + listed_b, + energy + +@experimental "long form" function plain_fn(x) + return x +end +@experimental "short form" short_fn(x) = x +@experimental "struct" struct Struct + v::Int +end +@experimental "mutable struct" mutable struct MutStruct + v::Int +end +@experimental "abstract type" abstract type AbstractKind end +@experimental "primitive type" primitive type PrimKind 8 end +@experimental "const" const CONSTANT = 1 +@experimental "plain assignment" ASSIGNED = 2 +@experimental "macro" macro marked_macro(x) + return esc(x) +end + +module Sub end +@experimental "a module can only be marked by name" Sub + +listed_a(x) = x +listed_b(x) = x +@experimental "declared, not attached" listed_a listed_b + +"Documented and settled." +documented_fn(x) = x + +# One name, several dispatch paths, only one in doubt — `QAtlas.fetch` in miniature. +struct Exact end +struct Numerical end +energy(::Exact, β::Float64) = β +energy(::Numerical, β::Float64) = β + 1e-12 +energy(::Numerical, β::Rational) = β + +end # module Declared + +@testset "every definition form can be marked" begin + got = Dict(mk.name => mk for mk in experimental(Declared)) + for (name, reason) in ( + :plain_fn => "long form", + :short_fn => "short form", + :Struct => "struct", + :MutStruct => "mutable struct", + :AbstractKind => "abstract type", + :PrimKind => "primitive type", + :CONSTANT => "const", + :ASSIGNED => "plain assignment", + Symbol("@marked_macro") => "macro", + :Sub => "a module can only be marked by name", + :listed_a => "declared, not attached", + :listed_b => "declared, not attached", + ) + @testset "$name" begin + @test haskey(got, name) + @test got[name].reason == reason + end + end +end + +@testset "nothing else is reported as marked" begin + # Control: rejects an `experimental()` that leaks every declared name. + got = Set(mk.name for mk in experimental(Declared)) + @test length(got) == 12 + for unmarked in (:documented_fn, :energy, :Exact, :Numerical) + @testset "$unmarked is not reported" begin + @test unmarked ∉ got + end + end +end + +@testset "the reason travels with every one of them" begin + for mk in experimental(Declared) + @test !isempty(strip(mk.reason)) + @test mk.mod === Declared + @test mk.line > 0 + @test String(mk.file) == @__FILE__ + end +end + +# ── the method-level unit ──────────────────────────────────────────────────────────────────── +# +# `Declared.energy` has three methods; marking the name marks all three, including the exact one. + +@testset "marking a name is the wrong unit when the name has many methods" begin + @test length(methods(Declared.energy)) == 3 + # Today the statement is about the name, so it over-claims. + @test !isexperimental(Declared, :energy) # not marked at all yet — see below +end + +@testset "a single dispatch path can be marked" begin + m = which(Declared.energy, Tuple{Declared.Numerical,Float64}) + @test_broken ExperimentalAPI.mark(m) isa Mark + @test_broken ExperimentalAPI.isexperimental(m) +end + +@testset "marking one method leaves its siblings alone" begin + numerical = which(Declared.energy, Tuple{Declared.Numerical,Float64}) + exact = which(Declared.energy, Tuple{Declared.Exact,Float64}) + rational = which(Declared.energy, Tuple{Declared.Numerical,Rational}) + @test numerical !== exact !== rational + @test_broken ExperimentalAPI.isexperimental(numerical) + @test_broken !ExperimentalAPI.isexperimental(exact) + @test_broken !ExperimentalAPI.isexperimental(rational) +end + +@testset "a method mark survives precompilation" begin + # Scope: the name-keyed registry already survives (`test/test_precompile.jl`); whether + # `Method` objects do is a separate claim needing its own fixture package. + @test_broken ExperimentalAPI.experimental_methods isa Function +end + +@testset "a method mark is queryable from a call site" begin + @test_broken ExperimentalAPI.isexperimental( + which(Declared.energy, Tuple{Declared.Numerical,Float64}) + ) +end + +# ── extensions ─────────────────────────────────────────────────────────────────────────────── +# +# An extension is a separate module: its public names are part of the surface a user sees, and +# invisible to `names(Package)`. + +@testset "a mark inside a package extension is reachable from the parent" begin + ext = Base.get_extension(ExperimentalAPI, :ExperimentalAPITestExt) + @test ext !== nothing + # Not `!isempty(...)`: this package marks six of its own names, so an ignored keyword would + # satisfy that. The claim is that a mark whose home is the extension comes back. + @test isempty(ExperimentalAPI.experimental(ext)) # today the extension declares none + @test_broken any( + mk -> mk.mod === ext, ExperimentalAPI.experimental(ExperimentalAPI; extensions=true) + ) +end + +@testset "every new Audit field keeps the partition invariant" begin + # Three files each add an `Audit` field behind `hasproperty` alone. `hasproperty` cannot see + # whether the partition still holds once all three land. + a = ExperimentalAPI.audit(ExperimentalAPI) + @test sort( + vcat(a.foreign, a.documented, a.unaccounted, setdiff(a.declared, a.documented)) + ) == a.surface + @test_broken ExperimentalAPI.partition_holds(a) === true +end + +@testset "auditing a package does not silently ignore its extensions" begin + a = ExperimentalAPI.audit(ExperimentalAPI) + # Scope: `audit` looks at one module today; whether extensions are included must be stated. + @test_broken hasproperty(a, :extensions) +end diff --git a/test/spec/test_spec_dispatch.jl b/test/spec/test_spec_dispatch.jl new file mode 100644 index 0000000..fe7f1b8 --- /dev/null +++ b/test/spec/test_spec_dispatch.jl @@ -0,0 +1,214 @@ +# One call site, several methods, only some of them marked. +# +# Scope: the call site the analysis cannot pin to one method. For a `Union`-typed or abstract +# argument `which(f, T)` throws, and an implementation that catches that and moves on reports +# `:clean` about a call that reaches a marked method half the time. That is the failure guarded +# here. + +using ExperimentalAPI: ExperimentalAPI, @experimental, experimental, isexperimental, mark +using Test + +module Dispatch + +using ExperimentalAPI + +public Exact, + Numerical, + Kind, + KA, + KB, + energy, + k, + only_settled, + either_way, + via_abstract, + via_invoke, + more_specific, + pair, + via_pair_clean, + via_pair_marked + +struct Exact end +struct Numerical end + +"Closed form; trustworthy." +energy(::Exact) = 1.0 +@experimental "no convergence proof below β ≈ 0.1" energy(::Numerical) = 1.0000001 + +abstract type Kind end +struct KA <: Kind end +struct KB <: Kind end +"Settled." +k(::KA) = 1.0 +@experimental "extrapolated, never cross-checked" k(::KB) = 2.0 + +# Control: can only ever reach the settled method. +only_settled(x::Exact) = energy(x) + +# The call site can reach EITHER method depending on the run-time type. +either_way(x::Union{Exact,Numerical}) = energy(x) +via_abstract(x::Kind) = k(x) + +# `invoke` pins a method dispatch would NOT pick: an `Int` goes to the unmarked `::Int` normally. +# Pinning `Tuple{Numerical}` instead would be satisfied by an analysis that ignores `invoke`. +via_invoke(x::Int) = invoke(more_specific, Tuple{Integer}, x) + +# Two arguments, as in `fetch(model, quantity)`: the marked combination is not reachable from +# either argument alone. +pair(::Exact, ::Exact) = 0.0 +pair(::Exact, ::Numerical) = 1.0 +@experimental "only this combination is unvalidated" pair(::Numerical, ::Numerical) = 2.0 +pair(::Numerical, ::Exact) = 3.0 +via_pair_clean(a::Exact, b::Exact) = pair(a, b) +via_pair_marked(a::Numerical, b::Numerical) = pair(a, b) + +# A more specific unmarked method shadows a marked less specific one. +"Settled, and more specific." +more_specific(::Int) = 0 +@experimental "the fallback is a placeholder" more_specific(::Integer) = 1 + +end # module Dispatch + +# ── the fixture really has the shape the file claims ───────────────────────────────────────── + +@testset "only some methods behind each name are marked" begin + marked = Set(mk.name for mk in experimental(Dispatch)) + @test :energy in marked + @test :k in marked + @test length(methods(Dispatch.energy)) == 2 + @test length(methods(Dispatch.k)) == 2 + # A name-keyed mark covers both methods, including the closed-form one. That is the problem. + @test isexperimental(Dispatch, :energy) +end + +@testset "the specificity premise the file rests on is true" begin + # The premise every `@test_broken` below rests on, and which nothing else would notice. + @test which(Dispatch.more_specific, Tuple{Int}).sig === + Tuple{typeof(Dispatch.more_specific),Int} + @test Dispatch.more_specific(5) == 0 # the UNMARKED, more specific method + @test Dispatch.more_specific(UInt8(5)) == 1 # falls through to the MARKED fallback + @test Dispatch.via_invoke(5) == 1 # invoke reaches what dispatch would not pick +end + +@testset "a branching call site has no unique method" begin + # `@test_throws Exception` would be satisfied by a typo raising `UndefVarError`, so pin the + # diagnosis rather than the failure. + @test which(Dispatch.energy, Tuple{Dispatch.Exact}) isa Method + for (f, T) in ( + (Dispatch.energy, Tuple{Union{Dispatch.Exact,Dispatch.Numerical}}), + (Dispatch.k, Tuple{Dispatch.Kind}), + ) + @testset "$T" begin + e = try + which(f, T) + nothing + catch err + err + end + @test e isa ErrorException + @test occursin("ambiguous", sprint(showerror, e)) + end + end +end + +@testset "attaching the mark at one method's definition marks the NAME today" begin + # The attached form reads as if it scoped the claim to one method; `_signame` throws the + # argument types away. So method-level marking needs a separate imperative route. + @test isexperimental(Dispatch, :energy) + @test mark(Dispatch, :energy).name === :energy # not a signature + @test_broken ExperimentalAPI.mark_method! isa Function +end + +# ── what the analysis has to say about each shape ──────────────────────────────────────────── + +@testset "a call site that can only reach settled methods is clean" begin + # Control: rejects a tool answering `:depends` for every multi-candidate call site. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Dispatch.only_settled, Tuple{Dispatch.Exact}) + ) === :clean +end + +@testset "a Union-typed call site that could reach a mark is not clean" begin + # Half the run-time values take the marked branch, so `:clean` is false, not conservative. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach( + Dispatch.either_way, Tuple{Union{Dispatch.Exact,Dispatch.Numerical}} + ), + ) !== :clean +end + +@testset "an abstract-typed call site that could reach a mark is not clean" begin + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Dispatch.via_abstract, Tuple{Dispatch.Kind}) + ) !== :clean +end + +@testset "the unresolvable call site is named, not just counted" begin + # Structured fields, not `occursin`: a short needle matches any boilerplate diagnostic. + @test_broken all( + u -> hasproperty(u, :file) && hasproperty(u, :line) && hasproperty(u, :callee), + ExperimentalAPI.reach(Dispatch.via_abstract, Tuple{Dispatch.Kind}).unresolved, + ) + @test_broken any( + u -> u.callee === :k, + ExperimentalAPI.reach(Dispatch.via_abstract, Tuple{Dispatch.Kind}).unresolved, + ) +end + +@testset "which() throwing must not be swallowed into :clean" begin + # The mistake this file exists to prevent: catching `which`, skipping the site, reporting + # the rest as clean. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Dispatch.via_abstract, Tuple{Dispatch.Kind}) + ) === :unknown +end + +# ── dispatch subtleties ────────────────────────────────────────────────────────────────────── + +@testset "invoke pins the method it names" begin + # Argument types alone cannot see `invoke` pinning a method dispatch would not pick. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Dispatch.via_invoke, Tuple{Dispatch.Numerical}) + ) === :depends +end + +@testset "a more specific unmarked method shadows a marked one" begin + # An `Int` never reaches the mark. `:depends` here is the name-level over-claim one level + # down. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Dispatch.more_specific, Tuple{Int}) + ) === :clean + # …and the call that does fall through is reported. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Dispatch.more_specific, Tuple{UInt8}) + ) === :depends +end + +@testset "a mark on one method does not leak to its siblings at a call site" begin + # Both call the same name and must get different verdicts. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Dispatch.only_settled, Tuple{Dispatch.Exact}) + ) !== ExperimentalAPI.verdict( + ExperimentalAPI.reach( + Dispatch.either_way, Tuple{Union{Dispatch.Exact,Dispatch.Numerical}} + ), + ) +end + +@testset "a marked combination is not reachable from either argument alone" begin + # Widening each argument independently would call both call sites `:depends`. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Dispatch.via_pair_clean, Tuple{Dispatch.Exact,Dispatch.Exact}) + ) === :clean + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach( + Dispatch.via_pair_marked, Tuple{Dispatch.Numerical,Dispatch.Numerical} + ), + ) === :depends +end + +@testset "the report says WHICH method was reached, not which name" begin + @test_broken first( + ExperimentalAPI.reach(Dispatch.either_way, Tuple{Dispatch.Numerical}).reached + ).method === which(Dispatch.energy, Tuple{Dispatch.Numerical}) +end diff --git a/test/spec/test_spec_docstring.jl b/test/spec/test_spec_docstring.jl new file mode 100644 index 0000000..0c1e992 --- /dev/null +++ b/test/spec/test_spec_docstring.jl @@ -0,0 +1,110 @@ +# A mark and a docstring are different accounts, and must coexist. +# +# Scope: a mark is never a substitute for prose. `Base.Experimental` is the precedent — Base +# marks an experimental surface AND documents it. Pinned by named entries rather than by a count, +# which moves with the Julia version and the counting rule. + +using ExperimentalAPI: + ExperimentalAPI, @experimental, audit, experimental, isdocumented, isexperimental, mark +using Test + +module Both + +using ExperimentalAPI + +public documented_and_marked, + marked_only, documented_only, silent, @marked_macro, separately + +""" +Documented, and declared unfinished. Both statements are true at once and neither replaces the +other: the prose says what it does, the mark says how much to trust it. +""" +@experimental "convergence not established at low β" documented_and_marked(x) = x + +@experimental "no prose yet" marked_only(x) = x + +"Documented and settled." +documented_only(x) = x + +silent(x) = x + +"A marked macro keeps its docstring." +@experimental "shape not settled" macro marked_macro(x) + return esc(x) +end + +# a docstring attached separately rather than adjacent to the definition +@experimental "declared here, documented below" separately(x) = x +@doc "Documented in a second statement." separately + +end # module Both + +@testset "Julia itself documents its experimental surface" begin + @test isdefined(Base, :Experimental) + for n in (Symbol("@optlevel"), Symbol("@compiler_options"), :Const) + @testset "Base.Experimental.$n" begin + @test isdefined(Base.Experimental, n) + @test Base.Docs.hasdoc(Base.Experimental, n) + end + end +end + +@testset "a docstring survives the macro" begin + # `Expr(:meta, :doc)` attaches the docstring to the definition rather than to the expanded + # block. Without it the two accounts are mutually exclusive in practice. + @test Base.Docs.hasdoc(Both, :documented_and_marked) + @test isexperimental(Both, :documented_and_marked) + @test occursin( + "Both statements are true at once", string(@doc Both.documented_and_marked) + ) +end + +@testset "a marked macro keeps its docstring" begin + @test Base.Docs.hasdoc(Both, Symbol("@marked_macro")) + @test isexperimental(Both, Symbol("@marked_macro")) +end + +@testset "a docstring attached in a separate statement also works" begin + @test Base.Docs.hasdoc(Both, :separately) + @test isexperimental(Both, :separately) +end + +@testset "the four combinations land where they should" begin + a = audit(Both) + @test :documented_and_marked in a.documented + @test :documented_and_marked in a.declared # both accounts, deliberately + @test :marked_only in a.declared + @test :marked_only ∉ a.documented + @test :documented_only in a.documented + @test :documented_only ∉ a.declared + @test :silent in a.unaccounted # neither — the finding +end + +@testset "a mark is not an excuse for missing prose" begin + # Scope: a marked name with no docstring is still undocumented. Today `declared` absorbs it. + a = audit(Both) + @test_broken :marked_only in a.undocumented + @test_broken hasproperty(a, :undocumented) +end + +@testset "the check can require a docstring regardless of the mark" begin + # Scope: a package adopting both this and Aqua must not have to choose between them. + @test_broken ExperimentalAPI.test_surface(Both; require_docstring=true) isa + ExperimentalAPI.Audit +end + +@testset "the reason is reachable from the rendered documentation" begin + # Scope: the reason must reach the docs site without the author typing it twice. + @test_broken ExperimentalAPI.docstring_note(Both, :documented_and_marked) isa + AbstractString +end + +@testset "what Documenter's checkdocs sees is not what audit sees" begin + # `checkdocs = :public` has no third answer; `audit` accepts a mark. Both are legitimate. + a = audit(Both) + undocumented_by_base = setdiff(Base.Docs.undocumented_names(Both), [nameof(Both)]) + @test :marked_only in undocumented_by_base # Base would flag it + @test :marked_only ∉ a.unaccounted # audit accepts the mark + @test :silent in undocumented_by_base + @test :silent in a.unaccounted # both flag this one +end diff --git a/test/spec/test_spec_foreign.jl b/test/spec/test_spec_foreign.jl new file mode 100644 index 0000000..974dfed --- /dev/null +++ b/test/spec/test_spec_foreign.jl @@ -0,0 +1,155 @@ +# Marking a method on somebody else's generic — the `QAtlas.fetch` case, refused outright today. +# +# Scope: `audit` files a name bound elsewhere under `foreign` and says nothing about the methods +# we contributed to it. Extending another package's generic is the normal Julia idiom, so a mark +# that cannot attach there cannot describe the surface that matters. + +using ExperimentalAPI: ExperimentalAPI, @experimental, audit, experimental, isexperimental +using Test + +module UpstreamGeneric +"The generic every downstream package extends." +fetch_value(model, quantity) = error("no method for $(typeof(model)), $(typeof(quantity))") +public fetch_value +end + +module Downstream + +using ExperimentalAPI +using ..UpstreamGeneric: UpstreamGeneric, fetch_value + +public Ising, Heisenberg, Energy, Susceptibility + +struct Ising end +struct Heisenberg end +struct Energy end +struct Susceptibility end + +# exact — trustworthy +UpstreamGeneric.fetch_value(::Ising, ::Energy) = -2.0 + +# numerically delicate — this is the one that should carry a mark +UpstreamGeneric.fetch_value(::Heisenberg, ::Energy) = -1.7724538509055159 + +# a whole family that is provisional +UpstreamGeneric.fetch_value(::Ising, ::Susceptibility) = 0.0 +UpstreamGeneric.fetch_value(::Heisenberg, ::Susceptibility) = 0.0 + +end # module Downstream + +@testset "the fixture really is the foreign-generic shape" begin + @test parentmodule(UpstreamGeneric.fetch_value) === UpstreamGeneric + @test length(methods(UpstreamGeneric.fetch_value)) == 5 + @test count(m -> m.module === Downstream, methods(UpstreamGeneric.fetch_value)) == 4 + @test :fetch_value ∉ ExperimentalAPI.surface(Downstream) # invisible to names() +end + +@testset "the macro currently refuses a qualified definition" begin + # Pinned so the change is visible when it happens. + @test_throws LoadError @eval module RefusedForeign + using ExperimentalAPI + using ..UpstreamGeneric + @experimental "why" UpstreamGeneric.fetch_value(::Int, ::Int) = 0 + end +end + +@testset "a method on a foreign generic can be marked" begin + # Ends in a Bool and checks what was marked — see `README.md` on `@eval module`. + @test_broken begin + @eval module MarkedForeign + using ExperimentalAPI + using ..UpstreamGeneric + struct Probe end + @experimental "provisional" UpstreamGeneric.fetch_value(::Probe, ::Probe) = 0 + end + !isempty(ExperimentalAPI.experimental_methods(Main.MarkedForeign)) + end +end + +@testset "marking one foreign method leaves the siblings alone" begin + exact = which(UpstreamGeneric.fetch_value, Tuple{Downstream.Ising,Downstream.Energy}) + delicate = which( + UpstreamGeneric.fetch_value, Tuple{Downstream.Heisenberg,Downstream.Energy} + ) + @test exact !== delicate + # The fixture cannot carry `@experimental` here — a qualified definition is refused today + # and the module would not load — so the test marks it through the future API. Otherwise this + # stays Broken even once method-level marking works. + @test_broken begin + ExperimentalAPI.mark_method!(delicate, "numerically delicate") + ExperimentalAPI.isexperimental(delicate) && !ExperimentalAPI.isexperimental(exact) + end +end + +@testset "the mark is stored in the module that WROTE the method" begin + # Not in `UpstreamGeneric`: a package cannot carry claims its dependents invented, and the + # mark must survive it being reloaded. `all(pred, [])` is `true`, so non-emptiness is part of + # the claim. + @test_broken !isempty(ExperimentalAPI.experimental_methods(Downstream)) && all( + mk -> mk.mod === Downstream, ExperimentalAPI.experimental_methods(Downstream) + ) +end + +@testset "the ownership query and the cross-module search are different verbs" begin + # "what this module owns" and "what anyone has marked on this generic" are both wanted, and + # one name for both means the reader cannot tell which they got. + @test_broken ExperimentalAPI.marks_on isa Function +end + +@testset "asking the generic finds marks contributed by every package" begin + @test_broken !isempty(ExperimentalAPI.experimental(UpstreamGeneric.fetch_value)) +end + +@testset "audit reports foreign methods this module owns" begin + # `foreign` means "bound elsewhere, not our problem". A method we wrote is the opposite. + a = audit(Downstream) + @test_broken hasproperty(a, :contributed_methods) + @test_broken length(a.contributed_methods) == 4 +end + +@testset "a contributed method with neither docstring nor mark is a finding" begin + @test_broken !isempty(ExperimentalAPI.unaccounted_methods(Downstream)) +end + +@testset "a docstring on a specific signature counts" begin + # Docstrings are keyed by signature, so "documented" is answerable per method. + @test_broken ExperimentalAPI.isdocumented( + which(UpstreamGeneric.fetch_value, Tuple{Downstream.Ising,Downstream.Energy}) + ) isa Bool +end + +@testset "marking a method does not make the foreign NAME experimental" begin + # Control: one dependent must not be able to label a whole generic unfinished. + @test !isexperimental(UpstreamGeneric, :fetch_value) +end + +@testset "propagation crosses the package boundary" begin + # The QAtlas -> analysis-script path: a third package reaching a marked method through the + # upstream generic. + caller(x) = + UpstreamGeneric.fetch_value(Downstream.Heisenberg(), Downstream.Energy()) + x + @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(caller, Tuple{Float64})) === + :depends +end + +@testset "a mark on a method of a function defined in Base is possible" begin + # "Base is off limits" would exclude a large part of every package's surface. + @test_broken begin + @eval module MarkedBase + using ExperimentalAPI + struct Widget end + @experimental "printing format not settled" Base.show(io::IO, ::Widget) = + print(io, "W") + end + !isempty(ExperimentalAPI.experimental_methods(Main.MarkedBase)) + end +end + +@testset "it stays refused when the mark cannot say WHICH method" begin + # A bare qualified name with no signature would mark every method of `Base.show` in the + # world. Guessing is worse than refusing. + @test_throws LoadError @eval module RefusedBareForeign + using ExperimentalAPI + @experimental "why" Base.show + end +end diff --git a/test/spec/test_spec_forms.jl b/test/spec/test_spec_forms.jl new file mode 100644 index 0000000..475d1a5 --- /dev/null +++ b/test/spec/test_spec_forms.jl @@ -0,0 +1,393 @@ +# The definition forms a real package hits on its second afternoon: kwargs, parametric +# signatures, callable structs, constructors, operators, stacked macros. +# +# Scope: each form either works, or the refusal names the alternative. Silently marking the wrong +# symbol is the outcome this file exists to prevent. + +using ExperimentalAPI: ExperimentalAPI, @experimental, experimental, isexperimental, mark +using Test + +module FormsSpec + +using ExperimentalAPI + +public kw_fn, where_fn, vararg_fn, ret_typed, Callable, Ctor, INTERP + +@experimental "keyword arguments" kw_fn(x; scale=1.0, kwargs...) = x * scale +@experimental "parametric" where_fn(x::T, y::T) where {T<:Real} = x + y +@experimental "varargs" vararg_fn(x, rest...) = x +@experimental "return type annotation" ret_typed(x)::Float64 = x + +struct Callable + k::Float64 +end +struct Ctor + v::Int +end + +# an interpolated reason +const WHY = "tolerance chosen by hand" +@experimental "$(WHY); see the sweep in issue 12" INTERP = 1e-8 + +end # module FormsSpec + +@testset "forms that already work" begin + got = Dict(mk.name => mk for mk in experimental(FormsSpec)) + for n in (:kw_fn, :where_fn, :vararg_fn, :ret_typed, :INTERP) + @testset "$n" begin + @test haskey(got, n) + @test !isempty(strip(got[n].reason)) + end + end + @test FormsSpec.kw_fn(2.0; scale=3.0) == 6.0 + @test FormsSpec.where_fn(1, 2) == 3 +end + +@testset "an interpolated reason is evaluated, not stored as source" begin + @test occursin("tolerance chosen by hand", mark(FormsSpec, :INTERP).reason) +end + +# ── forms that are not covered ─────────────────────────────────────────────────────────────── + +@testset "a callable struct is marked on the WRONG symbol today" begin + # `(c::C)(x)` has no function name; `_signame` walks the `::` and returns the argument name. + # The mark lands on `:c`, which is not a binding anywhere. + @eval module CallableMarked + using ExperimentalAPI + struct C + k::Float64 + end + @experimental "scaling rule provisional" (c::C)(x) = c.k * x + end + @test :c in [mk.name for mk in experimental(CallableMarked)] # today, and wrong + @test !isdefined(CallableMarked, :c) # marks a non-binding +end + +@testset "and the audit compounds it: C is reported as UNDECLARED" begin + # The second wrong signal: the audit reports a declaration for a name that does not exist AND + # a public name with no declaration, so it tells the author to declare what that line already + # declares. Both halves have to move together. + @eval module CallablePublic + using ExperimentalAPI + public C + struct C + k::Float64 + end + @experimental "scaling rule provisional" (c::C)(x) = c.k * x + end + a = ExperimentalAPI.audit(Main.CallablePublic) + @test :c in a.dangling # declared, but no such binding + @test :C in a.unaccounted # public, and — per the audit — never declared + @test Main.CallablePublic.C(2.0)(3.0) == 6.0 # the definition itself is fine; the mark is not +end + +@testset "a callable struct marks the type, or refuses" begin + # Either answer is defensible; the argument name is not. + @test_broken :C in [mk.name for mk in experimental(Main.CallableMarked)] +end + +@testset "…and fixing that clears BOTH signals" begin + # `_signame` returning `:C` is necessary but not sufficient — the audit is what the author + # reads. + a = ExperimentalAPI.audit(Main.CallablePublic) + @test_broken isempty(a.dangling) && :C ∉ a.unaccounted +end + +@testset "a constructor method is marked on the type" begin + # Already correct: the mark lands on `:T`. + @eval module CtorMarked + using ExperimentalAPI + struct T + v::Int + end + @experimental "validation not implemented" T(s::AbstractString) = T(parse(Int, s)) + end + got = Set(mk.name for mk in experimental(Main.CtorMarked)) + @test :T in got + # Control: an over-inclusive walk would mark the argument name here too. + @test :s ∉ got +end + +@testset "an inner constructor inside a marked struct is not separately marked" begin + # Included or excluded is a decision; silence is not. + @test_broken hasproperty(mark(FormsSpec, :Callable), :includes_constructors) +end + +@testset "an operator method can be marked" begin + # Each ends in a Bool and checks what was marked: `@eval module` returns a Module, which + # reports "non-Boolean" instead of the Unexpected Pass this directory relies on, and accepting + # the syntax while recording the wrong symbol is the defect above. + @test_broken begin + @eval module OpMarked + using ExperimentalAPI + struct V + x::Float64 + end + @experimental "no identity element yet" Base.:+(a::V, b::V) = V(a.x + b.x) + end + :+ in [mk.name for mk in experimental(Main.OpMarked)] + end +end + +@testset "a generated function can be marked" begin + @test_broken begin + @eval module GenMarked + using ExperimentalAPI + @experimental "generator is a prototype" @generated g(x) = :(x) + end + :g in [mk.name for mk in experimental(Main.GenMarked)] + end +end + +@testset "Base.@kwdef stacks with the mark" begin + # Two macros that both wrap a definition must compose in at least one order, and which one + # must be documented. + @test_broken begin + @eval module KwdefMarked + using ExperimentalAPI + @experimental "defaults are guesses" Base.@kwdef struct S + a::Int = 1 + end + end + :S in [mk.name for mk in experimental(Main.KwdefMarked)] + end +end + +@testset "@inline and the mark compose in both orders" begin + @test_broken begin + @eval module InlineMarked + using ExperimentalAPI + @experimental "kernel unverified" @inline f(x) = x + @inline @experimental "kernel unverified" g(x) = x + end + Set([mk.name for mk in experimental(Main.InlineMarked)]) == Set([:f, :g]) + end +end + +@testset "a definition produced by @eval can be marked by name" begin + # Metaprogrammed definitions cannot be attached to, so the name-list form must reach them. + @test_broken begin + @eval module EvalMarked + using ExperimentalAPI + for n in (:a, :b) + @eval $n(x) = x + end + @experimental "generated in a loop" a b + end + Set([mk.name for mk in experimental(Main.EvalMarked)]) == Set([:a, :b]) + end +end + +@testset "a mark inside a function body is refused" begin + # Refused by Julia, not by this package: `const` in local scope fails during lowering, before + # any emitted code runs, so no check of ours can intercept it. The one lever is where the + # error points, and the expansion carries the caller's `LineNumberNode`. + # + # The misuse must arrive from a FILE: written through `@eval` the message carries no location + # at all, so a location assertion made that way is vacuous. Built line by line so the + # formatter cannot shift line 4. + dir = mktempdir() + path = joinpath(dir, "caller_side.jl") + write( + path, + join( + [ + "module CallerSide", + "using ExperimentalAPI", + "function outer()", + " @experimental \"why\" inner(x) = x", + " return inner", + "end", + "end", + ], + "\n", + ), + ) + e = try + include(path) + nothing + catch err + err isa LoadError ? err.error : err + end + @test e isa ErrorException + msg = sprint(showerror, e) + @test occursin("unsupported `const` declaration", msg) + # The blame lands on the line the author wrote… + @test occursin("caller_side.jl:4", msg) + # …and nowhere in this package. Checked by file name: the repository path contains + # "ExperimentalAPI", so asserting on that word passes by accident. + @test !occursin("mark.jl", msg) +end + +@testset "the refusal names @experimental rather than leaking the emitted const" begin + # The message still names a `const` the author never wrote. Whether this is reachable is + # open: the only expansion avoiding `const` is `global`, which fails silently in local scope. + e = try + @eval module ClosureMarked2 + using ExperimentalAPI + function outer() + @experimental "why" inner(x) = x + return inner + end + end + nothing + catch err + err + end + @test_broken occursin("@experimental", sprint(showerror, e)) +end + +# ── metadata ───────────────────────────────────────────────────────────────────────────────── + +@testset "since must be a version, and the refusal must say so" begin + # Refused by accident: the field's conversion fails, with a message naming neither `since` + # nor `@experimental`. A deliberate check would also throw, so assert the diagnostic. + e = try + @eval module BadSince + using ExperimentalAPI + @experimental("why", since = "0.4.0", f(x) = x) + end + nothing + catch err + err isa LoadError ? err.error : err + end + @test e isa MethodError # today, and accidental + @test_broken occursin("since", sprint(showerror, e)) +end + +@testset "an unknown keyword is refused rather than ignored" begin + # A typo in a keyword name must not silently become part of the subject. + @test_throws LoadError @eval module BadKw + using ExperimentalAPI + @experimental("why", trackign = "u", f(x) = x) + end +end + +@testset "tracking is carried through to every report" begin + # Stored today; it has to survive into the audit and the record as well. + @test_broken ExperimentalAPI.audit(FormsSpec).tracking isa AbstractDict +end + +# Evaluate an expression in a fresh module and hand back the exception it raised, unwrapped. +function probe(ex) + m = Module() + Core.eval(m, :(using ExperimentalAPI)) + try + Core.eval(m, ex) + return nothing + catch err + return err isa LoadError ? err.error : err + end +end + +# ── writing it lazily ──────────────────────────────────────────────────────────────────────── +# +# Scope: what happens when the reason — the payload, and knowledge only the author has — is left +# out. Every lazy form is refused; two by accident, and two with a message pointing the wrong way. + +@testset "a bare @experimental is refused, and says which of the two is missing" begin + # Each case pins its own phrase: `@test_throws Exception` for all six would be satisfied by + # one generic message. + for (ex, needle) in ( + (:(@experimental), "needs a reason"), + (:(@experimental foo), "the reason comes first"), + (:(@experimental function f(x) + x + end), "the reason comes first"), + (:(@experimental struct S + v::Int + end), "the reason comes first"), + (:(@experimental f(x) = x), "nothing to mark"), + (:(@experimental const C = 1), "nothing to mark"), + ) + @testset "$(first(string(ex), 36))" begin + e = probe(ex) + @test e isa ArgumentError + @test occursin(needle, sprint(showerror, e)) + end + end +end + +@testset "an empty reason is refused, and the message says why" begin + for r in ("", " ", "\n\t ") + @testset "reason=$(repr(r))" begin + m = Module(:EmptyProbe) + Core.eval(m, :(using ExperimentalAPI)) + e = try + Core.eval(m, :(@experimental $r f(x) = x)) + nothing + catch err + err isa LoadError ? err.error : err + end + @test e isa ArgumentError + @test occursin("reason", sprint(showerror, e)) + end + end +end + +@testset "a non-string reason is refused by accident, not by a check" begin + # Refused by `strip` failing inside `_reason`, with a message naming neither `@experimental` + # nor `reason`. Same shape as the `since` case above. + for r in (:(:sym), 42) + @testset "reason=$(repr(r))" begin + e = probe(:(@experimental $r f(x) = x)) + @test e isa MethodError # today, and accidental + @test_broken occursin("reason", sprint(showerror, e)) + end + end +end + +@testset "forgetting the reason is diagnosed as a missing reason" begin + # The message says "nothing to mark" when a definition was given and the reason was not — + # pointing at the end of the call the author should not touch. + e = probe(:(@experimental f(x) = x)) + @test e isa ArgumentError + @test occursin("nothing to mark", sprint(showerror, e)) # today, and misleading + @test_broken occursin("reason", sprint(showerror, e)) +end + +@testset "nothing is marked when the macro refuses" begin + # Checks the module is left clean, not just that an exception came out. + m = Module(:RefusedProbe) + Core.eval(m, :(using ExperimentalAPI)) + try + Core.eval(m, :(@experimental f(x) = x)) + catch + end + @test isempty(experimental(m)) + @test !isdefined(m, :f) +end + +# ── same name, two places ──────────────────────────────────────────────────────────────────── + +@testset "the same name marked in two modules stays separate" begin + @eval module A1 + using ExperimentalAPI + public f + @experimental "reason A" f(x) = x + end + @eval module A2 + using ExperimentalAPI + public f + @experimental "reason B" f(x) = x + end + @test mark(A1, :f).reason == "reason A" + @test mark(A2, :f).reason == "reason B" +end + +@testset "re-marking with a different reason replaces rather than accumulates" begin + @eval module A3 + using ExperimentalAPI + public f + f(x) = x + @experimental "first" f + @experimental "second" f + end + @test count(mk -> mk.name === :f, experimental(A3)) == 1 + @test mark(A3, :f).reason == "second" +end + +@testset "the replacement is reported rather than silent" begin + # Last-write-wins is a decision, and it should be visible. + @test_broken ExperimentalAPI.superseded_marks isa Function +end diff --git a/test/spec/test_spec_integration.jl b/test/spec/test_spec_integration.jl new file mode 100644 index 0000000..6bca95f --- /dev/null +++ b/test/spec/test_spec_integration.jl @@ -0,0 +1,152 @@ +# Where the mark has to surface outside this package: docs, Aqua, releases, provenance, CI. +# +# Scope: a mark only `ExperimentalAPI` can read is a private note. Everything here is pure or +# touches a temporary file except the Documenter block, which needs a test dependency this +# package does not have — so nothing in this group is blocked on infrastructure. + +using ExperimentalAPI: ExperimentalAPI, @experimental, experimental +using Test + +module Shown + +using ExperimentalAPI + +public settled, provisional + +"Settled and documented." +settled(x) = x + +""" +Provisional. Documented, and declared unfinished. +""" +@experimental( + "the r,s branch has no reference value", + since = v"0.2.0", + tracking = "https://example.invalid/issues/9", + provisional(x) = x +) + +end # module Shown + +# ── documentation ──────────────────────────────────────────────────────────────────────────── + +@testset "the docs can render the mark without the author repeating it" begin + # Typed twice, the two drift and the machine-readable one loses. + @test_broken ExperimentalAPI.docstring_note(Shown, :provisional) isa AbstractString +end + +@testset "the rendered note carries the reason, the version and the tracking link" begin + for needle in ("reference value", "0.2.0", "example.invalid") + @test_broken occursin(needle, ExperimentalAPI.docstring_note(Shown, :provisional)) + end +end + +@testset "a Documenter block can list a module's marks" begin + # `@autodocs`-style. The only assertion here needing a new test dependency. + @test_broken ExperimentalAPI.DocumenterExt isa Module +end + +@testset "a settled name gets no note" begin + # Control: rejects a renderer that annotates everything. + @test_broken ExperimentalAPI.docstring_note(Shown, :settled) === nothing +end + +# ── Aqua ───────────────────────────────────────────────────────────────────────────────────── + +@testset "the audit composes with Aqua rather than competing" begin + # Aqua has no third answer; a package must be able to run both. + @test_broken ExperimentalAPI.aqua_compatible_names(Shown) isa AbstractVector +end + +# ── release ────────────────────────────────────────────────────────────────────────────────── +# +# One schema for every case below, mirroring `snapshot`'s two keys, so that "same shape, opposite +# verdict" is true — the fixtures differ only in the variable each one isolates. +function snap(stable, experimental) + return Dict("stable_methods" => stable, "experimental_methods" => experimental) +end + +const MARKED_METHOD = snap(String[], Dict("f(::Int)" => Dict("reason" => "r"))) +const PROMOTED_METHOD = snap(["f(::Int)"], Dict()) +const SETTLED_METHOD = snap(["f(::Int)"], Dict()) +const GONE_METHOD = snap(String[], Dict()) +const RESIGNED_METHOD = snap(["f(::Real)"], Dict()) + +@testset "the method snapshot mirrors the name snapshot's two keys" begin + # Diverging schemas mean `compare` and `compare_methods` cannot share a snapshot file. + for d in (MARKED_METHOD, PROMOTED_METHOD, SETTLED_METHOD, GONE_METHOD, RESIGNED_METHOD) + @test Set(keys(d)) == Set(["stable_methods", "experimental_methods"]) + end + @test Set(keys(ExperimentalAPI.snapshot(Shown))) ⊇ Set(["stable", "experimental"]) +end + +@testset "a snapshot records marks at method granularity" begin + # The schema change method-level marks force — which is why the release layer is itself + # declared experimental. + @test_broken haskey(ExperimentalAPI.snapshot(Shown), "experimental_methods") +end + +@testset "removing a marked METHOD is not breaking" begin + @test_broken !ExperimentalAPI.isbreaking( + ExperimentalAPI.compare_methods(MARKED_METHOD, PROMOTED_METHOD) + ) +end + +@testset "removing a SETTLED method is breaking" begin + # Control: same schema, differing only in whether the method was marked. + @test_broken ExperimentalAPI.isbreaking( + ExperimentalAPI.compare_methods(SETTLED_METHOD, GONE_METHOD) + ) +end + +@testset "a signature change to a settled method is reported as breaking" begin + # The blind spot `compare` admits to in its own docstring. + @test_broken ExperimentalAPI.isbreaking( + ExperimentalAPI.compare_methods(SETTLED_METHOD, RESIGNED_METHOD) + ) +end + +@testset "a keyword-only change is a blind spot here too" begin + # Keyword arguments live in a separate `kwcall` method, so a signature string cannot see a + # changed default any more than a name set can. Stated rather than discovered later. + @test_broken ExperimentalAPI.compare_methods_sees_keywords === true +end + +# ── the provenance record next to a result ─────────────────────────────────────────────────── + +@testset "a result file can carry the experimental dependencies of the run that made it" begin + # The end state: a figure's directory says which unvalidated code paths produced it. + @test_broken ExperimentalAPI.stamp(tempname(), () -> Shown.provisional(1)) isa + AbstractString +end + +@testset "the stamp is readable without loading the package that made it" begin + # Plain TOML or JSON: a year later the package may not resolve. The path goes through + # `stamp` first, since a bare `tempname()` throws for an unrelated reason. + @test_broken occursin( + "reference value", + read(ExperimentalAPI.stamp(tempname(), () -> Shown.provisional(1)), String), + ) +end + +@testset "a stamped result names the package versions involved" begin + @test_broken ExperimentalAPI.stamp_versions isa Function +end + +# ── CI gates ───────────────────────────────────────────────────────────────────────────────── + +@testset "CI can fail a PR that adds a mark without a tracking link" begin + # Whether `tracking` is required is a per-project decision, and must be expressible. + @test_broken ExperimentalAPI.test_surface(Shown; require_tracking=true) isa + ExperimentalAPI.Audit +end + +@testset "CI can fail a PR that increases the number of marks" begin + # The ratchet, in the shape `skip` already has. + @test_broken ExperimentalAPI.test_surface(Shown; max_marks=0) isa ExperimentalAPI.Audit +end + +@testset "a mark older than N releases is reported" begin + # `since` exists so a mark cannot quietly become permanent. Nothing reads it yet. + @test_broken ExperimentalAPI.stale_since(Shown, v"0.9.0") isa AbstractVector +end diff --git a/test/spec/test_spec_lifecycle.jl b/test/spec/test_spec_lifecycle.jl new file mode 100644 index 0000000..cee3b9d --- /dev/null +++ b/test/spec/test_spec_lifecycle.jl @@ -0,0 +1,181 @@ +# The mark's exit — when it may be removed — and entry points that are not a single function. +# +# Scope: a mark that can only ever be added is a decoration. The exit is what makes it a work +# item. The entry point has to be a module or a script, as `#print axioms` answers for any +# declaration and not only for one it is handed. + +using ExperimentalAPI: ExperimentalAPI, @experimental, audit, experimental, mark +using Test + +module Lifecycle + +using ExperimentalAPI + +public verified_now, still_unverified, tracked_but_unresolved, settled, consumer, entry + +# Reason discharged: reference value exists, suite exercises it. This one is removable. +@experimental( + "no reference value yet", + since = v"0.1.0", + tracking = "https://example.invalid/issues/1", + verified_now(β::Float64) = 2 * β +) + +# A mark whose reason still stands. +@experimental( + "convergence not established below β ≈ 0.1", + since = v"0.1.0", + still_unverified(β::Float64) = β * 1.0000001 +) + +# Without a third mark, the two above differ only in whether `tracking` is set, and a rule keyed +# on that alone would satisfy every assertion below. This one has a link and is still not ready. +@experimental( + "reference value exists but disagrees with the literature at the third digit", + since = v"0.1.0", + tracking = "https://example.invalid/issues/2", + tracked_but_unresolved(β::Float64) = β + 1e-9 +) + +"Settled from the start." +settled(β::Float64) = β + +# A caller that depends on the mark. Removing the mark must flip its verdict, and nothing else. +consumer(β::Float64) = verified_now(β) + settled(β) + +# The whole-module entry point: everything a user of this package can reach. +entry(β::Float64) = consumer(β) + still_unverified(β) + +end # module Lifecycle + +# ── the fixture can disagree ───────────────────────────────────────────────────────────────── + +@testset "two marks, one of which is ready to go and one of which is not" begin + names = Set(mk.name for mk in experimental(Lifecycle)) + @test :verified_now in names + @test :still_unverified in names + @test :settled ∉ names + @test Lifecycle.verified_now(0.5) == 1.0 # both are exercised by this suite… + @test Lifecycle.still_unverified(0.5) > 0.5 # …so coverage alone cannot separate them +end + +# ── end-to-end: an entry point that is not a single function ───────────────────────────────── + +@testset "a whole module can be the entry point" begin + # Function-by-function does not scale to a package. + @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(Lifecycle)) === :depends +end + +@testset "the module-level answer names which public entry points are affected" begin + # "Something in here is experimental" is not actionable at package scale. + @test_broken :entry in + [e.name for e in ExperimentalAPI.reach(Lifecycle).affected_entries] + # Control: `settled` reaches nothing marked and must not be listed. + @test_broken :settled ∉ + [e.name for e in ExperimentalAPI.reach(Lifecycle).affected_entries] +end + +@testset "a module with nothing marked comes back clean" begin + # Control for the two above. + @eval module CleanModule + "Settled." + f(x) = x + public f + end + @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(Main.CleanModule)) === :clean +end + +@testset "a script can be the entry point" begin + # The shape a researcher has: a file that produces a figure, not a package. The file must be + # written — `tempname()` alone throws regardless of the implementation. + path = tempname() + write(path, "1 + 1\n") + @test isfile(path) + @test_broken hasproperty(ExperimentalAPI.reach_script(path), :reached) +end + +# ── the exit: what licenses removing a mark ────────────────────────────────────────────────── + +@testset "the tool says a mark is ready to be removed, and why" begin + # Not "is this marked" but "may this stop being marked", with nameable evidence. + @test_broken ExperimentalAPI.ready_to_promote(Lifecycle, :verified_now) === true +end + +@testset "a mark whose reason still stands is NOT reported ready" begin + # Control: rejects a checker that says "ready" for everything. All three are exercised by + # this suite, so coverage cannot be the whole criterion. + @test_broken ExperimentalAPI.ready_to_promote(Lifecycle, :still_unverified) === false +end + +@testset "having a tracking link is not the same as being ready" begin + # Control: rejects a rule keyed on `tracking` alone. + @test mark(Lifecycle, :tracked_but_unresolved).tracking !== nothing + @test mark(Lifecycle, :verified_now).tracking !== nothing + @test mark(Lifecycle, :still_unverified).tracking === nothing + @test_broken ExperimentalAPI.ready_to_promote(Lifecycle, :tracked_but_unresolved) === + false +end + +@testset "removing a mark is reported as not breaking" begin + # Stated in the direction a person asks it: I am about to delete this line, is that a + # release event? + @test !ExperimentalAPI.isbreaking( + ExperimentalAPI.compare( + Dict( + "stable" => ["settled"], + "experimental" => Dict("verified_now" => Dict("reason" => "r")), + ), + Dict("stable" => ["settled", "verified_now"], "experimental" => Dict()), + ), + ) +end + +@testset "…but DELETING it outright still is" begin + # Control: promoting a mark and deleting the name both read as "the mark is gone", and only + # one is safe. + d = ExperimentalAPI.compare( + Dict("stable" => ["settled", "verified_now"], "experimental" => Dict()), + Dict("stable" => ["settled"], "experimental" => Dict()), + ) + @test d.removed_stable == [:verified_now] + @test ExperimentalAPI.isbreaking(d) +end + +@testset "removing the mark flips its callers, and only its callers" begin + # `consumer` becomes clean; `entry` does not, since it still reaches `still_unverified`. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Lifecycle.consumer, Tuple{Float64}; ignore=[:verified_now]) + ) === :clean + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Lifecycle.entry, Tuple{Float64}; ignore=[:verified_now]) + ) === :depends +end + +@testset "how long a mark has been standing is answerable" begin + # `since` is recorded and nothing reads it; a mark that never expires is just a label. + @test mark(Lifecycle, :still_unverified).since == v"0.1.0" + # The number the caller needs, not `isa Any` — which is true of `nothing` from a stub. + @test_broken ExperimentalAPI.age(Lifecycle, :still_unverified, v"0.9.0") == 8 +end + +@testset "the number of marks can only go down under a ratchet" begin + # Same shape as `test_surface`'s skip list. Not `isa Audit`: that comes back whether the cap + # was honoured or ignored, so assert what the cap does. + @test length(experimental(Lifecycle)) == 3 + @test_broken ExperimentalAPI.exceeds_mark_cap(Lifecycle, 1) === true + @test_broken ExperimentalAPI.exceeds_mark_cap(Lifecycle, 5) === false +end + +@testset "a mark removed while callers still depend on it is caught" begin + # Propagation read backwards: the line gets deleted because the author looked at the + # definition, not at who reaches it. + @test_broken :consumer in ExperimentalAPI.dependents(Lifecycle, :verified_now) + # Control: rejects a `dependents` that returns every public name. + @test_broken :settled ∉ ExperimentalAPI.dependents(Lifecycle, :verified_now) +end + +@testset "the exit works at method granularity too" begin + # The intersection neither this file nor `test_spec_dispatch.jl` covers: promoting one + # method must not promote its siblings. + @test_broken ExperimentalAPI.ready_to_promote isa Function +end diff --git a/test/spec/test_spec_profile.jl b/test/spec/test_spec_profile.jl new file mode 100644 index 0000000..7ddc9f4 --- /dev/null +++ b/test/spec/test_spec_profile.jl @@ -0,0 +1,404 @@ +# What a real run went through: which marked definitions it entered, how often, and how much of +# the run was spent inside them. +# +# Scope: two layers. Presence is detected by default and must cost nothing; counts, call sites +# and paths are opt-in. The measurements that put the boundary there are in `README.md`. + +using ExperimentalAPI: ExperimentalAPI, @experimental +using Profile: Profile +using Test + +module Sim + +using ExperimentalAPI + +public energy, correlator, driver, sweep, cold, Model + +struct Model + β::Float64 +end + +@experimental "convergence not established below β ≈ 0.1" energy(m::Model) = m.β * 1.0000001 +@experimental "edge cases at zero separation untested" correlator(m::Model, r::Int) = + m.β / (r + 1) +"Settled." +partition(m::Model) = exp(-m.β) + +inner(m::Model) = energy(m) + partition(m) +function driver(m::Model, n::Int) + s = 0.0 + for _ in 1:n + s += inner(m) + end + return s +end +function sweep(m::Model, n::Int) + s = 0.0 + for r in 1:n + s += correlator(m, r) + end + return s +end +@experimental "never exercised by the fixture" cold(m::Model) = m.β + +end # module Sim + +const M = Sim.Model(0.5) + +# ── the default layer: no opt-in ───────────────────────────────────────────────────────────── + +@testset "a run reports what it entered without being asked to record" begin + Sim.driver(M, 10) + @test_broken :energy in [h.name for h in ExperimentalAPI.entered()] +end + +@testset "the default answer is presence, and does not pretend to be counts" begin + # Scope: "at least once". A count field here would read as a measurement it did not make. + Sim.driver(M, 10) + @test_broken ExperimentalAPI.entered()[1].count === nothing +end + +@testset "a definition that was never entered is absent from the default answer too" begin + # Control: separates observed from enumerated. + Sim.driver(M, 10) + @test_broken :cold ∉ [h.name for h in ExperimentalAPI.entered()] +end + +# Observed from a child process: an `atexit` handler registered in this one would pass for a +# handler that prints nothing, and stdout alone would pass if the notice went to stderr. +module ChildRun + +const WARMED = Ref(false) + +function _cmd(code) + return `$(Base.julia_cmd()) --startup-file=no --project=$(Base.active_project()) -e $code` +end + +""" + output(code) -> String + +Everything a child `julia -e code` wrote, stdout and stderr merged. The first call warms the +precompilation cache, so a `Precompiling …` banner is never mistaken for the summary under test. +""" +function output(code::AbstractString) + if !WARMED[] + run( + pipeline( + ignorestatus(_cmd("using ExperimentalAPI")); stdout=devnull, stderr=devnull + ), + ) + WARMED[] = true + end + io = IOBuffer() + run(pipeline(ignorestatus(_cmd(code)); stdout=io, stderr=io)) + return String(take!(io)) +end + +# The two scripts differ only in the final call. +const ENTERS = """ +using ExperimentalAPI +module Child +using ExperimentalAPI +public energy +@experimental "convergence not established below beta" energy(x) = x * 1.0000001 +end +Child.energy(0.5) +""" + +const LOADS_ONLY = """ +using ExperimentalAPI +module Child +using ExperimentalAPI +public energy +@experimental "convergence not established below beta" energy(x) = x * 1.0000001 +end +""" + +end # module ChildRun + +@testset "the summary is printed at process exit" begin + @test_broken occursin("energy", ChildRun.output(ChildRun.ENTERS)) +end + +@testset "a process that loaded a mark but never entered it stays silent" begin + # Control: rejects a summary keyed on "this module has marks". + @test isempty(strip(ChildRun.output(ChildRun.LOADS_ONLY))) +end + +@testset "the summary carries the reason, not just the name" begin + @test_broken occursin("convergence not established", ExperimentalAPI.summary_text()) +end + +@testset "the default layer can be turned off" begin + # Scope: readable before `using` returns, so an environment variable rather than a call. + @test_broken ExperimentalAPI.detecting() === true +end + +# ── the opt-in layer: the basic question ───────────────────────────────────────────────────── + +@testset "a run reports which marked definitions it entered" begin + @test_broken :energy in + [h.name for h in ExperimentalAPI.record(() -> Sim.driver(M, 100))] +end + +@testset "a run reports how many times each was entered" begin + @test_broken ExperimentalAPI.record(() -> Sim.driver(M, 100))[1].count == 100 +end + +@testset "a marked definition the run never entered is absent, not zero" begin + # Control: separates observed from enumerated. + @test_broken :cold ∉ [h.name for h in ExperimentalAPI.record(() -> Sim.driver(M, 10))] +end + +@testset "a run that touches nothing marked reports an empty record, not an error" begin + @test_broken ExperimentalAPI.record(() -> sum(1:10)) == [] +end + +@testset "a record distinguishes 'touched nothing' from 'recording was off'" begin + # Both are an empty vector otherwise, and they mean opposite things. + @test_broken ExperimentalAPI.record(() -> sum(1:10)).enabled === true +end + +# ── granularity ────────────────────────────────────────────────────────────────────────────── + +@testset "attribution is to a method, not to a name" begin + @test_broken first(ExperimentalAPI.record(() -> Sim.driver(M, 10))).method isa Method +end + +@testset "two marked definitions in one run are reported separately" begin + @test_broken length( + ExperimentalAPI.record(() -> (Sim.driver(M, 10); Sim.sweep(M, 10))) + ) == 2 +end + +@testset "the call site that reached the mark is recorded" begin + # Scope: which part of the caller's own code to distrust, not just that a mark was hit. + @test_broken !isempty(first(ExperimentalAPI.record(() -> Sim.driver(M, 10))).callers) +end + +@testset "the whole path from the entry point is available" begin + @test_broken :driver in first(ExperimentalAPI.record(() -> Sim.driver(M, 10))).paths[1] +end + +# ── proportion, not just presence ──────────────────────────────────────────────────────────── + +@testset "the record says what fraction of the run was inside experimental code" begin + @test_broken 0.0 < + ExperimentalAPI.experimental_fraction( + ExperimentalAPI.record(() -> Sim.driver(M, 100_000)) + ) <= + 1.0 +end + +@testset "inclusive and exclusive time are distinguished" begin + # Scope: a marked wrapper over settled code is not a marked kernel. + @test_broken let h = first(ExperimentalAPI.record(() -> Sim.driver(M, 1000))) + h.inclusive >= h.exclusive + end +end + +# ── the floor: what may be emitted into the body ───────────────────────────────────────────── + +# Controls for the two testsets below. `!occursin(…)` is satisfied by an expansion that dropped +# the definition, so each check is also run against an expansion that does the forbidden thing. +# Kept in a module because a macro at the top level of a test file leaks into the whole shard. +module WrapControl + +"Wraps the call it is given — the shape the emitted flag must stay inside." +macro wrapping(_reason, def) + return esc(Expr(:function, def.args[1], Expr(:block, :(COUNTS[] += 1), def.args[2]))) +end + +"Puts a logging call in the body." +macro logging(reason, def) + return esc( + Expr( + :function, + def.args[1], + Expr(:block, :($Base.@warn $reason maxlog = 1), def.args[2]), + ), + ) +end + +""" + method_bodies(ex) -> Vector + +The body of every method definition inside an expanded expression, line numbers stripped. Walks +into `Expr(:escape, …)`, which is where a macro's own output lives. +""" +function method_bodies(ex) + out = Any[] + walk(x) = + if x isa Expr + if (x.head === :(=) || x.head === :function) && + x.args[1] isa Expr && + x.args[1].head === :call + push!(out, x.args[2]) + end + foreach(walk, x.args) + end + walk(Base.remove_linenums!(ex)) + return out +end + +end # module WrapControl + +@testset "today the expansion is untouched — which is what has to change" begin + # Scope: the default layer needs one statement in the body, so this equality must break. + # Compared at the AST: the expansion carries `Expr(:escape, …)` and prints differently. + bare = WrapControl.method_bodies(@macroexpand f(x) = x * 2) + marked = WrapControl.method_bodies(@macroexpand @experimental "why" f(x) = x * 2) + @test length(bare) == 1 + @test marked == bare + wrapped = WrapControl.method_bodies( + @macroexpand WrapControl.@wrapping "why" f(x) = x * 2 + ) + @test wrapped != bare + @test_broken length(marked[1].args) == length(bare[1].args) + 1 + @test Sim.driver(M, 3) ≈ 3 * (0.5 * 1.0000001 + exp(-0.5)) +end + +@testset "whatever is emitted, it is not a logging call" begin + # The bound on what the flag may cost: a logging call in the body stops the definition + # inlining whether or not it ever fires. + emitted = string(@macroexpand @experimental "why" f(x) = x * 2) + @test !occursin("CoreLogging", emitted) + @test occursin( + "CoreLogging", string(@macroexpand WrapControl.@logging "why" f(x) = x * 2) + ) +end + +# No wall-clock assertion here by decision, not omission: the figures separating these designs +# come from an idle machine, and the same thresholds on a shared CI runner would be a flake +# generator. CI checks the shape of the expansion; cost belongs in a benchmark. + +# ── mechanism constraints ──────────────────────────────────────────────────────────────────── + +@testset "recording survives inlining" begin + # Scope: marked definitions are usually small, so a mechanism needing `@noinline` is no + # mechanism. This is why the sampling route was rejected. + @test_broken ExperimentalAPI.record(() -> Sim.driver(M, 100))[1].count == 100 +end + +@testset "detection is on by default; counting is not" begin + @test_broken ExperimentalAPI.detecting() === true + @test_broken ExperimentalAPI.recording() === false +end + +@testset "the default layer's cost is stated, and it is the flag's cost" begin + # Not `>= 0`, which every number satisfies. Above a few percent it has become a counter. + @test_broken ExperimentalAPI.overhead_when_detecting() < 0.10 +end + +@testset "the opt-in layer's overhead is measured and reported, not discovered" begin + @test_broken ExperimentalAPI.record(() -> Sim.driver(M, 1000)).overhead isa Real +end + +@testset "recording nests without double counting" begin + @test_broken ExperimentalAPI.record( + () -> ExperimentalAPI.record(() -> Sim.driver(M, 10)) + )[1].count == 10 +end + +@testset "an exception inside the recorded block still yields a record" begin + @test_broken ExperimentalAPI.record(() -> error("boom"); rethrow=false) isa + AbstractVector +end + +# ── concurrency and distribution ───────────────────────────────────────────────────────────── + +@testset "the suite runs with more than one thread" begin + # `Threads.@threads for _ in 1:8` runs its body 8 times whatever `nthreads()` is, so without + # this the concurrency claims below would pass single-threaded. Set in `CI.yml`. + @test Threads.nthreads() > 1 +end + +@testset "hits from every thread are attributed" begin + # Scope: a racing counter is as wrong as a main-thread-only one, and reports a number. + threaded() = Threads.@threads for _ in 1:8 + Sim.driver(M, 100) + end + @test_broken ExperimentalAPI.record(threaded)[1].count == 800 +end + +@testset "per-thread storage is sized by maxthreadid, not nthreads" begin + # The interactive pool is counted separately, so `threadid()` exceeds `nthreads()` — an + # `nthreads()`-sized vector throws on the first hit from a REPL task. + @test Threads.maxthreadid() >= Threads.nthreads() + @test_broken ExperimentalAPI.record(() -> Sim.driver(M, 10)).slots >= + Threads.maxthreadid() +end + +@testset "a merged record is still a record" begin + # `isa AbstractVector` would let the merge return a plain `Vector` with no `.enabled`. + @test_broken hasproperty( + ExperimentalAPI.merge_records([ExperimentalAPI.record(() -> Sim.driver(M, 10))]), + :enabled, + ) +end + +@testset "records from separate processes merge into one" begin + @test_broken ExperimentalAPI.merge_records([ + ExperimentalAPI.record(() -> Sim.driver(M, 10)) for _ in 1:2 + ]) isa AbstractVector +end + +@testset "merging is associative and order-independent" begin + # Workers finish in arbitrary order; provenance must not depend on that. + @test_broken let a = ExperimentalAPI.record(() -> Sim.driver(M, 10)), + b = ExperimentalAPI.record(() -> Sim.sweep(M, 10)) + + ExperimentalAPI.merge_records([a, b]) == ExperimentalAPI.merge_records([b, a]) + end +end + +# ── coexistence with the profiler people already use ───────────────────────────────────────── + +@testset "recording does not disturb Profile" begin + @test_broken ExperimentalAPI.record(() -> Sim.driver(M, 10); with_profile=true) isa + AbstractVector +end + +@testset "an existing Profile buffer can be attributed after the fact" begin + # Scope: a twelve-hour run already profiled must not have to be run again. + @test_broken ExperimentalAPI.attribute(Profile.fetch()) isa AbstractVector +end + +# ── the output is evidence, not a printout ─────────────────────────────────────────────────── + +@testset "a record is serialisable" begin + @test_broken ExperimentalAPI.write_record( + tempname(), ExperimentalAPI.record(() -> Sim.driver(M, 10)) + ) isa AbstractString +end + +@testset "a serialised record round-trips" begin + @test_broken ExperimentalAPI.read_record( + ExperimentalAPI.write_record( + tempname(), ExperimentalAPI.record(() -> Sim.driver(M, 10)) + ), + ) isa AbstractVector +end + +@testset "a record names the versions it was taken against" begin + # `energy` being experimental in v0.3 says nothing about v0.9. + @test_broken ExperimentalAPI.record(() -> Sim.driver(M, 10)).versions isa AbstractDict +end + +@testset "the reason is carried into the record" begin + # Scope: readable a year later by someone who never saw the source. + @test_broken occursin( + "convergence", first(ExperimentalAPI.record(() -> Sim.driver(M, 10))).reason + ) +end + +# ── using it as a gate ─────────────────────────────────────────────────────────────────────── + +@testset "a run can be asserted to have touched nothing experimental" begin + @test_broken ExperimentalAPI.assert_clean(() -> 1 + 1) +end + +@testset "the assertion fails, naming the mark, when the run is not clean" begin + # Control: a gate that cannot be shown to fire is not a gate. + @test_broken !ExperimentalAPI.assert_clean(() -> Sim.driver(M, 10); throw=false) +end diff --git a/test/spec/test_spec_propagate.jl b/test/spec/test_spec_propagate.jl new file mode 100644 index 0000000..798a74d --- /dev/null +++ b/test/spec/test_spec_propagate.jl @@ -0,0 +1,261 @@ +# A caller that never names a marked thing still depends on it. +# +# Modelled on Lean's `sorry`, but Julia's call graph is not closed, so the answer is three-valued: +# +# :depends a marked definition is reachable +# :clean the whole call graph was resolved and nothing marked is in it +# :unknown some call site could not be resolved — the honest non-answer +# +# Scope: collapsing `:unknown` into `:clean` is the one failure this file exists to prevent. It is +# not a weaker claim, it is a false one. +# +# The mechanism is a `Core.Compiler.AbstractInterpreter` hooking `abstract_call_method`, because +# inference runs before inlining; `code_typed(...; optimize=true)` sees only `mul_float` and finds +# nothing. + +using ExperimentalAPI: ExperimentalAPI, @experimental, experimental +using Test + +module Chain + +using ExperimentalAPI + +public unstable, + solid, + mid_bad, + mid_good, + top_bad, + top_good, + top_arg, + top_nospec, + top_field, + top_table, + top_recursive, + top_mutual_a, + CONSTANT_BAD, + top_uses_const, + MarkedStruct, + top_constructs + +@experimental "convergence is not established below β ≈ 0.1" unstable(x::Float64) = + x * 1.0000001 +"Settled." +solid(x::Float64) = x * 2.0 + +# one hop +mid_bad(x::Float64) = unstable(x) + 1.0 +mid_good(x::Float64) = solid(x) + 1.0 + +# two hops — neither `top_*` mentions `unstable` +top_bad(x::Float64) = mid_bad(x) * 3 +top_good(x::Float64) = mid_good(x) * 3 + +# passed as a value — Julia specialises on `typeof(f)`, so this resolves +apply(f, x::Float64) = f(x) +top_arg(x::Float64) = apply(unstable, x) + +# `@nospecialize` — still resolves +top_nospec(@nospecialize(f), x::Float64) = f(x) + +# genuinely unresolvable: the callee is a value chosen at run time +struct Holder + f::Function +end +top_field(h::Holder, x::Float64) = h.f(x) + +const TABLE = Function[unstable, solid] +top_table(i::Int, x::Float64) = TABLE[i](x) + +# termination +top_recursive(n::Int, x::Float64) = n <= 0 ? unstable(x) : top_recursive(n - 1, x) + +# Deep enough that a depth limit must truncate. `top_recursive` calls `unstable` in its own body, +# so a depth test written against that one would pass for an implementation ignoring the keyword. +deep_5(x::Float64) = unstable(x) +deep_4(x::Float64) = deep_5(x) +deep_3(x::Float64) = deep_4(x) +deep_2(x::Float64) = deep_3(x) +deep_1(x::Float64) = deep_2(x) +top_mutual_a(n::Int, x::Float64) = n <= 0 ? unstable(x) : top_mutual_b(n - 1, x) +top_mutual_b(n::Int, x::Float64) = top_mutual_a(n - 1, x) + +# a marked const is not a call site — a different mechanism is needed to see its use +@experimental "the tolerance is a guess" const CONSTANT_BAD = 1e-8 +top_uses_const(x::Float64) = x + CONSTANT_BAD + +# a marked struct: construction is a call, field access is not +@experimental "layout not settled" struct MarkedStruct + v::Float64 +end +top_constructs(x::Float64) = MarkedStruct(x).v + +end # module Chain + +const ENTRY = Tuple{Float64} + +@testset "the mark itself is in place, so the fixture can disagree" begin + names = Set(mk.name for mk in experimental(Chain)) + @test :unstable in names + @test :CONSTANT_BAD in names + @test :MarkedStruct in names + @test :solid ∉ names # the negative control really is unmarked +end + +# ── the type the answer lives in ───────────────────────────────────────────────────────────── +# +# `Mark` and `Audit` are pinned nominally elsewhere in this directory. Without the same here, a +# `NamedTuple` with the right field names satisfies every assertion in four files. + +@testset "the result has a type, not just field names" begin + @test_broken ExperimentalAPI.reach(Chain.top_bad, ENTRY) isa ExperimentalAPI.Reach +end + +@testset "verdict is derived, never stored" begin + # A stored `verdict` makes `:clean` with a non-empty `.unresolved` representable, which is + # the one state this file forbids. Same rule as `isbreaking(d::Diff)`. + @test_broken !hasproperty(ExperimentalAPI.reach(Chain.top_bad, ENTRY), :verdict) +end + +@testset "a boolean gate exists alongside the three-valued answer" begin + # Every other verdict here is a named predicate, never a comparison the caller writes out. + @test_broken ExperimentalAPI.isclean(ExperimentalAPI.reach(Chain.top_good, ENTRY)) === + true + @test_broken ExperimentalAPI.isclean(ExperimentalAPI.reach(Chain.top_bad, ENTRY)) === + false +end + +@testset ":unknown absorbs when results are combined" begin + # `reach(Module)` folds every public entry into one answer, so the algebra must exist: one + # `:unknown` is not clean whatever the others say, and folding is order-independent. + @test_broken ExperimentalAPI.combine(:clean, :unknown) === :unknown + @test_broken ExperimentalAPI.combine(:depends, :unknown) === :depends + @test_broken ExperimentalAPI.combine(:clean, :depends) === :depends + @test_broken ExperimentalAPI.combine(:clean, :clean) === :clean + @test_broken ExperimentalAPI.combine(:unknown, :clean) === + ExperimentalAPI.combine(:clean, :unknown) +end + +# ── the core claim ─────────────────────────────────────────────────────────────────────────── + +@testset "a caller two hops away is reported as depending" begin + @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(Chain.top_bad, ENTRY)) === + :depends + @test_broken :unstable in + [mk.name for mk in ExperimentalAPI.reach(Chain.top_bad, ENTRY).reached] +end + +@testset "an equally deep caller with nothing marked is reported clean" begin + # Control: rejects a tool that always says `:depends`. + @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(Chain.top_good, ENTRY)) === + :clean + @test_broken isempty(ExperimentalAPI.reach(Chain.top_good, ENTRY).reached) +end + +@testset "a function passed as a value is still followed" begin + # Specialisation on `typeof(f)` resolves this; it is not a dynamic hole. + @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(Chain.top_arg, ENTRY)) === + :depends +end + +@testset "@nospecialize does not hide the callee" begin + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Chain.top_nospec, Tuple{typeof(Chain.unstable),Float64}) + ) === :depends +end + +# ── the honest non-answer ──────────────────────────────────────────────────────────────────── + +@testset "an abstract-typed callee field is :unknown, NOT :clean" begin + # `Holder.f::Function` can hold `unstable`, so `:clean` here is a lie. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Chain.top_field, Tuple{Chain.Holder,Float64}) + ) === :unknown + @test_broken !isempty( + ExperimentalAPI.reach(Chain.top_field, Tuple{Chain.Holder,Float64}).unresolved + ) +end + +@testset "a run-time table lookup is :unknown, NOT :clean" begin + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Chain.top_table, Tuple{Int,Float64}) + ) === :unknown + @test_broken !isempty( + ExperimentalAPI.reach(Chain.top_table, Tuple{Int,Float64}).unresolved + ) +end + +@testset "every unresolved site says where it is" begin + # "cannot tell" is actionable only if the user can go and look. + @test_broken all( + u -> hasproperty(u, :file) && hasproperty(u, :line), + ExperimentalAPI.reach(Chain.top_field, Tuple{Chain.Holder,Float64}).unresolved, + ) +end + +@testset "a depth limit reports :unknown rather than :clean" begin + # `:unknown` rather than `!== :clean`: the latter cannot separate "the limit truncated" from + # "the limit was ignored and the mark was found anyway". + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Chain.deep_1, ENTRY; maxdepth=2) + ) === :unknown + # …and without the limit it is found, so the fixture can disagree. + @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(Chain.deep_1, ENTRY)) === + :depends +end + +# ── termination ────────────────────────────────────────────────────────────────────────────── + +@testset "self-recursion terminates and still finds the mark" begin + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Chain.top_recursive, Tuple{Int,Float64}) + ) === :depends +end + +@testset "mutual recursion terminates" begin + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Chain.top_mutual_a, Tuple{Int,Float64}) + ) === :depends +end + +# ── things that are not calls ──────────────────────────────────────────────────────────────── + +@testset "a marked const is seen where it is used" begin + # A const is not a call site. Either the analysis reads globals out of the IR, or the case is + # declared out of scope — what it must not do is report `:clean`. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Chain.top_uses_const, ENTRY) + ) !== :clean +end + +@testset "a marked struct is seen at construction" begin + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Chain.top_constructs, ENTRY) + ) === :depends +end + +@testset "marking a module marks what it contains" begin + # Or it does not, stated. Either way a decision, not an omission. + @test_broken hasproperty(ExperimentalAPI.reach(Chain.top_bad, ENTRY), :through_modules) +end + +# ── across packages ────────────────────────────────────────────────────────────────────────── + +@testset "a mark in a dependency propagates into the dependent" begin + # Needs the fixture package, so only the API shape is pinned here. + @test_broken ExperimentalAPI.reach isa Function +end + +# ── cost ───────────────────────────────────────────────────────────────────────────────────── + +@testset "the macro emits nothing but the definition and one push" begin + # Look at the expansion: `Method.name` is the generic function's name, so filtering methods + # by what their bodies call is unconditionally empty. + emitted = string(@macroexpand @experimental "why" f(x) = x) + @test occursin("_mark!", emitted) # the one load-time effect + @test occursin("__EXPERIMENTAL_API_MARKS__", emitted) + for forbidden in ("reach", "verdict", "record", "abstract_call_method", "code_typed") + @testset "no $forbidden in the expansion" begin + @test !occursin(forbidden, emitted) + end + end +end diff --git a/test/spec/test_spec_verify.jl b/test/spec/test_spec_verify.jl new file mode 100644 index 0000000..27958b3 --- /dev/null +++ b/test/spec/test_spec_verify.jl @@ -0,0 +1,77 @@ +# How well a marked definition is exercised by the tests. +# +# Scope: joining the mark's `file`/`line` against `--code-coverage` counts. No new machinery, and +# it answers the worst case — unverified code that its own suite never runs. + +using ExperimentalAPI: ExperimentalAPI, @experimental, experimental +using Test + +module Covered + +using ExperimentalAPI + +public exercised, half_exercised, never_exercised + +@experimental "reference value not cross-checked" function exercised(x) + return x + 1 +end + +@experimental "the negative branch is untested" function half_exercised(x) + if x > 0 + return x + else + return -x # never reached by the suite below + end +end + +@experimental "shipped without ever being called" never_exercised(x) = x * 0 + +end # module Covered + +# Deliberately partial: a fully exercised fixture cannot tell a working join from one that +# always reports 100%. +@testset "the fixture is exercised only partly, on purpose" begin + @test Covered.exercised(1) == 2 + @test Covered.half_exercised(1) == 1 + # `half_exercised(-1)` is NOT called + # `never_exercised` is NOT called +end + +@testset "marks carry the location a coverage file is keyed by" begin + for mk in experimental(Covered) + @test isfile(String(mk.file)) + @test mk.line > 0 + + @test occursin("@experimental", readlines(String(mk.file))[mk.line]) + end +end + +@testset "a marked definition with no coverage at all is reported" begin + @test_broken :never_exercised in [mk.name for mk in ExperimentalAPI.unverified(Covered)] +end + +@testset "a marked definition that IS covered is not reported" begin + # Control: rejects a checker that reports everything. + @test_broken :exercised ∉ [mk.name for mk in ExperimentalAPI.unverified(Covered)] +end + +@testset "partial coverage is reported as partial, not as verified" begin + @test_broken 0.0 < ExperimentalAPI.coverage(Covered, :half_exercised) < 1.0 +end + +@testset "coverage is absent, not zero, when the run had none enabled" begin + # No `.cov` files without `--code-coverage`; reporting 0% then flags everything on every + # ordinary run. + @test_broken ExperimentalAPI.coverage(Covered, :exercised) === missing || + ExperimentalAPI.coverage(Covered, :exercised) isa Real +end + +@testset "a mark whose line no longer matches its definition is reported stale" begin + # An edit above the definition moves the code but not an existing coverage file. + @test_broken ExperimentalAPI.stale_marks(Covered) isa AbstractVector +end + +@testset "the verification report is data, not a printout" begin + # Same convention as `audit`: the number comes back on the normal return path. + @test_broken ExperimentalAPI.verification(Covered) isa AbstractVector +end diff --git a/test/test_spec_table.jl b/test/test_spec_table.jl new file mode 100644 index 0000000..ba7f568 --- /dev/null +++ b/test/test_spec_table.jl @@ -0,0 +1,76 @@ +# The table in `test/spec/README.md` is generated; this is what keeps it that way. +# +# Scope: generating it is only half a fix — a generator nobody runs drifts exactly as fast. + +using Test + +include("spec/summary.jl") + +const _SPEC_README = joinpath(@__DIR__, "spec", "README.md") +const _BEGIN = "" +const _END = "" + +# Git checks the README out with CRLF on Windows, so without this the comparison below is +# between line endings. +_lf(s::AbstractString) = replace(s, "\r\n" => "\n") + +@testset "the spec table is generated, not typed" begin + md = _lf(read(_SPEC_README, String)) + @test occursin(_BEGIN, md) + @test occursin(_END, md) + block = strip(split(split(md, _BEGIN)[2], _END)[1]) + # "a table is out of date" is not actionable without saying where the table comes from. + if block != _lf(SpecSummary.table()) + @info "test/spec/README.md is stale — regenerate with `julia --project=test test/spec/summary.jl`" + end + @test block == _lf(SpecSummary.table()) +end + +@testset "no spec file is missing from the table" begin + # The non-emptiness assertion is not redundant: a `readdir` returning nothing satisfies the + # set equality vacuously. + files = SpecSummary.spec_files() + @test length(files) >= 10 + @test Set(files) == Set(keys(SpecSummary.CONCERNS)) +end + +@testset "no spec file is missing from runtests.jl" begin + # `runtests.jl` includes spec files by name, so one can be listed in the table, never run, + # and still contribute to the published count. + driver = read(joinpath(@__DIR__, "runtests.jl"), String) + missing_from_driver = filter( + f -> !occursin("spec/$f", driver), SpecSummary.spec_files() + ) + @test isempty(missing_from_driver) +end + +@testset "the count is behaviours, and a loop cannot inflate it" begin + # The property that makes the measure worth publishing: a loop cannot inflate it. Pinned + # against a synthetic file so that editing a spec file does not change what this asserts. + path = joinpath(mktempdir(), "test_spec_synthetic.jl") + write( + path, + join( + [ + "@testset \"outer\" begin", # not a leaf: contains a testset + " for i in 1:100", + " @testset \"inner \$i\" begin", # one leaf, whatever the loop bound + " @test i == i", + " end", + " end", + "end", + "@testset \"specified only\" begin", # a leaf with no operating assertion + " for k in 1:50", + " @test_broken notyet(k)", + " end", + "end", + ], + "\n", + ), + ) + c = SpecSummary.counts(path) + @test c.behaviours == 2 # not 3, and not 100 + @test c.operating == 1 + @test c.specified == 1 + @test c.broken_assertions == 1 # one written line, not fifty executions +end