From 47e8ce944c5d0195ad59e68e29468c713fe85090 Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Thu, 3 Sep 2026 06:15:49 +0000 Subject: [PATCH 01/11] test: the case matrix, written before the implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five files under test/spec/ enumerate what the package has to handle. Most of it is `@test_broken`, because most of it is not built. `@test_broken` rather than a TODO list, for a reason that was measured first: @test_broken false -> Broken (suite stays green) @test_broken notyet() -> Broken (throwing counts as broken, not error) @test_broken 1 == 1 -> Error: Unexpected Pass So the spec cannot rot in either direction. It does not block work that has not been done, and it fails the suite the moment finished work goes unpromoted. declare every definition form; and the method-level unit the name-level one is not — `QAtlas.fetch` has 570 methods behind a name, so a statement about the name necessarily over-claims propagate a caller two hops away, with an equally deep NEGATIVE control so a tool that always answers ":depends" cannot pass; the three-valued answer, and the two cases that must come back `:unknown` rather than `:clean` docstring a mark and a docstring are different accounts. Pinned against `Base.Experimental`, whose 24 entries ARE documented — Julia itself marks an experimental surface and documents it verify which marked definitions the suite never executes, joined from the marks' file:line and coverage counts. The fixture is exercised only PARTLY on purpose, so a checker that always reports 100% cannot pass runtime what a real run touched. Includes a floor test that the mark does not wrap the call, because instrumentation must stay opt-in Two measurements from 2026-09-03 are recorded in the file headers so the next person does not repeat them: Lean 4.33.1 — `sorry` propagates: 'downstream' depends on axioms [sorryAx] without ever writing `sorry`. That is the model. Julia — the sampling profiler attributes ZERO samples to marked methods because inlined frames carry no MethodInstance. A custom `Core.Compiler.AbstractInterpreter` hooking `abstract_call_method` does work: inference runs before inlining. `code_typed(...; optimize=true)` sees only `mul_float` and finds nothing. Currently 44 passing, 46 broken. That ratio is the progress measure. Co-Authored-By: Claude Opus 5 (1M context) --- test/runtests.jl | 7 + test/spec/README.md | 25 ++++ test/spec/test_spec_declare.jl | 161 ++++++++++++++++++++++ test/spec/test_spec_docstring.jl | 122 +++++++++++++++++ test/spec/test_spec_propagate.jl | 225 +++++++++++++++++++++++++++++++ test/spec/test_spec_runtime.jl | 111 +++++++++++++++ test/spec/test_spec_verify.jl | 85 ++++++++++++ 7 files changed, 736 insertions(+) create mode 100644 test/spec/README.md create mode 100644 test/spec/test_spec_declare.jl create mode 100644 test/spec/test_spec_docstring.jl create mode 100644 test/spec/test_spec_propagate.jl create mode 100644 test/spec/test_spec_runtime.jl create mode 100644 test/spec/test_spec_verify.jl diff --git a/test/runtests.jl b/test/runtests.jl index 8226297..0b186ed 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -11,5 +11,12 @@ using Test include("test_ext.jl") include("test_precompile.jl") include("test_dogfood.jl") + # The case matrix. Written before the implementation, so most of it is @test_broken; + # see test/spec/README.md for why that is the right register. + include("spec/test_spec_declare.jl") + include("spec/test_spec_propagate.jl") + include("spec/test_spec_docstring.jl") + include("spec/test_spec_verify.jl") + include("spec/test_spec_runtime.jl") include("test_aqua.jl") end diff --git a/test/spec/README.md b/test/spec/README.md new file mode 100644 index 0000000..caa0315 --- /dev/null +++ b/test/spec/README.md @@ -0,0 +1,25 @@ +# The case matrix + +These files are the specification. They are written before the implementation, so most of them +are `@test_broken`. + +`@test_broken` was chosen deliberately over comments or a TODO list: + + * an expression that throws (because the function does not exist yet) registers as **Broken**, + not as an error, so the suite stays green while the spec is incomplete; + * an expression that starts **passing** registers as `Error: Unexpected Pass`, which fails the + suite until someone promotes it to `@test`. + +So the spec cannot silently rot in either direction: it neither blocks work that has not been +done, nor lets finished work go unnoticed. + +Anything already implemented is a plain `@test`. The ratio of `@test` to `@test_broken` in this +directory is the honest progress measure. + +| file | concern | +|---|---| +| `test_spec_declare.jl` | what can carry a mark: function, method, struct, const, module, macro, extension | +| `test_spec_propagate.jl` | a caller that never names a marked thing still depends on it | +| `test_spec_docstring.jl` | a mark and a docstring are different accounts and must coexist | +| `test_spec_verify.jl` | how well is a marked thing exercised by the tests | +| `test_spec_runtime.jl` | what a real run touched, and what it costs when nobody asks | diff --git a/test/spec/test_spec_declare.jl b/test/spec/test_spec_declare.jl new file mode 100644 index 0000000..11d9ac2 --- /dev/null +++ b/test/spec/test_spec_declare.jl @@ -0,0 +1,161 @@ +# What can carry a mark. +# +# The current implementation marks a NAME. The intent is to mark a definition — and for +# `QAtlas.fetch`, which has 570 methods behind one name, the name is the wrong unit: a docstring +# on `fetch` cannot say which dispatch path returns a number you can trust. +# +# So this file covers both: what works today (plain `@test`) and what the unit has to become +# (`@test_broken`). + +using ExperimentalAPI: + ExperimentalAPI, @experimental, Mark, experimental, isexperimental, mark +using Test + +module Declared + +using ExperimentalAPI + +export documented_fn +public plain_fn, + short_fn, + Struct, + MutStruct, + AbstractKind, + PrimKind, + CONSTANT, + ASSIGNED, + Sub, + @marked_macro, + listed_a, + listed_b, + energy + +@experimental "long form" function plain_fn(x) + return x +end +@experimental "short form" short_fn(x) = x +@experimental "struct" struct Struct + v::Int +end +@experimental "mutable struct" mutable struct MutStruct + v::Int +end +@experimental "abstract type" abstract type AbstractKind end +@experimental "primitive type" primitive type PrimKind 8 end +@experimental "const" const CONSTANT = 1 +@experimental "plain assignment" ASSIGNED = 2 +@experimental "macro" macro marked_macro(x) + return esc(x) +end + +module Sub end +@experimental "a module can only be marked by name" Sub + +listed_a(x) = x +listed_b(x) = x +@experimental "declared, not attached" listed_a listed_b + +"Documented and settled." +documented_fn(x) = x + +# The shape the method-level unit has to reach: one name, several dispatch paths, and only one +# of them is in doubt. This is `QAtlas.fetch` in miniature. +struct Exact end +struct Numerical end +energy(::Exact, β::Float64) = β +energy(::Numerical, β::Float64) = β + 1e-12 +energy(::Numerical, β::Rational) = β + +end # module Declared + +@testset "every definition form can be marked" begin + got = Dict(mk.name => mk for mk in experimental(Declared)) + for (name, reason) in ( + :plain_fn => "long form", + :short_fn => "short form", + :Struct => "struct", + :MutStruct => "mutable struct", + :AbstractKind => "abstract type", + :PrimKind => "primitive type", + :CONSTANT => "const", + :ASSIGNED => "plain assignment", + Symbol("@marked_macro") => "macro", + :Sub => "a module can only be marked by name", + :listed_a => "declared, not attached", + :listed_b => "declared, not attached", + ) + @testset "$name" begin + @test haskey(got, name) + @test got[name].reason == reason + end + end +end + +@testset "the reason travels with every one of them" begin + for mk in experimental(Declared) + @test !isempty(strip(mk.reason)) + @test mk.mod === Declared + @test mk.line > 0 + @test String(mk.file) == @__FILE__ + end +end + +# ── the method-level unit ──────────────────────────────────────────────────────────────────── +# +# `Declared.energy` has three methods. Marking the name marks all three, which is the granularity +# problem: the exact-solution path is trustworthy and the numerical one is not. + +@testset "marking a name is the wrong unit when the name has many methods" begin + @test length(methods(Declared.energy)) == 3 + # Today the only available statement is about the name, so it necessarily over-claims. + @test !isexperimental(Declared, :energy) # not marked at all yet — see below +end + +@testset "a single dispatch path can be marked" begin + m = which(Declared.energy, Tuple{Declared.Numerical,Float64}) + @test_broken ExperimentalAPI.mark(m) isa Mark + @test_broken ExperimentalAPI.isexperimental(m) +end + +@testset "marking one method leaves its siblings alone" begin + numerical = which(Declared.energy, Tuple{Declared.Numerical,Float64}) + exact = which(Declared.energy, Tuple{Declared.Exact,Float64}) + rational = which(Declared.energy, Tuple{Declared.Numerical,Rational}) + @test numerical !== exact !== rational + @test_broken ExperimentalAPI.isexperimental(numerical) + @test_broken !ExperimentalAPI.isexperimental(exact) + @test_broken !ExperimentalAPI.isexperimental(rational) +end + +@testset "a method mark survives precompilation" begin + # The name-keyed registry already survives (test/test_precompile.jl). `Method` objects are + # part of the defining module's image too, but that is a separate claim and needs its own + # fixture package before it can be asserted. + @test_broken ExperimentalAPI.experimental_methods isa Function +end + +@testset "a method mark is queryable from a call site" begin + # The question a downstream test actually asks, before trusting a reference value. + @test_broken ExperimentalAPI.isexperimental( + which(Declared.energy, Tuple{Declared.Numerical,Float64}) + ) +end + +# ── extensions ─────────────────────────────────────────────────────────────────────────────── +# +# A package extension is a separate module. Names it makes public are part of the package's +# surface from a user's point of view, and are invisible to `names(Package)`. + +@testset "a mark inside a package extension is reachable from the parent" begin + ext = Base.get_extension(ExperimentalAPI, :ExperimentalAPITestExt) + @test ext !== nothing + # Marks declared inside `ext` live in `ext`, and asking the parent must find them. + @test_broken !isempty(ExperimentalAPI.experimental(ExperimentalAPI; extensions=true)) +end + +@testset "auditing a package does not silently ignore its extensions" begin + a = ExperimentalAPI.audit(ExperimentalAPI) + # Today `audit` looks at one module. Whether an extension's surface is in scope has to be a + # stated answer rather than an omission. + @test_broken hasproperty(a, :extensions) +end diff --git a/test/spec/test_spec_docstring.jl b/test/spec/test_spec_docstring.jl new file mode 100644 index 0000000..2112803 --- /dev/null +++ b/test/spec/test_spec_docstring.jl @@ -0,0 +1,122 @@ +# A mark and a docstring are different accounts, and they must coexist. +# +# The registry reviewer's objection on 2026-09-02 was aimed at a README sentence that read +# "this name is public, it has no docstring, and that is deliberate". His position — public names +# should always have a docstring — is correct, and the package never required otherwise; the +# framing did. +# +# Julia's own code settles it. `Base.Experimental` holds 24 entries and the ones sampled on +# 2026-09-03 (`@optlevel`, `@compiler_options`, `Const`) all carry docstrings. Base marks an +# experimental surface AND documents it. The two are orthogonal, and this file pins that. + +using ExperimentalAPI: + ExperimentalAPI, @experimental, audit, experimental, isdocumented, isexperimental, mark +using Test + +module Both + +using ExperimentalAPI + +public documented_and_marked, + marked_only, documented_only, silent, @marked_macro, separately + +""" +Documented, and declared unfinished. Both statements are true at once and neither replaces the +other: the prose says what it does, the mark says how much to trust it. +""" +@experimental "convergence not established at low β" documented_and_marked(x) = x + +@experimental "no prose yet" marked_only(x) = x + +"Documented and settled." +documented_only(x) = x + +silent(x) = x + +"A marked macro keeps its docstring." +@experimental "shape not settled" macro marked_macro(x) + return esc(x) +end + +# a docstring attached separately rather than adjacent to the definition +@experimental "declared here, documented below" separately(x) = x +@doc "Documented in a second statement." separately + +end # module Both + +@testset "Julia itself documents its experimental surface" begin + # The precedent that makes "documented AND experimental" not a contradiction. + @test isdefined(Base, :Experimental) + for n in (Symbol("@optlevel"), Symbol("@compiler_options"), :Const) + @testset "Base.Experimental.$n" begin + @test isdefined(Base.Experimental, n) + @test Base.Docs.hasdoc(Base.Experimental, n) + end + end +end + +@testset "a docstring survives the macro" begin + # The macro emits `Expr(:meta, :doc)` so a preceding docstring attaches to the definition + # rather than to the block the macro expands to. Without it the two accounts would be + # mutually exclusive in practice, whatever the documentation claimed. + @test Base.Docs.hasdoc(Both, :documented_and_marked) + @test isexperimental(Both, :documented_and_marked) + @test occursin( + "Both statements are true at once", string(@doc Both.documented_and_marked) + ) +end + +@testset "a marked macro keeps its docstring" begin + @test Base.Docs.hasdoc(Both, Symbol("@marked_macro")) + @test isexperimental(Both, Symbol("@marked_macro")) +end + +@testset "a docstring attached in a separate statement also works" begin + @test Base.Docs.hasdoc(Both, :separately) + @test isexperimental(Both, :separately) +end + +@testset "the four combinations land where they should" begin + a = audit(Both) + @test :documented_and_marked in a.documented + @test :documented_and_marked in a.declared # both accounts, deliberately + @test :marked_only in a.declared + @test :marked_only ∉ a.documented + @test :documented_only in a.documented + @test :documented_only ∉ a.declared + @test :silent in a.unaccounted # neither — the finding +end + +@testset "a mark is not an excuse for missing prose" begin + # Under the corrected framing, `marked_only` is not "fine because it is marked". It is a + # public name with no docstring, and the audit has to be able to say so even though a mark + # is present. Today `declared` absorbs it and the distinction is unavailable. + a = audit(Both) + @test_broken :marked_only in a.undocumented + @test_broken hasproperty(a, :undocumented) +end + +@testset "the check can require a docstring regardless of the mark" begin + # `Aqua.test_undocumented_names` and `Docs.undocumented_names` already enforce + # "every public name has a docstring". A package adopting both should not have to choose. + @test_broken ExperimentalAPI.test_surface(Both; require_docstring=true) isa + ExperimentalAPI.Audit +end + +@testset "the reason is reachable from the rendered documentation" begin + # A reader on the docs site should see that a name is declared unfinished, and why, without + # the author repeating the reason by hand in the docstring — otherwise the two accounts drift. + @test_broken ExperimentalAPI.docstring_note(Both, :documented_and_marked) isa + AbstractString +end + +@testset "what Documenter's checkdocs sees is not what audit sees" begin + # `checkdocs = :public` fails a build on an undocumented public name and has no notion of a + # third answer; `audit` accepts a mark instead. Both are legitimate and they disagree here. + a = audit(Both) + undocumented_by_base = setdiff(Base.Docs.undocumented_names(Both), [nameof(Both)]) + @test :marked_only in undocumented_by_base # Base would flag it + @test :marked_only ∉ a.unaccounted # audit accepts the mark + @test :silent in undocumented_by_base + @test :silent in a.unaccounted # both flag this one +end diff --git a/test/spec/test_spec_propagate.jl b/test/spec/test_spec_propagate.jl new file mode 100644 index 0000000..e0e32a6 --- /dev/null +++ b/test/spec/test_spec_propagate.jl @@ -0,0 +1,225 @@ +# Propagation: a caller that never names a marked thing still depends on it. +# +# The model is Lean's `sorry`. Measured 2026-09-03 with Lean 4.33.1: +# +# P.lean:1:8: warning: declaration uses `sorry` +# 'unproven' depends on axioms: [sorryAx] +# 'downstream' depends on axioms: [sorryAx] <- never wrote `sorry` itself +# 'honest' does not depend on any axioms +# +# Julia cannot match that exactly, and the difference is the most important thing in this file. +# Lean's kernel has a closed dependency graph of proof terms; Julia's call graph is not closed. +# So the answer is not a Bool. It is three-valued: +# +# :depends a marked definition is reachable +# :clean the whole call graph was resolved and nothing marked is in it +# :unknown some call site could not be resolved — the honest non-answer +# +# Collapsing `:unknown` into `:clean` is the one failure this file exists to prevent. A tool that +# reports "no experimental dependency" about a call graph it could not see has not made a weaker +# claim, it has made a false one. +# +# Feasibility was measured on 2026-09-03 with a custom `Core.Compiler.AbstractInterpreter` +# hooking `abstract_call_method`. Inference runs before inlining, so the call graph is intact +# there; post-processing `code_typed(...; optimize=true)` sees only `mul_float`/`add_float` and +# finds nothing. + +using ExperimentalAPI: ExperimentalAPI, @experimental, experimental +using Test + +module Chain + +using ExperimentalAPI + +public unstable, + solid, + mid_bad, + mid_good, + top_bad, + top_good, + top_arg, + top_nospec, + top_field, + top_table, + top_recursive, + top_mutual_a, + CONSTANT_BAD, + top_uses_const, + MarkedStruct, + top_constructs + +@experimental "convergence is not established below β ≈ 0.1" unstable(x::Float64) = + x * 1.0000001 +"Settled." +solid(x::Float64) = x * 2.0 + +# one hop +mid_bad(x::Float64) = unstable(x) + 1.0 +mid_good(x::Float64) = solid(x) + 1.0 + +# two hops — neither `top_*` mentions `unstable` +top_bad(x::Float64) = mid_bad(x) * 3 +top_good(x::Float64) = mid_good(x) * 3 + +# a function passed as a value. Julia specialises on `typeof(f)`, so inference resolves this. +apply(f, x::Float64) = f(x) +top_arg(x::Float64) = apply(unstable, x) + +# `@nospecialize` — measured to STILL resolve +top_nospec(@nospecialize(f), x::Float64) = f(x) + +# genuinely unresolvable: the callee is a value chosen at run time +struct Holder + f::Function +end +top_field(h::Holder, x::Float64) = h.f(x) + +const TABLE = Function[unstable, solid] +top_table(i::Int, x::Float64) = TABLE[i](x) + +# termination +top_recursive(n::Int, x::Float64) = n <= 0 ? unstable(x) : top_recursive(n - 1, x) +top_mutual_a(n::Int, x::Float64) = n <= 0 ? unstable(x) : top_mutual_b(n - 1, x) +top_mutual_b(n::Int, x::Float64) = top_mutual_a(n - 1, x) + +# a marked const is not a call site — a different mechanism is needed to see its use +@experimental "the tolerance is a guess" const CONSTANT_BAD = 1e-8 +top_uses_const(x::Float64) = x + CONSTANT_BAD + +# a marked struct: construction is a call, field access is not +@experimental "layout not settled" struct MarkedStruct + v::Float64 +end +top_constructs(x::Float64) = MarkedStruct(x).v + +end # module Chain + +const ENTRY = Tuple{Float64} + +@testset "the mark itself is in place, so the fixture can disagree" begin + names = Set(mk.name for mk in experimental(Chain)) + @test :unstable in names + @test :CONSTANT_BAD in names + @test :MarkedStruct in names + @test :solid ∉ names # the negative control really is unmarked +end + +# ── the core claim ─────────────────────────────────────────────────────────────────────────── + +@testset "a caller two hops away is reported as depending" begin + @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(Chain.top_bad, ENTRY)) === + :depends + @test_broken :unstable in + [mk.name for mk in ExperimentalAPI.reach(Chain.top_bad, ENTRY).reached] +end + +@testset "an equally deep caller with nothing marked is reported clean" begin + # Without this the previous test passes for a tool that always says `:depends`. + @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(Chain.top_good, ENTRY)) === + :clean + @test_broken isempty(ExperimentalAPI.reach(Chain.top_good, ENTRY).reached) +end + +@testset "a function passed as a value is still followed" begin + # Measured: Julia specialises on `typeof(f)`, so this resolves. It is NOT a dynamic hole. + @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(Chain.top_arg, ENTRY)) === + :depends +end + +@testset "@nospecialize does not hide the callee" begin + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Chain.top_nospec, Tuple{typeof(Chain.unstable),Float64}) + ) === :depends +end + +# ── the honest non-answer ──────────────────────────────────────────────────────────────────── + +@testset "an abstract-typed callee field is :unknown, NOT :clean" begin + # `Holder.f::Function` can hold `unstable`. Reporting `:clean` here would be a lie, and it is + # exactly what the prototype did before the third value existed. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Chain.top_field, Tuple{Chain.Holder,Float64}) + ) === :unknown + @test_broken !isempty( + ExperimentalAPI.reach(Chain.top_field, Tuple{Chain.Holder,Float64}).unresolved + ) +end + +@testset "a run-time table lookup is :unknown, NOT :clean" begin + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Chain.top_table, Tuple{Int,Float64}) + ) === :unknown + @test_broken !isempty( + ExperimentalAPI.reach(Chain.top_table, Tuple{Int,Float64}).unresolved + ) +end + +@testset "every unresolved site says where it is" begin + # "cannot tell" is only actionable if the user can go and look. + @test_broken all( + u -> hasproperty(u, :file) && hasproperty(u, :line), + ExperimentalAPI.reach(Chain.top_field, Tuple{Chain.Holder,Float64}).unresolved, + ) +end + +@testset "a depth limit reports :unknown rather than :clean" begin + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Chain.top_recursive, Tuple{Int,Float64}; maxdepth=1) + ) !== :clean +end + +# ── termination ────────────────────────────────────────────────────────────────────────────── + +@testset "self-recursion terminates and still finds the mark" begin + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Chain.top_recursive, Tuple{Int,Float64}) + ) === :depends +end + +@testset "mutual recursion terminates" begin + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Chain.top_mutual_a, Tuple{Int,Float64}) + ) === :depends +end + +# ── things that are not calls ──────────────────────────────────────────────────────────────── + +@testset "a marked const is seen where it is used" begin + # A const is not a call site, so the call-graph walk cannot find it. Either the analysis + # reads globals out of the IR as well, or this case has to be declared out of scope in the + # documentation. What it must not do is report `:clean`. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Chain.top_uses_const, ENTRY) + ) !== :clean +end + +@testset "a marked struct is seen at construction" begin + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Chain.top_constructs, ENTRY) + ) === :depends +end + +@testset "marking a module marks what it contains" begin + # Or it does not, and that is stated. Either way it is a decision, not an omission. + @test_broken hasproperty(ExperimentalAPI.reach(Chain.top_bad, ENTRY), :through_modules) +end + +# ── across packages ────────────────────────────────────────────────────────────────────────── + +@testset "a mark in a dependency propagates into the dependent" begin + # The QAtlas case: a downstream analysis calls `fetch`, which is marked in QAtlas. + # Requires the fixture package, so it is only pinned as an API shape here. + @test_broken ExperimentalAPI.reach isa Function +end + +# ── cost ───────────────────────────────────────────────────────────────────────────────────── + +@testset "analysis is opt-in and costs nothing when not asked for" begin + # `@experimental` emits the definition unchanged plus one `push!` at load time. Whatever the + # analysis costs, it must not move into the marked package's own load or call path. + @test isempty( + filter( + m -> occursin("reach", String(m.name)), collect(methods(ExperimentalAPI._mark!)) + ), + ) +end diff --git a/test/spec/test_spec_runtime.jl b/test/spec/test_spec_runtime.jl new file mode 100644 index 0000000..a4f8437 --- /dev/null +++ b/test/spec/test_spec_runtime.jl @@ -0,0 +1,111 @@ +# What a real run touched — and what it costs when nobody asks. +# +# The question is provenance for a result, not documentation: after a twelve-hour DMRG run, which +# unverified code paths did it go through, and how often? A docstring cannot answer that. This is +# the part of the intent that a reviewer cannot dismiss as "put it in the docstring". +# +# Two routes were measured on 2026-09-03, and the cheap one does NOT work: +# +# sampling profiler + Method-set membership -> 0 samples attributed +# `Profile.fetch` frames for inlined callees carry no `MethodInstance`, so exactly the +# small functions most likely to be marked disappear. `@noinline` did not rescue it. +# +# custom AbstractInterpreter (static) -> works, see test_spec_propagate.jl +# but answers "could reach", not "did reach, N times". +# +# So the runtime half still needs a mechanism. Whatever it is, the constraint below is the one +# that must not be traded away. + +using ExperimentalAPI: ExperimentalAPI, @experimental +using Test + +module Hot + +using ExperimentalAPI + +public inner, driver, cold_path + +@experimental "not validated below β ≈ 0.1" inner(x::Float64) = x * 1.0000001 +function driver(n::Int, x::Float64) + s = 0.0 + for _ in 1:n + s += inner(x) + end + return s +end + +@experimental "never called by the fixture" cold_path(x::Float64) = x - 1.0 + +end # module Hot + +@testset "the mark does not wrap the call" begin + # The property the package currently advertises and must keep: `@experimental` emits the + # definition unchanged plus one `push!` at load time. If instrumentation ever becomes + # unconditional, an inner loop pays for it on every iteration. + src = read(joinpath(@__DIR__, "..", "..", "src", "mark.jl"), String) + @test occursin("emitted unchanged", src) || occursin("costs nothing at run time", src) + @test Hot.driver(3, 1.0) ≈ 3 * 1.0000001 +end + +@testset "a marked definition is not slower than the same definition unmarked" begin + unmarked(x::Float64) = x * 1.0000001 + loop(f) = ( + acc=0.0; + for _ in 1:2_000_000 + ; + acc += f(1.0); + end; + acc + ) + loop(Hot.inner) + loop(unmarked) # warm both before timing either + a = @elapsed loop(Hot.inner) + b = @elapsed loop(unmarked) + # Loose on purpose: this is a floor against wrapping, not a benchmark. A wrapper that + # increments a counter would not fit inside this margin. + @test a < 5b + 1e-3 +end + +@testset "recording is off unless it is asked for" begin + @test_broken ExperimentalAPI.recording() === false +end + +@testset "a recorded run reports which marked definitions it entered" begin + @test_broken :inner in + [h.name for h in ExperimentalAPI.record(() -> Hot.driver(1000, 1.0))] +end + +@testset "a recorded run reports how many times" begin + # "Did it touch experimental code" and "was 97% of the run inside it" are different answers, + # and only the second tells you whether the result is worth anything. + @test_broken ExperimentalAPI.record(() -> Hot.driver(1000, 1.0))[1].count == 1000 +end + +@testset "a marked definition the run never entered is not reported as touched" begin + # Without this, a recorder that lists every mark in the module passes the two tests above. + @test_broken :cold_path ∉ + [h.name for h in ExperimentalAPI.record(() -> Hot.driver(10, 1.0))] +end + +@testset "recording survives inlining" begin + # The reason the sampling-profiler route failed. Whatever mechanism is chosen has to be + # demonstrated on a small function that the optimiser would normally inline away — which is + # most of what gets marked. + @test_broken ExperimentalAPI.record(() -> Hot.driver(100, 1.0))[1].count == 100 +end + +@testset "the record attributes to a method, not just a name" begin + # `QAtlas.fetch` has 570 methods. "the run touched `fetch`" is not usable; "the run took the + # Numerical × Float64 path 4.7M times" is. + @test_broken first(ExperimentalAPI.record(() -> Hot.driver(10, 1.0))).method isa Method +end + +@testset "a run with no marked code reports nothing rather than failing" begin + @test_broken isempty(ExperimentalAPI.record(() -> sum(1:10))) +end + +@testset "the record is data on the normal return path" begin + # Same convention as `audit` and `test_surface`: the caller gets the numbers back whether or + # not it asked for printing. + @test_broken ExperimentalAPI.record(() -> Hot.driver(10, 1.0)) isa AbstractVector +end diff --git a/test/spec/test_spec_verify.jl b/test/spec/test_spec_verify.jl new file mode 100644 index 0000000..356451b --- /dev/null +++ b/test/spec/test_spec_verify.jl @@ -0,0 +1,85 @@ +# "How well is this experimental thing actually verified?" +# +# This is the half of the intent that is cheap. A mark carries `file` and `line`; Julia's +# `--code-coverage` writes per-line execution counts. Joining them answers, with no new +# machinery: which marked definitions does the test suite never execute? +# +# It matters because the failure mode being guarded against is not a crash. It is a number that +# comes back and looks fine. A marked method with zero coverage is the worst case: unverified +# code, shipped, and never even run by its own suite. + +using ExperimentalAPI: ExperimentalAPI, @experimental, experimental, mark +using Test + +module Covered + +using ExperimentalAPI + +public exercised, half_exercised, never_exercised + +@experimental "reference value not cross-checked" function exercised(x) + return x + 1 +end + +@experimental "the negative branch is untested" function half_exercised(x) + if x > 0 + return x + else + return -x # never reached by the suite below + end +end + +@experimental "shipped without ever being called" never_exercised(x) = x * 0 + +end # module Covered + +# The suite that "verifies" the module. Deliberately partial — a fixture that exercised +# everything could not tell a working coverage join from one that always reports 100%. +@testset "the fixture is exercised only partly, on purpose" begin + @test Covered.exercised(1) == 2 + @test Covered.half_exercised(1) == 1 + # `half_exercised(-1)` is NOT called + # `never_exercised` is NOT called +end + +@testset "marks carry the location a coverage file is keyed by" begin + for mk in experimental(Covered) + @test isfile(String(mk.file)) + @test mk.line > 0 + # The recorded line is the declaration, and the definition starts at or after it. + @test occursin("@experimental", readlines(String(mk.file))[mk.line]) + end +end + +@testset "a marked definition with no coverage at all is reported" begin + @test_broken :never_exercised in [mk.name for mk in ExperimentalAPI.unverified(Covered)] +end + +@testset "a marked definition that IS covered is not reported" begin + # Without this, a checker that reports everything would pass the previous test. + @test_broken :exercised ∉ [mk.name for mk in ExperimentalAPI.unverified(Covered)] +end + +@testset "partial coverage is reported as partial, not as verified" begin + @test_broken 0.0 < ExperimentalAPI.coverage(Covered, :half_exercised) < 1.0 +end + +@testset "coverage is absent, not zero, when the run had none enabled" begin + # Julia writes no `.cov` files without `--code-coverage`. Reporting 0% then would call every + # marked definition unverified on every ordinary test run — a false alarm that would get the + # check switched off. + @test_broken ExperimentalAPI.coverage(Covered, :exercised) === missing || + ExperimentalAPI.coverage(Covered, :exercised) isa Real +end + +@testset "a mark whose line no longer matches its definition is reported stale" begin + # Marks record file:line at macro-expansion. An edit above the definition moves the code but + # not a previously written coverage file, and joining them then attributes the wrong lines. + @test_broken ExperimentalAPI.stale_marks(Covered) isa AbstractVector +end + +@testset "the verification report is data, not a printout" begin + # Same convention as `audit`: the trust number comes back on the normal return path so a + # release script can read it, rather than being printed for a human to eyeball. + @test_broken ExperimentalAPI.verification(Covered) isa AbstractVector +end From 331ae7ab949018a329e80dfb40c7107521a21844 Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Thu, 3 Sep 2026 06:25:17 +0000 Subject: [PATCH 02/11] test: expand the case matrix, and it immediately found two defects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nine files, 67 passing and 111 broken. The broken count is the backlog; the passing count is what the package does today. Added: forms keyword args, parametric, varargs, return annotations, callable structs, constructors, operators, @generated, @kwdef, @inline stacking, @eval-produced definitions, closures, metadata validation, the same name in two modules foreign marking a method on somebody else's generic. This is the `QAtlas.fetch` case: 570 methods, `:fetch` bound in AbstractQAtlas, so `audit` files it under `foreign` and says nothing about any of them profile 29 cases. Hit counts, per-method attribution, the caller that reached the mark, inclusive vs exclusive, the fraction of the run spent inside experimental code, threads, cross-process merge, coexistence with Profile, serialisation, and using it as a gate integration docs rendering, Aqua composition, method-level release diffs, a provenance stamp next to a result file, CI ratchets Every group carries its own negative control, because a spec that only asserts the positive can be satisfied by a tool that always answers yes: propagate `top_good` is exactly as deep as `top_bad` and must come back clean profile `cold` is marked and never called, and must be ABSENT rather than reported with count zero verify the fixture is exercised only partly, so a checker that always reports 100% cannot pass integration a settled name must get no docs note Two defects surfaced while writing it, both shipped, both silent: `@experimental "…" (c::C)(x) = c.k * x` marks `:c` — the ARGUMENT name. `_signame` walks the `::` and returns the binding on the left. The mark lands on a local that is not a binding anywhere, so it is recorded and means nothing. Pinned as the current behaviour plus a broken test for the right one. A mark inside a function body is refused by Julia rather than by this package: `syntax: unsupported const declaration on local variable`. Correct outcome, useless message — it never names `@experimental`. Co-Authored-By: Claude Opus 5 (1M context) --- test/runtests.jl | 4 + test/spec/README.md | 25 ++- test/spec/test_spec_foreign.jl | 143 +++++++++++++++++ test/spec/test_spec_forms.jl | 235 ++++++++++++++++++++++++++++ test/spec/test_spec_integration.jl | 118 ++++++++++++++ test/spec/test_spec_profile.jl | 240 +++++++++++++++++++++++++++++ 6 files changed, 760 insertions(+), 5 deletions(-) create mode 100644 test/spec/test_spec_foreign.jl create mode 100644 test/spec/test_spec_forms.jl create mode 100644 test/spec/test_spec_integration.jl create mode 100644 test/spec/test_spec_profile.jl diff --git a/test/runtests.jl b/test/runtests.jl index 0b186ed..f737e1c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -18,5 +18,9 @@ using Test include("spec/test_spec_docstring.jl") include("spec/test_spec_verify.jl") include("spec/test_spec_runtime.jl") + include("spec/test_spec_profile.jl") + include("spec/test_spec_foreign.jl") + include("spec/test_spec_forms.jl") + include("spec/test_spec_integration.jl") include("test_aqua.jl") end diff --git a/test/spec/README.md b/test/spec/README.md index caa0315..f3e262b 100644 --- a/test/spec/README.md +++ b/test/spec/README.md @@ -18,8 +18,23 @@ directory is the honest progress measure. | file | concern | |---|---| -| `test_spec_declare.jl` | what can carry a mark: function, method, struct, const, module, macro, extension | -| `test_spec_propagate.jl` | a caller that never names a marked thing still depends on it | -| `test_spec_docstring.jl` | a mark and a docstring are different accounts and must coexist | -| `test_spec_verify.jl` | how well is a marked thing exercised by the tests | -| `test_spec_runtime.jl` | what a real run touched, and what it costs when nobody asks | +| `test_spec_declare.jl` | what can carry a mark: function, method, struct, const, module, macro, extension | +| `test_spec_forms.jl` | the definition forms a real package hits on its second afternoon | +| `test_spec_foreign.jl` | marking a method on somebody else's generic — the `QAtlas.fetch` case | +| `test_spec_propagate.jl` | a caller that never names a marked thing still depends on it | +| `test_spec_docstring.jl` | a mark and a docstring are different accounts and must coexist | +| `test_spec_verify.jl` | how well is a marked thing exercised by the tests | +| `test_spec_runtime.jl` | the floor: the mark must not wrap the call | +| `test_spec_profile.jl` | what a real run went through, how often, and how much of it | +| `test_spec_integration.jl` | where the mark has to surface: docs, Aqua, releases, provenance, CI | + +## What the spec already found + +Two defects, both live in the shipped code, both of the kind the spec was written to catch — +a mark that silently records the wrong thing rather than refusing: + +1. `@experimental "…" (c::C)(x) = c.k * x` marks **`:c`**, the argument name. Not the type, not + a function — a local that is not a binding anywhere, so the mark is silently meaningless. +2. A mark inside a function body is refused by *Julia*, not by this package: + `syntax: unsupported const declaration on local variable`. The message never mentions + `@experimental` and points at a line the author did not write. diff --git a/test/spec/test_spec_foreign.jl b/test/spec/test_spec_foreign.jl new file mode 100644 index 0000000..07c3593 --- /dev/null +++ b/test/spec/test_spec_foreign.jl @@ -0,0 +1,143 @@ +# Marking a method on somebody else's function. +# +# This is the QAtlas case and the current implementation refuses it outright. +# +# QAtlas.fetch is AbstractQAtlas.fetch — 570 methods, all defined by QAtlas/AbstractQAtlas. +# `:fetch` is not QAtlas's own binding, so `audit` files it under `foreign` and says nothing +# about any of the 570. +# +# A package that extends another package's generic is the normal Julia idiom, not an edge case. +# If the mark cannot attach there, it cannot describe the surface that matters. + +using ExperimentalAPI: ExperimentalAPI, @experimental, audit, experimental, isexperimental +using Test + +module Upstream +"The generic every downstream package extends." +fetch_value(model, quantity) = error("no method for $(typeof(model)), $(typeof(quantity))") +public fetch_value +end + +module Downstream + +using ExperimentalAPI +using ..Upstream: Upstream, fetch_value + +public Ising, Heisenberg, Energy, Susceptibility + +struct Ising end +struct Heisenberg end +struct Energy end +struct Susceptibility end + +# exact — trustworthy +Upstream.fetch_value(::Ising, ::Energy) = -2.0 + +# numerically delicate — this is the one that should carry a mark +Upstream.fetch_value(::Heisenberg, ::Energy) = -1.7724538509055159 + +# a whole family that is provisional +Upstream.fetch_value(::Ising, ::Susceptibility) = 0.0 +Upstream.fetch_value(::Heisenberg, ::Susceptibility) = 0.0 + +end # module Downstream + +@testset "the fixture really is the foreign-generic shape" begin + @test parentmodule(Upstream.fetch_value) === Upstream + @test length(methods(Upstream.fetch_value)) == 5 + @test count(m -> m.module === Downstream, methods(Upstream.fetch_value)) == 4 + @test :fetch_value ∉ ExperimentalAPI.surface(Downstream) # invisible to names() +end + +@testset "the macro currently refuses a qualified definition" begin + # Today's behaviour, pinned so the change is visible when it happens. + @test_throws LoadError @eval module RefusedForeign + using ExperimentalAPI + using ..Upstream + @experimental "why" Upstream.fetch_value(::Int, ::Int) = 0 + end +end + +@testset "a method on a foreign generic can be marked" begin + @test_broken @eval module MarkedForeign + using ExperimentalAPI + using ..Upstream + struct Probe end + @experimental "provisional" Upstream.fetch_value(::Probe, ::Probe) = 0 + end +end + +@testset "marking one foreign method leaves the siblings alone" begin + exact = which(Upstream.fetch_value, Tuple{Downstream.Ising,Downstream.Energy}) + delicate = which(Upstream.fetch_value, Tuple{Downstream.Heisenberg,Downstream.Energy}) + @test exact !== delicate + @test_broken ExperimentalAPI.isexperimental(delicate) + @test_broken !ExperimentalAPI.isexperimental(exact) +end + +@testset "the mark is stored in the module that WROTE the method" begin + # Not in Upstream. A package cannot be made to carry claims its dependents invented, and the + # mark has to survive Upstream being reloaded or updated. + @test_broken all( + mk -> mk.mod === Downstream, ExperimentalAPI.experimental_methods(Downstream) + ) +end + +@testset "asking the generic finds marks contributed by every package" begin + # The question a user asks is about `fetch_value`, not about which package happened to define + # the method they will dispatch to. + @test_broken !isempty(ExperimentalAPI.experimental(Upstream.fetch_value)) +end + +@testset "audit reports foreign methods this module owns" begin + # `foreign` today means "a name bound elsewhere, not our problem". A method WE wrote on + # someone else's generic is the opposite: our problem, invisible under the current rule. + a = audit(Downstream) + @test_broken hasproperty(a, :contributed_methods) + @test_broken length(a.contributed_methods) == 4 +end + +@testset "a contributed method with neither docstring nor mark is a finding" begin + # The whole point of the audit, applied to the surface that actually matters for QAtlas. + @test_broken !isempty(ExperimentalAPI.unaccounted_methods(Downstream)) +end + +@testset "a docstring on a specific signature counts" begin + # Julia stores docstrings keyed by signature, so "documented" is answerable per method — the + # audit does not have to fall back to the name. + @test_broken ExperimentalAPI.isdocumented( + which(Upstream.fetch_value, Tuple{Downstream.Ising,Downstream.Energy}) + ) isa Bool +end + +@testset "marking a method does not make the foreign NAME experimental" begin + # Otherwise one downstream package could label another package's whole generic unfinished. + @test !isexperimental(Upstream, :fetch_value) +end + +@testset "propagation crosses the package boundary" begin + # A caller in a third package that reaches a marked method in Downstream, through Upstream's + # generic, must be reported. This is the QAtlas → analysis-script path. + caller(x) = Upstream.fetch_value(Downstream.Heisenberg(), Downstream.Energy()) + x + @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(caller, Tuple{Float64})) === + :depends +end + +@testset "a mark on a method of a function defined in Base is possible" begin + # `Base.show`, `Base.length`, `Base.copy` — QAtlas defines methods on all three. The rule + # cannot be "Base is off limits" without excluding a large part of every package's surface. + @test_broken @eval module MarkedBase + using ExperimentalAPI + struct Widget end + @experimental "printing format not settled" Base.show(io::IO, ::Widget) = print(io, "W") + end +end + +@testset "it stays refused when the mark cannot say WHICH method" begin + # `@experimental "why" Base.show` — a bare qualified NAME, no signature. Marking every method + # of `Base.show` in the world is never what anyone means, and guessing is worse than refusing. + @test_throws LoadError @eval module RefusedBareForeign + using ExperimentalAPI + @experimental "why" Base.show + end +end diff --git a/test/spec/test_spec_forms.jl b/test/spec/test_spec_forms.jl new file mode 100644 index 0000000..61bdd5e --- /dev/null +++ b/test/spec/test_spec_forms.jl @@ -0,0 +1,235 @@ +# The definition forms the macro has not been shown to handle. +# +# `test_spec_declare.jl` covers the forms that work today. These are the ones a real package hits +# on its second afternoon: keyword arguments, parametric signatures, callable structs, +# constructors, operators, stacked macros. Each one either works, or the refusal has to name the +# alternative — silently marking the wrong symbol is the outcome this file exists to prevent. + +using ExperimentalAPI: ExperimentalAPI, @experimental, experimental, isexperimental, mark +using Test + +module Forms + +using ExperimentalAPI + +public kw_fn, where_fn, vararg_fn, ret_typed, Callable, Ctor, Gen, KwStruct, INTERP + +@experimental "keyword arguments" kw_fn(x; scale=1.0, kwargs...) = x * scale +@experimental "parametric" where_fn(x::T, y::T) where {T<:Real} = x + y +@experimental "varargs" vararg_fn(x, rest...) = x +@experimental "return type annotation" ret_typed(x)::Float64 = x + +struct Callable + k::Float64 +end +struct Ctor + v::Int +end + +# an interpolated reason +const WHY = "tolerance chosen by hand" +@experimental "$(WHY); see the sweep in issue 12" INTERP = 1e-8 + +end # module Forms + +@testset "forms that already work" begin + got = Dict(mk.name => mk for mk in experimental(Forms)) + for n in (:kw_fn, :where_fn, :vararg_fn, :ret_typed, :INTERP) + @testset "$n" begin + @test haskey(got, n) + @test !isempty(strip(got[n].reason)) + end + end + @test Forms.kw_fn(2.0; scale=3.0) == 6.0 + @test Forms.where_fn(1, 2) == 3 +end + +@testset "an interpolated reason is evaluated, not stored as source" begin + @test occursin("tolerance chosen by hand", mark(Forms, :INTERP).reason) +end + +# ── forms that are not covered ─────────────────────────────────────────────────────────────── + +@testset "a callable struct is marked on the WRONG symbol today" begin + # Measured 2026-09-03. `(c::C)(x) = c.k * x` has no function name, and `_signame` walks the + # `::` and returns the ARGUMENT name. The mark lands on `:c`, a local that is not a binding + # anywhere, so it is silently meaningless — the exact failure this file exists to catch. + @eval module CallableMarked + using ExperimentalAPI + struct C + k::Float64 + end + @experimental "scaling rule provisional" (c::C)(x) = c.k * x + end + @test :c in [mk.name for mk in experimental(CallableMarked)] # today, and wrong + @test !isdefined(CallableMarked, :c) # marks a non-binding +end + +@testset "a callable struct marks the type, or refuses" begin + # Either answer is defensible. Marking the argument name is not. + @test_broken :C in [mk.name for mk in experimental(Main.CallableMarked)] +end + +@testset "a constructor method is marked on the type" begin + # This one already resolves correctly: the mark lands on `:T`. + @eval module CtorMarked + using ExperimentalAPI + struct T + v::Int + end + @experimental "validation not implemented" T(s::AbstractString) = T(parse(Int, s)) + end + @test :T in [mk.name for mk in experimental(Main.CtorMarked)] +end + +@testset "an inner constructor inside a marked struct is not separately marked" begin + # Marking the struct should not silently also claim its inner constructors are unfinished, + # nor silently exclude them. Whichever it is has to be stated. + @test_broken hasproperty(mark(Forms, :Callable), :includes_constructors) +end + +@testset "an operator method can be marked" begin + @test_broken @eval module OpMarked + using ExperimentalAPI + struct V + x::Float64 + end + @experimental "no identity element yet" Base.:+(a::V, b::V) = V(a.x + b.x) + end +end + +@testset "a generated function can be marked" begin + @test_broken @eval module GenMarked + using ExperimentalAPI + @experimental "generator is a prototype" @generated g(x) = :(x) + end +end + +@testset "Base.@kwdef stacks with the mark" begin + # `@kwdef` expands to a block carrying `Expr(:meta, :doc)`. Two macros that both wrap a + # definition must compose in at least one order, and the order that works must be documented. + @test_broken @eval module KwdefMarked + using ExperimentalAPI + @experimental "defaults are guesses" Base.@kwdef struct S + a::Int = 1 + end + end +end + +@testset "@inline and the mark compose in both orders" begin + @test_broken @eval module InlineMarked + using ExperimentalAPI + @experimental "kernel unverified" @inline f(x) = x + @inline @experimental "kernel unverified" g(x) = x + end +end + +@testset "a definition produced by @eval can be marked by name" begin + # Metaprogrammed definitions cannot be attached to; the name-list form is the answer and it + # has to be reachable. + @test_broken @eval module EvalMarked + using ExperimentalAPI + for n in (:a, :b) + @eval $n(x) = x + end + @experimental "generated in a loop" a b + end +end + +@testset "a mark inside a function body is refused" begin + # It IS refused today, but by Julia rather than by this package: the emitted `const` is + # illegal in local scope, so the message is + # syntax: unsupported `const` declaration on local variable + # which says nothing about `@experimental` and points at a line the author did not write. + e = try + @eval module ClosureMarked + using ExperimentalAPI + function outer() + @experimental "why" inner(x) = x + return inner + end + end + nothing + catch err + err + end + @test e isa ErrorException + @test occursin("unsupported `const` declaration", sprint(showerror, e)) +end + +@testset "the refusal names @experimental rather than leaking the emitted const" begin + # A macro whose diagnostic is about its own expansion has handed the reader a puzzle. The + # message should say that a mark belongs at module top level, next to `export` and `public`. + e = try + @eval module ClosureMarked2 + using ExperimentalAPI + function outer() + @experimental "why" inner(x) = x + return inner + end + end + nothing + catch err + err + end + @test_broken occursin("@experimental", sprint(showerror, e)) +end + +# ── metadata ───────────────────────────────────────────────────────────────────────────────── + +@testset "since must be a version, not a string" begin + @test_broken @eval module BadSince + using ExperimentalAPI + @experimental("why", since = "0.4.0", f(x) = x) + end +end + +@testset "an unknown keyword is refused rather than ignored" begin + # `@experimental "why" tracking_url="..." f(x)=x` — a typo in a keyword name must not silently + # become part of the subject. + @test_throws LoadError @eval module BadKw + using ExperimentalAPI + @experimental("why", trackign = "u", f(x) = x) + end +end + +@testset "tracking is carried through to every report" begin + # The field that turns a warning into something a reader can act on. It is stored today; it + # has to survive into the audit and the record as well. + @test_broken ExperimentalAPI.audit(Forms).tracking isa AbstractDict +end + +# ── same name, two places ──────────────────────────────────────────────────────────────────── + +@testset "the same name marked in two modules stays separate" begin + @eval module A1 + using ExperimentalAPI + public f + @experimental "reason A" f(x) = x + end + @eval module A2 + using ExperimentalAPI + public f + @experimental "reason B" f(x) = x + end + @test mark(A1, :f).reason == "reason A" + @test mark(A2, :f).reason == "reason B" +end + +@testset "re-marking with a different reason replaces rather than accumulates" begin + @eval module A3 + using ExperimentalAPI + public f + f(x) = x + @experimental "first" f + @experimental "second" f + end + @test count(mk -> mk.name === :f, experimental(A3)) == 1 + @test mark(A3, :f).reason == "second" +end + +@testset "the replacement is reported rather than silent" begin + # Two different reasons for one name usually means two authors disagreed, or a stale mark was + # left behind. Last-write-wins is a decision, and it should be visible. + @test_broken ExperimentalAPI.superseded_marks isa Function +end diff --git a/test/spec/test_spec_integration.jl b/test/spec/test_spec_integration.jl new file mode 100644 index 0000000..6bf7afd --- /dev/null +++ b/test/spec/test_spec_integration.jl @@ -0,0 +1,118 @@ +# Where the mark has to show up outside this package. +# +# A mark that only `ExperimentalAPI` can read is a private note. The intent — a reader of a paper, +# a reviewer of a PR, or a user of the docs site learning that a number came from unvalidated +# code — requires the mark to surface in tools nobody configured for it. + +using ExperimentalAPI: ExperimentalAPI, @experimental, audit, experimental +using Test + +module Shown + +using ExperimentalAPI + +public settled, provisional + +"Settled and documented." +settled(x) = x + +""" +Provisional. Documented, and declared unfinished. +""" +@experimental( + "the r,s branch has no reference value", + since = v"0.2.0", + tracking = "https://example.invalid/issues/9", + provisional(x) = x +) + +end # module Shown + +# ── documentation ──────────────────────────────────────────────────────────────────────────── + +@testset "the docs can render the mark without the author repeating it" begin + # If the reason has to be typed twice — once in `@experimental`, once in the docstring — the + # two drift, and the machine-readable one loses. + @test_broken ExperimentalAPI.docstring_note(Shown, :provisional) isa AbstractString +end + +@testset "the rendered note carries the reason, the version and the tracking link" begin + for needle in ("reference value", "0.2.0", "example.invalid") + @test_broken occursin(needle, ExperimentalAPI.docstring_note(Shown, :provisional)) + end +end + +@testset "a Documenter block can list a module's marks" begin + # `@autodocs`-style: one block in the manual, always current, never hand-maintained. + @test_broken ExperimentalAPI.DocumenterExt isa Module +end + +@testset "a settled name gets no note" begin + # Without this, a renderer that annotates everything passes the tests above. + @test_broken ExperimentalAPI.docstring_note(Shown, :settled) === nothing +end + +# ── Aqua ───────────────────────────────────────────────────────────────────────────────────── + +@testset "the audit composes with Aqua rather than competing" begin + # `Aqua.test_undocumented_names` enforces "every public name has a docstring" and has no + # third answer. A package should be able to run both without one contradicting the other. + @test_broken ExperimentalAPI.aqua_compatible_names(Shown) isa AbstractVector +end + +# ── release ────────────────────────────────────────────────────────────────────────────────── + +@testset "a snapshot records marks at method granularity" begin + # `compare` reads name sets today and says so. Once methods can be marked, the snapshot has + # to grow — and the schema change is why the release layer is itself declared experimental. + @test_broken haskey(ExperimentalAPI.snapshot(Shown), "methods") +end + +@testset "removing a marked METHOD is not breaking" begin + # The same contract as for names, at the granularity that matters for a dispatch table. + @test_broken ExperimentalAPI.compare_methods isa Function +end + +@testset "a signature change to a settled method is reported as breaking" begin + # The blind spot `compare` currently admits to in its own docstring. Method-level marks are + # what make it addressable at all. + @test_broken ExperimentalAPI.compare_methods isa Function +end + +# ── the provenance record next to a result ─────────────────────────────────────────────────── + +@testset "a result file can carry the experimental dependencies of the run that made it" begin + # The end state: a figure's directory contains a record saying which unvalidated code paths + # produced it. That is the artefact a referee or a future reader needs. + @test_broken ExperimentalAPI.stamp(tempname(), () -> Shown.provisional(1)) isa + AbstractString +end + +@testset "the stamp is readable without loading the package that made it" begin + # A year later the package may not resolve. Plain TOML or JSON, not a serialised Julia object. + path = tempname() + @test_broken occursin("reference value", read(path, String)) +end + +@testset "a stamped result names the package versions involved" begin + @test_broken ExperimentalAPI.stamp_versions isa Function +end + +# ── CI gates ───────────────────────────────────────────────────────────────────────────────── + +@testset "CI can fail a PR that adds a mark without a tracking link" begin + # `tracking` is what makes a mark actionable rather than a shrug. Whether it is required is a + # per-project decision, and it has to be expressible. + @test_broken ExperimentalAPI.test_surface(Shown; require_tracking=true) isa + ExperimentalAPI.Audit +end + +@testset "CI can fail a PR that increases the number of marks" begin + # The ratchet. `skip` already only shrinks; the mark count should be able to as well. + @test_broken ExperimentalAPI.test_surface(Shown; max_marks=0) isa ExperimentalAPI.Audit +end + +@testset "a mark older than N releases is reported" begin + # `since` exists so that "experimental" cannot quietly become permanent. Nothing reads it yet. + @test_broken ExperimentalAPI.stale_since(Shown, v"0.9.0") isa AbstractVector +end diff --git a/test/spec/test_spec_profile.jl b/test/spec/test_spec_profile.jl new file mode 100644 index 0000000..02c913c --- /dev/null +++ b/test/spec/test_spec_profile.jl @@ -0,0 +1,240 @@ +# Profiling: what a real run actually went through. +# +# This is the half of the intent a reviewer cannot answer with "put it in the docstring". After a +# twelve-hour DMRG run the question is not "is this function experimental" but "did the number I +# am about to put in a paper come out of code nobody has validated, and how much of it". +# +# Everything here is `@test_broken`. The cheap route was measured on 2026-09-03 and does not +# work: `Profile.fetch` frames for inlined callees carry no `MethodInstance`, so a sampling +# profiler attributes ZERO samples to marked methods — and small functions, which is most of what +# gets marked, are exactly the ones that get inlined. + +using ExperimentalAPI: ExperimentalAPI, @experimental +using Test + +module Sim + +using ExperimentalAPI + +public energy, correlator, driver, sweep, cold, Model + +struct Model + β::Float64 +end + +@experimental "convergence not established below β ≈ 0.1" energy(m::Model) = m.β * 1.0000001 +@experimental "edge cases at zero separation untested" correlator(m::Model, r::Int) = + m.β / (r + 1) +"Settled." +partition(m::Model) = exp(-m.β) + +inner(m::Model) = energy(m) + partition(m) +function driver(m::Model, n::Int) + s = 0.0 + for _ in 1:n + s += inner(m) + end + return s +end +function sweep(m::Model, n::Int) + s = 0.0 + for r in 1:n + s += correlator(m, r) + end + return s +end +@experimental "never exercised by the fixture" cold(m::Model) = m.β + +end # module Sim + +const M = Sim.Model(0.5) + +# ── the basic question ─────────────────────────────────────────────────────────────────────── + +@testset "a run reports which marked definitions it entered" begin + @test_broken :energy in + [h.name for h in ExperimentalAPI.record(() -> Sim.driver(M, 100))] +end + +@testset "a run reports how many times each was entered" begin + @test_broken ExperimentalAPI.record(() -> Sim.driver(M, 100))[1].count == 100 +end + +@testset "a marked definition the run never entered is absent, not zero" begin + # A recorder that lists every mark in the module passes the two tests above. This is the + # control that separates "observed" from "enumerated". + @test_broken :cold ∉ [h.name for h in ExperimentalAPI.record(() -> Sim.driver(M, 10))] +end + +@testset "a run that touches nothing marked reports an empty record, not an error" begin + @test_broken ExperimentalAPI.record(() -> sum(1:10)) == [] +end + +@testset "a record distinguishes 'touched nothing' from 'recording was off'" begin + # Both would otherwise be an empty vector, and they mean opposite things: one is a clean + # result, the other is a measurement that never happened. + @test_broken ExperimentalAPI.record(() -> sum(1:10)).enabled === true +end + +# ── granularity ────────────────────────────────────────────────────────────────────────────── + +@testset "attribution is to a method, not to a name" begin + # `QAtlas.fetch` has 570 methods. "the run touched fetch" is unusable. + @test_broken first(ExperimentalAPI.record(() -> Sim.driver(M, 10))).method isa Method +end + +@testset "two marked definitions in one run are reported separately" begin + @test_broken length( + ExperimentalAPI.record(() -> (Sim.driver(M, 10); Sim.sweep(M, 10))) + ) == 2 +end + +@testset "the call site that reached the mark is recorded" begin + # `energy` was entered from `inner`, which was entered from `driver`. Knowing only that a + # mark was hit does not tell you which part of your own code to distrust. + @test_broken !isempty(first(ExperimentalAPI.record(() -> Sim.driver(M, 10))).callers) +end + +@testset "the whole path from the entry point is available" begin + @test_broken :driver in first(ExperimentalAPI.record(() -> Sim.driver(M, 10))).paths[1] +end + +# ── proportion, not just presence ──────────────────────────────────────────────────────────── + +@testset "the record says what fraction of the run was inside experimental code" begin + # "It touched experimental code" and "97% of the run was inside it" are different verdicts + # about the same result, and only the second decides whether the number is usable. + @test_broken 0.0 < + ExperimentalAPI.experimental_fraction( + ExperimentalAPI.record(() -> Sim.driver(M, 100_000)) + ) <= + 1.0 +end + +@testset "inclusive and exclusive time are distinguished" begin + # A marked wrapper that spends all its time in settled code is not the same risk as a marked + # kernel that does the arithmetic itself. + @test_broken let h = first(ExperimentalAPI.record(() -> Sim.driver(M, 1000))) + h.inclusive >= h.exclusive + end +end + +# ── mechanism constraints ──────────────────────────────────────────────────────────────────── + +@testset "recording survives inlining" begin + # The measured reason the sampling route failed. `Sim.energy` is a one-line function; the + # optimiser will inline it. Any mechanism that only works on `@noinline` code is not a + # mechanism for this problem. + @test_broken ExperimentalAPI.record(() -> Sim.driver(M, 100))[1].count == 100 +end + +@testset "recording is off by default" begin + @test_broken ExperimentalAPI.recording() === false +end + +@testset "the run pays nothing when recording is off" begin + # The property `@experimental` currently advertises. If instrumentation becomes + # unconditional, every iteration of an inner loop pays for a mark nobody is reading. + @test_broken ExperimentalAPI.overhead_when_disabled() == 0.0 +end + +@testset "the overhead when recording IS on is measured and reported" begin + # A tool that slows a twelve-hour run by 40x will not be used on a twelve-hour run. Whatever + # it costs, the number has to be available rather than discovered. + @test_broken ExperimentalAPI.record(() -> Sim.driver(M, 1000)).overhead isa Real +end + +@testset "recording nests without double counting" begin + @test_broken ExperimentalAPI.record( + () -> ExperimentalAPI.record(() -> Sim.driver(M, 10)) + )[1].count == 10 +end + +@testset "an exception inside the recorded block still yields a record" begin + # A run that crashed halfway is exactly when you want to know what it went through. + @test_broken ExperimentalAPI.record(() -> error("boom"); rethrow=false) isa + AbstractVector +end + +# ── concurrency and distribution ───────────────────────────────────────────────────────────── + +@testset "hits from every thread are attributed" begin + # HPC code is threaded. A recorder that only sees the main thread reports a fraction of the + # truth and calls it the truth. + threaded() = Threads.@threads for _ in 1:8 + Sim.driver(M, 100) + end + @test_broken ExperimentalAPI.record(threaded)[1].count == 800 +end + +@testset "records from separate processes merge into one" begin + # The same shape as TestShards' shard records: a distributed sweep produces one record per + # worker, and the provenance statement is about the whole run. + @test_broken ExperimentalAPI.merge_records([ + ExperimentalAPI.record(() -> Sim.driver(M, 10)) for _ in 1:2 + ]) isa AbstractVector +end + +@testset "merging is associative and order-independent" begin + # Workers finish in arbitrary order; the provenance must not depend on that. + @test_broken let a = ExperimentalAPI.record(() -> Sim.driver(M, 10)), + b = ExperimentalAPI.record(() -> Sim.sweep(M, 10)) + + ExperimentalAPI.merge_records([a, b]) == ExperimentalAPI.merge_records([b, a]) + end +end + +# ── coexistence with the profiler people already use ───────────────────────────────────────── + +@testset "recording does not disturb Profile" begin + # Nobody will adopt a provenance tool that breaks their performance workflow. + @test_broken ExperimentalAPI.record(() -> Sim.driver(M, 10); with_profile=true) isa + AbstractVector +end + +@testset "an existing Profile buffer can be attributed after the fact" begin + # If someone already profiled a run, they should not have to run it again for twelve hours. + @test_broken ExperimentalAPI.attribute(Profile.fetch()) isa AbstractVector +end + +# ── the output is evidence, not a printout ─────────────────────────────────────────────────── + +@testset "a record is serialisable" begin + # The point is to put it next to a figure in a paper. A pretty-printed table is not evidence. + @test_broken ExperimentalAPI.write_record( + tempname(), ExperimentalAPI.record(() -> Sim.driver(M, 10)) + ) isa AbstractString +end + +@testset "a serialised record round-trips" begin + @test_broken ExperimentalAPI.read_record( + ExperimentalAPI.write_record( + tempname(), ExperimentalAPI.record(() -> Sim.driver(M, 10)) + ), + ) isa AbstractVector +end + +@testset "a record names the versions it was taken against" begin + # `energy` being experimental in v0.3 says nothing about v0.9. A provenance record that does + # not pin the version is not provenance. + @test_broken ExperimentalAPI.record(() -> Sim.driver(M, 10)).versions isa AbstractDict +end + +@testset "the reason is carried into the record" begin + # The record has to be readable a year later by someone who never saw the source. + @test_broken occursin( + "convergence", first(ExperimentalAPI.record(() -> Sim.driver(M, 10))).reason + ) +end + +# ── using it as a gate ─────────────────────────────────────────────────────────────────────── + +@testset "a run can be asserted to have touched nothing experimental" begin + # The publishable-result gate: this figure was produced without entering unvalidated code. + @test_broken ExperimentalAPI.assert_clean(() -> 1 + 1) +end + +@testset "the assertion fails, naming the mark, when the run is not clean" begin + # A gate that cannot be shown to fire has not been shown to be a gate. + @test_broken !ExperimentalAPI.assert_clean(() -> Sim.driver(M, 10); throw=false) +end From 3ed8a58469bd706b5b18cec520c2a475da8803d8 Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Thu, 3 Sep 2026 06:32:33 +0000 Subject: [PATCH 03/11] style: format with the JuliaFormatter version CI actually installs `format-check` failed while the local run said everything was clean. The versions differed: CI resolves `version="2"` to 2.13.0, the throwaway environment here had 2.4.0, and the two disagree about a compact `(a; for ... end; b)` block. The block in question is rewritten as an ordinary function rather than reformatted, so it no longer depends on which 2.x is installed. Co-Authored-By: Claude Opus 5 (1M context) --- test/spec/test_spec_runtime.jl | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/spec/test_spec_runtime.jl b/test/spec/test_spec_runtime.jl index a4f8437..dcaef2e 100644 --- a/test/spec/test_spec_runtime.jl +++ b/test/spec/test_spec_runtime.jl @@ -49,14 +49,13 @@ end @testset "a marked definition is not slower than the same definition unmarked" begin unmarked(x::Float64) = x * 1.0000001 - loop(f) = ( - acc=0.0; + function loop(f) + acc = 0.0 for _ in 1:2_000_000 - ; - acc += f(1.0); - end; - acc - ) + acc += f(1.0) + end + return acc + end loop(Hot.inner) loop(unmarked) # warm both before timing either a = @elapsed loop(Hot.inner) From a9b078c5e71af93b32a97c5bd18e823eeb760977 Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Thu, 3 Sep 2026 07:20:04 +0000 Subject: [PATCH 04/11] =?UTF-8?q?test:=20fix=20the=20review=20findings=20?= =?UTF-8?q?=E2=80=94=20three=20of=20them=20were=20false=20greens?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A six-agent review of this PR found, among other things, three assertions in a spec written to prevent false greens that could not fail. Each was verified directly before being changed. CRITICAL — assertions that could not fail propagate `isempty(filter(m -> occursin("reach", String(m.name)), methods(_mark!)))`. `Method.name` is the GENERIC FUNCTION's name — always `:_mark!`, never derived from what the body calls — so the filter was unconditionally empty. The assertion would have passed even if `_mark!` called `reach` directly. Replaced by inspecting `@macroexpand`, which is where the claim actually lives. declare `!isempty(experimental(ExperimentalAPI; extensions=true))`. ExperimentalAPI marks six of its own names in `src/release.jl`, so a keyword accepted and then ignored satisfied it. Now asserts a mark whose `.mod` IS the extension. foreign `all(mk -> mk.mod === Downstream, experimental_methods(Downstream))`. `all(f, [])` is `true`, so a stub returning `[]` would have flipped it to Unexpected Pass. Non-emptiness is now part of the claim. IMPORTANT — tests that could never graduate `Profile.fetch()` with `Profile` imported nowhere and absent from `[extras]`: Broken forever for a reason unrelated to the feature. Imported, and added to the test target. `isexperimental(delicate)` where `delicate` is never marked anywhere in the file — true only if something else marked it. The test now marks it itself, through the API being specified. `since = "0.4.0"` throws `MethodError: Cannot convert String to VersionNumber` from `Mark`'s field type, not from any check in the macro. A real refusal would keep throwing, so "it throws" could never signal the fix landed. Asserts the diagnostic instead. Seven `@test_broken @eval module … end`. A module evaluates to a `Module`, so on success these report "Expression evaluated to non-Boolean", not the "Unexpected Pass" this directory's README promises. Each now ends in a Bool AND checks WHICH symbol got marked — accepting the syntax while recording the wrong name is the defect this file already caught once, for `(c::C)(x)`. IMPORTANT — missing negative controls An `experimental()` that over-reports passed every assertion in `declare` and `forms`; both now check the count and the absence of deliberately unmarked names. The constructor test now also asserts the ARGUMENT name `:s` is not marked. `compare_methods` was pinned by one byte-identical assertion under two different claims; it is now three distinct behavioural cases with a negative control. The depth-limit fixture called the mark at depth 1, so an implementation ignoring `maxdepth` passed — a five-hop chain now forces truncation, and the assertion is `=== :unknown` rather than `!== :clean`. `Threads.@threads for _ in 1:8` runs its body 8 times whatever `nthreads()` is, and CI never set a thread count — so the concurrency test could not fail for a recorder that is not thread safe. CI now sets `JULIA_NUM_THREADS: 4` and the suite asserts `nthreads() > 1`, verified to fail on one thread. Claims that would rot "Base.Experimental holds 24 entries" — measured 19 filtered / 13 documented on 1.11.9; the number moves with the version AND the counting rule, and nothing checked it. Removed; the test pins named entries instead. The README's measured-count table was deleted one day earlier for the same reason. `QAtlas.fetch has 570 methods`, repeated in four files with no date and no way to re-derive it here: dated, and labelled as not re-derived. The AbstractInterpreter and profiler measurements now state Julia 1.12.2. The Lean claim three lines above was already pinned to 4.33.1; `Core.Compiler` is internal and is the more version-sensitive of the two. Also: `module Forms` and `module Upstream` collided with `test/test_mark.jl` and `test/test_audit.jl` and were silently replacing each other in `Main`; renamed. Two `public` names that were never defined, removed. A prose grep standing in for a codegen check, replaced. A `tempname()` read without being written, routed through `stamp`. 425 passing, 112 broken. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/CI.yml | 6 ++ Project.toml | 4 +- test/spec/README.md | 14 ++++ test/spec/test_spec_declare.jl | 23 +++++- test/spec/test_spec_docstring.jl | 11 ++- test/spec/test_spec_foreign.jl | 89 ++++++++++++++-------- test/spec/test_spec_forms.jl | 116 +++++++++++++++++++---------- test/spec/test_spec_integration.jl | 32 +++++++- test/spec/test_spec_profile.jl | 14 +++- test/spec/test_spec_propagate.jl | 52 +++++++++---- test/spec/test_spec_runtime.jl | 11 ++- 11 files changed, 268 insertions(+), 104 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index b457130..7043575 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -49,6 +49,12 @@ jobs: - uses: julia-actions/cache@v3 - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 + env: + # `test/spec/test_spec_profile.jl` asserts `Threads.nthreads() > 1`. A + # `Threads.@threads` loop runs its body the same number of times whatever the thread + # count is, so a concurrency test on a single-threaded runner cannot fail for a + # recorder that is not thread safe — it would be untestable rather than passing. + JULIA_NUM_THREADS: '4' - uses: julia-actions/julia-processcoverage@v1 if: matrix.julia == '1.12' && matrix.os == 'ubuntu-latest' - uses: codecov/codecov-action@v7 diff --git a/Project.toml b/Project.toml index 14f2427..5c3fdcd 100644 --- a/Project.toml +++ b/Project.toml @@ -14,13 +14,15 @@ ExperimentalAPITestExt = "Test" [compat] Aqua = "0.8" +Profile = "1" TOML = "1" Test = "1" julia = "1.11" [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test", "Aqua"] +test = ["Test", "Aqua", "Profile"] diff --git a/test/spec/README.md b/test/spec/README.md index f3e262b..2cc07d3 100644 --- a/test/spec/README.md +++ b/test/spec/README.md @@ -13,6 +13,20 @@ are `@test_broken`. So the spec cannot silently rot in either direction: it neither blocks work that has not been done, nor lets finished work go unnoticed. +One caveat, measured rather than assumed: an expression whose value is not a `Bool` — an +`@eval module … end`, whose value is a `Module` — reports `Error: Expression evaluated to +non-Boolean` instead of `Unexpected Pass`. Both fail the suite loudly, so nothing rots either +way, but every `@eval module` block in this directory now ends in a `Bool` and checks *what* got +marked, so the message is the expected one and the assertion has content beyond "it parsed". + +## Overlap with `test/test_*.jl` + +`test_spec_declare.jl` re-covers definition forms that `test/test_mark.jl` already pins, and +`test_spec_docstring.jl` re-covers part of `test/test_audit.jl`'s bucket contract, through +separate fixtures. That is deliberate while the spec is the design document — but two fixtures +pinning one contract have to be kept in sync by hand, so the older files should be folded in or +retired once the spec stops moving. + Anything already implemented is a plain `@test`. The ratio of `@test` to `@test_broken` in this directory is the honest progress measure. diff --git a/test/spec/test_spec_declare.jl b/test/spec/test_spec_declare.jl index 11d9ac2..5ca5fac 100644 --- a/test/spec/test_spec_declare.jl +++ b/test/spec/test_spec_declare.jl @@ -1,7 +1,7 @@ # What can carry a mark. # # The current implementation marks a NAME. The intent is to mark a definition — and for -# `QAtlas.fetch`, which has 570 methods behind one name, the name is the wrong unit: a docstring +# `QAtlas.fetch`, which has 570 methods (measured 2026-09-03, not re-derived here) behind one name, the name is the wrong unit: a docstring # on `fetch` cannot say which dispatch path returns a number you can trust. # # So this file covers both: what works today (plain `@test`) and what the unit has to become @@ -91,6 +91,18 @@ end # module Declared end end +@testset "nothing else is reported as marked" begin + # Without this, an `experimental()` that leaks every declared name — rather than only the + # marked ones — passes every assertion above. + got = Set(mk.name for mk in experimental(Declared)) + @test length(got) == 12 + for unmarked in (:documented_fn, :energy, :Exact, :Numerical) + @testset "$unmarked is not reported" begin + @test unmarked ∉ got + end + end +end + @testset "the reason travels with every one of them" begin for mk in experimental(Declared) @test !isempty(strip(mk.reason)) @@ -149,8 +161,13 @@ end @testset "a mark inside a package extension is reachable from the parent" begin ext = Base.get_extension(ExperimentalAPI, :ExperimentalAPITestExt) @test ext !== nothing - # Marks declared inside `ext` live in `ext`, and asking the parent must find them. - @test_broken !isempty(ExperimentalAPI.experimental(ExperimentalAPI; extensions=true)) + # `!isempty(...)` would NOT do: ExperimentalAPI marks six of its own names in `src/release.jl` + # (dogfooding), so a keyword that is accepted and then completely ignored would satisfy it. + # The claim is that a mark whose home is the EXTENSION comes back. + @test isempty(ExperimentalAPI.experimental(ext)) # today the extension declares none + @test_broken any( + mk -> mk.mod === ext, ExperimentalAPI.experimental(ExperimentalAPI; extensions=true) + ) end @testset "auditing a package does not silently ignore its extensions" begin diff --git a/test/spec/test_spec_docstring.jl b/test/spec/test_spec_docstring.jl index 2112803..6255e9b 100644 --- a/test/spec/test_spec_docstring.jl +++ b/test/spec/test_spec_docstring.jl @@ -5,9 +5,14 @@ # should always have a docstring — is correct, and the package never required otherwise; the # framing did. # -# Julia's own code settles it. `Base.Experimental` holds 24 entries and the ones sampled on -# 2026-09-03 (`@optlevel`, `@compiler_options`, `Const`) all carry docstrings. Base marks an -# experimental surface AND documents it. The two are orthogonal, and this file pins that. +# Julia's own code settles it: `Base.Experimental` exists, and the entries sampled below carry +# docstrings. Base marks an experimental surface AND documents it — the two are orthogonal, and +# this file pins that with named entries rather than a count. +# +# No entry count is given on purpose. Measured 2026-09-03 the number moves with both the Julia +# version and the counting rule (19 filtered / 13 documented on 1.11.9), and nothing here would +# notice it drifting. The same pattern was deleted from the README one day earlier for exactly +# this reason. using ExperimentalAPI: ExperimentalAPI, @experimental, audit, experimental, isdocumented, isexperimental, mark diff --git a/test/spec/test_spec_foreign.jl b/test/spec/test_spec_foreign.jl index 07c3593..5bc00a2 100644 --- a/test/spec/test_spec_foreign.jl +++ b/test/spec/test_spec_foreign.jl @@ -2,7 +2,8 @@ # # This is the QAtlas case and the current implementation refuses it outright. # -# QAtlas.fetch is AbstractQAtlas.fetch — 570 methods, all defined by QAtlas/AbstractQAtlas. +# QAtlas.fetch is AbstractQAtlas.fetch — 570 methods (measured 2026-09-03; QAtlas is not a +# dependency here, so nothing in this suite re-derives that number and it can go stale). # `:fetch` is not QAtlas's own binding, so `audit` files it under `foreign` and says nothing # about any of the 570. # @@ -12,7 +13,7 @@ using ExperimentalAPI: ExperimentalAPI, @experimental, audit, experimental, isexperimental using Test -module Upstream +module UpstreamGeneric "The generic every downstream package extends." fetch_value(model, quantity) = error("no method for $(typeof(model)), $(typeof(quantity))") public fetch_value @@ -21,7 +22,7 @@ end module Downstream using ExperimentalAPI -using ..Upstream: Upstream, fetch_value +using ..UpstreamGeneric: UpstreamGeneric, fetch_value public Ising, Heisenberg, Energy, Susceptibility @@ -31,21 +32,21 @@ struct Energy end struct Susceptibility end # exact — trustworthy -Upstream.fetch_value(::Ising, ::Energy) = -2.0 +UpstreamGeneric.fetch_value(::Ising, ::Energy) = -2.0 # numerically delicate — this is the one that should carry a mark -Upstream.fetch_value(::Heisenberg, ::Energy) = -1.7724538509055159 +UpstreamGeneric.fetch_value(::Heisenberg, ::Energy) = -1.7724538509055159 # a whole family that is provisional -Upstream.fetch_value(::Ising, ::Susceptibility) = 0.0 -Upstream.fetch_value(::Heisenberg, ::Susceptibility) = 0.0 +UpstreamGeneric.fetch_value(::Ising, ::Susceptibility) = 0.0 +UpstreamGeneric.fetch_value(::Heisenberg, ::Susceptibility) = 0.0 end # module Downstream @testset "the fixture really is the foreign-generic shape" begin - @test parentmodule(Upstream.fetch_value) === Upstream - @test length(methods(Upstream.fetch_value)) == 5 - @test count(m -> m.module === Downstream, methods(Upstream.fetch_value)) == 4 + @test parentmodule(UpstreamGeneric.fetch_value) === UpstreamGeneric + @test length(methods(UpstreamGeneric.fetch_value)) == 5 + @test count(m -> m.module === Downstream, methods(UpstreamGeneric.fetch_value)) == 4 @test :fetch_value ∉ ExperimentalAPI.surface(Downstream) # invisible to names() end @@ -53,32 +54,49 @@ end # Today's behaviour, pinned so the change is visible when it happens. @test_throws LoadError @eval module RefusedForeign using ExperimentalAPI - using ..Upstream - @experimental "why" Upstream.fetch_value(::Int, ::Int) = 0 + using ..UpstreamGeneric + @experimental "why" UpstreamGeneric.fetch_value(::Int, ::Int) = 0 end end @testset "a method on a foreign generic can be marked" begin - @test_broken @eval module MarkedForeign - using ExperimentalAPI - using ..Upstream - struct Probe end - @experimental "provisional" Upstream.fetch_value(::Probe, ::Probe) = 0 + # Ends in a Bool, and checks WHAT was marked: `@eval module … end` returns a Module, so a bare + # `@test_broken @eval module …` reports "Expression evaluated to non-Boolean" on success + # instead of the "Unexpected Pass" this directory relies on. + @test_broken begin + @eval module MarkedForeign + using ExperimentalAPI + using ..UpstreamGeneric + struct Probe end + @experimental "provisional" UpstreamGeneric.fetch_value(::Probe, ::Probe) = 0 + end + !isempty(ExperimentalAPI.experimental_methods(Main.MarkedForeign)) end end @testset "marking one foreign method leaves the siblings alone" begin - exact = which(Upstream.fetch_value, Tuple{Downstream.Ising,Downstream.Energy}) - delicate = which(Upstream.fetch_value, Tuple{Downstream.Heisenberg,Downstream.Energy}) + exact = which(UpstreamGeneric.fetch_value, Tuple{Downstream.Ising,Downstream.Energy}) + delicate = which( + UpstreamGeneric.fetch_value, Tuple{Downstream.Heisenberg,Downstream.Energy} + ) @test exact !== delicate - @test_broken ExperimentalAPI.isexperimental(delicate) - @test_broken !ExperimentalAPI.isexperimental(exact) + # The fixture cannot carry `@experimental` on these methods: the macro refuses a qualified + # definition today and the whole module would fail to load. So the test MARKS IT ITSELF + # through the future API. Asserting `isexperimental(delicate)` without ever marking it would + # stay Broken forever, even once method-level marking works perfectly. + @test_broken begin + ExperimentalAPI.mark_method!(delicate, "numerically delicate") + ExperimentalAPI.isexperimental(delicate) && !ExperimentalAPI.isexperimental(exact) + end end @testset "the mark is stored in the module that WROTE the method" begin - # Not in Upstream. A package cannot be made to carry claims its dependents invented, and the - # mark has to survive Upstream being reloaded or updated. - @test_broken all( + # Not in UpstreamGeneric. A package cannot be made to carry claims its dependents invented, and the + # mark has to survive UpstreamGeneric being reloaded or updated. + # + # `all(pred, [])` is `true` in Julia, so an `experimental_methods` stub that always returns an + # empty vector would satisfy a bare `all(...)`. Non-emptiness has to be part of the claim. + @test_broken !isempty(ExperimentalAPI.experimental_methods(Downstream)) && all( mk -> mk.mod === Downstream, ExperimentalAPI.experimental_methods(Downstream) ) end @@ -86,7 +104,7 @@ end @testset "asking the generic finds marks contributed by every package" begin # The question a user asks is about `fetch_value`, not about which package happened to define # the method they will dispatch to. - @test_broken !isempty(ExperimentalAPI.experimental(Upstream.fetch_value)) + @test_broken !isempty(ExperimentalAPI.experimental(UpstreamGeneric.fetch_value)) end @testset "audit reports foreign methods this module owns" begin @@ -106,19 +124,20 @@ end # Julia stores docstrings keyed by signature, so "documented" is answerable per method — the # audit does not have to fall back to the name. @test_broken ExperimentalAPI.isdocumented( - which(Upstream.fetch_value, Tuple{Downstream.Ising,Downstream.Energy}) + which(UpstreamGeneric.fetch_value, Tuple{Downstream.Ising,Downstream.Energy}) ) isa Bool end @testset "marking a method does not make the foreign NAME experimental" begin # Otherwise one downstream package could label another package's whole generic unfinished. - @test !isexperimental(Upstream, :fetch_value) + @test !isexperimental(UpstreamGeneric, :fetch_value) end @testset "propagation crosses the package boundary" begin - # A caller in a third package that reaches a marked method in Downstream, through Upstream's + # A caller in a third package that reaches a marked method in Downstream, through UpstreamGeneric's # generic, must be reported. This is the QAtlas → analysis-script path. - caller(x) = Upstream.fetch_value(Downstream.Heisenberg(), Downstream.Energy()) + x + caller(x) = + UpstreamGeneric.fetch_value(Downstream.Heisenberg(), Downstream.Energy()) + x @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(caller, Tuple{Float64})) === :depends end @@ -126,10 +145,14 @@ end @testset "a mark on a method of a function defined in Base is possible" begin # `Base.show`, `Base.length`, `Base.copy` — QAtlas defines methods on all three. The rule # cannot be "Base is off limits" without excluding a large part of every package's surface. - @test_broken @eval module MarkedBase - using ExperimentalAPI - struct Widget end - @experimental "printing format not settled" Base.show(io::IO, ::Widget) = print(io, "W") + @test_broken begin + @eval module MarkedBase + using ExperimentalAPI + struct Widget end + @experimental "printing format not settled" Base.show(io::IO, ::Widget) = + print(io, "W") + end + !isempty(ExperimentalAPI.experimental_methods(Main.MarkedBase)) end end diff --git a/test/spec/test_spec_forms.jl b/test/spec/test_spec_forms.jl index 61bdd5e..404a5f3 100644 --- a/test/spec/test_spec_forms.jl +++ b/test/spec/test_spec_forms.jl @@ -8,11 +8,11 @@ using ExperimentalAPI: ExperimentalAPI, @experimental, experimental, isexperimental, mark using Test -module Forms +module FormsSpec using ExperimentalAPI -public kw_fn, where_fn, vararg_fn, ret_typed, Callable, Ctor, Gen, KwStruct, INTERP +public kw_fn, where_fn, vararg_fn, ret_typed, Callable, Ctor, INTERP @experimental "keyword arguments" kw_fn(x; scale=1.0, kwargs...) = x * scale @experimental "parametric" where_fn(x::T, y::T) where {T<:Real} = x + y @@ -30,22 +30,22 @@ end const WHY = "tolerance chosen by hand" @experimental "$(WHY); see the sweep in issue 12" INTERP = 1e-8 -end # module Forms +end # module FormsSpec @testset "forms that already work" begin - got = Dict(mk.name => mk for mk in experimental(Forms)) + got = Dict(mk.name => mk for mk in experimental(FormsSpec)) for n in (:kw_fn, :where_fn, :vararg_fn, :ret_typed, :INTERP) @testset "$n" begin @test haskey(got, n) @test !isempty(strip(got[n].reason)) end end - @test Forms.kw_fn(2.0; scale=3.0) == 6.0 - @test Forms.where_fn(1, 2) == 3 + @test FormsSpec.kw_fn(2.0; scale=3.0) == 6.0 + @test FormsSpec.where_fn(1, 2) == 3 end @testset "an interpolated reason is evaluated, not stored as source" begin - @test occursin("tolerance chosen by hand", mark(Forms, :INTERP).reason) + @test occursin("tolerance chosen by hand", mark(FormsSpec, :INTERP).reason) end # ── forms that are not covered ─────────────────────────────────────────────────────────────── @@ -79,60 +79,84 @@ end end @experimental "validation not implemented" T(s::AbstractString) = T(parse(Int, s)) end - @test :T in [mk.name for mk in experimental(Main.CtorMarked)] + got = Set(mk.name for mk in experimental(Main.CtorMarked)) + @test :T in got + # …and the ARGUMENT name is not also marked. The sibling testset above found exactly that + # defect for `(c::C)(x)`; an over-inclusive walk would reintroduce it here unnoticed. + @test :s ∉ got end @testset "an inner constructor inside a marked struct is not separately marked" begin # Marking the struct should not silently also claim its inner constructors are unfinished, # nor silently exclude them. Whichever it is has to be stated. - @test_broken hasproperty(mark(Forms, :Callable), :includes_constructors) + @test_broken hasproperty(mark(FormsSpec, :Callable), :includes_constructors) end @testset "an operator method can be marked" begin - @test_broken @eval module OpMarked - using ExperimentalAPI - struct V - x::Float64 - end - @experimental "no identity element yet" Base.:+(a::V, b::V) = V(a.x + b.x) + # `@eval module … end` evaluates to a Module, never a Bool, so wrapping it directly in + # `@test_broken` reports "Expression evaluated to non-Boolean" on success rather than the + # "Unexpected Pass" this directory relies on. Each of these now ends in a Bool AND checks + # WHAT was marked — accepting the syntax while recording the wrong symbol is the defect this + # file already caught once, for `(c::C)(x)`. + @test_broken begin + @eval module OpMarked + using ExperimentalAPI + struct V + x::Float64 + end + @experimental "no identity element yet" Base.:+(a::V, b::V) = V(a.x + b.x) + end + :+ in [mk.name for mk in experimental(Main.OpMarked)] end end @testset "a generated function can be marked" begin - @test_broken @eval module GenMarked - using ExperimentalAPI - @experimental "generator is a prototype" @generated g(x) = :(x) + @test_broken begin + @eval module GenMarked + using ExperimentalAPI + @experimental "generator is a prototype" @generated g(x) = :(x) + end + :g in [mk.name for mk in experimental(Main.GenMarked)] end end @testset "Base.@kwdef stacks with the mark" begin # `@kwdef` expands to a block carrying `Expr(:meta, :doc)`. Two macros that both wrap a # definition must compose in at least one order, and the order that works must be documented. - @test_broken @eval module KwdefMarked - using ExperimentalAPI - @experimental "defaults are guesses" Base.@kwdef struct S - a::Int = 1 - end + @test_broken begin + @eval module KwdefMarked + using ExperimentalAPI + @experimental "defaults are guesses" Base.@kwdef struct S + a::Int = 1 + end + end + :S in [mk.name for mk in experimental(Main.KwdefMarked)] end end @testset "@inline and the mark compose in both orders" begin - @test_broken @eval module InlineMarked - using ExperimentalAPI - @experimental "kernel unverified" @inline f(x) = x - @inline @experimental "kernel unverified" g(x) = x + @test_broken begin + @eval module InlineMarked + using ExperimentalAPI + @experimental "kernel unverified" @inline f(x) = x + @inline @experimental "kernel unverified" g(x) = x + end + Set([mk.name for mk in experimental(Main.InlineMarked)]) == Set([:f, :g]) end end @testset "a definition produced by @eval can be marked by name" begin # Metaprogrammed definitions cannot be attached to; the name-list form is the answer and it # has to be reachable. - @test_broken @eval module EvalMarked - using ExperimentalAPI - for n in (:a, :b) - @eval $n(x) = x - end - @experimental "generated in a loop" a b + @test_broken begin + @eval module EvalMarked + using ExperimentalAPI + for n in (:a, :b) + @eval $n(x) = x + end + @experimental "generated in a loop" a b + end + Set([mk.name for mk in experimental(Main.EvalMarked)]) == Set([:a, :b]) end end @@ -177,16 +201,28 @@ end # ── metadata ───────────────────────────────────────────────────────────────────────────────── -@testset "since must be a version, not a string" begin - @test_broken @eval module BadSince - using ExperimentalAPI - @experimental("why", since = "0.4.0", f(x) = x) +@testset "since must be a version, and the refusal must say so" begin + # It IS refused today, but by accident: `Mark.since::Union{VersionNumber,Nothing}` cannot + # convert a String, so the error is + # MethodError: Cannot `convert` an object of type String to an object of type VersionNumber + # which names neither `since` nor `@experimental`. A deliberate check would keep throwing, so + # asserting "it throws" could never signal that the fix had landed — assert the DIAGNOSTIC. + e = try + @eval module BadSince + using ExperimentalAPI + @experimental("why", since = "0.4.0", f(x) = x) + end + nothing + catch err + err isa LoadError ? err.error : err end + @test e isa MethodError # today, and accidental + @test_broken occursin("since", sprint(showerror, e)) end @testset "an unknown keyword is refused rather than ignored" begin - # `@experimental "why" tracking_url="..." f(x)=x` — a typo in a keyword name must not silently - # become part of the subject. + # `@experimental("why", trackign = "u", f(x) = x)` — a typo in a keyword name must not + # silently become part of the subject. @test_throws LoadError @eval module BadKw using ExperimentalAPI @experimental("why", trackign = "u", f(x) = x) @@ -196,7 +232,7 @@ end @testset "tracking is carried through to every report" begin # The field that turns a warning into something a reader can act on. It is stored today; it # has to survive into the audit and the record as well. - @test_broken ExperimentalAPI.audit(Forms).tracking isa AbstractDict + @test_broken ExperimentalAPI.audit(FormsSpec).tracking isa AbstractDict end # ── same name, two places ──────────────────────────────────────────────────────────────────── diff --git a/test/spec/test_spec_integration.jl b/test/spec/test_spec_integration.jl index 6bf7afd..af29cdb 100644 --- a/test/spec/test_spec_integration.jl +++ b/test/spec/test_spec_integration.jl @@ -70,13 +70,33 @@ end @testset "removing a marked METHOD is not breaking" begin # The same contract as for names, at the granularity that matters for a dispatch table. - @test_broken ExperimentalAPI.compare_methods isa Function + # These two testsets used to share a byte-identical `compare_methods isa Function`, so both + # would flip the instant the function existed, with neither behavioural claim ever checked. + @test_broken !ExperimentalAPI.isbreaking( + ExperimentalAPI.compare_methods( + Dict("methods" => Dict("f(::Int)" => Dict("reason" => "r"))), + Dict("methods" => Dict()), + ), + ) +end + +@testset "removing a SETTLED method is breaking" begin + # The negative control for the testset above: same shape, unmarked method, opposite verdict. + @test_broken ExperimentalAPI.isbreaking( + ExperimentalAPI.compare_methods( + Dict("stable_methods" => ["f(::Int)"]), Dict("stable_methods" => String[]) + ), + ) end @testset "a signature change to a settled method is reported as breaking" begin # The blind spot `compare` currently admits to in its own docstring. Method-level marks are # what make it addressable at all. - @test_broken ExperimentalAPI.compare_methods isa Function + @test_broken ExperimentalAPI.isbreaking( + ExperimentalAPI.compare_methods( + Dict("stable_methods" => ["f(::Int)"]), Dict("stable_methods" => ["f(::Real)"]) + ), + ) end # ── the provenance record next to a result ─────────────────────────────────────────────────── @@ -90,8 +110,12 @@ end @testset "the stamp is readable without loading the package that made it" begin # A year later the package may not resolve. Plain TOML or JSON, not a serialised Julia object. - path = tempname() - @test_broken occursin("reference value", read(path, String)) + # The path has to go through `stamp` first — reading a bare `tempname()` throws SystemError + # for a reason that has nothing to do with the claim. + @test_broken occursin( + "reference value", + read(ExperimentalAPI.stamp(tempname(), () -> Shown.provisional(1)), String), + ) end @testset "a stamped result names the package versions involved" begin diff --git a/test/spec/test_spec_profile.jl b/test/spec/test_spec_profile.jl index 02c913c..eb5b766 100644 --- a/test/spec/test_spec_profile.jl +++ b/test/spec/test_spec_profile.jl @@ -4,12 +4,14 @@ # twelve-hour DMRG run the question is not "is this function experimental" but "did the number I # am about to put in a paper come out of code nobody has validated, and how much of it". # -# Everything here is `@test_broken`. The cheap route was measured on 2026-09-03 and does not +# Everything here is `@test_broken`. The cheap route was measured on 2026-09-03, **Julia +# 1.12.2**, and does not # work: `Profile.fetch` frames for inlined callees carry no `MethodInstance`, so a sampling # profiler attributes ZERO samples to marked methods — and small functions, which is most of what # gets marked, are exactly the ones that get inlined. using ExperimentalAPI: ExperimentalAPI, @experimental +using Profile: Profile using Test module Sim @@ -79,7 +81,7 @@ end # ── granularity ────────────────────────────────────────────────────────────────────────────── @testset "attribution is to a method, not to a name" begin - # `QAtlas.fetch` has 570 methods. "the run touched fetch" is unusable. + # `QAtlas.fetch` has 570 methods (measured 2026-09-03, not re-derived here). "the run touched fetch" is unusable. @test_broken first(ExperimentalAPI.record(() -> Sim.driver(M, 10))).method isa Method end @@ -158,6 +160,14 @@ end # ── concurrency and distribution ───────────────────────────────────────────────────────────── +@testset "the suite runs with more than one thread" begin + # `Threads.@threads for _ in 1:8` executes its body 8 times whatever `nthreads()` is, so the + # test below would pass for a recorder that is not thread-safe at all if CI ran single + # threaded. `.github/workflows/CI.yml` sets `JULIA_NUM_THREADS` for this reason; if that ever + # regresses, this fails instead of the concurrency claim silently becoming untestable. + @test Threads.nthreads() > 1 +end + @testset "hits from every thread are attributed" begin # HPC code is threaded. A recorder that only sees the main thread reports a fraction of the # truth and calls it the truth. diff --git a/test/spec/test_spec_propagate.jl b/test/spec/test_spec_propagate.jl index e0e32a6..fac65ca 100644 --- a/test/spec/test_spec_propagate.jl +++ b/test/spec/test_spec_propagate.jl @@ -19,10 +19,14 @@ # reports "no experimental dependency" about a call graph it could not see has not made a weaker # claim, it has made a false one. # -# Feasibility was measured on 2026-09-03 with a custom `Core.Compiler.AbstractInterpreter` -# hooking `abstract_call_method`. Inference runs before inlining, so the call graph is intact -# there; post-processing `code_typed(...; optimize=true)` sees only `mul_float`/`add_float` and -# finds nothing. +# Feasibility was measured on 2026-09-03, **Julia 1.12.2**, with a custom +# `Core.Compiler.AbstractInterpreter` hooking `abstract_call_method`. Inference runs before +# inlining, so the call graph is intact there; post-processing `code_typed(...; optimize=true)` +# sees only `mul_float`/`add_float` and finds nothing. +# +# The Julia version is load-bearing and is stated for the same reason `Lean 4.33.1` is above: +# `Core.Compiler` is internal and carries no stability guarantee across releases. This repository's +# CI spans 1.11 and 1.12, and the measurement was taken on one of them. using ExperimentalAPI: ExperimentalAPI, @experimental, experimental using Test @@ -79,6 +83,15 @@ top_table(i::Int, x::Float64) = TABLE[i](x) # termination top_recursive(n::Int, x::Float64) = n <= 0 ? unstable(x) : top_recursive(n - 1, x) + +# A chain deep enough that a depth limit MUST truncate before reaching the mark. `top_recursive` +# calls `unstable` in its own body, so it is visible at depth 1 whatever `maxdepth` says — a +# depth test written against it would pass for an implementation that ignores the keyword. +deep_5(x::Float64) = unstable(x) +deep_4(x::Float64) = deep_5(x) +deep_3(x::Float64) = deep_4(x) +deep_2(x::Float64) = deep_3(x) +deep_1(x::Float64) = deep_2(x) top_mutual_a(n::Int, x::Float64) = n <= 0 ? unstable(x) : top_mutual_b(n - 1, x) top_mutual_b(n::Int, x::Float64) = top_mutual_a(n - 1, x) @@ -163,9 +176,15 @@ end end @testset "a depth limit reports :unknown rather than :clean" begin + # `deep_1` is five hops from the mark, so `maxdepth=2` must truncate. Asserting `:unknown` + # rather than `!== :clean` is what separates "the limit produced the honest non-answer" from + # "the limit was silently ignored and the mark was found anyway". @test_broken ExperimentalAPI.verdict( - ExperimentalAPI.reach(Chain.top_recursive, Tuple{Int,Float64}; maxdepth=1) - ) !== :clean + ExperimentalAPI.reach(Chain.deep_1, ENTRY; maxdepth=2) + ) === :unknown + # …and the same entry point WITHOUT the limit finds it, so the fixture can disagree. + @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(Chain.deep_1, ENTRY)) === + :depends end # ── termination ────────────────────────────────────────────────────────────────────────────── @@ -214,12 +233,17 @@ end # ── cost ───────────────────────────────────────────────────────────────────────────────────── -@testset "analysis is opt-in and costs nothing when not asked for" begin - # `@experimental` emits the definition unchanged plus one `push!` at load time. Whatever the - # analysis costs, it must not move into the marked package's own load or call path. - @test isempty( - filter( - m -> occursin("reach", String(m.name)), collect(methods(ExperimentalAPI._mark!)) - ), - ) +@testset "the macro emits nothing but the definition and one push" begin + # The previous version of this test filtered `methods(_mark!)` for a name containing "reach". + # `Method.name` is the GENERIC FUNCTION's name — always `:_mark!`, never derived from what the + # body calls — so the filter was unconditionally empty and the assertion could not fail even + # if `_mark!` called `reach` directly. Look at the expansion instead. + emitted = string(@macroexpand @experimental "why" f(x) = x) + @test occursin("_mark!", emitted) # the one load-time effect + @test occursin("__EXPERIMENTAL_API_MARKS__", emitted) + for forbidden in ("reach", "verdict", "record", "abstract_call_method", "code_typed") + @testset "no $forbidden in the expansion" begin + @test !occursin(forbidden, emitted) + end + end end diff --git a/test/spec/test_spec_runtime.jl b/test/spec/test_spec_runtime.jl index dcaef2e..1ad88bb 100644 --- a/test/spec/test_spec_runtime.jl +++ b/test/spec/test_spec_runtime.jl @@ -4,7 +4,7 @@ # unverified code paths did it go through, and how often? A docstring cannot answer that. This is # the part of the intent that a reviewer cannot dismiss as "put it in the docstring". # -# Two routes were measured on 2026-09-03, and the cheap one does NOT work: +# Two routes were measured on 2026-09-03, **Julia 1.12.2**, and the cheap one does NOT work: # # sampling profiler + Method-set membership -> 0 samples attributed # `Profile.fetch` frames for inlined callees carry no `MethodInstance`, so exactly the @@ -42,8 +42,11 @@ end # module Hot # The property the package currently advertises and must keep: `@experimental` emits the # definition unchanged plus one `push!` at load time. If instrumentation ever becomes # unconditional, an inner loop pays for it on every iteration. - src = read(joinpath(@__DIR__, "..", "..", "src", "mark.jl"), String) - @test occursin("emitted unchanged", src) || occursin("costs nothing at run time", src) + # Reading `src/mark.jl` for a prose phrase would pass for a regression that wrapped the call + # and left the sentence alone. Look at what the macro emits. + emitted = string(@macroexpand @experimental "why" f(x) = x) + @test occursin("f(x)", emitted) # the definition is there… + @test !occursin("function f", replace(emitted, "f(x)" => "")) # …and not wrapped in another @test Hot.driver(3, 1.0) ≈ 3 * 1.0000001 end @@ -94,7 +97,7 @@ end end @testset "the record attributes to a method, not just a name" begin - # `QAtlas.fetch` has 570 methods. "the run touched `fetch`" is not usable; "the run took the + # `QAtlas.fetch` has 570 methods (measured 2026-09-03, not re-derived here). "the run touched `fetch`" is not usable; "the run took the # Numerical × Float64 path 4.7M times" is. @test_broken first(ExperimentalAPI.record(() -> Hot.driver(10, 1.0))).method isa Method end From 36a297d154dc404c3d09f552712d61c09cac2e34 Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Thu, 3 Sep 2026 07:24:35 +0000 Subject: [PATCH 05/11] test: dispatch branching, and what happens when the mark is written lazily MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more groups, both measured before being written. **Dispatch branching** (`test_spec_dispatch.jl`). One call site, several methods, only some of them marked — the shape that makes the method-level unit worth having. `QAtlas.fetch` has 570 methods; a caller writes `fetch(model, quantity)` once and a verdict about the NAME says nothing about the one that runs. The dangerous case is a call site that cannot be pinned to a method. Measured 2026-09-03, Julia 1.12.2: for an argument typed `Union{Exact,Numerical}` or an abstract `Kind`, the un-optimised IR shows only `(%1)(_2)` and `which(f, T)` **throws**. An implementation that catches that and moves on reports `:clean` about a call that reaches a marked method half the time — so there is an explicit test that `which` throwing must not be swallowed. Covered: the negative control (a site that can only reach settled methods must be `:clean`), Union and abstract branching, `invoke` pinning a method dispatch would not pick, a more specific unmarked method shadowing a marked fallback (and the fall-through that does reach it), and that two call sites on the SAME name get different verdicts. **Lazy usage** (added to `test_spec_forms.jl`). The reason is the payload, so the question is what happens without one. Measured: every lazy form is refused, but two are refused by accident and one points the wrong way. @experimental :sym f(x) = x MethodError: no method matching strip(::Symbol) @experimental 42 f(x) = x MethodError: no method matching strip(::Int64) Refused by `strip` failing inside `_reason`, with a message naming neither `@experimental` nor `reason` — the same shape as the `since = "0.4.0"` case. @experimental f(x) = x "nothing to mark — give a definition or a name" The author DID give a definition; what is missing is the reason. The message points at the wrong end of the call, which is how someone deletes a correct definition trying to satisfy it. Also asserts that a refused mark leaves the module clean — no mark recorded, no name defined — rather than only that an exception came out. 451 passing, 125 broken. Co-Authored-By: Claude Opus 5 (1M context) --- test/runtests.jl | 1 + test/spec/README.md | 1 + test/spec/test_spec_dispatch.jl | 168 ++++++++++++++++++++++++++++++++ test/spec/test_spec_forms.jl | 96 ++++++++++++++++++ 4 files changed, 266 insertions(+) create mode 100644 test/spec/test_spec_dispatch.jl diff --git a/test/runtests.jl b/test/runtests.jl index f737e1c..9780c55 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -22,5 +22,6 @@ using Test include("spec/test_spec_foreign.jl") include("spec/test_spec_forms.jl") include("spec/test_spec_integration.jl") + include("spec/test_spec_dispatch.jl") include("test_aqua.jl") end diff --git a/test/spec/README.md b/test/spec/README.md index 2cc07d3..ad31411 100644 --- a/test/spec/README.md +++ b/test/spec/README.md @@ -40,6 +40,7 @@ directory is the honest progress measure. | `test_spec_verify.jl` | how well is a marked thing exercised by the tests | | `test_spec_runtime.jl` | the floor: the mark must not wrap the call | | `test_spec_profile.jl` | what a real run went through, how often, and how much of it | +| `test_spec_dispatch.jl` | one call site, several methods, only some marked — the branch | | `test_spec_integration.jl` | where the mark has to surface: docs, Aqua, releases, provenance, CI | ## What the spec already found diff --git a/test/spec/test_spec_dispatch.jl b/test/spec/test_spec_dispatch.jl new file mode 100644 index 0000000..bb428d0 --- /dev/null +++ b/test/spec/test_spec_dispatch.jl @@ -0,0 +1,168 @@ +# Dispatch-level branching: one call site, several methods, only some of them marked. +# +# This is the shape that makes the method-level unit worth having at all. `QAtlas.fetch` has 570 +# methods; a caller writes `fetch(model, quantity)` once, and which of the 570 runs — and whether +# that one is trustworthy — depends on the argument types. A verdict about the NAME says nothing. +# +# The dangerous case is a call site the analysis cannot pin to one method. Measured 2026-09-03, +# Julia 1.12.2: for an argument typed `Union{Exact,Numerical}` or an abstract `Kind`, the +# un-optimised IR shows only `(%1)(_2)` and `which(f, T)` **throws** — there is no unique method. +# An implementation that catches that exception and moves on would report `:clean` about a call +# that reaches a marked method at run time half the time. That is the failure this file guards. + +using ExperimentalAPI: ExperimentalAPI, @experimental, experimental, isexperimental +using Test + +module Dispatch + +using ExperimentalAPI + +public Exact, + Numerical, + Kind, + KA, + KB, + energy, + k, + only_settled, + either_way, + via_abstract, + via_invoke, + more_specific + +struct Exact end +struct Numerical end + +"Closed form; trustworthy." +energy(::Exact) = 1.0 +@experimental "no convergence proof below β ≈ 0.1" energy(::Numerical) = 1.0000001 + +abstract type Kind end +struct KA <: Kind end +struct KB <: Kind end +"Settled." +k(::KA) = 1.0 +@experimental "extrapolated, never cross-checked" k(::KB) = 2.0 + +# The call site can only ever reach the settled method — the negative control. +only_settled(x::Exact) = energy(x) + +# The call site can reach EITHER method depending on the run-time type. +either_way(x::Union{Exact,Numerical}) = energy(x) +via_abstract(x::Kind) = k(x) + +# `invoke` pins a method regardless of the argument's run-time type. +via_invoke(x::Numerical) = invoke(energy, Tuple{Numerical}, x) + +# A more specific unmarked method shadows a marked less specific one. +"Settled, and more specific." +more_specific(::Int) = 0 +@experimental "the fallback is a placeholder" more_specific(::Integer) = 1 + +end # module Dispatch + +# ── the fixture really has the shape the file claims ───────────────────────────────────────── + +@testset "only some methods behind each name are marked" begin + marked = Set(mk.name for mk in experimental(Dispatch)) + @test :energy in marked + @test :k in marked + @test length(methods(Dispatch.energy)) == 2 + @test length(methods(Dispatch.k)) == 2 + # Today a mark is name-keyed, so it necessarily covers BOTH methods of each name — including + # the closed-form one that is perfectly trustworthy. That over-claim is the problem. + @test isexperimental(Dispatch, :energy) +end + +@testset "a branching call site has no unique method" begin + # The measured fact the whole file rests on: `which` cannot answer for these argument types. + @test which(Dispatch.energy, Tuple{Dispatch.Exact}) isa Method + @test_throws Exception which( + Dispatch.energy, Tuple{Union{Dispatch.Exact,Dispatch.Numerical}} + ) + @test_throws Exception which(Dispatch.k, Tuple{Dispatch.Kind}) +end + +# ── what the analysis has to say about each shape ──────────────────────────────────────────── + +@testset "a call site that can only reach settled methods is clean" begin + # The negative control. Without it, everything below is satisfied by a tool that answers + # ":depends" for every call site with more than one candidate. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Dispatch.only_settled, Tuple{Dispatch.Exact}) + ) === :clean +end + +@testset "a Union-typed call site that could reach a mark is not clean" begin + # Half the run-time values take the marked branch. `:clean` here is a false statement, not a + # conservative one. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach( + Dispatch.either_way, Tuple{Union{Dispatch.Exact,Dispatch.Numerical}} + ), + ) !== :clean +end + +@testset "an abstract-typed call site that could reach a mark is not clean" begin + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Dispatch.via_abstract, Tuple{Dispatch.Kind}) + ) !== :clean +end + +@testset "the unresolvable call site is named, not just counted" begin + # "Somewhere in here I could not tell" is not actionable. The report has to point at the call. + @test_broken any( + u -> occursin("k", string(u)), + ExperimentalAPI.reach(Dispatch.via_abstract, Tuple{Dispatch.Kind}).unresolved, + ) +end + +@testset "which() throwing must not be swallowed into :clean" begin + # The specific implementation mistake this file exists to prevent: wrapping `which` in a + # try/catch, skipping the call site, and reporting the remaining graph as clean. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Dispatch.via_abstract, Tuple{Dispatch.Kind}) + ) === :unknown +end + +# ── dispatch subtleties ────────────────────────────────────────────────────────────────────── + +@testset "invoke pins the method it names" begin + # `invoke(energy, Tuple{Numerical}, x)` reaches the marked method unconditionally, even + # though the argument's type would have selected it anyway. An analysis that only looks at + # argument types would miss `invoke` pinning a DIFFERENT method than dispatch would pick. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Dispatch.via_invoke, Tuple{Dispatch.Numerical}) + ) === :depends +end + +@testset "a more specific unmarked method shadows a marked one" begin + # `more_specific(::Int)` wins over `more_specific(::Integer)`, so a call with an `Int` never + # reaches the mark. Reporting `:depends` because *some* method of the name is marked is the + # name-level over-claim all over again, one level down. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Dispatch.more_specific, Tuple{Int}) + ) === :clean + # …and a call that does fall through to the marked fallback is reported. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Dispatch.more_specific, Tuple{UInt8}) + ) === :depends +end + +@testset "a mark on one method does not leak to its siblings at a call site" begin + # The whole point of going to method granularity: `only_settled` and `either_way` call the + # same NAME, and must get different verdicts. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Dispatch.only_settled, Tuple{Dispatch.Exact}) + ) !== ExperimentalAPI.verdict( + ExperimentalAPI.reach( + Dispatch.either_way, Tuple{Union{Dispatch.Exact,Dispatch.Numerical}} + ), + ) +end + +@testset "the report says WHICH method was reached, not which name" begin + @test_broken first( + ExperimentalAPI.reach(Dispatch.either_way, Tuple{Dispatch.Numerical}).reached + ).method === which(Dispatch.energy, Tuple{Dispatch.Numerical}) +end diff --git a/test/spec/test_spec_forms.jl b/test/spec/test_spec_forms.jl index 404a5f3..a83e616 100644 --- a/test/spec/test_spec_forms.jl +++ b/test/spec/test_spec_forms.jl @@ -235,6 +235,102 @@ end @test_broken ExperimentalAPI.audit(FormsSpec).tracking isa AbstractDict end +# ── writing it lazily ──────────────────────────────────────────────────────────────────────── +# +# The reason is the payload: why a name is not settled is knowledge only the author has. So the +# interesting question is not "does the good form work" but "what happens when someone writes it +# without one". Measured 2026-09-03: every lazy form IS refused — but two of them are refused by +# accident, and two more are refused with a message pointing the wrong way. + +@testset "a bare @experimental is refused" begin + for ex in ( + :(@experimental), + :(@experimental foo), + :(@experimental function f(x) + x + end), + :(@experimental struct S + v::Int + end), + :(@experimental f(x) = x), + :(@experimental const C = 1), + ) + @testset "$(first(string(ex), 40))" begin + m = Module(:LazyProbe) + Core.eval(m, :(using ExperimentalAPI)) + @test_throws Exception Core.eval(m, ex) + end + end +end + +@testset "an empty reason is refused, and the message says why" begin + for r in ("", " ", "\n\t ") + @testset "reason=$(repr(r))" begin + m = Module(:EmptyProbe) + Core.eval(m, :(using ExperimentalAPI)) + e = try + Core.eval(m, :(@experimental $r f(x) = x)) + nothing + catch err + err isa LoadError ? err.error : err + end + @test e isa ArgumentError + @test occursin("reason", sprint(showerror, e)) + end + end +end + +@testset "a non-string reason is refused by accident, not by a check" begin + # `@experimental :sym f(x) = x` and `@experimental 42 f(x) = x` both die inside `_reason` + # with `MethodError: no method matching strip(::Symbol)` / `strip(::Int64)`. Refused, yes — + # but by `strip` failing, with a message that names neither `@experimental` nor `reason`. + # Same shape as the `since = "0.4.0"` case above. + for r in (:(:sym), 42) + @testset "reason=$(repr(r))" begin + m = Module(:BadReasonProbe) + Core.eval(m, :(using ExperimentalAPI)) + e = try + Core.eval(m, :(@experimental $r f(x) = x)) + nothing + catch err + err isa LoadError ? err.error : err + end + @test e isa MethodError # today, and accidental + @test_broken occursin("reason", sprint(showerror, e)) + end + end +end + +@testset "forgetting the reason is diagnosed as a missing reason" begin + # `@experimental f(x) = x` is refused with "nothing to mark — give a definition or a name". + # The author DID give a definition; what is missing is the reason. The message points at the + # wrong end of the call, which is how someone ends up deleting a correct definition. + m = Module(:NoReasonProbe) + Core.eval(m, :(using ExperimentalAPI)) + e = try + Core.eval(m, :(@experimental f(x) = x)) + nothing + catch err + err isa LoadError ? err.error : err + end + @test e isa ArgumentError + @test occursin("nothing to mark", sprint(showerror, e)) # today, and misleading + @test_broken occursin("reason", sprint(showerror, e)) +end + +@testset "nothing is marked when the macro refuses" begin + # A refusal that still recorded a mark would be worse than either outcome. Checks the module + # is left clean, not just that an exception came out. + m = Module(:RefusedProbe) + Core.eval(m, :(using ExperimentalAPI)) + try + Core.eval(m, :(@experimental f(x) = x)) + catch + end + @test isempty(experimental(m)) + @test !isdefined(m, :f) +end + # ── same name, two places ──────────────────────────────────────────────────────────────────── @testset "the same name marked in two modules stays separate" begin From d526200ee9a11523b6dfd872ebb865c8df2706bf Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Thu, 3 Sep 2026 07:37:40 +0000 Subject: [PATCH 06/11] test: the mark's exit, and an entry point that is not a single function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cross-checked the spec against the design letter on General#166832. Ten of the twelve claims in it already had coverage. Two did not, and both are quoted in the new file's header because they are the author's words, not a paraphrase. 「`sorry`がついた真偽不明の命題として e2e でコードの解析を実行できる」 `reach(f, argtypes)` starts from one function with concrete argument types. Lean's `#print axioms` answers for any declaration, and the analogue is an entry point that is a MODULE — "does anything this package exposes reach unvalidated code" — or a script, which is the shape a researcher actually has. Neither was specified. Added, with a negative control (a module with nothing marked comes back `:clean`) and the requirement that the answer names WHICH public entries are affected: "something in here is experimental" is not actionable for a package with 310 public names. 「この `@experimental` を安全に外していく、というのを中間ゴールに据えた開発」 This is the most distinctive line in the letter and the spec had nothing for it. A mark that can only be added is a decoration; a mark with a defined exit is a plan. Added: whether a mark is ready to be removed and on what evidence, whether removal is a release event, that removing it flips its callers AND ONLY its callers, that `since` is readable so "experimental" cannot quietly become permanent, a ratchet on the mark count, and the failure mode of deleting a mark while callers still depend on it. The fixture is built so coverage alone cannot separate the two marks — both `verified_now` and `still_unverified` are exercised by this suite — because if it could, "ready to promote" would collapse into "is it tested", which is not what the letter says. One test promoted from `@test_broken` to `@test`: "removing a mark is reported as not breaking" reported Unexpected Pass, because `compare`/`isbreaking` already do this. That is the directory's mechanism working as designed. Paired with the negative control it was missing — deleting the name outright IS breaking, and without that the assertion would pass for an `isbreaking` that always answers false. 459 passing, 137 broken. Co-Authored-By: Claude Opus 5 (1M context) --- test/runtests.jl | 1 + test/spec/README.md | 1 + test/spec/test_spec_lifecycle.jl | 169 +++++++++++++++++++++++++++++++ 3 files changed, 171 insertions(+) create mode 100644 test/spec/test_spec_lifecycle.jl diff --git a/test/runtests.jl b/test/runtests.jl index 9780c55..272d809 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -23,5 +23,6 @@ using Test include("spec/test_spec_forms.jl") include("spec/test_spec_integration.jl") include("spec/test_spec_dispatch.jl") + include("spec/test_spec_lifecycle.jl") include("test_aqua.jl") end diff --git a/test/spec/README.md b/test/spec/README.md index ad31411..ea36cb8 100644 --- a/test/spec/README.md +++ b/test/spec/README.md @@ -41,6 +41,7 @@ directory is the honest progress measure. | `test_spec_runtime.jl` | the floor: the mark must not wrap the call | | `test_spec_profile.jl` | what a real run went through, how often, and how much of it | | `test_spec_dispatch.jl` | one call site, several methods, only some marked — the branch | +| `test_spec_lifecycle.jl` | the mark's EXIT, and an entry point that is a module rather than a function | | `test_spec_integration.jl` | where the mark has to surface: docs, Aqua, releases, provenance, CI | ## What the spec already found diff --git a/test/spec/test_spec_lifecycle.jl b/test/spec/test_spec_lifecycle.jl new file mode 100644 index 0000000..90edd7f --- /dev/null +++ b/test/spec/test_spec_lifecycle.jl @@ -0,0 +1,169 @@ +# The mark has an exit, and something has to say when it may be taken. +# +# From the design letter on General#166832: +# +# 「この `@experimental` を安全に外していく、というのを中間ゴールに据えた開発が可能になります」 +# +# That is the part that makes this a work item rather than a permanent label. A mark that can only +# ever be added is a decoration; a mark with a defined exit is a plan. Nothing in the rest of this +# directory covers the exit, so it is here. +# +# The other half is end-to-end analysis. The letter's reading of Lean is that `sorry` lets the +# whole development be checked with the unproven proposition still in it — +# +# 「`sorry`がついた真偽不明の命題として e2e でコードの解析を実行できる」 +# +# — and `#print axioms` answers for any declaration, not only for one you hand it. The analogue +# here is an entry point that is a MODULE or a script, not just a function with argument types. + +using ExperimentalAPI: ExperimentalAPI, @experimental, audit, experimental, mark +using Test + +module Lifecycle + +using ExperimentalAPI + +public verified_now, still_unverified, settled, consumer, entry + +# A mark whose reason has been discharged: the reference value now exists and the test suite +# exercises it. This is the one that should be removable. +@experimental( + "no reference value yet", + since = v"0.1.0", + tracking = "https://example.invalid/issues/1", + verified_now(β::Float64) = 2 * β +) + +# A mark whose reason still stands. +@experimental( + "convergence not established below β ≈ 0.1", + since = v"0.1.0", + still_unverified(β::Float64) = β * 1.0000001 +) + +"Settled from the start." +settled(β::Float64) = β + +# A caller that depends on the mark. Removing the mark must flip its verdict, and nothing else. +consumer(β::Float64) = verified_now(β) + settled(β) + +# The whole-module entry point: everything a user of this package can reach. +entry(β::Float64) = consumer(β) + still_unverified(β) + +end # module Lifecycle + +# ── the fixture can disagree ───────────────────────────────────────────────────────────────── + +@testset "two marks, one of which is ready to go and one of which is not" begin + names = Set(mk.name for mk in experimental(Lifecycle)) + @test :verified_now in names + @test :still_unverified in names + @test :settled ∉ names + @test Lifecycle.verified_now(0.5) == 1.0 # both are exercised by this suite… + @test Lifecycle.still_unverified(0.5) > 0.5 # …so coverage alone cannot separate them +end + +# ── end-to-end: an entry point that is not a single function ───────────────────────────────── + +@testset "a whole module can be the entry point" begin + # `#print axioms` answers for any declaration; the analogue is "does anything this package + # exposes reach unvalidated code". Asking function-by-function does not scale to a package. + @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(Lifecycle)) === :depends +end + +@testset "the module-level answer names which public entry points are affected" begin + # "Something in here is experimental" is not actionable for a package with 310 public names. + @test_broken :entry in + [e.name for e in ExperimentalAPI.reach(Lifecycle).affected_entries] +end + +@testset "a module with nothing marked comes back clean" begin + # The negative control for the two above. + @eval module CleanModule + "Settled." + f(x) = x + public f + end + @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(Main.CleanModule)) === :clean +end + +@testset "a script can be the entry point" begin + # The shape a researcher actually has: not a package, a file that produces a figure. + @test_broken ExperimentalAPI.reach_script(tempname()) isa Any +end + +# ── the exit: what licenses removing a mark ────────────────────────────────────────────────── + +@testset "the tool says a mark is ready to be removed, and why" begin + # Not "is this marked" but "may this stop being marked". The evidence has to be nameable: + # the reason discharged, the definition exercised, a reference value present. + @test_broken ExperimentalAPI.ready_to_promote(Lifecycle, :verified_now) === true +end + +@testset "a mark whose reason still stands is NOT reported ready" begin + # Without this, a checker that says "ready" for everything passes the test above. Both + # definitions in the fixture are exercised by this suite, so coverage cannot be the whole + # criterion — which is the point. + @test_broken ExperimentalAPI.ready_to_promote(Lifecycle, :still_unverified) === false +end + +@testset "removing a mark is reported as not breaking" begin + # `compare` already treats experimental → stable as non-breaking for names. The exit needs it + # stated in the direction a person asks the question: I am about to delete this line, is that + # a release event? + # Already implemented — promoted from @test_broken after the suite reported Unexpected Pass, + # which is the mechanism this directory exists for. + @test !ExperimentalAPI.isbreaking( + ExperimentalAPI.compare( + Dict( + "stable" => ["settled"], + "experimental" => Dict("verified_now" => Dict("reason" => "r")), + ), + Dict("stable" => ["settled", "verified_now"], "experimental" => Dict()), + ), + ) +end + +@testset "…but DELETING it outright still is" begin + # The negative control. Promoting a mark and deleting the name are both "the mark is gone" to + # a careless reading, and only one of them is safe. Without this the test above would pass for + # an `isbreaking` that always answers false. + d = ExperimentalAPI.compare( + Dict("stable" => ["settled", "verified_now"], "experimental" => Dict()), + Dict("stable" => ["settled"], "experimental" => Dict()), + ) + @test d.removed_stable == [:verified_now] + @test ExperimentalAPI.isbreaking(d) +end + +@testset "removing the mark flips its callers, and only its callers" begin + # The propagation half of the exit. After `verified_now` is promoted, `consumer` becomes + # clean; `entry` does not, because it still reaches `still_unverified`. A tool that flips + # everything, or nothing, fails one of these. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Lifecycle.consumer, Tuple{Float64}; ignore=[:verified_now]) + ) === :clean + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Lifecycle.entry, Tuple{Float64}; ignore=[:verified_now]) + ) === :depends +end + +@testset "how long a mark has been standing is answerable" begin + # `since` is recorded and nothing reads it. "Experimental" that never expires is just a label, + # and the letter's framing — an intermediate goal — needs the clock to be visible. + @test mark(Lifecycle, :still_unverified).since == v"0.1.0" + @test_broken ExperimentalAPI.age(Lifecycle, :still_unverified, v"0.9.0") isa Any +end + +@testset "the number of marks can only go down under a ratchet" begin + # The mechanism that makes "an intermediate goal" real rather than aspirational, and the same + # shape as `test_surface`'s skip list, which already only shrinks. + @test_broken ExperimentalAPI.test_surface(Lifecycle; max_marks=1) isa + ExperimentalAPI.Audit +end + +@testset "a mark removed while callers still depend on it is caught" begin + # The failure mode of removing one carelessly: the line is deleted because the author looked + # at the definition, not at who reaches it. That is what propagation is for, read backwards. + @test_broken !isempty(ExperimentalAPI.dependents(Lifecycle, :verified_now)) +end From ba44093f8fa3b9d91a2ddc46f583e7378ed566da Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Thu, 3 Sep 2026 08:02:34 +0000 Subject: [PATCH 07/11] =?UTF-8?q?fix:=20second=20review=20round=20?= =?UTF-8?q?=E2=80=94=20including=20one=20defect=20in=20the=20shipped=20cod?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six agents, two rounds. This round found a defect in `src/`, three false greens I introduced in the files added since round one, and a set of design-level gaps in the API the spec pins by assertion. Every finding was verified directly before being acted on. SHIPPED CODE `Mark` had no inner constructor, so `Mark(Main, :x, "", …)` built an empty-reason mark. The check lived only in `_reason`, which only the macro calls — and `Mark` is `public`. The reason is the payload, so the invariant now lives in the type: every future construction route (a method-level `mark_method!`, a deserialised snapshot) gets it without remembering to ask. FALSE GREENS, SAME CLASSES AS ROUND ONE, REINTRODUCED IN THE NEW FILES `isa Any` twice in the lifecycle spec. `x isa Any` is true of every Julia value, so both would have reported Unexpected Pass for a stub returning `nothing`. Replaced with the property the caller actually needs. `reach_script(tempname())` — `tempname()` creates no file, so this would have thrown `SystemError` forever for a reason unrelated to the feature. Exactly the defect fixed one commit earlier for `stamp`. The file is written now. `@test_throws Exception which(...)` — satisfied by a typo raising `UndefVarError` as well as by the ambiguity the file is about. Pinned to `ErrorException` with "ambiguous" in the message, measured. FIXTURES THAT VARIED ON THE WRONG AXIS The two lifecycle marks differed only in whether `tracking` was set, so `ready_to_promote(m,n) = mark(m,n).tracking !== nothing` — a rule with no relationship to "has the reason been discharged" — satisfied both assertions. A third mark now carries a tracking link and is still not ready. The `invoke` fixture pinned a method ordinary dispatch would have picked anyway, so an invoke-blind analysis passed. It now forces the marked `::Integer` fallback from an `Int`, which dispatch sends to the unmarked `::Int`. `occursin("k", string(u))` — a one-character needle matching "unknown call site" and "package boundary" alike. Asks for structured fields now. `compare_methods` used `"methods"` in one testset and `"stable_methods"` in the other two, so "same shape, opposite verdict" was false. One schema, mirroring `snapshot`'s `"stable"`/`"experimental"` pair. Six lazy-usage forms all checked with `@test_throws Exception` collapse into three message templates; one generic `ArgumentError("invalid usage")` passed all six, in a section whose stated purpose is that the message points the right way. Each case pins its phrase. `max_marks`, `affected_entries` and `dependents` were pinned by shape or non-emptiness only — an ignored keyword, or a walk reporting every public name, passed all three. Each now has the exclusion its fixture already contained but never used. DESIGN THE SPEC WAS PINNING BY ACCIDENT `Reach` appeared nowhere: the result was fixed by field name alone, so a NamedTuple satisfied four files and `verdict` could be duck-typed. Now pinned nominally, with `verdict` required to be DERIVED rather than stored — following `isbreaking(d::Diff)`, which is why a `Diff` cannot claim "not breaking" while carrying a removal. `:clean` with a non-empty `.unresolved` is the state this whole area exists to forbid. Every existing verdict in this package is a named predicate — `isbreaking`, `isexperimental`, `isdocumented` — never a comparison the caller writes out. The spec hand-wrote `verdict(...) === :clean` 26 times. A boolean gate is now specified alongside. `:unknown`'s algebra was never stated, though `reach(Module)` cannot be implemented without folding verdicts. Specified: `:unknown` absorbs `:clean`, `:depends` absorbs `:unknown`, and folding is order-independent. `Audit` was growing three fields pinned by `hasproperty` from three files written without cross-referencing each other; the partition invariant they pressure is now asserted. Attaching `@experimental` at one method's definition marks the NAME — `_signame` throws the argument types away. The dispatch fixture read as if it scoped. Stated, with `mark_method!` named as the actual route. STRUCTURE `test_spec_runtime.jl` is gone. Seven of its ten testsets restated `test_spec_profile.jl` on a strictly smaller fixture; its two real measurements are folded in. Four unused imports removed, three thrice-stated comments reduced to one, and the repeated Module+eval+LoadError plumbing in the forms spec is a `probe` helper. 486 passing, 150 broken. Co-Authored-By: Claude Opus 5 (1M context) --- src/mark.jl | 12 +++ test/runtests.jl | 1 - test/spec/README.md | 1 - test/spec/test_spec_declare.jl | 17 ++++- test/spec/test_spec_dispatch.jl | 87 ++++++++++++++++++++-- test/spec/test_spec_foreign.jl | 12 ++- test/spec/test_spec_forms.jl | 63 ++++++++-------- test/spec/test_spec_integration.jl | 50 +++++++++---- test/spec/test_spec_lifecycle.jl | 66 +++++++++++++++-- test/spec/test_spec_profile.jl | 47 +++++++++++- test/spec/test_spec_propagate.jl | 41 +++++++++++ test/spec/test_spec_runtime.jl | 113 ----------------------------- test/spec/test_spec_verify.jl | 2 +- 13 files changed, 329 insertions(+), 183 deletions(-) delete mode 100644 test/spec/test_spec_runtime.jl diff --git a/src/mark.jl b/src/mark.jl index ca6f149..818c913 100644 --- a/src/mark.jl +++ b/src/mark.jl @@ -35,6 +35,18 @@ struct Mark tracking::Union{String,Nothing} file::Symbol line::Int + + # The reason is the payload, so "it may not be empty" belongs in the type rather than in one + # code path. Without this the check lived only in `_reason`, which only the macro calls — + # `Mark(Main, :x, "", nothing, nothing, :f, 1)` built an empty-reason mark quite happily, and + # `Mark` is `public`. Every future construction route (a method-level `mark_method!`, a + # deserialised snapshot) gets the invariant for free now instead of remembering to ask. + function Mark(mod, name, reason, since, tracking, file, line) + r = String(strip(reason)) + isempty(r) && + throw(ArgumentError("a Mark's reason may not be empty — it is the payload")) + return new(mod, name, r, since, tracking, file, line) + end end function Base.show(io::IO, m::Mark) diff --git a/test/runtests.jl b/test/runtests.jl index 272d809..d2adfab 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -17,7 +17,6 @@ using Test include("spec/test_spec_propagate.jl") include("spec/test_spec_docstring.jl") include("spec/test_spec_verify.jl") - include("spec/test_spec_runtime.jl") include("spec/test_spec_profile.jl") include("spec/test_spec_foreign.jl") include("spec/test_spec_forms.jl") diff --git a/test/spec/README.md b/test/spec/README.md index ea36cb8..448f9e6 100644 --- a/test/spec/README.md +++ b/test/spec/README.md @@ -38,7 +38,6 @@ directory is the honest progress measure. | `test_spec_propagate.jl` | a caller that never names a marked thing still depends on it | | `test_spec_docstring.jl` | a mark and a docstring are different accounts and must coexist | | `test_spec_verify.jl` | how well is a marked thing exercised by the tests | -| `test_spec_runtime.jl` | the floor: the mark must not wrap the call | | `test_spec_profile.jl` | what a real run went through, how often, and how much of it | | `test_spec_dispatch.jl` | one call site, several methods, only some marked — the branch | | `test_spec_lifecycle.jl` | the mark's EXIT, and an entry point that is a module rather than a function | diff --git a/test/spec/test_spec_declare.jl b/test/spec/test_spec_declare.jl index 5ca5fac..20d4d1f 100644 --- a/test/spec/test_spec_declare.jl +++ b/test/spec/test_spec_declare.jl @@ -1,14 +1,13 @@ # What can carry a mark. # # The current implementation marks a NAME. The intent is to mark a definition — and for -# `QAtlas.fetch`, which has 570 methods (measured 2026-09-03, not re-derived here) behind one name, the name is the wrong unit: a docstring +# `QAtlas.fetch`, which has 570 methods behind one name (see `test_spec_foreign.jl`), the name is the wrong unit: a docstring # on `fetch` cannot say which dispatch path returns a number you can trust. # # So this file covers both: what works today (plain `@test`) and what the unit has to become # (`@test_broken`). -using ExperimentalAPI: - ExperimentalAPI, @experimental, Mark, experimental, isexperimental, mark +using ExperimentalAPI: ExperimentalAPI, @experimental, Mark, experimental, isexperimental using Test module Declared @@ -170,6 +169,18 @@ end ) end +@testset "every new Audit field keeps the partition invariant" begin + # Three files each pin a new `Audit` field with `hasproperty` alone — `:extensions` here, + # `:undocumented` in the docstring spec, `:contributed_methods` in the foreign spec — written + # without cross-referencing each other. `hasproperty` is blind to whether the property the + # type exists for still holds once three fields are bolted on from three directions. + a = ExperimentalAPI.audit(ExperimentalAPI) + @test sort( + vcat(a.foreign, a.documented, a.unaccounted, setdiff(a.declared, a.documented)) + ) == a.surface + @test_broken ExperimentalAPI.partition_holds(a) === true +end + @testset "auditing a package does not silently ignore its extensions" begin a = ExperimentalAPI.audit(ExperimentalAPI) # Today `audit` looks at one module. Whether an extension's surface is in scope has to be a diff --git a/test/spec/test_spec_dispatch.jl b/test/spec/test_spec_dispatch.jl index bb428d0..37c0373 100644 --- a/test/spec/test_spec_dispatch.jl +++ b/test/spec/test_spec_dispatch.jl @@ -10,7 +10,7 @@ # An implementation that catches that exception and moves on would report `:clean` about a call # that reaches a marked method at run time half the time. That is the failure this file guards. -using ExperimentalAPI: ExperimentalAPI, @experimental, experimental, isexperimental +using ExperimentalAPI: ExperimentalAPI, @experimental, experimental, isexperimental, mark using Test module Dispatch @@ -28,7 +28,10 @@ public Exact, either_way, via_abstract, via_invoke, - more_specific + more_specific, + pair, + via_pair_clean, + via_pair_marked struct Exact end struct Numerical end @@ -51,8 +54,21 @@ only_settled(x::Exact) = energy(x) either_way(x::Union{Exact,Numerical}) = energy(x) via_abstract(x::Kind) = k(x) -# `invoke` pins a method regardless of the argument's run-time type. -via_invoke(x::Numerical) = invoke(energy, Tuple{Numerical}, x) +# `invoke` pins a method DISPATCH WOULD NOT PICK. `invoke(energy, Tuple{Numerical}, x)` would +# not discriminate: `x::Numerical` selects the marked method anyway, so an analysis that ignores +# `invoke` entirely still gets the right answer by accident. Forcing the marked `::Integer` +# fallback from an `Int` — which ordinary dispatch sends to the unmarked `::Int` — does. +via_invoke(x::Int) = invoke(more_specific, Tuple{Integer}, x) + +# Two arguments, which is the shape the motivating example actually has: `fetch(model, quantity)`. +# Specificity differs per position, so the marked combination is not reachable from either +# argument alone. +pair(::Exact, ::Exact) = 0.0 +pair(::Exact, ::Numerical) = 1.0 +@experimental "only this combination is unvalidated" pair(::Numerical, ::Numerical) = 2.0 +pair(::Numerical, ::Exact) = 3.0 +via_pair_clean(a::Exact, b::Exact) = pair(a, b) +via_pair_marked(a::Numerical, b::Numerical) = pair(a, b) # A more specific unmarked method shadows a marked less specific one. "Settled, and more specific." @@ -74,13 +90,47 @@ end # module Dispatch @test isexperimental(Dispatch, :energy) end +@testset "the specificity premise the file rests on is true" begin + # Stated only in a comment until now, and exercised solely inside `@test_broken` blocks gated + # behind a `reach` that does not exist — so nothing would have noticed if it were false. + @test which(Dispatch.more_specific, Tuple{Int}).sig === + Tuple{typeof(Dispatch.more_specific),Int} + @test Dispatch.more_specific(5) == 0 # the UNMARKED, more specific method + @test Dispatch.more_specific(UInt8(5)) == 1 # falls through to the MARKED fallback + @test Dispatch.via_invoke(5) == 1 # invoke reaches what dispatch would not pick +end + @testset "a branching call site has no unique method" begin # The measured fact the whole file rests on: `which` cannot answer for these argument types. + # `@test_throws Exception` would be satisfied by a typo in the fixture raising `UndefVarError` + # just as well as by the ambiguity this file is about, so pin the diagnosis, not the failure. @test which(Dispatch.energy, Tuple{Dispatch.Exact}) isa Method - @test_throws Exception which( - Dispatch.energy, Tuple{Union{Dispatch.Exact,Dispatch.Numerical}} + for (f, T) in ( + (Dispatch.energy, Tuple{Union{Dispatch.Exact,Dispatch.Numerical}}), + (Dispatch.k, Tuple{Dispatch.Kind}), ) - @test_throws Exception which(Dispatch.k, Tuple{Dispatch.Kind}) + @testset "$T" begin + e = try + which(f, T) + nothing + catch err + err + end + @test e isa ErrorException + @test occursin("ambiguous", sprint(showerror, e)) + end + end +end + +@testset "attaching the mark at one method's definition marks the NAME today" begin + # Line 38 above reads as if it scopes the claim to `energy(::Numerical)`. It does not: + # `_signame` walks a `:call` head straight to the base Symbol and throws the argument types + # away. So method-level marking cannot come from the attached form as written — it has to + # arrive through a separate imperative route (`mark_method!`), and that is a design decision + # the spec should state rather than leave implied. + @test isexperimental(Dispatch, :energy) + @test mark(Dispatch, :energy).name === :energy # not a signature + @test_broken ExperimentalAPI.mark_method! isa Function end # ── what the analysis has to say about each shape ──────────────────────────────────────────── @@ -111,8 +161,15 @@ end @testset "the unresolvable call site is named, not just counted" begin # "Somewhere in here I could not tell" is not actionable. The report has to point at the call. + # NOT `occursin("k", string(u))`: a one-character needle matches "unknown call site", + # "package boundary" and "backtrace unavailable" alike, so one generic boilerplate diagnostic + # would satisfy it. Ask for the structured fields, the way `test_spec_propagate.jl` does. + @test_broken all( + u -> hasproperty(u, :file) && hasproperty(u, :line) && hasproperty(u, :callee), + ExperimentalAPI.reach(Dispatch.via_abstract, Tuple{Dispatch.Kind}).unresolved, + ) @test_broken any( - u -> occursin("k", string(u)), + u -> u.callee === :k, ExperimentalAPI.reach(Dispatch.via_abstract, Tuple{Dispatch.Kind}).unresolved, ) end @@ -161,6 +218,20 @@ end ) end +@testset "a marked combination is not reachable from either argument alone" begin + # `pair(::Exact,::Exact)` and `pair(::Numerical,::Exact)` are settled; only + # `pair(::Numerical,::Numerical)` is marked. An analysis that widens each argument + # independently would call both call sites `:depends`. + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach(Dispatch.via_pair_clean, Tuple{Dispatch.Exact,Dispatch.Exact}) + ) === :clean + @test_broken ExperimentalAPI.verdict( + ExperimentalAPI.reach( + Dispatch.via_pair_marked, Tuple{Dispatch.Numerical,Dispatch.Numerical} + ), + ) === :depends +end + @testset "the report says WHICH method was reached, not which name" begin @test_broken first( ExperimentalAPI.reach(Dispatch.either_way, Tuple{Dispatch.Numerical}).reached diff --git a/test/spec/test_spec_foreign.jl b/test/spec/test_spec_foreign.jl index 5bc00a2..0048d0f 100644 --- a/test/spec/test_spec_foreign.jl +++ b/test/spec/test_spec_foreign.jl @@ -60,9 +60,7 @@ end end @testset "a method on a foreign generic can be marked" begin - # Ends in a Bool, and checks WHAT was marked: `@eval module … end` returns a Module, so a bare - # `@test_broken @eval module …` reports "Expression evaluated to non-Boolean" on success - # instead of the "Unexpected Pass" this directory relies on. + # Ends in a Bool and checks WHAT was marked — same caveat as `spec/README.md`. @test_broken begin @eval module MarkedForeign using ExperimentalAPI @@ -101,6 +99,14 @@ end ) end +@testset "the ownership query and the cross-module search are different verbs" begin + # `experimental(Module)` answers "what does this module own". `experimental(fetch_value)` has + # to answer "what has any package anywhere marked on any method of this generic". Both are + # wanted, but collapsing them onto one name means a reader cannot tell which they get without + # knowing the argument's type. + @test_broken ExperimentalAPI.marks_on isa Function +end + @testset "asking the generic finds marks contributed by every package" begin # The question a user asks is about `fetch_value`, not about which package happened to define # the method they will dispatch to. diff --git a/test/spec/test_spec_forms.jl b/test/spec/test_spec_forms.jl index a83e616..6785ee7 100644 --- a/test/spec/test_spec_forms.jl +++ b/test/spec/test_spec_forms.jl @@ -235,6 +235,20 @@ end @test_broken ExperimentalAPI.audit(FormsSpec).tracking isa AbstractDict end +# Evaluate an expression in a fresh module and hand back the exception it raised, unwrapped. +# The `Module(...)` + `Core.eval` + `LoadError`-stripping plumbing was written out five times +# before this; it carries none of the claim, so it is a helper rather than part of each test. +function probe(ex) + m = Module() + Core.eval(m, :(using ExperimentalAPI)) + try + Core.eval(m, ex) + return nothing + catch err + return err isa LoadError ? err.error : err + end +end + # ── writing it lazily ──────────────────────────────────────────────────────────────────────── # # The reason is the payload: why a name is not settled is knowledge only the author has. So the @@ -242,23 +256,26 @@ end # without one". Measured 2026-09-03: every lazy form IS refused — but two of them are refused by # accident, and two more are refused with a message pointing the wrong way. -@testset "a bare @experimental is refused" begin - for ex in ( - :(@experimental), - :(@experimental foo), - :(@experimental function f(x) +@testset "a bare @experimental is refused, and says which of the two is missing" begin + # `@test_throws Exception` for all six would be satisfied by one generic + # `ArgumentError("@experimental: invalid usage")`. The section exists to check that the + # message points the right way, so each case pins the phrase it should carry. + for (ex, needle) in ( + (:(@experimental), "needs a reason"), + (:(@experimental foo), "the reason comes first"), + (:(@experimental function f(x) x - end), - :(@experimental struct S + end), "the reason comes first"), + (:(@experimental struct S v::Int - end), - :(@experimental f(x) = x), - :(@experimental const C = 1), + end), "the reason comes first"), + (:(@experimental f(x) = x), "nothing to mark"), + (:(@experimental const C = 1), "nothing to mark"), ) - @testset "$(first(string(ex), 40))" begin - m = Module(:LazyProbe) - Core.eval(m, :(using ExperimentalAPI)) - @test_throws Exception Core.eval(m, ex) + @testset "$(first(string(ex), 36))" begin + e = probe(ex) + @test e isa ArgumentError + @test occursin(needle, sprint(showerror, e)) end end end @@ -287,14 +304,7 @@ end # Same shape as the `since = "0.4.0"` case above. for r in (:(:sym), 42) @testset "reason=$(repr(r))" begin - m = Module(:BadReasonProbe) - Core.eval(m, :(using ExperimentalAPI)) - e = try - Core.eval(m, :(@experimental $r f(x) = x)) - nothing - catch err - err isa LoadError ? err.error : err - end + e = probe(:(@experimental $r f(x) = x)) @test e isa MethodError # today, and accidental @test_broken occursin("reason", sprint(showerror, e)) end @@ -305,14 +315,7 @@ end # `@experimental f(x) = x` is refused with "nothing to mark — give a definition or a name". # The author DID give a definition; what is missing is the reason. The message points at the # wrong end of the call, which is how someone ends up deleting a correct definition. - m = Module(:NoReasonProbe) - Core.eval(m, :(using ExperimentalAPI)) - e = try - Core.eval(m, :(@experimental f(x) = x)) - nothing - catch err - err isa LoadError ? err.error : err - end + e = probe(:(@experimental f(x) = x)) @test e isa ArgumentError @test occursin("nothing to mark", sprint(showerror, e)) # today, and misleading @test_broken occursin("reason", sprint(showerror, e)) diff --git a/test/spec/test_spec_integration.jl b/test/spec/test_spec_integration.jl index af29cdb..5a9a902 100644 --- a/test/spec/test_spec_integration.jl +++ b/test/spec/test_spec_integration.jl @@ -4,7 +4,7 @@ # a reviewer of a PR, or a user of the docs site learning that a number came from unvalidated # code — requires the mark to surface in tools nobody configured for it. -using ExperimentalAPI: ExperimentalAPI, @experimental, audit, experimental +using ExperimentalAPI: ExperimentalAPI, @experimental, experimental using Test module Shown @@ -61,31 +61,46 @@ end end # ── release ────────────────────────────────────────────────────────────────────────────────── +# +# One schema for every case below, mirroring `snapshot`'s `"stable"` / `"experimental"` pair. An +# earlier version used `"methods"` in the first testset and `"stable_methods"` in the other two, +# so "same shape, opposite verdict" was false: the fixtures differed in more than the variable +# each one isolates, and no test ever supplied both keys at once. +function snap(stable, experimental) + return Dict("stable_methods" => stable, "experimental_methods" => experimental) +end + +const MARKED_METHOD = snap(String[], Dict("f(::Int)" => Dict("reason" => "r"))) +const PROMOTED_METHOD = snap(["f(::Int)"], Dict()) +const SETTLED_METHOD = snap(["f(::Int)"], Dict()) +const GONE_METHOD = snap(String[], Dict()) +const RESIGNED_METHOD = snap(["f(::Real)"], Dict()) + +@testset "the method snapshot mirrors the name snapshot's two keys" begin + # If the two schemas diverge, `compare` and `compare_methods` cannot share a snapshot file. + for d in (MARKED_METHOD, PROMOTED_METHOD, SETTLED_METHOD, GONE_METHOD, RESIGNED_METHOD) + @test Set(keys(d)) == Set(["stable_methods", "experimental_methods"]) + end + @test Set(keys(ExperimentalAPI.snapshot(Shown))) ⊇ Set(["stable", "experimental"]) +end @testset "a snapshot records marks at method granularity" begin # `compare` reads name sets today and says so. Once methods can be marked, the snapshot has # to grow — and the schema change is why the release layer is itself declared experimental. - @test_broken haskey(ExperimentalAPI.snapshot(Shown), "methods") + @test_broken haskey(ExperimentalAPI.snapshot(Shown), "experimental_methods") end @testset "removing a marked METHOD is not breaking" begin # The same contract as for names, at the granularity that matters for a dispatch table. - # These two testsets used to share a byte-identical `compare_methods isa Function`, so both - # would flip the instant the function existed, with neither behavioural claim ever checked. @test_broken !ExperimentalAPI.isbreaking( - ExperimentalAPI.compare_methods( - Dict("methods" => Dict("f(::Int)" => Dict("reason" => "r"))), - Dict("methods" => Dict()), - ), + ExperimentalAPI.compare_methods(MARKED_METHOD, PROMOTED_METHOD) ) end @testset "removing a SETTLED method is breaking" begin - # The negative control for the testset above: same shape, unmarked method, opposite verdict. + # The negative control: the same schema, differing only in whether the method was marked. @test_broken ExperimentalAPI.isbreaking( - ExperimentalAPI.compare_methods( - Dict("stable_methods" => ["f(::Int)"]), Dict("stable_methods" => String[]) - ), + ExperimentalAPI.compare_methods(SETTLED_METHOD, GONE_METHOD) ) end @@ -93,12 +108,17 @@ end # The blind spot `compare` currently admits to in its own docstring. Method-level marks are # what make it addressable at all. @test_broken ExperimentalAPI.isbreaking( - ExperimentalAPI.compare_methods( - Dict("stable_methods" => ["f(::Int)"]), Dict("stable_methods" => ["f(::Real)"]) - ), + ExperimentalAPI.compare_methods(SETTLED_METHOD, RESIGNED_METHOD) ) end +@testset "a keyword-only change is a blind spot here too" begin + # `Tuple`-based signatures have no room for keyword arguments — they live in a separate + # `kwcall` method — so a change to a default or a kwarg name is invisible to a signature + # string just as it is to a name set. Stated rather than discovered later. + @test_broken ExperimentalAPI.compare_methods_sees_keywords === true +end + # ── the provenance record next to a result ─────────────────────────────────────────────────── @testset "a result file can carry the experimental dependencies of the run that made it" begin diff --git a/test/spec/test_spec_lifecycle.jl b/test/spec/test_spec_lifecycle.jl index 90edd7f..0e13a6d 100644 --- a/test/spec/test_spec_lifecycle.jl +++ b/test/spec/test_spec_lifecycle.jl @@ -23,7 +23,7 @@ module Lifecycle using ExperimentalAPI -public verified_now, still_unverified, settled, consumer, entry +public verified_now, still_unverified, tracked_but_unresolved, settled, consumer, entry # A mark whose reason has been discharged: the reference value now exists and the test suite # exercises it. This is the one that should be removable. @@ -41,6 +41,18 @@ public verified_now, still_unverified, settled, consumer, entry still_unverified(β::Float64) = β * 1.0000001 ) +# A third mark, and the reason it is here: without it, `verified_now` and `still_unverified` +# differ ONLY in whether `tracking` is set, so `ready_to_promote(m, n) = mark(m,n).tracking !== +# nothing` — a rule with no relationship at all to "has the reason been discharged" — satisfies +# both assertions below. This one HAS a tracking link and is still not ready, which breaks that +# shortcut. The fixture has to vary on the axis under test, not on a neighbouring one. +@experimental( + "reference value exists but disagrees with the literature at the third digit", + since = v"0.1.0", + tracking = "https://example.invalid/issues/2", + tracked_but_unresolved(β::Float64) = β + 1e-9 +) + "Settled from the start." settled(β::Float64) = β @@ -75,6 +87,10 @@ end # "Something in here is experimental" is not actionable for a package with 310 public names. @test_broken :entry in [e.name for e in ExperimentalAPI.reach(Lifecycle).affected_entries] + # …and `settled` reaches nothing marked, so it must NOT be listed. Without this, a walk that + # reports every public name whenever the module contains any mark at all passes. + @test_broken :settled ∉ + [e.name for e in ExperimentalAPI.reach(Lifecycle).affected_entries] end @testset "a module with nothing marked comes back clean" begin @@ -89,7 +105,15 @@ end @testset "a script can be the entry point" begin # The shape a researcher actually has: not a package, a file that produces a figure. - @test_broken ExperimentalAPI.reach_script(tempname()) isa Any + # + # Two traps avoided here. `tempname()` returns a path and creates NO file, so reading it + # throws `SystemError` forever regardless of the implementation — the same defect fixed one + # commit earlier for `stamp`. And `isa Any` is true of every Julia value, so it would have + # reported Unexpected Pass for a no-op returning `nothing`. + path = tempname() + write(path, "1 + 1\n") + @test isfile(path) + @test_broken hasproperty(ExperimentalAPI.reach_script(path), :reached) end # ── the exit: what licenses removing a mark ────────────────────────────────────────────────── @@ -101,12 +125,23 @@ end end @testset "a mark whose reason still stands is NOT reported ready" begin - # Without this, a checker that says "ready" for everything passes the test above. Both + # Without this, a checker that says "ready" for everything passes the test above. All three # definitions in the fixture are exercised by this suite, so coverage cannot be the whole # criterion — which is the point. @test_broken ExperimentalAPI.ready_to_promote(Lifecycle, :still_unverified) === false end +@testset "having a tracking link is not the same as being ready" begin + # `tracked_but_unresolved` carries a tracking link and is still not ready. Without this, + # `ready_to_promote(m, n) = mark(m, n).tracking !== nothing` — which has nothing to do with + # the criteria the comments name — satisfies both testsets above. + @test mark(Lifecycle, :tracked_but_unresolved).tracking !== nothing + @test mark(Lifecycle, :verified_now).tracking !== nothing + @test mark(Lifecycle, :still_unverified).tracking === nothing + @test_broken ExperimentalAPI.ready_to_promote(Lifecycle, :tracked_but_unresolved) === + false +end + @testset "removing a mark is reported as not breaking" begin # `compare` already treats experimental → stable as non-breaking for names. The exit needs it # stated in the direction a person asks the question: I am about to delete this line, is that @@ -152,18 +187,35 @@ end # `since` is recorded and nothing reads it. "Experimental" that never expires is just a label, # and the letter's framing — an intermediate goal — needs the clock to be visible. @test mark(Lifecycle, :still_unverified).since == v"0.1.0" - @test_broken ExperimentalAPI.age(Lifecycle, :still_unverified, v"0.9.0") isa Any + # Not `isa Any` — that is true of every value, including `nothing` from a stub. Assert the + # number the caller actually needs: eight minor releases have passed since v0.1.0. + @test_broken ExperimentalAPI.age(Lifecycle, :still_unverified, v"0.9.0") == 8 end @testset "the number of marks can only go down under a ratchet" begin # The mechanism that makes "an intermediate goal" real rather than aspirational, and the same # shape as `test_surface`'s skip list, which already only shrinks. - @test_broken ExperimentalAPI.test_surface(Lifecycle; max_marks=1) isa - ExperimentalAPI.Audit + # + # `isa Audit` will NOT do: `test_surface` is documented to return the audit on the normal + # return path whether the testset passed or not, so an implementation that accepts + # `max_marks` and ignores it satisfies that. Assert what the cap actually does. + @test length(experimental(Lifecycle)) == 3 + @test_broken ExperimentalAPI.exceeds_mark_cap(Lifecycle, 1) === true + @test_broken ExperimentalAPI.exceeds_mark_cap(Lifecycle, 5) === false end @testset "a mark removed while callers still depend on it is caught" begin # The failure mode of removing one carelessly: the line is deleted because the author looked # at the definition, not at who reaches it. That is what propagation is for, read backwards. - @test_broken !isempty(ExperimentalAPI.dependents(Lifecycle, :verified_now)) + @test_broken :consumer in ExperimentalAPI.dependents(Lifecycle, :verified_now) + # `settled` calls nothing marked, so it is nobody's dependent. Without this, a `dependents` + # that returns every public name passes. + @test_broken :settled ∉ ExperimentalAPI.dependents(Lifecycle, :verified_now) +end + +@testset "the exit works at method granularity too" begin + # `test_spec_dispatch.jl` argues that a name-keyed mark over-claims when a name has several + # methods. The exit has the same problem read backwards: promoting one method of a name must + # not promote its siblings. Neither file tests the intersection, so it is stated here. + @test_broken ExperimentalAPI.ready_to_promote isa Function end diff --git a/test/spec/test_spec_profile.jl b/test/spec/test_spec_profile.jl index eb5b766..cb2c0e5 100644 --- a/test/spec/test_spec_profile.jl +++ b/test/spec/test_spec_profile.jl @@ -81,7 +81,7 @@ end # ── granularity ────────────────────────────────────────────────────────────────────────────── @testset "attribution is to a method, not to a name" begin - # `QAtlas.fetch` has 570 methods (measured 2026-09-03, not re-derived here). "the run touched fetch" is unusable. + # `QAtlas.fetch` has 570 methods — see `test_spec_foreign.jl`. "the run touched fetch" is unusable. @test_broken first(ExperimentalAPI.record(() -> Sim.driver(M, 10))).method isa Method end @@ -121,6 +121,41 @@ end end end +# ── the floor: the mark must not wrap the call ─────────────────────────────────────────────── +# +# Folded in from what used to be `test_spec_runtime.jl`. Its other eight testsets restated this +# file's claims on a strictly smaller fixture; only these two measured anything of their own. + +@testset "the mark does not wrap the call" begin + # The property the package currently advertises and must keep: `@experimental` emits the + # definition unchanged plus one `push!` at load time. If instrumentation ever becomes + # unconditional, an inner loop pays for it on every iteration. + # Reading `src/mark.jl` for a prose phrase would pass for a regression that wrapped the call + # and left the sentence alone. Look at what the macro emits. + emitted = string(@macroexpand @experimental "why" f(x) = x) + @test occursin("f(x)", emitted) # the definition is there… + @test !occursin("function f", replace(emitted, "f(x)" => "")) # …and not wrapped in another + @test Sim.driver(M, 3) ≈ 3 * (0.5 * 1.0000001 + exp(-0.5)) +end + +@testset "a marked definition is not slower than the same definition unmarked" begin + unmarked(x::Float64) = x * 1.0000001 + function loop(f) + acc = 0.0 + for _ in 1:2_000_000 + acc += f(1.0) + end + return acc + end + loop(x -> Sim.energy(Sim.Model(x))) + loop(unmarked) # warm both before timing either + a = @elapsed loop(x -> Sim.energy(Sim.Model(x))) + b = @elapsed loop(unmarked) + # Loose on purpose: this is a floor against wrapping, not a benchmark. A wrapper that + # increments a counter would not fit inside this margin. + @test a < 5b + 1e-3 +end + # ── mechanism constraints ──────────────────────────────────────────────────────────────────── @testset "recording survives inlining" begin @@ -177,6 +212,16 @@ end @test_broken ExperimentalAPI.record(threaded)[1].count == 800 end +@testset "a merged record is still a record" begin + # `merge_records(...) isa AbstractVector` is the only constraint today, so a `record()` you + # can ask `.overhead` of, merged with another, may legally come back as a plain `Vector` you + # cannot. Losing the type across a verb's own merge is avoidable. + @test_broken hasproperty( + ExperimentalAPI.merge_records([ExperimentalAPI.record(() -> Sim.driver(M, 10))]), + :enabled, + ) +end + @testset "records from separate processes merge into one" begin # The same shape as TestShards' shard records: a distributed sweep produces one record per # worker, and the provenance statement is about the whole run. diff --git a/test/spec/test_spec_propagate.jl b/test/spec/test_spec_propagate.jl index fac65ca..1c2f39d 100644 --- a/test/spec/test_spec_propagate.jl +++ b/test/spec/test_spec_propagate.jl @@ -117,6 +117,47 @@ const ENTRY = Tuple{Float64} @test :solid ∉ names # the negative control really is unmarked end +# ── the type the answer lives in ───────────────────────────────────────────────────────────── +# +# `Mark` and `Audit` are both pinned nominally somewhere in this directory (`isa Mark`, +# `isa Audit`). The propagation result is not: before these tests it was pinned purely by field +# name, so a `NamedTuple` with `.reached`/`.unresolved` satisfied every assertion in four files +# and `verdict` could be duck-typed on `hasproperty`. + +@testset "the result has a type, not just field names" begin + @test_broken ExperimentalAPI.reach(Chain.top_bad, ENTRY) isa ExperimentalAPI.Reach +end + +@testset "verdict is derived, never stored" begin + # `isbreaking(d::Diff)` is a pure function over `d.removed_stable`/`d.demoted`, never a cached + # field — which is why a `Diff` cannot claim "not breaking" while carrying a removal. Same + # rule here: if `verdict` were a stored field, `:clean` with a non-empty `.unresolved` would + # become representable, and that is the one state this whole file exists to forbid. + @test_broken !hasproperty(ExperimentalAPI.reach(Chain.top_bad, ENTRY), :verdict) +end + +@testset "a boolean gate exists alongside the three-valued answer" begin + # Every existing verdict in this package is a named predicate — `isbreaking`, + # `isexperimental`, `isdocumented` — never a comparison the caller writes out. The spec + # hand-writes `verdict(...) === :clean` and friends 26 times, which is the smell. + @test_broken ExperimentalAPI.isclean(ExperimentalAPI.reach(Chain.top_good, ENTRY)) === + true + @test_broken ExperimentalAPI.isclean(ExperimentalAPI.reach(Chain.top_bad, ENTRY)) === + false +end + +@testset ":unknown absorbs when results are combined" begin + # `reach(Module)` has to fold the verdicts of every public entry into one answer, so the + # algebra has to exist. It is stated here rather than discovered: a module with one + # `:unknown` entry is not clean, whatever the others say, and folding is order-independent. + @test_broken ExperimentalAPI.combine(:clean, :unknown) === :unknown + @test_broken ExperimentalAPI.combine(:depends, :unknown) === :depends + @test_broken ExperimentalAPI.combine(:clean, :depends) === :depends + @test_broken ExperimentalAPI.combine(:clean, :clean) === :clean + @test_broken ExperimentalAPI.combine(:unknown, :clean) === + ExperimentalAPI.combine(:clean, :unknown) +end + # ── the core claim ─────────────────────────────────────────────────────────────────────────── @testset "a caller two hops away is reported as depending" begin diff --git a/test/spec/test_spec_runtime.jl b/test/spec/test_spec_runtime.jl deleted file mode 100644 index 1ad88bb..0000000 --- a/test/spec/test_spec_runtime.jl +++ /dev/null @@ -1,113 +0,0 @@ -# What a real run touched — and what it costs when nobody asks. -# -# The question is provenance for a result, not documentation: after a twelve-hour DMRG run, which -# unverified code paths did it go through, and how often? A docstring cannot answer that. This is -# the part of the intent that a reviewer cannot dismiss as "put it in the docstring". -# -# Two routes were measured on 2026-09-03, **Julia 1.12.2**, and the cheap one does NOT work: -# -# sampling profiler + Method-set membership -> 0 samples attributed -# `Profile.fetch` frames for inlined callees carry no `MethodInstance`, so exactly the -# small functions most likely to be marked disappear. `@noinline` did not rescue it. -# -# custom AbstractInterpreter (static) -> works, see test_spec_propagate.jl -# but answers "could reach", not "did reach, N times". -# -# So the runtime half still needs a mechanism. Whatever it is, the constraint below is the one -# that must not be traded away. - -using ExperimentalAPI: ExperimentalAPI, @experimental -using Test - -module Hot - -using ExperimentalAPI - -public inner, driver, cold_path - -@experimental "not validated below β ≈ 0.1" inner(x::Float64) = x * 1.0000001 -function driver(n::Int, x::Float64) - s = 0.0 - for _ in 1:n - s += inner(x) - end - return s -end - -@experimental "never called by the fixture" cold_path(x::Float64) = x - 1.0 - -end # module Hot - -@testset "the mark does not wrap the call" begin - # The property the package currently advertises and must keep: `@experimental` emits the - # definition unchanged plus one `push!` at load time. If instrumentation ever becomes - # unconditional, an inner loop pays for it on every iteration. - # Reading `src/mark.jl` for a prose phrase would pass for a regression that wrapped the call - # and left the sentence alone. Look at what the macro emits. - emitted = string(@macroexpand @experimental "why" f(x) = x) - @test occursin("f(x)", emitted) # the definition is there… - @test !occursin("function f", replace(emitted, "f(x)" => "")) # …and not wrapped in another - @test Hot.driver(3, 1.0) ≈ 3 * 1.0000001 -end - -@testset "a marked definition is not slower than the same definition unmarked" begin - unmarked(x::Float64) = x * 1.0000001 - function loop(f) - acc = 0.0 - for _ in 1:2_000_000 - acc += f(1.0) - end - return acc - end - loop(Hot.inner) - loop(unmarked) # warm both before timing either - a = @elapsed loop(Hot.inner) - b = @elapsed loop(unmarked) - # Loose on purpose: this is a floor against wrapping, not a benchmark. A wrapper that - # increments a counter would not fit inside this margin. - @test a < 5b + 1e-3 -end - -@testset "recording is off unless it is asked for" begin - @test_broken ExperimentalAPI.recording() === false -end - -@testset "a recorded run reports which marked definitions it entered" begin - @test_broken :inner in - [h.name for h in ExperimentalAPI.record(() -> Hot.driver(1000, 1.0))] -end - -@testset "a recorded run reports how many times" begin - # "Did it touch experimental code" and "was 97% of the run inside it" are different answers, - # and only the second tells you whether the result is worth anything. - @test_broken ExperimentalAPI.record(() -> Hot.driver(1000, 1.0))[1].count == 1000 -end - -@testset "a marked definition the run never entered is not reported as touched" begin - # Without this, a recorder that lists every mark in the module passes the two tests above. - @test_broken :cold_path ∉ - [h.name for h in ExperimentalAPI.record(() -> Hot.driver(10, 1.0))] -end - -@testset "recording survives inlining" begin - # The reason the sampling-profiler route failed. Whatever mechanism is chosen has to be - # demonstrated on a small function that the optimiser would normally inline away — which is - # most of what gets marked. - @test_broken ExperimentalAPI.record(() -> Hot.driver(100, 1.0))[1].count == 100 -end - -@testset "the record attributes to a method, not just a name" begin - # `QAtlas.fetch` has 570 methods (measured 2026-09-03, not re-derived here). "the run touched `fetch`" is not usable; "the run took the - # Numerical × Float64 path 4.7M times" is. - @test_broken first(ExperimentalAPI.record(() -> Hot.driver(10, 1.0))).method isa Method -end - -@testset "a run with no marked code reports nothing rather than failing" begin - @test_broken isempty(ExperimentalAPI.record(() -> sum(1:10))) -end - -@testset "the record is data on the normal return path" begin - # Same convention as `audit` and `test_surface`: the caller gets the numbers back whether or - # not it asked for printing. - @test_broken ExperimentalAPI.record(() -> Hot.driver(10, 1.0)) isa AbstractVector -end diff --git a/test/spec/test_spec_verify.jl b/test/spec/test_spec_verify.jl index 356451b..42e6f0a 100644 --- a/test/spec/test_spec_verify.jl +++ b/test/spec/test_spec_verify.jl @@ -8,7 +8,7 @@ # comes back and looks fine. A marked method with zero coverage is the worst case: unverified # code, shipped, and never even run by its own suite. -using ExperimentalAPI: ExperimentalAPI, @experimental, experimental, mark +using ExperimentalAPI: ExperimentalAPI, @experimental, experimental using Test module Covered From 2008f7514730a9d39d7cb3e73e9c11d5a2f2959c Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Thu, 3 Sep 2026 08:20:22 +0000 Subject: [PATCH 08/11] =?UTF-8?q?fix:=20address=20the=20review=20=E2=80=94?= =?UTF-8?q?=20generate=20the=20table,=20count=20behaviours,=20fix=20where?= =?UTF-8?q?=20the=20refusal=20points?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five findings, all reproduced before being acted on. 1. The description had drifted from its own branch, inside the change that argues prose drifts and tests do not. Fixed at the source rather than by retyping it: `test/spec/summary.jl` generates the table by reading the directory, `test/test_spec_table.jl` pins README against it, and both a spec file missing from `CONCERNS` and one missing from `runtests.jl` are now errors. The hand-written table lost `dispatch` and `lifecycle` exactly that way. 2. "Every group has a negative control" was present tense about controls that are themselves `@test_broken`. The README now separates specified from operating, per group, and says which fixture premises are pinned live so the specified controls are not resting on nothing. 3. The published measure was the `@test`:`@test_broken` ratio, which moves without any implementation progress: 36 assertion lines in `test_spec_declare.jl` run 91 assertions because several sit inside a loop over the fixture's marks. The measure is now distinct behaviours — one per leaf `@testset` — and `test_spec_table.jl` pins that a 100-iteration loop counts as one behaviour, not one hundred. 4. Defect 1 emits two wrong signals, not one. `(c::C)(x) = c.k * x` marks `:c`, so the audit reports `:c` dangling AND `:C` unaccounted — it tells the author to declare the very thing that line declares. Both halves are pinned, with the corrected behaviour as one broken test asserting they clear together. 5. Defect 2 pointed into this package: `ExperimentalAPI/src/mark.jl:219`, reading as a bug here rather than a misuse. `const` in local scope fails during lowering, before any emitted code runs, so no check of ours can intercept it — but the expansion is now built with `Expr` and carries the caller's `LineNumberNode`, so the message names the line the author wrote. The remaining half (naming `@experimental`) stays broken, with the reason it may be unreachable. Also: the wall-clock ratio in `test_spec_profile.jl` was measuring the fixture, not the mark. Its two arms did different work — `Sim.energy(Sim.Model(x))` against a bare call — and it failed at 11.8 ms vs 2.1 ms. Making the arms identical does not rescue it: the ratio over eight trials on an idle machine ran 0.79 to 3.03, so the 5x threshold sat inside the noise. Replaced with an exact allocation check, with the structural `@macroexpand` test named as the stronger of the two. Smaller: `test/spec/README.md` is linked from the top-level README, and `test_spec_integration.jl` records which of its assertions need a test dependency this package does not have (one: Documenter) so the group is not assumed to be blocked on infrastructure. Co-Authored-By: Claude Opus 5 --- README.md | 8 ++ src/mark.jl | 15 ++- test/runtests.jl | 1 + test/spec/README.md | 86 +++++++++++---- test/spec/summary.jl | 167 +++++++++++++++++++++++++++++ test/spec/test_spec_forms.jl | 88 ++++++++++++--- test/spec/test_spec_integration.jl | 12 +++ test/spec/test_spec_profile.jl | 44 ++++++-- test/test_spec_table.jl | 83 ++++++++++++++ 9 files changed, 455 insertions(+), 49 deletions(-) create mode 100644 test/spec/summary.jl create mode 100644 test/test_spec_table.jl diff --git a/README.md b/README.md index 358f2d9..77e9959 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,14 @@ reads as if it had none: - **Whether a name appears in your guide, README or docs site.** Docstring presence is not documentation-page presence, and those two gaps are usually different sets. +## Where this is going + +`test/spec/` is the specification for the rest: the propagation, profiling and lifecycle work is +written there as tests before it is implemented, so it cannot drift from the code. Most of it is +`@test_broken` today, and [`test/spec/README.md`](test/spec/README.md) explains why that register +was chosen, which of the negative controls are actually running, and the two defects the exercise +already found in the shipped code. + ## License MIT diff --git a/src/mark.jl b/src/mark.jl index 818c913..f54e1a9 100644 --- a/src/mark.jl +++ b/src/mark.jl @@ -214,12 +214,17 @@ macro experimental(args...) # because the `_mark!` calls below READ that binding and Julia 1.12 forbids reading a binding # created in the same world age. marks = esc(MARKS_BINDING) - init = :( - if !$(isdefined)($__module__, $(QuoteNode(MARKS_BINDING))) - const $marks = $(Mark)[] - end - ) src = __source__ + # Built with `Expr` rather than quoted, so that no `LineNumberNode` from THIS file ends up in + # the expansion. `const` in local scope is a lowering error this macro cannot catch — it fires + # before any code it emits runs — so the only lever left is where the error points. Quoted, it + # named `src/mark.jl` and read as a bug in this package; the caller's own line is the line the + # author can act on. See `test/spec/test_spec_forms.jl`, "a mark inside a function body". + init = Expr( + :if, + :(!$(isdefined)($__module__, $(QuoteNode(MARKS_BINDING)))), + Expr(:block, src, Expr(:const, Expr(:(=), marks, :($(Mark)[])))), + ) records = [ :($(_mark!)( $marks, diff --git a/test/runtests.jl b/test/runtests.jl index d2adfab..a4e9c83 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -23,5 +23,6 @@ using Test include("spec/test_spec_integration.jl") include("spec/test_spec_dispatch.jl") include("spec/test_spec_lifecycle.jl") + include("test_spec_table.jl") include("test_aqua.jl") end diff --git a/test/spec/README.md b/test/spec/README.md index 448f9e6..bbed166 100644 --- a/test/spec/README.md +++ b/test/spec/README.md @@ -27,21 +27,64 @@ separate fixtures. That is deliberate while the spec is the design document — pinning one contract have to be kept in sync by hand, so the older files should be folded in or retired once the spec stops moving. -Anything already implemented is a plain `@test`. The ratio of `@test` to `@test_broken` in this -directory is the honest progress measure. - -| file | concern | -|---|---| -| `test_spec_declare.jl` | what can carry a mark: function, method, struct, const, module, macro, extension | -| `test_spec_forms.jl` | the definition forms a real package hits on its second afternoon | -| `test_spec_foreign.jl` | marking a method on somebody else's generic — the `QAtlas.fetch` case | -| `test_spec_propagate.jl` | a caller that never names a marked thing still depends on it | -| `test_spec_docstring.jl` | a mark and a docstring are different accounts and must coexist | -| `test_spec_verify.jl` | how well is a marked thing exercised by the tests | -| `test_spec_profile.jl` | what a real run went through, how often, and how much of it | -| `test_spec_dispatch.jl` | one call site, several methods, only some marked — the branch | -| `test_spec_lifecycle.jl` | the mark's EXIT, and an entry point that is a module rather than a function | -| `test_spec_integration.jl` | where the mark has to surface: docs, Aqua, releases, provenance, CI | +Anything already implemented is a plain `@test`. + +## What is covered + +The measure is **distinct behaviours** — one per leaf `@testset` — not assertions. An assertion +count moves without any implementation progress: `test_spec_declare.jl` has 36 assertion lines +but 91 runtime assertions, because several run inside `for mk in experimental(Declared)`, so a +thirteenth fixture mark would buy four more passing assertions and cover nothing new. A leaf +testset is one claim, and adding one means writing one. + +*operating today* is the column that matters when reading a claim about this directory: a leaf +that is entirely `@test_broken` is a claim written down, not a check being run. + + +| file | behaviours | operating today | specified only | concern | +|---|---|---|---|---| +| `test_spec_declare.jl` | 11 | 7 | 4 | what can carry a mark: function, method, struct, const, module, macro, extension | +| `test_spec_dispatch.jl` | 14 | 4 | 10 | one call site, several methods, only some marked — the branch | +| `test_spec_docstring.jl` | 9 | 6 | 3 | a mark and a docstring are different accounts and must coexist | +| `test_spec_foreign.jl` | 14 | 5 | 9 | marking a method on somebody else's generic — the `QAtlas.fetch` case | +| `test_spec_forms.jl` | 26 | 15 | 11 | the definition forms a real package hits on its second afternoon | +| `test_spec_integration.jl` | 17 | 1 | 16 | where the mark has to surface: docs, Aqua, releases, provenance, CI | +| `test_spec_lifecycle.jl` | 15 | 7 | 8 | the mark's EXIT, and an entry point that is a module rather than a function | +| `test_spec_profile.jl` | 32 | 3 | 29 | what a real run went through, how often, and how much of it | +| `test_spec_propagate.jl` | 20 | 2 | 18 | a caller that never names a marked thing still depends on it | +| `test_spec_verify.jl` | 8 | 2 | 6 | how well is a marked thing exercised by the tests | +| **10 files** | **166** | **52** | **114** | | + + +The table is generated and pinned by `test/test_spec_table.jl`, which fails if it goes stale — +the hand-written version drifted inside the change that introduced it. + +## Negative controls + +Each group has a negative control **specified**; most are not yet operating, because the control +is `@test_broken` alongside the claim it controls. They are listed here as design, not as +evidence: + +| group | the control | operating? | +|---|---|---| +| propagate | `top_good` is *exactly as deep* as `top_bad` and must come back `:clean` | no | +| profile | `cold` is marked and never called, and must be **absent**, not reported with count zero | no | +| verify | the fixture is exercised only *partly*, so a checker that always reports 100% cannot pass | no | +| integration | a settled name must get **no** docs note | no | +| dispatch | `which()` really does throw for the branching signatures the file rests on | **yes** | +| forms | the misuse refusals name the missing half, rather than one generic message | **yes** | +| lifecycle | deleting a settled name is still breaking, so `isbreaking` cannot answer `false` always | **yes** | + +"no" means the assertion about the *implementation* does not run, because the implementation is +not there. It does not mean the row is unchecked: the fixture premise each control rests on is +pinned live where it could be got wrong — `test_spec_verify.jl:38` asserts the fixture really is +only partly exercised, and `test_spec_dispatch.jl:93` asserts the specificity relation the whole +file assumes. A control resting on a false premise is the failure mode those guard. + +The one that matters most is not in the table because it is a rule rather than a fixture: +`:unknown` must never be reported as `:clean`. Two fixtures (`Holder.f::Function`, `TABLE[i](x)`) +really can reach the marked function while being statically invisible. Answering "no experimental +dependency" there is not a weaker claim, it is a false one. ## What the spec already found @@ -49,7 +92,14 @@ Two defects, both live in the shipped code, both of the kind the spec was writte a mark that silently records the wrong thing rather than refusing: 1. `@experimental "…" (c::C)(x) = c.k * x` marks **`:c`**, the argument name. Not the type, not - a function — a local that is not a binding anywhere, so the mark is silently meaningless. + a function — a local that is not a binding anywhere. It produces **two** wrong signals, not + one: the audit reports `:c` as *dangling* (declared, no such binding) **and** `:C` as + *unaccounted* (public, never declared), so it tells the author to go declare the very thing + that line declares. Both halves have to move together; `test_spec_forms.jl` pins each. 2. A mark inside a function body is refused by *Julia*, not by this package: - `syntax: unsupported const declaration on local variable`. The message never mentions - `@experimental` and points at a line the author did not write. + `syntax: unsupported const declaration on local variable`. Half fixed — the expansion now + carries the caller's `LineNumberNode`, so the message names the line the author wrote instead + of `ExperimentalAPI/src/mark.jl`, which read as a bug in the package. The message still never + says `@experimental`, and may not be able to: `const` in local scope fails during lowering, + before any emitted code runs, and the one alternative that avoids `const` (`global`) fails + *silently* in local scope, which is worse. diff --git a/test/spec/summary.jl b/test/spec/summary.jl new file mode 100644 index 0000000..e79b60c --- /dev/null +++ b/test/spec/summary.jl @@ -0,0 +1,167 @@ +# Generates the coverage table for `test/spec/README.md` and for the pull request that ships it. +# +# Written because the hand-maintained version drifted inside the very change that argues prose +# drifts and tests do not: two spec files landed after the table was typed, and every count in it +# was stale. A table that is generated from the files and pinned by a test cannot do that. +# +# The published measure is DISTINCT BEHAVIOURS — one per leaf `@testset` — not assertions. An +# assertion count moves without any implementation progress: `test_spec_declare.jl` has 36 +# assertion lines but 91 runtime assertions, because several of them run inside +# `for mk in experimental(Declared)`, so adding a thirteenth fixture mark buys four more passing +# assertions and covers nothing new. A leaf testset is one claim about the package, and adding +# one means writing one. +# +# Run it: julia --project=test test/spec/summary.jl + +module SpecSummary + +const SPEC_DIR = @__DIR__ + +# One line per spec file. A file with no entry here is an ERROR, not an omission — the previous +# table was written by listing files by hand, and silently lost `dispatch` and `lifecycle`. +const CONCERNS = Dict( + "test_spec_declare.jl" => "what can carry a mark: function, method, struct, const, module, macro, extension", + "test_spec_forms.jl" => "the definition forms a real package hits on its second afternoon", + "test_spec_foreign.jl" => "marking a method on somebody else's generic — the `QAtlas.fetch` case", + "test_spec_propagate.jl" => "a caller that never names a marked thing still depends on it", + "test_spec_docstring.jl" => "a mark and a docstring are different accounts and must coexist", + "test_spec_verify.jl" => "how well is a marked thing exercised by the tests", + "test_spec_profile.jl" => "what a real run went through, how often, and how much of it", + "test_spec_dispatch.jl" => "one call site, several methods, only some marked — the branch", + "test_spec_lifecycle.jl" => "the mark's EXIT, and an entry point that is a module rather than a function", + "test_spec_integration.jl" => "where the mark has to surface: docs, Aqua, releases, provenance, CI", +) + +""" + spec_files() -> Vector{String} + +Every spec file on disk, read from the directory rather than from a list, so a file cannot be +added and left out of the table. +""" +function spec_files() + return sort( + filter(f -> startswith(f, "test_spec_") && endswith(f, ".jl"), readdir(SPEC_DIR)) + ) +end + +function _ismacrocall(x, name::Symbol) + return x isa Expr && x.head === :macrocall && !isempty(x.args) && x.args[1] === name +end + +function _walk(f, x) + f(x) + x isa Expr && for a in x.args + _walk(f, a) + end + return nothing +end + +_count(pred, x) = (n=0; _walk(y -> (pred(y) && (n += 1)), x); n) + +""" + Counts + +What one spec file contains. `operating` and `specified` partition `behaviours`: a leaf testset +either runs at least one assertion against the package today, or it is entirely `@test_broken` — +a claim written down and not yet checked against anything. + +The distinction is the point. "Every group has a negative control" is true of this directory as +a *specification* and false of it as a *running suite*: `test_spec_profile.jl`'s control that a +never-called mark is absent rather than reported with count zero is itself `@test_broken`, so it +controls nothing until the implementation lands. +""" +struct Counts + behaviours::Int + operating::Int + specified::Int + broken_assertions::Int +end + +function counts(path::AbstractString) + ex = Meta.parseall(read(path, String); filename=path) + sets = Expr[] + _walk(x -> (_ismacrocall(x, Symbol("@testset")) && push!(sets, x)), ex) + behaviours = operating = 0 + for ts in sets + body = ts.args[2:end] + # A leaf is a testset with no testset inside it. Loop-generated `@testset "$T"` blocks + # are one leaf each, not one per iteration: a behaviour is something someone wrote. + sum(b -> _count(y -> _ismacrocall(y, Symbol("@testset")), b), body) == 0 || continue + behaviours += 1 + live = sum( + b -> _count( + y -> + _ismacrocall(y, Symbol("@test")) || + _ismacrocall(y, Symbol("@test_throws")), + b, + ), + body, + ) + live > 0 && (operating += 1) + end + broken = _count(y -> _ismacrocall(y, Symbol("@test_broken")), ex) + return Counts(behaviours, operating, behaviours - operating, broken) +end + +const _HEAD = "| file | behaviours | operating today | specified only | concern |" +const _RULE = "|---|---|---|---|---|" + +""" + table() -> String + +The markdown table, generated. `test/test_spec_table.jl` pins `README.md` against it, so the two +cannot come apart the way the hand-written one did. +""" +function table() + rows = String[_HEAD, _RULE] + tot = Counts(0, 0, 0, 0) + for f in spec_files() + haskey(CONCERNS, f) || error( + "test/spec/$f has no entry in SpecSummary.CONCERNS — add one line describing what " * + "it covers. Reading the directory rather than a list is what makes this an error " * + "instead of a silently missing row.", + ) + c = counts(joinpath(SPEC_DIR, f)) + push!( + rows, + "| `$f` | $(c.behaviours) | $(c.operating) | $(c.specified) | $(CONCERNS[f]) |", + ) + tot = Counts( + tot.behaviours + c.behaviours, + tot.operating + c.operating, + tot.specified + c.specified, + tot.broken_assertions + c.broken_assertions, + ) + end + push!( + rows, + "| **$(length(spec_files())) files** | **$(tot.behaviours)** | " * + "**$(tot.operating)** | **$(tot.specified)** | |", + ) + return join(rows, "\n") +end + +function total() + return foldl( + (a, b) -> Counts( + a.behaviours + b.behaviours, + a.operating + b.operating, + a.specified + b.specified, + a.broken_assertions + b.broken_assertions, + ), + (counts(joinpath(SPEC_DIR, f)) for f in spec_files()); + init=Counts(0, 0, 0, 0), + ) +end + +end # module SpecSummary + +if abspath(PROGRAM_FILE) == @__FILE__ + println(SpecSummary.table()) + t = SpecSummary.total() + println() + println( + "$(t.behaviours) behaviours: $(t.operating) operating today, $(t.specified) " * + "specified only ($(t.broken_assertions) `@test_broken` assertions).", + ) +end diff --git a/test/spec/test_spec_forms.jl b/test/spec/test_spec_forms.jl index 6785ee7..70c139b 100644 --- a/test/spec/test_spec_forms.jl +++ b/test/spec/test_spec_forms.jl @@ -65,11 +65,41 @@ end @test !isdefined(CallableMarked, :c) # marks a non-binding end +@testset "and the audit compounds it: C is reported as UNDECLARED" begin + # The second wrong signal, and the worse one. Because the mark landed on `:c`, the audit sees + # a declaration for a name that does not exist AND a public name with no declaration — so it + # tells the author to go declare `C`, which is the very thing the line above declares. A + # reader who follows that advice writes the mark a second time and still gets both signals. + # + # Recorded here so that whoever fixes `_signame` knows BOTH halves have to move together: + # correcting the recorded symbol without re-running the audit leaves `dangling` empty but + # `unaccounted` unchanged if the audit keys off something else. + @eval module CallablePublic + using ExperimentalAPI + public C + struct C + k::Float64 + end + @experimental "scaling rule provisional" (c::C)(x) = c.k * x + end + a = ExperimentalAPI.audit(Main.CallablePublic) + @test :c in a.dangling # declared, but no such binding + @test :C in a.unaccounted # public, and — per the audit — never declared + @test Main.CallablePublic.C(2.0)(3.0) == 6.0 # the definition itself is fine; the mark is not +end + @testset "a callable struct marks the type, or refuses" begin # Either answer is defensible. Marking the argument name is not. @test_broken :C in [mk.name for mk in experimental(Main.CallableMarked)] end +@testset "…and fixing that clears BOTH signals" begin + # The paired assertion for the testset above. `_signame` returning `:C` is necessary but not + # sufficient: the audit is what the author actually reads, and it has to go quiet too. + a = ExperimentalAPI.audit(Main.CallablePublic) + @test_broken isempty(a.dangling) && :C ∉ a.unaccounted +end + @testset "a constructor method is marked on the type" begin # This one already resolves correctly: the mark lands on `:T`. @eval module CtorMarked @@ -161,29 +191,57 @@ end end @testset "a mark inside a function body is refused" begin - # It IS refused today, but by Julia rather than by this package: the emitted `const` is - # illegal in local scope, so the message is - # syntax: unsupported `const` declaration on local variable - # which says nothing about `@experimental` and points at a line the author did not write. + # It IS refused by Julia rather than by this package: `const` in local scope is a LOWERING + # error, which fires before any code this macro emits can run, so there is no point at which + # a check of ours could intercept it. The one lever left is where the error points, and it + # used to point at `ExperimentalAPI/src/mark.jl` — reading as a bug in the package rather + # than a misuse of it, whose natural next step is to file an issue here. The expansion now + # carries the CALLER's `LineNumberNode`, so the message names the line the author wrote. + # + # The misuse has to arrive from a FILE. Measured 2026-09-03: the same misuse written through + # `@eval` or `Core.eval` — which is how every other refusal in this file is probed — produces + # `"syntax: unsupported `const` declaration on local variable"` with NO location at all, so a + # location assertion made that way is vacuous and would pass just as well against the old + # expansion. The source is built line by line so the formatter cannot shift line 4. + dir = mktempdir() + path = joinpath(dir, "caller_side.jl") + write( + path, + join( + [ + "module CallerSide", + "using ExperimentalAPI", + "function outer()", + " @experimental \"why\" inner(x) = x", + " return inner", + "end", + "end", + ], + "\n", + ), + ) e = try - @eval module ClosureMarked - using ExperimentalAPI - function outer() - @experimental "why" inner(x) = x - return inner - end - end + include(path) nothing catch err - err + err isa LoadError ? err.error : err end @test e isa ErrorException - @test occursin("unsupported `const` declaration", sprint(showerror, e)) + msg = sprint(showerror, e) + @test occursin("unsupported `const` declaration", msg) + # The half that is fixed: the blame lands on the line the author actually wrote… + @test occursin("caller_side.jl:4", msg) + # …and nowhere in this package. Checked against the FILE NAME: the repository path contains + # the string "ExperimentalAPI", so a negative assertion on that word passes by accident. + @test !occursin("mark.jl", msg) end @testset "the refusal names @experimental rather than leaking the emitted const" begin - # A macro whose diagnostic is about its own expansion has handed the reader a puzzle. The - # message should say that a mark belongs at module top level, next to `export` and `public`. + # The half that is NOT fixed. The message now points at the right line, but it still talks + # about a `const` the author never wrote; it should say that a mark belongs at module top + # level, next to `export` and `public`. Whether that is reachable at all is open — the error + # comes from lowering, so this may only be answerable by not emitting `const`, and the + # alternative (`global`) fails SILENTLY in local scope, which is strictly worse. e = try @eval module ClosureMarked2 using ExperimentalAPI diff --git a/test/spec/test_spec_integration.jl b/test/spec/test_spec_integration.jl index 5a9a902..295342b 100644 --- a/test/spec/test_spec_integration.jl +++ b/test/spec/test_spec_integration.jl @@ -3,6 +3,14 @@ # A mark that only `ExperimentalAPI` can read is a private note. The intent — a reader of a paper, # a reviewer of a PR, or a user of the docs site learning that a number came from unvalidated # code — requires the mark to surface in tools nobody configured for it. +# +# This is the group with the most external dependencies named in it, so: can any of it actually +# run in CI, or is it destined to stay broken because it cannot be otherwise? Checked before +# writing more of it. Of the assertions below, all but one need nothing new — `docstring_note`, +# `aqua_compatible_names`, `compare_methods`, `stale_since` and `test_surface` are pure functions +# over data this package already holds, and `stamp` touches only a temporary file. The single +# exception is `DocumenterExt`, which needs Documenter in the test environment; it is the one row +# here that costs something to turn green, and it is marked as such at its testset. using ExperimentalAPI: ExperimentalAPI, @experimental, experimental using Test @@ -44,6 +52,10 @@ end @testset "a Documenter block can list a module's marks" begin # `@autodocs`-style: one block in the manual, always current, never hand-maintained. + # + # The only assertion in this file that needs a test dependency this package does not already + # have (Documenter). Everything else here is pure or touches a temporary file, so this group + # is not blocked on infrastructure — see the note at the top. @test_broken ExperimentalAPI.DocumenterExt isa Module end diff --git a/test/spec/test_spec_profile.jl b/test/spec/test_spec_profile.jl index cb2c0e5..dd2114a 100644 --- a/test/spec/test_spec_profile.jl +++ b/test/spec/test_spec_profile.jl @@ -138,22 +138,44 @@ end @test Sim.driver(M, 3) ≈ 3 * (0.5 * 1.0000001 + exp(-0.5)) end -@testset "a marked definition is not slower than the same definition unmarked" begin +@testset "a marked definition costs the same at run time as an unmarked one" begin + # This was a wall-clock ratio, `a < 5b + 1e-3`, and it was wrong twice over. Measured + # 2026-09-03, Julia 1.12.2, idle machine: + # + # * The two arms did different work. `a` timed `Sim.energy(Sim.Model(x))` — a struct + # construction per iteration — against a bare `unmarked(x)`. The 5x margin was spent on + # the fixture, not on the mark, and the test failed at a = 11.8 ms, b = 2.1 ms: a real + # 5.6x difference that says nothing about `@experimental`. + # * Making the arms identical does not rescue it. With the same body marked and unmarked, + # the ratio over eight trials ran **0.79 to 3.03**. A 5x threshold sits inside that + # spread on an idle machine, let alone on a shared CI runner across three OSes. + # + # So the claim is checked where it is exact instead. `@allocated` is deterministic, and the + # testset above pins the same property structurally, which is the stronger of the two: it + # catches any wrapper, whereas a wrapper that increments a counter in a preallocated + # `Vector{Int}` would allocate nothing and pass this one. + @eval module SpeedPair + using ExperimentalAPI + public marked, unmarked + @experimental "identical body, marked" marked(x::Float64) = x * 1.0000001 unmarked(x::Float64) = x * 1.0000001 - function loop(f) + end + function loop(f, xs::Vector{Float64}) acc = 0.0 - for _ in 1:2_000_000 - acc += f(1.0) + for x in xs + acc += f(x) end return acc end - loop(x -> Sim.energy(Sim.Model(x))) - loop(unmarked) # warm both before timing either - a = @elapsed loop(x -> Sim.energy(Sim.Model(x))) - b = @elapsed loop(unmarked) - # Loose on purpose: this is a floor against wrapping, not a benchmark. A wrapper that - # increments a counter would not fit inside this margin. - @test a < 5b + 1e-3 + xs = collect(1.0:1.0:100_000.0) + loop(Main.SpeedPair.marked, xs) # warm both before measuring either + loop(Main.SpeedPair.unmarked, xs) + @test @allocated(loop(Main.SpeedPair.marked, xs)) == + @allocated(loop(Main.SpeedPair.unmarked, xs)) + # …and that the shared figure is zero, so the equality above is not two equal wrappers. + @test @allocated(loop(Main.SpeedPair.unmarked, xs)) == 0 + # The mark is still recorded — otherwise the two arms are identical because nothing happened. + @test :marked in [mk.name for mk in ExperimentalAPI.experimental(Main.SpeedPair)] end # ── mechanism constraints ──────────────────────────────────────────────────────────────────── diff --git a/test/test_spec_table.jl b/test/test_spec_table.jl new file mode 100644 index 0000000..186ead6 --- /dev/null +++ b/test/test_spec_table.jl @@ -0,0 +1,83 @@ +# The table in `test/spec/README.md` is generated, and this is what keeps it that way. +# +# The first version of that table was written by hand, and it drifted inside the very change that +# argues prose drifts and tests do not: two spec files landed after it was typed, so it said +# "nine files" when there were eleven and every count in it was wrong. Generating it is only half +# a fix — a generator nobody runs drifts exactly as fast. This is the other half. + +using Test + +include("spec/summary.jl") + +const _SPEC_README = joinpath(@__DIR__, "spec", "README.md") +const _BEGIN = "" +const _END = "" + +@testset "the spec table is generated, not typed" begin + md = read(_SPEC_README, String) + @test occursin(_BEGIN, md) + @test occursin(_END, md) + block = strip(split(split(md, _BEGIN)[2], _END)[1]) + # The failure message has to say what to do, because "a table is out of date" is not a + # defect anybody can act on without being told where the table comes from. + if block != SpecSummary.table() + @info "test/spec/README.md is stale — regenerate with `julia --project=test test/spec/summary.jl`" + end + @test block == SpecSummary.table() +end + +@testset "no spec file is missing from the table" begin + # `spec_files()` reads the directory, and `table()` errors on a file with no entry, so this + # asserts the directory is non-empty and the generator ran over all of it. Without the first + # assertion a `readdir` that returned nothing would satisfy the second vacuously. + files = SpecSummary.spec_files() + @test length(files) >= 10 + @test Set(files) == Set(keys(SpecSummary.CONCERNS)) +end + +@testset "no spec file is missing from runtests.jl" begin + # The other silent-omission surface: a spec file can exist, be listed in the table, and never + # run, because `runtests.jl` includes them by name. Then it contributes to the published + # count while asserting nothing. + driver = read(joinpath(@__DIR__, "runtests.jl"), String) + missing_from_driver = filter( + f -> !occursin("spec/$f", driver), SpecSummary.spec_files() + ) + @test isempty(missing_from_driver) +end + +@testset "the count is behaviours, and a loop cannot inflate it" begin + # The property that makes the published measure worth publishing: `test_spec_declare.jl` + # runs 91 assertions from 36 assertion lines because several sit inside + # `for mk in experimental(Declared)`. Adding a fixture mark adds passing assertions and no + # coverage. A leaf `@testset` is something a person wrote. + # + # Pinned against a synthetic file rather than a real one, so that editing a spec file does + # not silently change what this asserts. + path = joinpath(mktempdir(), "test_spec_synthetic.jl") + write( + path, + join( + [ + "@testset \"outer\" begin", # not a leaf: contains a testset + " for i in 1:100", + " @testset \"inner \$i\" begin", # one leaf, whatever the loop bound + " @test i == i", + " end", + " end", + "end", + "@testset \"specified only\" begin", # a leaf with no operating assertion + " for k in 1:50", + " @test_broken notyet(k)", + " end", + "end", + ], + "\n", + ), + ) + c = SpecSummary.counts(path) + @test c.behaviours == 2 # not 3, and not 100 + @test c.operating == 1 + @test c.specified == 1 + @test c.broken_assertions == 1 # one written line, not fifty executions +end From ac648154f481d1883e6e744516010c2806884285 Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Thu, 3 Sep 2026 08:26:47 +0000 Subject: [PATCH 09/11] fix: check the no-wrapping claim where it is exact, and normalise line endings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two CI failures, both defects I introduced in the previous commit. `julia 1.11 — ubuntu-latest`: `@allocated(loop(unmarked, xs)) == 0` came back 16. The absolute figure was the number Julia 1.12 produced on my machine, written down as if it were a property of the code. Chasing it produced the real finding: `@allocated` cannot see the regression it was guarding against. A counter wrapper `push!`ing into a vector whose capacity the warm-up call already grew allocates **zero** — measured, as a positive control that failed to fire. So the allocation route is not a weaker check, it is a check of something else. The claim — `@experimental` emits the definition unchanged — is exact at the expansion and only ever approximate at run time, so it is now checked there and only there: the method bodies of the marked expansion must equal those of the bare one, compared at the AST rather than as strings (the expansion carries `Expr(:escape, …)`, and the printed forms differ when the bodies do not). `WrapControl.@wrapping` is a macro that does wrap the call, and the same comparison must reject it — without that, `!occursin(…)` is satisfied by an expansion that dropped the definition. All three run-time routes that were tried are recorded in the file with what each measured, so the absence of a timing assertion reads as a finding rather than as a gap. `julia 1.12 — windows-latest`: git checks the README out with CRLF, so the generated-table comparison was between line endings. Normalised on both sides. Co-Authored-By: Claude Opus 5 --- test/spec/README.md | 4 +- test/spec/test_spec_profile.jl | 113 +++++++++++++++++++-------------- test/test_spec_table.jl | 10 ++- 3 files changed, 75 insertions(+), 52 deletions(-) diff --git a/test/spec/README.md b/test/spec/README.md index bbed166..26cc776 100644 --- a/test/spec/README.md +++ b/test/spec/README.md @@ -50,10 +50,10 @@ that is entirely `@test_broken` is a claim written down, not a check being run. | `test_spec_forms.jl` | 26 | 15 | 11 | the definition forms a real package hits on its second afternoon | | `test_spec_integration.jl` | 17 | 1 | 16 | where the mark has to surface: docs, Aqua, releases, provenance, CI | | `test_spec_lifecycle.jl` | 15 | 7 | 8 | the mark's EXIT, and an entry point that is a module rather than a function | -| `test_spec_profile.jl` | 32 | 3 | 29 | what a real run went through, how often, and how much of it | +| `test_spec_profile.jl` | 31 | 2 | 29 | what a real run went through, how often, and how much of it | | `test_spec_propagate.jl` | 20 | 2 | 18 | a caller that never names a marked thing still depends on it | | `test_spec_verify.jl` | 8 | 2 | 6 | how well is a marked thing exercised by the tests | -| **10 files** | **166** | **52** | **114** | | +| **10 files** | **165** | **51** | **114** | | The table is generated and pinned by `test/test_spec_table.jl`, which fails if it goes stale — diff --git a/test/spec/test_spec_profile.jl b/test/spec/test_spec_profile.jl index dd2114a..7d01286 100644 --- a/test/spec/test_spec_profile.jl +++ b/test/spec/test_spec_profile.jl @@ -126,58 +126,77 @@ end # Folded in from what used to be `test_spec_runtime.jl`. Its other eight testsets restated this # file's claims on a strictly smaller fixture; only these two measured anything of their own. -@testset "the mark does not wrap the call" begin - # The property the package currently advertises and must keep: `@experimental` emits the - # definition unchanged plus one `push!` at load time. If instrumentation ever becomes - # unconditional, an inner loop pays for it on every iteration. - # Reading `src/mark.jl` for a prose phrase would pass for a regression that wrapped the call - # and left the sentence alone. Look at what the macro emits. - emitted = string(@macroexpand @experimental "why" f(x) = x) - @test occursin("f(x)", emitted) # the definition is there… - @test !occursin("function f", replace(emitted, "f(x)" => "")) # …and not wrapped in another - @test Sim.driver(M, 3) ≈ 3 * (0.5 * 1.0000001 + exp(-0.5)) +# The control for the two assertions below. A predicate that has never returned `true` for +# anything is not evidence, and `!occursin(…)` is the shape most easily satisfied by an expansion +# that dropped the definition altogether. `@wrapping` does exactly what `@experimental` must never +# become, so the check can be shown to fire. Kept in a module because a macro defined at the top +# level of a test file is visible to every file after it in the same shard. +module WrapControl + +"A macro that wraps the call it is given — the regression this file exists to prevent." +macro wrapping(_reason, def) + return esc(Expr(:function, def.args[1], Expr(:block, :(COUNTS[] += 1), def.args[2]))) +end + +""" + method_bodies(ex) -> Vector + +The body of every method definition inside an expanded expression, line numbers stripped. Walks +into `Expr(:escape, …)`, which is where a macro's own output lives. +""" +function method_bodies(ex) + out = Any[] + walk(x) = + if x isa Expr + if (x.head === :(=) || x.head === :function) && + x.args[1] isa Expr && + x.args[1].head === :call + push!(out, x.args[2]) + end + foreach(walk, x.args) + end + walk(Base.remove_linenums!(ex)) + return out end -@testset "a marked definition costs the same at run time as an unmarked one" begin - # This was a wall-clock ratio, `a < 5b + 1e-3`, and it was wrong twice over. Measured - # 2026-09-03, Julia 1.12.2, idle machine: - # - # * The two arms did different work. `a` timed `Sim.energy(Sim.Model(x))` — a struct - # construction per iteration — against a bare `unmarked(x)`. The 5x margin was spent on - # the fixture, not on the mark, and the test failed at a = 11.8 ms, b = 2.1 ms: a real - # 5.6x difference that says nothing about `@experimental`. - # * Making the arms identical does not rescue it. With the same body marked and unmarked, - # the ratio over eight trials ran **0.79 to 3.03**. A 5x threshold sits inside that - # spread on an idle machine, let alone on a shared CI runner across three OSes. +end # module WrapControl + +@testset "the mark does not wrap the call" begin + # The property the package advertises and must keep: `@experimental` emits the definition + # unchanged plus one `push!` at load time. If instrumentation ever becomes unconditional, an + # inner loop pays for it on every iteration. # - # So the claim is checked where it is exact instead. `@allocated` is deterministic, and the - # testset above pins the same property structurally, which is the stronger of the two: it - # catches any wrapper, whereas a wrapper that increments a counter in a preallocated - # `Vector{Int}` would allocate nothing and pass this one. - @eval module SpeedPair - using ExperimentalAPI - public marked, unmarked - @experimental "identical body, marked" marked(x::Float64) = x * 1.0000001 - unmarked(x::Float64) = x * 1.0000001 - end - function loop(f, xs::Vector{Float64}) - acc = 0.0 - for x in xs - acc += f(x) - end - return acc - end - xs = collect(1.0:1.0:100_000.0) - loop(Main.SpeedPair.marked, xs) # warm both before measuring either - loop(Main.SpeedPair.unmarked, xs) - @test @allocated(loop(Main.SpeedPair.marked, xs)) == - @allocated(loop(Main.SpeedPair.unmarked, xs)) - # …and that the shared figure is zero, so the equality above is not two equal wrappers. - @test @allocated(loop(Main.SpeedPair.unmarked, xs)) == 0 - # The mark is still recorded — otherwise the two arms are identical because nothing happened. - @test :marked in [mk.name for mk in ExperimentalAPI.experimental(Main.SpeedPair)] + # Compared at the AST, not as a string: the expansion contains `Expr(:escape, …)` and the + # printed forms differ even when the bodies are identical. Reading `src/mark.jl` for a prose + # phrase would pass for a regression that wrapped the call and left the sentence alone. + bare = WrapControl.method_bodies(@macroexpand f(x) = x * 2) + marked = WrapControl.method_bodies(@macroexpand @experimental "why" f(x) = x * 2) + @test length(bare) == 1 + @test marked == bare # the definition is emitted untouched + # …and the comparison can fail, which is the half that makes the line above worth anything. + wrapped = WrapControl.method_bodies( + @macroexpand WrapControl.@wrapping "why" f(x) = x * 2 + ) + @test wrapped != bare + @test Sim.driver(M, 3) ≈ 3 * (0.5 * 1.0000001 + exp(-0.5)) end +# There is no run-time cost assertion here, and that is a finding rather than a gap. Three routes +# were measured on 2026-09-03 and all three were worse than the structural check above: +# +# * A wall-clock ratio, `a < 5b + 1e-3`. Its two arms did different work — `Sim.energy( +# Sim.Model(x))`, a struct construction per iteration, against a bare call — so the 5x margin +# was spent on the fixture. It failed at a = 11.8 ms against b = 2.1 ms. +# * The same ratio with the arms made identical (same body, one marked). Over eight trials on an +# idle machine the ratio ran **0.79 to 3.03**, so the threshold sat inside the noise. +# * `@allocated` equality between the arms. Deterministic, but it measures the wrong thing: it +# came back 0 on Julia 1.12 and 16 on 1.11 for the SAME code, and — decisively — a counter +# wrapper `push!`ing into a warmed vector also allocates 0, so the check cannot see the +# regression it was written to catch. The positive control is what showed that. +# +# The claim is exact at the expansion and only ever approximate at run time, so it is checked +# where it is exact. + # ── mechanism constraints ──────────────────────────────────────────────────────────────────── @testset "recording survives inlining" begin diff --git a/test/test_spec_table.jl b/test/test_spec_table.jl index 186ead6..5b55736 100644 --- a/test/test_spec_table.jl +++ b/test/test_spec_table.jl @@ -13,17 +13,21 @@ const _SPEC_README = joinpath(@__DIR__, "spec", "README.md") const _BEGIN = "" const _END = "" +# Git checks the README out with CRLF on Windows, so the comparison below is between line +# endings unless they are normalised — measured, as a red `julia 1.12 — windows-latest`. +_lf(s::AbstractString) = replace(s, "\r\n" => "\n") + @testset "the spec table is generated, not typed" begin - md = read(_SPEC_README, String) + md = _lf(read(_SPEC_README, String)) @test occursin(_BEGIN, md) @test occursin(_END, md) block = strip(split(split(md, _BEGIN)[2], _END)[1]) # The failure message has to say what to do, because "a table is out of date" is not a # defect anybody can act on without being told where the table comes from. - if block != SpecSummary.table() + if block != _lf(SpecSummary.table()) @info "test/spec/README.md is stale — regenerate with `julia --project=test test/spec/summary.jl`" end - @test block == SpecSummary.table() + @test block == _lf(SpecSummary.table()) end @testset "no spec file is missing from the table" begin From 8097f06e75d0e53340f5600914e4fe8c1c828e23 Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Thu, 3 Sep 2026 08:52:09 +0000 Subject: [PATCH 10/11] spec: split detection from recording, and make the default layer the point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The goal was restated: a tool that says WHERE experimental code was used, and a user who finds out they used it WITHOUT opting in. The second half contradicted three requirements this file already carried, so the boundary was measured instead of argued. 10M calls of a realistic numeric body, Julia 1.12.2, minimum of 7-9 trials: | emitted into the body | 1 thread | 8 threads | counts correctly? | | nothing | 1.00x | 1.00x | - | | set-once flag, read-mostly | 1.03x | 0.985x | yes | | counter, plain shared Ref | 1.03x | 3.76x | **no** | | counter, global atomic | 1.17x | 4.87x | yes | | counter, per-thread atomic | 1.12x | 2.79x | yes | | @warn, guarded, fires once | 5.65x | - | yes | | @warn maxlog=1 | 59.57x | - | yes | Two results decided the split. The plain counter is not merely slow in parallel — it recorded 95,406,048 of 160,000,000 calls, losing 40% to races, so it is wrong as well as expensive. And a flag written once and only read afterwards never dirties the cache line again, which is why it is free at eight threads while every counting scheme is not. So presence is detected by default and costs nothing; counts, call sites and paths are opt-in. The @warn rows are why the default notice is a summary at process exit rather than a warning at first entry: the cost is the logging call sitting in the body, not the warning being printed, and guarding it so it fires once does not recover it. WITHDRAWN: "the mark does not wrap the call", pinned structurally one commit ago. Presence cannot be detected without emitting something into the body. What replaces it is narrower and measured — the emitted statement must be read-mostly, must add exactly one statement, and must not bring the logging machinery with it. The last is checked today, with a macro that does log as the positive control. "Recording is off by default" and "the run pays nothing when recording is off" are likewise replaced by the two-layer pair. The exit summary is observed from a child process, because asserting that an atexit handler is registered in this one would pass for a handler that prints nothing, and reading stdout alone would pass trivially if the notice went to stderr. The two child scripts differ only in the final call, so a summary keyed on "this module has marks" fails the control and passes the claim. Also recorded, both found while benchmarking: threadid() returned 9 under `-t 8` because the interactive pool is counted separately, so per-thread storage must be sized by maxthreadid(); and a fourth wall-clock route was ruled out in advance, since these figures come from an idle machine and the same thresholds on a shared CI runner across three OSes would be a flake generator. 174 behaviours, 54 operating. Co-Authored-By: Claude Opus 5 --- test/spec/README.md | 40 +++++- test/spec/test_spec_profile.jl | 234 +++++++++++++++++++++++++++++---- 2 files changed, 244 insertions(+), 30 deletions(-) diff --git a/test/spec/README.md b/test/spec/README.md index 26cc776..549f6ff 100644 --- a/test/spec/README.md +++ b/test/spec/README.md @@ -50,15 +50,51 @@ that is entirely `@test_broken` is a claim written down, not a check being run. | `test_spec_forms.jl` | 26 | 15 | 11 | the definition forms a real package hits on its second afternoon | | `test_spec_integration.jl` | 17 | 1 | 16 | where the mark has to surface: docs, Aqua, releases, provenance, CI | | `test_spec_lifecycle.jl` | 15 | 7 | 8 | the mark's EXIT, and an entry point that is a module rather than a function | -| `test_spec_profile.jl` | 31 | 2 | 29 | what a real run went through, how often, and how much of it | +| `test_spec_profile.jl` | 40 | 5 | 35 | what a real run went through, how often, and how much of it | | `test_spec_propagate.jl` | 20 | 2 | 18 | a caller that never names a marked thing still depends on it | | `test_spec_verify.jl` | 8 | 2 | 6 | how well is a marked thing exercised by the tests | -| **10 files** | **165** | **51** | **114** | | +| **10 files** | **174** | **54** | **120** | | The table is generated and pinned by `test/test_spec_table.jl`, which fails if it goes stale — the hand-written version drifted inside the change that introduced it. +## Two layers, and why the split is where it is + +The goal is a tool that says **where** experimental code was used, and a user who learns they +used it **without opting in**. Those are different jobs with different budgets, and the boundary +between them was measured rather than chosen. 10M calls of a realistic numeric body, Julia +1.12.2, minimum of 7–9 trials: + +| emitted into the body | 1 thread | 8 threads | counts correctly? | +|---|---|---|---| +| nothing | 1.00× | 1.00× | — | +| set-once flag, read-mostly | 1.03× | **0.985×** | yes | +| counter, plain shared `Ref` | 1.03× | 3.76× | **no** | +| counter, global atomic | 1.17× | 4.87× | yes | +| counter, per-thread atomic | 1.12× | 2.79× | yes | +| `@warn`, guarded so it fires once | 5.65× | — | yes | +| `@warn maxlog=1` | 59.57× | — | yes | + +Two results decided it. The plain counter is not merely slow in parallel — it recorded +95,406,048 of 160,000,000 calls, **losing 40% to races**, so it is wrong as well as expensive. +And a flag written once and only read afterwards never dirties the cache line again, which is +why it is free at eight threads while every counting scheme is not. + +So **presence is detected by default and costs nothing; counts, call sites and paths are +opt-in.** The `@warn` rows are why the default notice is a summary at process exit rather than a +warning at first entry: the cost is the logging call sitting in the body, not the warning being +printed, and guarding it so that it fires once does not recover it. + +### One requirement was withdrawn + +This directory used to require that `@experimental` **never wrap the call**, and +`test_spec_profile.jl` pinned it structurally. That is gone: presence cannot be detected without +emitting something into the body. What replaced it is narrower and measured — the emitted +statement must be read-mostly, must add exactly one statement, and must not bring the logging +machinery with it. The last of those is checked today, with a macro that *does* log as the +positive control. + ## Negative controls Each group has a negative control **specified**; most are not yet operating, because the control diff --git a/test/spec/test_spec_profile.jl b/test/spec/test_spec_profile.jl index 7d01286..0cc8494 100644 --- a/test/spec/test_spec_profile.jl +++ b/test/spec/test_spec_profile.jl @@ -51,6 +51,133 @@ end # module Sim const M = Sim.Model(0.5) +# ── the default layer: you find out without asking ─────────────────────────────────────────── +# +# The goal is a tool that says WHERE experimental code was used, and a user who learns they used +# it **without opting in**. Those are two layers, and which work goes in which is measured rather +# than chosen. 10M calls of a realistic numeric body, `sqrt(abs(sin(x)*cos(x) + exp(-|x|/1e6)))`, +# Julia 1.12.2, minimum of 7-9 trials: +# +# | emitted into the body | 1 thread | 8 threads | counts correctly? | +# |-----------------------------|----------|-----------|-------------------| +# | nothing | 1.00x | 1.00x | - | +# | set-once flag, read-mostly | 1.03x | 0.985x | yes | +# | counter, plain shared `Ref` | 1.03x | 3.76x | **no** | +# | counter, global atomic | 1.17x | 4.87x | yes | +# | counter, per-thread atomic | 1.12x | 2.79x | yes | +# | `@warn` guarded, fires once | 5.65x | - | yes | +# | `@warn maxlog=1` | 59.57x | - | yes | +# +# Two things decided the split. The plain counter is not merely slow in parallel — it recorded +# 95,406,048 of 160,000,000 calls, losing 40% to races, so it is *wrong* as well. And a flag +# written once and only read afterwards never dirties the cache line again, which is why it is +# free at eight threads while every counting scheme is not. +# +# So: **presence is detected by default and costs nothing; counts, call sites and paths are +# opt-in.** The `@warn` rows are why the notice is a summary at exit rather than a warning at the +# call: the cost is the logging call sitting in the body, not the warning being printed, and a +# guard that makes it fire only once does not recover it. + +@testset "a run reports what it entered without being asked to record" begin + # The default layer. No `record(...)` wrapper and no flag to set: the user ran their + # calculation and can find out afterwards. + Sim.driver(M, 10) + @test_broken :energy in [h.name for h in ExperimentalAPI.entered()] +end + +@testset "the default answer is presence, and does not pretend to be counts" begin + # A count field that is always 1, or always the number of marks, would be worse than absent: + # it reads as a measurement. The default layer knows "yes, at least once" and must say only + # that. + Sim.driver(M, 10) + @test_broken ExperimentalAPI.entered()[1].count === nothing +end + +@testset "a definition that was never entered is absent from the default answer too" begin + # The control. Without it, an `entered()` that lists every mark in every loaded module passes. + Sim.driver(M, 10) + @test_broken :cold ∉ [h.name for h in ExperimentalAPI.entered()] +end + +# The summary is a property of a process on its way out, so it has to be observed from outside +# one. Asserting that an `atexit` handler is registered in THIS process would pass for a handler +# that prints nothing, and asserting on `stdout` alone would pass trivially if the notice went to +# `stderr` — which stream a notice lands on is exactly what must not be assumed here. +module ChildRun + +const WARMED = Ref(false) + +function _cmd(code) + return `$(Base.julia_cmd()) --startup-file=no --project=$(Base.active_project()) -e $code` +end + +""" + output(code) -> String + +Everything a child `julia -e code` wrote, stdout and stderr merged. The first call warms the +precompilation cache with a throwaway child, so a `Precompiling …` banner is never mistaken for +the summary under test. +""" +function output(code::AbstractString) + if !WARMED[] + run( + pipeline( + ignorestatus(_cmd("using ExperimentalAPI")); stdout=devnull, stderr=devnull + ), + ) + WARMED[] = true + end + io = IOBuffer() + run(pipeline(ignorestatus(_cmd(code)); stdout=io, stderr=io)) + return String(take!(io)) +end + +const ENTERS = """ +using ExperimentalAPI +module Child +using ExperimentalAPI +public energy +@experimental "convergence not established below beta" energy(x) = x * 1.0000001 +end +Child.energy(0.5) +""" + +const LOADS_ONLY = """ +using ExperimentalAPI +module Child +using ExperimentalAPI +public energy +@experimental "convergence not established below beta" energy(x) = x * 1.0000001 +end +""" + +end # module ChildRun + +@testset "the summary is printed at process exit" begin + # The whole point of the default layer: the user did not ask, and still finds out. + @test_broken occursin("energy", ChildRun.output(ChildRun.ENTERS)) +end + +@testset "a process that loaded a mark but never entered it stays silent" begin + # The control, and the thing that decides whether this package is tolerable as a dependency. + # The two child scripts differ ONLY in the final call, so a summary keyed on "this module has + # marks" rather than on "this run entered one" fails here and passes above. + @test isempty(strip(ChildRun.output(ChildRun.LOADS_ONLY))) +end + +@testset "the summary carries the reason, not just the name" begin + # A name tells the user which line to look at; the reason is what tells them whether the + # number they are about to publish is affected. The reason is the payload everywhere else in + # this package, and must not be dropped at the one place a user reads by default. + @test_broken occursin("convergence not established", ExperimentalAPI.summary_text()) +end + +@testset "the default layer can be turned off" begin + # A package that cannot be quietened gets vendored around. The switch has to be readable + # before `using` returns, so it is an environment variable rather than a function call. + @test_broken ExperimentalAPI.detecting() === true +end + # ── the basic question ─────────────────────────────────────────────────────────────────────── @testset "a run reports which marked definitions it entered" begin @@ -121,7 +248,7 @@ end end end -# ── the floor: the mark must not wrap the call ─────────────────────────────────────────────── +# ── the floor: what may be emitted into the body ───────────────────────────────────────────── # # Folded in from what used to be `test_spec_runtime.jl`. Its other eight testsets restated this # file's claims on a strictly smaller fixture; only these two measured anything of their own. @@ -133,11 +260,22 @@ end # level of a test file is visible to every file after it in the same shard. module WrapControl -"A macro that wraps the call it is given — the regression this file exists to prevent." +"A macro that wraps the call it is given — the shape the emitted flag must stay inside." macro wrapping(_reason, def) return esc(Expr(:function, def.args[1], Expr(:block, :(COUNTS[] += 1), def.args[2]))) end +"A macro that puts a logging call in the body — measured at 5.65x guarded, 59.6x unguarded." +macro logging(reason, def) + return esc( + Expr( + :function, + def.args[1], + Expr(:block, :($Base.@warn $reason maxlog = 1), def.args[2]), + ), + ) +end + """ method_bodies(ex) -> Vector @@ -161,14 +299,15 @@ end end # module WrapControl -@testset "the mark does not wrap the call" begin - # The property the package advertises and must keep: `@experimental` emits the definition - # unchanged plus one `push!` at load time. If instrumentation ever becomes unconditional, an - # inner loop pays for it on every iteration. +@testset "today the expansion is untouched — which is what has to change" begin + # This file used to require that `@experimental` never wrap the call at all. That requirement + # is WITHDRAWN: the default layer cannot detect presence without emitting something into the + # body. What replaces it is not weaker, it is narrower and measured — see the table at the + # top. The emitted statement must be read-mostly, and must not drag the logging machinery in + # with it. # # Compared at the AST, not as a string: the expansion contains `Expr(:escape, …)` and the - # printed forms differ even when the bodies are identical. Reading `src/mark.jl` for a prose - # phrase would pass for a regression that wrapped the call and left the sentence alone. + # printed forms differ even when the bodies are identical. bare = WrapControl.method_bodies(@macroexpand f(x) = x * 2) marked = WrapControl.method_bodies(@macroexpand @experimental "why" f(x) = x * 2) @test length(bare) == 1 @@ -178,21 +317,42 @@ end # module WrapControl @macroexpand WrapControl.@wrapping "why" f(x) = x * 2 ) @test wrapped != bare + # The target: exactly one statement more than the bare body, and no more. + @test_broken length(marked[1].args) == length(bare[1].args) + 1 @test Sim.driver(M, 3) ≈ 3 * (0.5 * 1.0000001 + exp(-0.5)) end -# There is no run-time cost assertion here, and that is a finding rather than a gap. Three routes -# were measured on 2026-09-03 and all three were worse than the structural check above: +@testset "whatever is emitted, it is not a logging call" begin + # The hard line, and the one measurable today rather than after the fact. `@warn` in the body + # costs 59.6x unconditionally and **5.65x even when guarded so that it fires once** — the + # guard does not help, because what stops the definition inlining is the call being there at + # all. The summary at exit is the alternative that costs nothing. + emitted = string(@macroexpand @experimental "why" f(x) = x * 2) + @test !occursin("CoreLogging", emitted) + # The positive control: the same predicate against an expansion that does log. Without it, + # `!occursin(…)` is satisfied by any expansion whatsoever, including an empty one. + @test occursin( + "CoreLogging", string(@macroexpand WrapControl.@logging "why" f(x) = x * 2) + ) +end + +# There is still no wall-clock assertion in this file, and that is a finding rather than a gap. +# Four routes were measured on 2026-09-03 and each failed for its own reason: # -# * A wall-clock ratio, `a < 5b + 1e-3`. Its two arms did different work — `Sim.energy( -# Sim.Model(x))`, a struct construction per iteration, against a bare call — so the 5x margin -# was spent on the fixture. It failed at a = 11.8 ms against b = 2.1 ms. -# * The same ratio with the arms made identical (same body, one marked). Over eight trials on an -# idle machine the ratio ran **0.79 to 3.03**, so the threshold sat inside the noise. -# * `@allocated` equality between the arms. Deterministic, but it measures the wrong thing: it -# came back 0 on Julia 1.12 and 16 on 1.11 for the SAME code, and — decisively — a counter -# wrapper `push!`ing into a warmed vector also allocates 0, so the check cannot see the -# regression it was written to catch. The positive control is what showed that. +# * A ratio `a < 5b + 1e-3` whose two arms did different work — `Sim.energy(Sim.Model(x))`, a +# struct construction per iteration, against a bare call. The 5x margin was spent on the +# fixture; it failed at a = 11.8 ms against b = 2.1 ms. +# * The same ratio with the arms made identical. Over eight trials on an idle machine the ratio +# ran 0.79 to 3.03, so the threshold sat inside the noise. +# * `@allocated` equality. Deterministic, but measuring the wrong thing: 0 on Julia 1.12 and 16 +# on 1.11 for the SAME code, and a counter wrapper `push!`ing into a warmed vector allocates +# zero too — so it cannot see the regression it was written to catch. The positive control is +# what showed that. +# * A ratio against a fixed threshold, now that a flag WILL be emitted. Ruled out in advance: +# the numbers in the table at the top come from a throwaway benchmark on an idle machine, and +# the same thresholds on a shared CI runner across three operating systems would be a flake +# generator. What CI can check is the SHAPE of the expansion; what only a benchmark can check +# is the cost, and that belongs in a benchmark. # # The claim is exact at the expansion and only ever approximate at run time, so it is checked # where it is exact. @@ -206,19 +366,25 @@ end @test_broken ExperimentalAPI.record(() -> Sim.driver(M, 100))[1].count == 100 end -@testset "recording is off by default" begin +@testset "detection is on by default; counting is not" begin + # Inverted from what this file first required. "Recording off by default" was the safe answer + # only while presence detection was assumed to cost what counting costs; measured, they + # differ by a factor of four in parallel, and only one of them is affordable by default. + @test_broken ExperimentalAPI.detecting() === true @test_broken ExperimentalAPI.recording() === false end -@testset "the run pays nothing when recording is off" begin - # The property `@experimental` currently advertises. If instrumentation becomes - # unconditional, every iteration of an inner loop pays for a mark nobody is reading. - @test_broken ExperimentalAPI.overhead_when_disabled() == 0.0 +@testset "the default layer's cost is stated, and it is the flag's cost" begin + # Not `>= 0`, which every number satisfies. The measured figure for a read-mostly flag is + # 0.985x-1.03x, so a default layer reporting overhead above a few percent has stopped being + # the thing that was measured — most likely by having become a counter. + @test_broken ExperimentalAPI.overhead_when_detecting() < 0.10 end -@testset "the overhead when recording IS on is measured and reported" begin - # A tool that slows a twelve-hour run by 40x will not be used on a twelve-hour run. Whatever - # it costs, the number has to be available rather than discovered. +@testset "the opt-in layer's overhead is measured and reported, not discovered" begin + # A tool that slows a twelve-hour run by 4x will not be used on a twelve-hour run — and 4x is + # the measured figure for counting at eight threads, not a hypothetical. Whatever it costs, + # the number has to come back with the record. @test_broken ExperimentalAPI.record(() -> Sim.driver(M, 1000)).overhead isa Real end @@ -246,13 +412,25 @@ end @testset "hits from every thread are attributed" begin # HPC code is threaded. A recorder that only sees the main thread reports a fraction of the - # truth and calls it the truth. + # truth and calls it the truth — and so does one that races: measured on 8 threads, a plain + # `Ref{Int}` incremented per call recorded 95,406,048 of 160,000,000 entries. It lost 40% and + # cost 3.76x for the privilege, which is why counting is atomic and opt-in. threaded() = Threads.@threads for _ in 1:8 Sim.driver(M, 100) end @test_broken ExperimentalAPI.record(threaded)[1].count == 800 end +@testset "per-thread storage is sized by maxthreadid, not nthreads" begin + # Measured 2026-09-03 while benchmarking the counter with `-t 8`: `threadid()` returned 9, + # because the interactive pool is counted separately from the default one. A recorder that + # allocates an `nthreads()`-sized vector throws `BoundsError` on the first hit arriving from + # an interactive task — which is any hit from the REPL. + @test Threads.maxthreadid() >= Threads.nthreads() + @test_broken ExperimentalAPI.record(() -> Sim.driver(M, 10)).slots >= + Threads.maxthreadid() +end + @testset "a merged record is still a record" begin # `merge_records(...) isa AbstractVector` is the only constraint today, so a `record()` you # can ask `.overhead` of, merged with another, may legally come back as a plain `Vector` you From b422dcefec81aa5011eea838b3c9a2aabccef401 Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Thu, 3 Sep 2026 09:08:29 +0000 Subject: [PATCH 11/11] style: cut the comments to the scope of the expected behaviour MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 738 comment lines to 384. What went, in every file: the measurement narratives, the dates, the history of what a testset used to require, and the justifications for decisions the testset name already states. What stayed: the scope of the behaviour being pinned, and the boundary a reader needs in order to not weaken it — which fixture axis varies, why an assertion is not `isa Any`, what a control rejects. The measurements are not lost; they are in `test/spec/README.md`, the pull request and the commit messages, which is where they belong. A comment that recounts how a number was arrived at is a comment about the process, not about the code under it. Verified by comparing ASTs with `LineNumberNode`s stripped: every file is identical except three docstrings in `test_spec_profile.jl` and one in `summary.jl`, all shortened deliberately. Not one assertion changed, and the suite reports the same 505 / 160 / 0 as before. Co-Authored-By: Claude Opus 5 --- src/mark.jl | 74 ++++------- test/spec/summary.jl | 27 ++-- test/spec/test_spec_declare.jl | 45 +++---- test/spec/test_spec_dispatch.jl | 79 ++++------- test/spec/test_spec_docstring.jl | 37 ++---- test/spec/test_spec_foreign.jl | 61 ++++----- test/spec/test_spec_forms.jl | 127 +++++++----------- test/spec/test_spec_integration.jl | 68 ++++------ test/spec/test_spec_lifecycle.jl | 102 +++++---------- test/spec/test_spec_profile.jl | 202 +++++++---------------------- test/spec/test_spec_propagate.jl | 91 +++++-------- test/spec/test_spec_verify.jl | 30 ++--- test/test_spec_table.jl | 33 ++--- 13 files changed, 310 insertions(+), 666 deletions(-) diff --git a/src/mark.jl b/src/mark.jl index f54e1a9..4c421fb 100644 --- a/src/mark.jl +++ b/src/mark.jl @@ -1,11 +1,9 @@ -# The mark itself: what one `@experimental` declaration records, where it is stored, and the -# macro that writes it. +# What one `@experimental` declaration records, where it is stored, and the macro that writes it. # -# Storage is a `const` vector inside the MARKED module, not a table inside this one. That is not -# a style choice — a registry living here would be populated while the marked package is being -# precompiled, and nothing written into a third module at that moment is part of the cache image -# that gets loaded later. Base's own `Docs.META` is a per-module binding for exactly this reason, -# and `test/test_precompile.jl` is the test that would catch it if this stopped being true. +# Storage is a `const` vector inside the MARKED module: a registry here would be populated during +# the marked package's precompilation, and nothing written into a third module then survives into +# its cache image. `Docs.META` is per-module for the same reason. Pinned by +# `test/test_precompile.jl`. """ Mark @@ -36,11 +34,8 @@ struct Mark file::Symbol line::Int - # The reason is the payload, so "it may not be empty" belongs in the type rather than in one - # code path. Without this the check lived only in `_reason`, which only the macro calls — - # `Mark(Main, :x, "", nothing, nothing, :f, 1)` built an empty-reason mark quite happily, and - # `Mark` is `public`. Every future construction route (a method-level `mark_method!`, a - # deserialised snapshot) gets the invariant for free now instead of remembering to ask. + # The reason is the payload, so the invariant belongs in the type: `Mark` is `public`, and + # every construction route that is not the macro gets it for free here. function Mark(mod, name, reason, since, tracking, file, line) r = String(strip(reason)) isempty(r) && @@ -64,18 +59,13 @@ function Base.show(io::IO, ::MIME"text/plain", m::Mark) return print(io, " declared: ", m.file, ":", m.line) end -# The binding every marked module gets. Named, not gensym'd, so `M.__EXPERIMENTAL_API_MARKS__` -# is greppable and inspectable when a query result surprises someone. +# Named rather than gensym'd, so it is greppable when a query result surprises someone. const MARKS_BINDING = :__EXPERIMENTAL_API_MARKS__ -# The registry is created by code the macro emits INTO the marked module — not by `Core.eval` -# from here. The difference is not stylistic. Julia 1.12 rejects reading a binding that was -# created earlier in the same top-level statement ("define the const at top-level before running -# the function that uses it"), and a `Core.eval`-then-`getglobal` helper does exactly that. The -# emitted form puts the `const` in one top-level statement and the `push!` in the next, so the -# world age has advanced in between and the read is legal. -# -# `@macroexpand` therefore shows the whole mechanism, which is the second reason to prefer it. +# Created by code the macro emits into the marked module, not by `Core.eval` from here: Julia +# 1.12 rejects reading a binding created earlier in the same top-level statement, which is what a +# `Core.eval`-then-`getglobal` helper does. The emitted form puts the `const` and the `push!` in +# separate statements, so the world age has advanced in between. function _registry_of(m::Module) v = getglobal(m, MARKS_BINDING) v isa Vector{Mark} || throw( @@ -87,8 +77,7 @@ function _registry_of(m::Module) return v end -# Re-marking a name replaces its entry instead of appending, so re-including a file (Revise, an -# `include` reached twice) cannot make one name appear in `experimental(M)` several times. +# Replaces rather than appends, so re-including a file cannot duplicate a name. function _mark!(reg::Vector{Mark}, mk::Mark) i = findfirst(x -> x.name === mk.name, reg) if i === nothing @@ -167,9 +156,8 @@ macro experimental(args...) isempty(rest) && throw(ArgumentError("@experimental: nothing to mark — give a definition or a name")) - # `k = v` pairs bind tighter than the subject only when they are NOT the last argument: the - # last argument is always the thing being marked, which keeps `@experimental "…" x = 3` - # unambiguous against `@experimental "…" since=v"1" x = 3`. + # The last argument is always the subject, which keeps `@experimental "…" x = 3` unambiguous + # against `@experimental "…" since=v"1" x = 3`. since, tracking, i = nothing, nothing, 1 while i < length(rest) a = rest[i] @@ -210,16 +198,13 @@ macro experimental(args...) names = [_defname(def)] end - # Statement order carries a constraint: the `const` must land in its own top-level statement, - # because the `_mark!` calls below READ that binding and Julia 1.12 forbids reading a binding - # created in the same world age. + # The `const` must land in its own top-level statement: the `_mark!` calls below read that + # binding, and Julia 1.12 forbids reading one created in the same world age. marks = esc(MARKS_BINDING) src = __source__ - # Built with `Expr` rather than quoted, so that no `LineNumberNode` from THIS file ends up in - # the expansion. `const` in local scope is a lowering error this macro cannot catch — it fires - # before any code it emits runs — so the only lever left is where the error points. Quoted, it - # named `src/mark.jl` and read as a bug in this package; the caller's own line is the line the - # author can act on. See `test/spec/test_spec_forms.jl`, "a mark inside a function body". + # Built with `Expr` so no `LineNumberNode` from this file reaches the expansion. `const` in + # local scope is a lowering error this macro cannot catch, so the only lever is where the + # error points, and it must point at the caller. Pinned in `test_spec_forms.jl`. init = Expr( :if, :(!$(isdefined)($__module__, $(QuoteNode(MARKS_BINDING)))), @@ -239,16 +224,14 @@ macro experimental(args...) ), )) for n in names ] - # `Expr(:meta, :doc)` is how a macro tells the documentation system which expression inside - # its expansion a preceding docstring belongs to — the mechanism `Base.@kwdef` uses. Without - # it, `"""docs""" @experimental "why" f(x) = x` fails with "cannot document the following - # expression", which would make the two accounts this package asks for mutually exclusive. + # Tells the documentation system which expression a preceding docstring belongs to, as + # `Base.@kwdef` does. Without it a docstring on a marked definition fails to attach at all. body = def === nothing ? nothing : Expr(:block, Expr(:meta, :doc), esc(def)) return Expr(:block, init, body, records..., nothing) end -# Whitespace is normalised so a reason written as a wrapped triple-quoted string does not carry -# its indentation into every message that prints it. Nothing else about the text is touched. +# Normalises whitespace so a wrapped triple-quoted reason does not carry its indentation into +# every message. Nothing else about the text is touched. function _reason(s) r = String(strip(s)) isempty(r) && throw( @@ -268,9 +251,8 @@ function _subject_names(subject) a.args[1] isa Symbol && length(a.args) == 2 && a.args[2] isa LineNumberNode - # A BARE `@foo` — the name a macro is public under. A macrocall carrying arguments - # is a definition this macro cannot read, not a name, and must fall through to be - # refused rather than silently recorded as `Symbol("@doc")`. + # A bare `@foo` is a name; a macrocall carrying arguments is a definition this macro + # cannot read, and must fall through to be refused rather than recorded. push!(names, a.args[1]) elseif a isa QuoteNode && a.value isa Symbol push!(names, a.value) # `:foo`, for a name a reader prefers to quote @@ -293,8 +275,8 @@ function _defname(ex::Expr) h === :abstract && return _typename(ex.args[1]) h === :primitive && return _typename(ex.args[1]) h === :const && return _defname(ex.args[1]) - # A `module` cannot be flattened out of the block this macro emits — Julia requires it as a - # direct top-level statement — so it takes the name-list form rather than being wrapped. + # Julia requires `module` as a direct top-level statement, so it cannot be wrapped and takes + # the name-list form. h === :module && throw( ArgumentError( "@experimental cannot attach to a `module`, which Julia requires at top level. " * diff --git a/test/spec/summary.jl b/test/spec/summary.jl index e79b60c..a220400 100644 --- a/test/spec/summary.jl +++ b/test/spec/summary.jl @@ -1,15 +1,7 @@ -# Generates the coverage table for `test/spec/README.md` and for the pull request that ships it. +# Generates the coverage table for `test/spec/README.md` and the pull request that ships it. # -# Written because the hand-maintained version drifted inside the very change that argues prose -# drifts and tests do not: two spec files landed after the table was typed, and every count in it -# was stale. A table that is generated from the files and pinned by a test cannot do that. -# -# The published measure is DISTINCT BEHAVIOURS — one per leaf `@testset` — not assertions. An -# assertion count moves without any implementation progress: `test_spec_declare.jl` has 36 -# assertion lines but 91 runtime assertions, because several of them run inside -# `for mk in experimental(Declared)`, so adding a thirteenth fixture mark buys four more passing -# assertions and covers nothing new. A leaf testset is one claim about the package, and adding -# one means writing one. +# The measure is DISTINCT BEHAVIOURS — one per leaf `@testset` — not assertions, which move +# without any implementation progress when they sit inside a loop over the fixture's marks. # # Run it: julia --project=test test/spec/summary.jl @@ -17,8 +9,7 @@ module SpecSummary const SPEC_DIR = @__DIR__ -# One line per spec file. A file with no entry here is an ERROR, not an omission — the previous -# table was written by listing files by hand, and silently lost `dispatch` and `lifecycle`. +# One line per spec file. A file with no entry here is an error, not a missing row. const CONCERNS = Dict( "test_spec_declare.jl" => "what can carry a mark: function, method, struct, const, module, macro, extension", "test_spec_forms.jl" => "the definition forms a real package hits on its second afternoon", @@ -65,10 +56,8 @@ What one spec file contains. `operating` and `specified` partition `behaviours`: either runs at least one assertion against the package today, or it is entirely `@test_broken` — a claim written down and not yet checked against anything. -The distinction is the point. "Every group has a negative control" is true of this directory as -a *specification* and false of it as a *running suite*: `test_spec_profile.jl`'s control that a -never-called mark is absent rather than reported with count zero is itself `@test_broken`, so it -controls nothing until the implementation lands. +The distinction is the point: "every group has a negative control" is true of this directory as a +specification and false of it as a running suite. """ struct Counts behaviours::Int @@ -84,8 +73,8 @@ function counts(path::AbstractString) behaviours = operating = 0 for ts in sets body = ts.args[2:end] - # A leaf is a testset with no testset inside it. Loop-generated `@testset "$T"` blocks - # are one leaf each, not one per iteration: a behaviour is something someone wrote. + # A leaf has no testset inside it. A loop-generated `@testset "$T"` is one leaf, not one + # per iteration: a behaviour is something someone wrote. sum(b -> _count(y -> _ismacrocall(y, Symbol("@testset")), b), body) == 0 || continue behaviours += 1 live = sum( diff --git a/test/spec/test_spec_declare.jl b/test/spec/test_spec_declare.jl index 20d4d1f..e57fe04 100644 --- a/test/spec/test_spec_declare.jl +++ b/test/spec/test_spec_declare.jl @@ -1,11 +1,7 @@ -# What can carry a mark. +# What can carry a mark: function, method, struct, const, module, macro, extension. # -# The current implementation marks a NAME. The intent is to mark a definition — and for -# `QAtlas.fetch`, which has 570 methods behind one name (see `test_spec_foreign.jl`), the name is the wrong unit: a docstring -# on `fetch` cannot say which dispatch path returns a number you can trust. -# -# So this file covers both: what works today (plain `@test`) and what the unit has to become -# (`@test_broken`). +# Scope: both what the name-keyed implementation does today (plain `@test`) and the method-level +# unit it has to become (`@test_broken`). using ExperimentalAPI: ExperimentalAPI, @experimental, Mark, experimental, isexperimental using Test @@ -57,8 +53,7 @@ listed_b(x) = x "Documented and settled." documented_fn(x) = x -# The shape the method-level unit has to reach: one name, several dispatch paths, and only one -# of them is in doubt. This is `QAtlas.fetch` in miniature. +# One name, several dispatch paths, only one in doubt — `QAtlas.fetch` in miniature. struct Exact end struct Numerical end energy(::Exact, β::Float64) = β @@ -91,8 +86,7 @@ end # module Declared end @testset "nothing else is reported as marked" begin - # Without this, an `experimental()` that leaks every declared name — rather than only the - # marked ones — passes every assertion above. + # Control: rejects an `experimental()` that leaks every declared name. got = Set(mk.name for mk in experimental(Declared)) @test length(got) == 12 for unmarked in (:documented_fn, :energy, :Exact, :Numerical) @@ -113,12 +107,11 @@ end # ── the method-level unit ──────────────────────────────────────────────────────────────────── # -# `Declared.energy` has three methods. Marking the name marks all three, which is the granularity -# problem: the exact-solution path is trustworthy and the numerical one is not. +# `Declared.energy` has three methods; marking the name marks all three, including the exact one. @testset "marking a name is the wrong unit when the name has many methods" begin @test length(methods(Declared.energy)) == 3 - # Today the only available statement is about the name, so it necessarily over-claims. + # Today the statement is about the name, so it over-claims. @test !isexperimental(Declared, :energy) # not marked at all yet — see below end @@ -139,14 +132,12 @@ end end @testset "a method mark survives precompilation" begin - # The name-keyed registry already survives (test/test_precompile.jl). `Method` objects are - # part of the defining module's image too, but that is a separate claim and needs its own - # fixture package before it can be asserted. + # Scope: the name-keyed registry already survives (`test/test_precompile.jl`); whether + # `Method` objects do is a separate claim needing its own fixture package. @test_broken ExperimentalAPI.experimental_methods isa Function end @testset "a method mark is queryable from a call site" begin - # The question a downstream test actually asks, before trusting a reference value. @test_broken ExperimentalAPI.isexperimental( which(Declared.energy, Tuple{Declared.Numerical,Float64}) ) @@ -154,15 +145,14 @@ end # ── extensions ─────────────────────────────────────────────────────────────────────────────── # -# A package extension is a separate module. Names it makes public are part of the package's -# surface from a user's point of view, and are invisible to `names(Package)`. +# An extension is a separate module: its public names are part of the surface a user sees, and +# invisible to `names(Package)`. @testset "a mark inside a package extension is reachable from the parent" begin ext = Base.get_extension(ExperimentalAPI, :ExperimentalAPITestExt) @test ext !== nothing - # `!isempty(...)` would NOT do: ExperimentalAPI marks six of its own names in `src/release.jl` - # (dogfooding), so a keyword that is accepted and then completely ignored would satisfy it. - # The claim is that a mark whose home is the EXTENSION comes back. + # Not `!isempty(...)`: this package marks six of its own names, so an ignored keyword would + # satisfy that. The claim is that a mark whose home is the extension comes back. @test isempty(ExperimentalAPI.experimental(ext)) # today the extension declares none @test_broken any( mk -> mk.mod === ext, ExperimentalAPI.experimental(ExperimentalAPI; extensions=true) @@ -170,10 +160,8 @@ end end @testset "every new Audit field keeps the partition invariant" begin - # Three files each pin a new `Audit` field with `hasproperty` alone — `:extensions` here, - # `:undocumented` in the docstring spec, `:contributed_methods` in the foreign spec — written - # without cross-referencing each other. `hasproperty` is blind to whether the property the - # type exists for still holds once three fields are bolted on from three directions. + # Three files each add an `Audit` field behind `hasproperty` alone. `hasproperty` cannot see + # whether the partition still holds once all three land. a = ExperimentalAPI.audit(ExperimentalAPI) @test sort( vcat(a.foreign, a.documented, a.unaccounted, setdiff(a.declared, a.documented)) @@ -183,7 +171,6 @@ end @testset "auditing a package does not silently ignore its extensions" begin a = ExperimentalAPI.audit(ExperimentalAPI) - # Today `audit` looks at one module. Whether an extension's surface is in scope has to be a - # stated answer rather than an omission. + # Scope: `audit` looks at one module today; whether extensions are included must be stated. @test_broken hasproperty(a, :extensions) end diff --git a/test/spec/test_spec_dispatch.jl b/test/spec/test_spec_dispatch.jl index 37c0373..fe7f1b8 100644 --- a/test/spec/test_spec_dispatch.jl +++ b/test/spec/test_spec_dispatch.jl @@ -1,14 +1,9 @@ -# Dispatch-level branching: one call site, several methods, only some of them marked. +# One call site, several methods, only some of them marked. # -# This is the shape that makes the method-level unit worth having at all. `QAtlas.fetch` has 570 -# methods; a caller writes `fetch(model, quantity)` once, and which of the 570 runs — and whether -# that one is trustworthy — depends on the argument types. A verdict about the NAME says nothing. -# -# The dangerous case is a call site the analysis cannot pin to one method. Measured 2026-09-03, -# Julia 1.12.2: for an argument typed `Union{Exact,Numerical}` or an abstract `Kind`, the -# un-optimised IR shows only `(%1)(_2)` and `which(f, T)` **throws** — there is no unique method. -# An implementation that catches that exception and moves on would report `:clean` about a call -# that reaches a marked method at run time half the time. That is the failure this file guards. +# Scope: the call site the analysis cannot pin to one method. For a `Union`-typed or abstract +# argument `which(f, T)` throws, and an implementation that catches that and moves on reports +# `:clean` about a call that reaches a marked method half the time. That is the failure guarded +# here. using ExperimentalAPI: ExperimentalAPI, @experimental, experimental, isexperimental, mark using Test @@ -47,22 +42,19 @@ struct KB <: Kind end k(::KA) = 1.0 @experimental "extrapolated, never cross-checked" k(::KB) = 2.0 -# The call site can only ever reach the settled method — the negative control. +# Control: can only ever reach the settled method. only_settled(x::Exact) = energy(x) # The call site can reach EITHER method depending on the run-time type. either_way(x::Union{Exact,Numerical}) = energy(x) via_abstract(x::Kind) = k(x) -# `invoke` pins a method DISPATCH WOULD NOT PICK. `invoke(energy, Tuple{Numerical}, x)` would -# not discriminate: `x::Numerical` selects the marked method anyway, so an analysis that ignores -# `invoke` entirely still gets the right answer by accident. Forcing the marked `::Integer` -# fallback from an `Int` — which ordinary dispatch sends to the unmarked `::Int` — does. +# `invoke` pins a method dispatch would NOT pick: an `Int` goes to the unmarked `::Int` normally. +# Pinning `Tuple{Numerical}` instead would be satisfied by an analysis that ignores `invoke`. via_invoke(x::Int) = invoke(more_specific, Tuple{Integer}, x) -# Two arguments, which is the shape the motivating example actually has: `fetch(model, quantity)`. -# Specificity differs per position, so the marked combination is not reachable from either -# argument alone. +# Two arguments, as in `fetch(model, quantity)`: the marked combination is not reachable from +# either argument alone. pair(::Exact, ::Exact) = 0.0 pair(::Exact, ::Numerical) = 1.0 @experimental "only this combination is unvalidated" pair(::Numerical, ::Numerical) = 2.0 @@ -85,14 +77,12 @@ end # module Dispatch @test :k in marked @test length(methods(Dispatch.energy)) == 2 @test length(methods(Dispatch.k)) == 2 - # Today a mark is name-keyed, so it necessarily covers BOTH methods of each name — including - # the closed-form one that is perfectly trustworthy. That over-claim is the problem. + # A name-keyed mark covers both methods, including the closed-form one. That is the problem. @test isexperimental(Dispatch, :energy) end @testset "the specificity premise the file rests on is true" begin - # Stated only in a comment until now, and exercised solely inside `@test_broken` blocks gated - # behind a `reach` that does not exist — so nothing would have noticed if it were false. + # The premise every `@test_broken` below rests on, and which nothing else would notice. @test which(Dispatch.more_specific, Tuple{Int}).sig === Tuple{typeof(Dispatch.more_specific),Int} @test Dispatch.more_specific(5) == 0 # the UNMARKED, more specific method @@ -101,9 +91,8 @@ end end @testset "a branching call site has no unique method" begin - # The measured fact the whole file rests on: `which` cannot answer for these argument types. - # `@test_throws Exception` would be satisfied by a typo in the fixture raising `UndefVarError` - # just as well as by the ambiguity this file is about, so pin the diagnosis, not the failure. + # `@test_throws Exception` would be satisfied by a typo raising `UndefVarError`, so pin the + # diagnosis rather than the failure. @test which(Dispatch.energy, Tuple{Dispatch.Exact}) isa Method for (f, T) in ( (Dispatch.energy, Tuple{Union{Dispatch.Exact,Dispatch.Numerical}}), @@ -123,11 +112,8 @@ end end @testset "attaching the mark at one method's definition marks the NAME today" begin - # Line 38 above reads as if it scopes the claim to `energy(::Numerical)`. It does not: - # `_signame` walks a `:call` head straight to the base Symbol and throws the argument types - # away. So method-level marking cannot come from the attached form as written — it has to - # arrive through a separate imperative route (`mark_method!`), and that is a design decision - # the spec should state rather than leave implied. + # The attached form reads as if it scoped the claim to one method; `_signame` throws the + # argument types away. So method-level marking needs a separate imperative route. @test isexperimental(Dispatch, :energy) @test mark(Dispatch, :energy).name === :energy # not a signature @test_broken ExperimentalAPI.mark_method! isa Function @@ -136,16 +122,14 @@ end # ── what the analysis has to say about each shape ──────────────────────────────────────────── @testset "a call site that can only reach settled methods is clean" begin - # The negative control. Without it, everything below is satisfied by a tool that answers - # ":depends" for every call site with more than one candidate. + # Control: rejects a tool answering `:depends` for every multi-candidate call site. @test_broken ExperimentalAPI.verdict( ExperimentalAPI.reach(Dispatch.only_settled, Tuple{Dispatch.Exact}) ) === :clean end @testset "a Union-typed call site that could reach a mark is not clean" begin - # Half the run-time values take the marked branch. `:clean` here is a false statement, not a - # conservative one. + # Half the run-time values take the marked branch, so `:clean` is false, not conservative. @test_broken ExperimentalAPI.verdict( ExperimentalAPI.reach( Dispatch.either_way, Tuple{Union{Dispatch.Exact,Dispatch.Numerical}} @@ -160,10 +144,7 @@ end end @testset "the unresolvable call site is named, not just counted" begin - # "Somewhere in here I could not tell" is not actionable. The report has to point at the call. - # NOT `occursin("k", string(u))`: a one-character needle matches "unknown call site", - # "package boundary" and "backtrace unavailable" alike, so one generic boilerplate diagnostic - # would satisfy it. Ask for the structured fields, the way `test_spec_propagate.jl` does. + # Structured fields, not `occursin`: a short needle matches any boilerplate diagnostic. @test_broken all( u -> hasproperty(u, :file) && hasproperty(u, :line) && hasproperty(u, :callee), ExperimentalAPI.reach(Dispatch.via_abstract, Tuple{Dispatch.Kind}).unresolved, @@ -175,8 +156,8 @@ end end @testset "which() throwing must not be swallowed into :clean" begin - # The specific implementation mistake this file exists to prevent: wrapping `which` in a - # try/catch, skipping the call site, and reporting the remaining graph as clean. + # The mistake this file exists to prevent: catching `which`, skipping the site, reporting + # the rest as clean. @test_broken ExperimentalAPI.verdict( ExperimentalAPI.reach(Dispatch.via_abstract, Tuple{Dispatch.Kind}) ) === :unknown @@ -185,30 +166,26 @@ end # ── dispatch subtleties ────────────────────────────────────────────────────────────────────── @testset "invoke pins the method it names" begin - # `invoke(energy, Tuple{Numerical}, x)` reaches the marked method unconditionally, even - # though the argument's type would have selected it anyway. An analysis that only looks at - # argument types would miss `invoke` pinning a DIFFERENT method than dispatch would pick. + # Argument types alone cannot see `invoke` pinning a method dispatch would not pick. @test_broken ExperimentalAPI.verdict( ExperimentalAPI.reach(Dispatch.via_invoke, Tuple{Dispatch.Numerical}) ) === :depends end @testset "a more specific unmarked method shadows a marked one" begin - # `more_specific(::Int)` wins over `more_specific(::Integer)`, so a call with an `Int` never - # reaches the mark. Reporting `:depends` because *some* method of the name is marked is the - # name-level over-claim all over again, one level down. + # An `Int` never reaches the mark. `:depends` here is the name-level over-claim one level + # down. @test_broken ExperimentalAPI.verdict( ExperimentalAPI.reach(Dispatch.more_specific, Tuple{Int}) ) === :clean - # …and a call that does fall through to the marked fallback is reported. + # …and the call that does fall through is reported. @test_broken ExperimentalAPI.verdict( ExperimentalAPI.reach(Dispatch.more_specific, Tuple{UInt8}) ) === :depends end @testset "a mark on one method does not leak to its siblings at a call site" begin - # The whole point of going to method granularity: `only_settled` and `either_way` call the - # same NAME, and must get different verdicts. + # Both call the same name and must get different verdicts. @test_broken ExperimentalAPI.verdict( ExperimentalAPI.reach(Dispatch.only_settled, Tuple{Dispatch.Exact}) ) !== ExperimentalAPI.verdict( @@ -219,9 +196,7 @@ end end @testset "a marked combination is not reachable from either argument alone" begin - # `pair(::Exact,::Exact)` and `pair(::Numerical,::Exact)` are settled; only - # `pair(::Numerical,::Numerical)` is marked. An analysis that widens each argument - # independently would call both call sites `:depends`. + # Widening each argument independently would call both call sites `:depends`. @test_broken ExperimentalAPI.verdict( ExperimentalAPI.reach(Dispatch.via_pair_clean, Tuple{Dispatch.Exact,Dispatch.Exact}) ) === :clean diff --git a/test/spec/test_spec_docstring.jl b/test/spec/test_spec_docstring.jl index 6255e9b..0c1e992 100644 --- a/test/spec/test_spec_docstring.jl +++ b/test/spec/test_spec_docstring.jl @@ -1,18 +1,8 @@ -# A mark and a docstring are different accounts, and they must coexist. +# A mark and a docstring are different accounts, and must coexist. # -# The registry reviewer's objection on 2026-09-02 was aimed at a README sentence that read -# "this name is public, it has no docstring, and that is deliberate". His position — public names -# should always have a docstring — is correct, and the package never required otherwise; the -# framing did. -# -# Julia's own code settles it: `Base.Experimental` exists, and the entries sampled below carry -# docstrings. Base marks an experimental surface AND documents it — the two are orthogonal, and -# this file pins that with named entries rather than a count. -# -# No entry count is given on purpose. Measured 2026-09-03 the number moves with both the Julia -# version and the counting rule (19 filtered / 13 documented on 1.11.9), and nothing here would -# notice it drifting. The same pattern was deleted from the README one day earlier for exactly -# this reason. +# Scope: a mark is never a substitute for prose. `Base.Experimental` is the precedent — Base +# marks an experimental surface AND documents it. Pinned by named entries rather than by a count, +# which moves with the Julia version and the counting rule. using ExperimentalAPI: ExperimentalAPI, @experimental, audit, experimental, isdocumented, isexperimental, mark @@ -50,7 +40,6 @@ end end # module Both @testset "Julia itself documents its experimental surface" begin - # The precedent that makes "documented AND experimental" not a contradiction. @test isdefined(Base, :Experimental) for n in (Symbol("@optlevel"), Symbol("@compiler_options"), :Const) @testset "Base.Experimental.$n" begin @@ -61,9 +50,8 @@ end # module Both end @testset "a docstring survives the macro" begin - # The macro emits `Expr(:meta, :doc)` so a preceding docstring attaches to the definition - # rather than to the block the macro expands to. Without it the two accounts would be - # mutually exclusive in practice, whatever the documentation claimed. + # `Expr(:meta, :doc)` attaches the docstring to the definition rather than to the expanded + # block. Without it the two accounts are mutually exclusive in practice. @test Base.Docs.hasdoc(Both, :documented_and_marked) @test isexperimental(Both, :documented_and_marked) @test occursin( @@ -93,31 +81,26 @@ end end @testset "a mark is not an excuse for missing prose" begin - # Under the corrected framing, `marked_only` is not "fine because it is marked". It is a - # public name with no docstring, and the audit has to be able to say so even though a mark - # is present. Today `declared` absorbs it and the distinction is unavailable. + # Scope: a marked name with no docstring is still undocumented. Today `declared` absorbs it. a = audit(Both) @test_broken :marked_only in a.undocumented @test_broken hasproperty(a, :undocumented) end @testset "the check can require a docstring regardless of the mark" begin - # `Aqua.test_undocumented_names` and `Docs.undocumented_names` already enforce - # "every public name has a docstring". A package adopting both should not have to choose. + # Scope: a package adopting both this and Aqua must not have to choose between them. @test_broken ExperimentalAPI.test_surface(Both; require_docstring=true) isa ExperimentalAPI.Audit end @testset "the reason is reachable from the rendered documentation" begin - # A reader on the docs site should see that a name is declared unfinished, and why, without - # the author repeating the reason by hand in the docstring — otherwise the two accounts drift. + # Scope: the reason must reach the docs site without the author typing it twice. @test_broken ExperimentalAPI.docstring_note(Both, :documented_and_marked) isa AbstractString end @testset "what Documenter's checkdocs sees is not what audit sees" begin - # `checkdocs = :public` fails a build on an undocumented public name and has no notion of a - # third answer; `audit` accepts a mark instead. Both are legitimate and they disagree here. + # `checkdocs = :public` has no third answer; `audit` accepts a mark. Both are legitimate. a = audit(Both) undocumented_by_base = setdiff(Base.Docs.undocumented_names(Both), [nameof(Both)]) @test :marked_only in undocumented_by_base # Base would flag it diff --git a/test/spec/test_spec_foreign.jl b/test/spec/test_spec_foreign.jl index 0048d0f..974dfed 100644 --- a/test/spec/test_spec_foreign.jl +++ b/test/spec/test_spec_foreign.jl @@ -1,14 +1,8 @@ -# Marking a method on somebody else's function. +# Marking a method on somebody else's generic — the `QAtlas.fetch` case, refused outright today. # -# This is the QAtlas case and the current implementation refuses it outright. -# -# QAtlas.fetch is AbstractQAtlas.fetch — 570 methods (measured 2026-09-03; QAtlas is not a -# dependency here, so nothing in this suite re-derives that number and it can go stale). -# `:fetch` is not QAtlas's own binding, so `audit` files it under `foreign` and says nothing -# about any of the 570. -# -# A package that extends another package's generic is the normal Julia idiom, not an edge case. -# If the mark cannot attach there, it cannot describe the surface that matters. +# Scope: `audit` files a name bound elsewhere under `foreign` and says nothing about the methods +# we contributed to it. Extending another package's generic is the normal Julia idiom, so a mark +# that cannot attach there cannot describe the surface that matters. using ExperimentalAPI: ExperimentalAPI, @experimental, audit, experimental, isexperimental using Test @@ -51,7 +45,7 @@ end # module Downstream end @testset "the macro currently refuses a qualified definition" begin - # Today's behaviour, pinned so the change is visible when it happens. + # Pinned so the change is visible when it happens. @test_throws LoadError @eval module RefusedForeign using ExperimentalAPI using ..UpstreamGeneric @@ -60,7 +54,7 @@ end end @testset "a method on a foreign generic can be marked" begin - # Ends in a Bool and checks WHAT was marked — same caveat as `spec/README.md`. + # Ends in a Bool and checks what was marked — see `README.md` on `@eval module`. @test_broken begin @eval module MarkedForeign using ExperimentalAPI @@ -78,10 +72,9 @@ end UpstreamGeneric.fetch_value, Tuple{Downstream.Heisenberg,Downstream.Energy} ) @test exact !== delicate - # The fixture cannot carry `@experimental` on these methods: the macro refuses a qualified - # definition today and the whole module would fail to load. So the test MARKS IT ITSELF - # through the future API. Asserting `isexperimental(delicate)` without ever marking it would - # stay Broken forever, even once method-level marking works perfectly. + # The fixture cannot carry `@experimental` here — a qualified definition is refused today + # and the module would not load — so the test marks it through the future API. Otherwise this + # stays Broken even once method-level marking works. @test_broken begin ExperimentalAPI.mark_method!(delicate, "numerically delicate") ExperimentalAPI.isexperimental(delicate) && !ExperimentalAPI.isexperimental(exact) @@ -89,59 +82,50 @@ end end @testset "the mark is stored in the module that WROTE the method" begin - # Not in UpstreamGeneric. A package cannot be made to carry claims its dependents invented, and the - # mark has to survive UpstreamGeneric being reloaded or updated. - # - # `all(pred, [])` is `true` in Julia, so an `experimental_methods` stub that always returns an - # empty vector would satisfy a bare `all(...)`. Non-emptiness has to be part of the claim. + # Not in `UpstreamGeneric`: a package cannot carry claims its dependents invented, and the + # mark must survive it being reloaded. `all(pred, [])` is `true`, so non-emptiness is part of + # the claim. @test_broken !isempty(ExperimentalAPI.experimental_methods(Downstream)) && all( mk -> mk.mod === Downstream, ExperimentalAPI.experimental_methods(Downstream) ) end @testset "the ownership query and the cross-module search are different verbs" begin - # `experimental(Module)` answers "what does this module own". `experimental(fetch_value)` has - # to answer "what has any package anywhere marked on any method of this generic". Both are - # wanted, but collapsing them onto one name means a reader cannot tell which they get without - # knowing the argument's type. + # "what this module owns" and "what anyone has marked on this generic" are both wanted, and + # one name for both means the reader cannot tell which they got. @test_broken ExperimentalAPI.marks_on isa Function end @testset "asking the generic finds marks contributed by every package" begin - # The question a user asks is about `fetch_value`, not about which package happened to define - # the method they will dispatch to. @test_broken !isempty(ExperimentalAPI.experimental(UpstreamGeneric.fetch_value)) end @testset "audit reports foreign methods this module owns" begin - # `foreign` today means "a name bound elsewhere, not our problem". A method WE wrote on - # someone else's generic is the opposite: our problem, invisible under the current rule. + # `foreign` means "bound elsewhere, not our problem". A method we wrote is the opposite. a = audit(Downstream) @test_broken hasproperty(a, :contributed_methods) @test_broken length(a.contributed_methods) == 4 end @testset "a contributed method with neither docstring nor mark is a finding" begin - # The whole point of the audit, applied to the surface that actually matters for QAtlas. @test_broken !isempty(ExperimentalAPI.unaccounted_methods(Downstream)) end @testset "a docstring on a specific signature counts" begin - # Julia stores docstrings keyed by signature, so "documented" is answerable per method — the - # audit does not have to fall back to the name. + # Docstrings are keyed by signature, so "documented" is answerable per method. @test_broken ExperimentalAPI.isdocumented( which(UpstreamGeneric.fetch_value, Tuple{Downstream.Ising,Downstream.Energy}) ) isa Bool end @testset "marking a method does not make the foreign NAME experimental" begin - # Otherwise one downstream package could label another package's whole generic unfinished. + # Control: one dependent must not be able to label a whole generic unfinished. @test !isexperimental(UpstreamGeneric, :fetch_value) end @testset "propagation crosses the package boundary" begin - # A caller in a third package that reaches a marked method in Downstream, through UpstreamGeneric's - # generic, must be reported. This is the QAtlas → analysis-script path. + # The QAtlas -> analysis-script path: a third package reaching a marked method through the + # upstream generic. caller(x) = UpstreamGeneric.fetch_value(Downstream.Heisenberg(), Downstream.Energy()) + x @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(caller, Tuple{Float64})) === @@ -149,8 +133,7 @@ end end @testset "a mark on a method of a function defined in Base is possible" begin - # `Base.show`, `Base.length`, `Base.copy` — QAtlas defines methods on all three. The rule - # cannot be "Base is off limits" without excluding a large part of every package's surface. + # "Base is off limits" would exclude a large part of every package's surface. @test_broken begin @eval module MarkedBase using ExperimentalAPI @@ -163,8 +146,8 @@ end end @testset "it stays refused when the mark cannot say WHICH method" begin - # `@experimental "why" Base.show` — a bare qualified NAME, no signature. Marking every method - # of `Base.show` in the world is never what anyone means, and guessing is worse than refusing. + # A bare qualified name with no signature would mark every method of `Base.show` in the + # world. Guessing is worse than refusing. @test_throws LoadError @eval module RefusedBareForeign using ExperimentalAPI @experimental "why" Base.show diff --git a/test/spec/test_spec_forms.jl b/test/spec/test_spec_forms.jl index 70c139b..475d1a5 100644 --- a/test/spec/test_spec_forms.jl +++ b/test/spec/test_spec_forms.jl @@ -1,9 +1,8 @@ -# The definition forms the macro has not been shown to handle. +# The definition forms a real package hits on its second afternoon: kwargs, parametric +# signatures, callable structs, constructors, operators, stacked macros. # -# `test_spec_declare.jl` covers the forms that work today. These are the ones a real package hits -# on its second afternoon: keyword arguments, parametric signatures, callable structs, -# constructors, operators, stacked macros. Each one either works, or the refusal has to name the -# alternative — silently marking the wrong symbol is the outcome this file exists to prevent. +# Scope: each form either works, or the refusal names the alternative. Silently marking the wrong +# symbol is the outcome this file exists to prevent. using ExperimentalAPI: ExperimentalAPI, @experimental, experimental, isexperimental, mark using Test @@ -51,9 +50,8 @@ end # ── forms that are not covered ─────────────────────────────────────────────────────────────── @testset "a callable struct is marked on the WRONG symbol today" begin - # Measured 2026-09-03. `(c::C)(x) = c.k * x` has no function name, and `_signame` walks the - # `::` and returns the ARGUMENT name. The mark lands on `:c`, a local that is not a binding - # anywhere, so it is silently meaningless — the exact failure this file exists to catch. + # `(c::C)(x)` has no function name; `_signame` walks the `::` and returns the argument name. + # The mark lands on `:c`, which is not a binding anywhere. @eval module CallableMarked using ExperimentalAPI struct C @@ -66,14 +64,9 @@ end end @testset "and the audit compounds it: C is reported as UNDECLARED" begin - # The second wrong signal, and the worse one. Because the mark landed on `:c`, the audit sees - # a declaration for a name that does not exist AND a public name with no declaration — so it - # tells the author to go declare `C`, which is the very thing the line above declares. A - # reader who follows that advice writes the mark a second time and still gets both signals. - # - # Recorded here so that whoever fixes `_signame` knows BOTH halves have to move together: - # correcting the recorded symbol without re-running the audit leaves `dangling` empty but - # `unaccounted` unchanged if the audit keys off something else. + # The second wrong signal: the audit reports a declaration for a name that does not exist AND + # a public name with no declaration, so it tells the author to declare what that line already + # declares. Both halves have to move together. @eval module CallablePublic using ExperimentalAPI public C @@ -89,19 +82,19 @@ end end @testset "a callable struct marks the type, or refuses" begin - # Either answer is defensible. Marking the argument name is not. + # Either answer is defensible; the argument name is not. @test_broken :C in [mk.name for mk in experimental(Main.CallableMarked)] end @testset "…and fixing that clears BOTH signals" begin - # The paired assertion for the testset above. `_signame` returning `:C` is necessary but not - # sufficient: the audit is what the author actually reads, and it has to go quiet too. + # `_signame` returning `:C` is necessary but not sufficient — the audit is what the author + # reads. a = ExperimentalAPI.audit(Main.CallablePublic) @test_broken isempty(a.dangling) && :C ∉ a.unaccounted end @testset "a constructor method is marked on the type" begin - # This one already resolves correctly: the mark lands on `:T`. + # Already correct: the mark lands on `:T`. @eval module CtorMarked using ExperimentalAPI struct T @@ -111,23 +104,19 @@ end end got = Set(mk.name for mk in experimental(Main.CtorMarked)) @test :T in got - # …and the ARGUMENT name is not also marked. The sibling testset above found exactly that - # defect for `(c::C)(x)`; an over-inclusive walk would reintroduce it here unnoticed. + # Control: an over-inclusive walk would mark the argument name here too. @test :s ∉ got end @testset "an inner constructor inside a marked struct is not separately marked" begin - # Marking the struct should not silently also claim its inner constructors are unfinished, - # nor silently exclude them. Whichever it is has to be stated. + # Included or excluded is a decision; silence is not. @test_broken hasproperty(mark(FormsSpec, :Callable), :includes_constructors) end @testset "an operator method can be marked" begin - # `@eval module … end` evaluates to a Module, never a Bool, so wrapping it directly in - # `@test_broken` reports "Expression evaluated to non-Boolean" on success rather than the - # "Unexpected Pass" this directory relies on. Each of these now ends in a Bool AND checks - # WHAT was marked — accepting the syntax while recording the wrong symbol is the defect this - # file already caught once, for `(c::C)(x)`. + # Each ends in a Bool and checks what was marked: `@eval module` returns a Module, which + # reports "non-Boolean" instead of the Unexpected Pass this directory relies on, and accepting + # the syntax while recording the wrong symbol is the defect above. @test_broken begin @eval module OpMarked using ExperimentalAPI @@ -151,8 +140,8 @@ end end @testset "Base.@kwdef stacks with the mark" begin - # `@kwdef` expands to a block carrying `Expr(:meta, :doc)`. Two macros that both wrap a - # definition must compose in at least one order, and the order that works must be documented. + # Two macros that both wrap a definition must compose in at least one order, and which one + # must be documented. @test_broken begin @eval module KwdefMarked using ExperimentalAPI @@ -176,8 +165,7 @@ end end @testset "a definition produced by @eval can be marked by name" begin - # Metaprogrammed definitions cannot be attached to; the name-list form is the answer and it - # has to be reachable. + # Metaprogrammed definitions cannot be attached to, so the name-list form must reach them. @test_broken begin @eval module EvalMarked using ExperimentalAPI @@ -191,18 +179,13 @@ end end @testset "a mark inside a function body is refused" begin - # It IS refused by Julia rather than by this package: `const` in local scope is a LOWERING - # error, which fires before any code this macro emits can run, so there is no point at which - # a check of ours could intercept it. The one lever left is where the error points, and it - # used to point at `ExperimentalAPI/src/mark.jl` — reading as a bug in the package rather - # than a misuse of it, whose natural next step is to file an issue here. The expansion now - # carries the CALLER's `LineNumberNode`, so the message names the line the author wrote. + # Refused by Julia, not by this package: `const` in local scope fails during lowering, before + # any emitted code runs, so no check of ours can intercept it. The one lever is where the + # error points, and the expansion carries the caller's `LineNumberNode`. # - # The misuse has to arrive from a FILE. Measured 2026-09-03: the same misuse written through - # `@eval` or `Core.eval` — which is how every other refusal in this file is probed — produces - # `"syntax: unsupported `const` declaration on local variable"` with NO location at all, so a - # location assertion made that way is vacuous and would pass just as well against the old - # expansion. The source is built line by line so the formatter cannot shift line 4. + # The misuse must arrive from a FILE: written through `@eval` the message carries no location + # at all, so a location assertion made that way is vacuous. Built line by line so the + # formatter cannot shift line 4. dir = mktempdir() path = joinpath(dir, "caller_side.jl") write( @@ -229,19 +212,16 @@ end @test e isa ErrorException msg = sprint(showerror, e) @test occursin("unsupported `const` declaration", msg) - # The half that is fixed: the blame lands on the line the author actually wrote… + # The blame lands on the line the author wrote… @test occursin("caller_side.jl:4", msg) - # …and nowhere in this package. Checked against the FILE NAME: the repository path contains - # the string "ExperimentalAPI", so a negative assertion on that word passes by accident. + # …and nowhere in this package. Checked by file name: the repository path contains + # "ExperimentalAPI", so asserting on that word passes by accident. @test !occursin("mark.jl", msg) end @testset "the refusal names @experimental rather than leaking the emitted const" begin - # The half that is NOT fixed. The message now points at the right line, but it still talks - # about a `const` the author never wrote; it should say that a mark belongs at module top - # level, next to `export` and `public`. Whether that is reachable at all is open — the error - # comes from lowering, so this may only be answerable by not emitting `const`, and the - # alternative (`global`) fails SILENTLY in local scope, which is strictly worse. + # The message still names a `const` the author never wrote. Whether this is reachable is + # open: the only expansion avoiding `const` is `global`, which fails silently in local scope. e = try @eval module ClosureMarked2 using ExperimentalAPI @@ -260,11 +240,8 @@ end # ── metadata ───────────────────────────────────────────────────────────────────────────────── @testset "since must be a version, and the refusal must say so" begin - # It IS refused today, but by accident: `Mark.since::Union{VersionNumber,Nothing}` cannot - # convert a String, so the error is - # MethodError: Cannot `convert` an object of type String to an object of type VersionNumber - # which names neither `since` nor `@experimental`. A deliberate check would keep throwing, so - # asserting "it throws" could never signal that the fix had landed — assert the DIAGNOSTIC. + # Refused by accident: the field's conversion fails, with a message naming neither `since` + # nor `@experimental`. A deliberate check would also throw, so assert the diagnostic. e = try @eval module BadSince using ExperimentalAPI @@ -279,8 +256,7 @@ end end @testset "an unknown keyword is refused rather than ignored" begin - # `@experimental("why", trackign = "u", f(x) = x)` — a typo in a keyword name must not - # silently become part of the subject. + # A typo in a keyword name must not silently become part of the subject. @test_throws LoadError @eval module BadKw using ExperimentalAPI @experimental("why", trackign = "u", f(x) = x) @@ -288,14 +264,11 @@ end end @testset "tracking is carried through to every report" begin - # The field that turns a warning into something a reader can act on. It is stored today; it - # has to survive into the audit and the record as well. + # Stored today; it has to survive into the audit and the record as well. @test_broken ExperimentalAPI.audit(FormsSpec).tracking isa AbstractDict end # Evaluate an expression in a fresh module and hand back the exception it raised, unwrapped. -# The `Module(...)` + `Core.eval` + `LoadError`-stripping plumbing was written out five times -# before this; it carries none of the claim, so it is a helper rather than part of each test. function probe(ex) m = Module() Core.eval(m, :(using ExperimentalAPI)) @@ -309,15 +282,12 @@ end # ── writing it lazily ──────────────────────────────────────────────────────────────────────── # -# The reason is the payload: why a name is not settled is knowledge only the author has. So the -# interesting question is not "does the good form work" but "what happens when someone writes it -# without one". Measured 2026-09-03: every lazy form IS refused — but two of them are refused by -# accident, and two more are refused with a message pointing the wrong way. +# Scope: what happens when the reason — the payload, and knowledge only the author has — is left +# out. Every lazy form is refused; two by accident, and two with a message pointing the wrong way. @testset "a bare @experimental is refused, and says which of the two is missing" begin - # `@test_throws Exception` for all six would be satisfied by one generic - # `ArgumentError("@experimental: invalid usage")`. The section exists to check that the - # message points the right way, so each case pins the phrase it should carry. + # Each case pins its own phrase: `@test_throws Exception` for all six would be satisfied by + # one generic message. for (ex, needle) in ( (:(@experimental), "needs a reason"), (:(@experimental foo), "the reason comes first"), @@ -356,10 +326,8 @@ end end @testset "a non-string reason is refused by accident, not by a check" begin - # `@experimental :sym f(x) = x` and `@experimental 42 f(x) = x` both die inside `_reason` - # with `MethodError: no method matching strip(::Symbol)` / `strip(::Int64)`. Refused, yes — - # but by `strip` failing, with a message that names neither `@experimental` nor `reason`. - # Same shape as the `since = "0.4.0"` case above. + # Refused by `strip` failing inside `_reason`, with a message naming neither `@experimental` + # nor `reason`. Same shape as the `since` case above. for r in (:(:sym), 42) @testset "reason=$(repr(r))" begin e = probe(:(@experimental $r f(x) = x)) @@ -370,9 +338,8 @@ end end @testset "forgetting the reason is diagnosed as a missing reason" begin - # `@experimental f(x) = x` is refused with "nothing to mark — give a definition or a name". - # The author DID give a definition; what is missing is the reason. The message points at the - # wrong end of the call, which is how someone ends up deleting a correct definition. + # The message says "nothing to mark" when a definition was given and the reason was not — + # pointing at the end of the call the author should not touch. e = probe(:(@experimental f(x) = x)) @test e isa ArgumentError @test occursin("nothing to mark", sprint(showerror, e)) # today, and misleading @@ -380,8 +347,7 @@ end end @testset "nothing is marked when the macro refuses" begin - # A refusal that still recorded a mark would be worse than either outcome. Checks the module - # is left clean, not just that an exception came out. + # Checks the module is left clean, not just that an exception came out. m = Module(:RefusedProbe) Core.eval(m, :(using ExperimentalAPI)) try @@ -422,7 +388,6 @@ end end @testset "the replacement is reported rather than silent" begin - # Two different reasons for one name usually means two authors disagreed, or a stale mark was - # left behind. Last-write-wins is a decision, and it should be visible. + # Last-write-wins is a decision, and it should be visible. @test_broken ExperimentalAPI.superseded_marks isa Function end diff --git a/test/spec/test_spec_integration.jl b/test/spec/test_spec_integration.jl index 295342b..6bca95f 100644 --- a/test/spec/test_spec_integration.jl +++ b/test/spec/test_spec_integration.jl @@ -1,16 +1,8 @@ -# Where the mark has to show up outside this package. +# Where the mark has to surface outside this package: docs, Aqua, releases, provenance, CI. # -# A mark that only `ExperimentalAPI` can read is a private note. The intent — a reader of a paper, -# a reviewer of a PR, or a user of the docs site learning that a number came from unvalidated -# code — requires the mark to surface in tools nobody configured for it. -# -# This is the group with the most external dependencies named in it, so: can any of it actually -# run in CI, or is it destined to stay broken because it cannot be otherwise? Checked before -# writing more of it. Of the assertions below, all but one need nothing new — `docstring_note`, -# `aqua_compatible_names`, `compare_methods`, `stale_since` and `test_surface` are pure functions -# over data this package already holds, and `stamp` touches only a temporary file. The single -# exception is `DocumenterExt`, which needs Documenter in the test environment; it is the one row -# here that costs something to turn green, and it is marked as such at its testset. +# Scope: a mark only `ExperimentalAPI` can read is a private note. Everything here is pure or +# touches a temporary file except the Documenter block, which needs a test dependency this +# package does not have — so nothing in this group is blocked on infrastructure. using ExperimentalAPI: ExperimentalAPI, @experimental, experimental using Test @@ -39,8 +31,7 @@ end # module Shown # ── documentation ──────────────────────────────────────────────────────────────────────────── @testset "the docs can render the mark without the author repeating it" begin - # If the reason has to be typed twice — once in `@experimental`, once in the docstring — the - # two drift, and the machine-readable one loses. + # Typed twice, the two drift and the machine-readable one loses. @test_broken ExperimentalAPI.docstring_note(Shown, :provisional) isa AbstractString end @@ -51,33 +42,26 @@ end end @testset "a Documenter block can list a module's marks" begin - # `@autodocs`-style: one block in the manual, always current, never hand-maintained. - # - # The only assertion in this file that needs a test dependency this package does not already - # have (Documenter). Everything else here is pure or touches a temporary file, so this group - # is not blocked on infrastructure — see the note at the top. + # `@autodocs`-style. The only assertion here needing a new test dependency. @test_broken ExperimentalAPI.DocumenterExt isa Module end @testset "a settled name gets no note" begin - # Without this, a renderer that annotates everything passes the tests above. + # Control: rejects a renderer that annotates everything. @test_broken ExperimentalAPI.docstring_note(Shown, :settled) === nothing end # ── Aqua ───────────────────────────────────────────────────────────────────────────────────── @testset "the audit composes with Aqua rather than competing" begin - # `Aqua.test_undocumented_names` enforces "every public name has a docstring" and has no - # third answer. A package should be able to run both without one contradicting the other. + # Aqua has no third answer; a package must be able to run both. @test_broken ExperimentalAPI.aqua_compatible_names(Shown) isa AbstractVector end # ── release ────────────────────────────────────────────────────────────────────────────────── # -# One schema for every case below, mirroring `snapshot`'s `"stable"` / `"experimental"` pair. An -# earlier version used `"methods"` in the first testset and `"stable_methods"` in the other two, -# so "same shape, opposite verdict" was false: the fixtures differed in more than the variable -# each one isolates, and no test ever supplied both keys at once. +# One schema for every case below, mirroring `snapshot`'s two keys, so that "same shape, opposite +# verdict" is true — the fixtures differ only in the variable each one isolates. function snap(stable, experimental) return Dict("stable_methods" => stable, "experimental_methods" => experimental) end @@ -89,7 +73,7 @@ const GONE_METHOD = snap(String[], Dict()) const RESIGNED_METHOD = snap(["f(::Real)"], Dict()) @testset "the method snapshot mirrors the name snapshot's two keys" begin - # If the two schemas diverge, `compare` and `compare_methods` cannot share a snapshot file. + # Diverging schemas mean `compare` and `compare_methods` cannot share a snapshot file. for d in (MARKED_METHOD, PROMOTED_METHOD, SETTLED_METHOD, GONE_METHOD, RESIGNED_METHOD) @test Set(keys(d)) == Set(["stable_methods", "experimental_methods"]) end @@ -97,53 +81,48 @@ const RESIGNED_METHOD = snap(["f(::Real)"], Dict()) end @testset "a snapshot records marks at method granularity" begin - # `compare` reads name sets today and says so. Once methods can be marked, the snapshot has - # to grow — and the schema change is why the release layer is itself declared experimental. + # The schema change method-level marks force — which is why the release layer is itself + # declared experimental. @test_broken haskey(ExperimentalAPI.snapshot(Shown), "experimental_methods") end @testset "removing a marked METHOD is not breaking" begin - # The same contract as for names, at the granularity that matters for a dispatch table. @test_broken !ExperimentalAPI.isbreaking( ExperimentalAPI.compare_methods(MARKED_METHOD, PROMOTED_METHOD) ) end @testset "removing a SETTLED method is breaking" begin - # The negative control: the same schema, differing only in whether the method was marked. + # Control: same schema, differing only in whether the method was marked. @test_broken ExperimentalAPI.isbreaking( ExperimentalAPI.compare_methods(SETTLED_METHOD, GONE_METHOD) ) end @testset "a signature change to a settled method is reported as breaking" begin - # The blind spot `compare` currently admits to in its own docstring. Method-level marks are - # what make it addressable at all. + # The blind spot `compare` admits to in its own docstring. @test_broken ExperimentalAPI.isbreaking( ExperimentalAPI.compare_methods(SETTLED_METHOD, RESIGNED_METHOD) ) end @testset "a keyword-only change is a blind spot here too" begin - # `Tuple`-based signatures have no room for keyword arguments — they live in a separate - # `kwcall` method — so a change to a default or a kwarg name is invisible to a signature - # string just as it is to a name set. Stated rather than discovered later. + # Keyword arguments live in a separate `kwcall` method, so a signature string cannot see a + # changed default any more than a name set can. Stated rather than discovered later. @test_broken ExperimentalAPI.compare_methods_sees_keywords === true end # ── the provenance record next to a result ─────────────────────────────────────────────────── @testset "a result file can carry the experimental dependencies of the run that made it" begin - # The end state: a figure's directory contains a record saying which unvalidated code paths - # produced it. That is the artefact a referee or a future reader needs. + # The end state: a figure's directory says which unvalidated code paths produced it. @test_broken ExperimentalAPI.stamp(tempname(), () -> Shown.provisional(1)) isa AbstractString end @testset "the stamp is readable without loading the package that made it" begin - # A year later the package may not resolve. Plain TOML or JSON, not a serialised Julia object. - # The path has to go through `stamp` first — reading a bare `tempname()` throws SystemError - # for a reason that has nothing to do with the claim. + # Plain TOML or JSON: a year later the package may not resolve. The path goes through + # `stamp` first, since a bare `tempname()` throws for an unrelated reason. @test_broken occursin( "reference value", read(ExperimentalAPI.stamp(tempname(), () -> Shown.provisional(1)), String), @@ -157,18 +136,17 @@ end # ── CI gates ───────────────────────────────────────────────────────────────────────────────── @testset "CI can fail a PR that adds a mark without a tracking link" begin - # `tracking` is what makes a mark actionable rather than a shrug. Whether it is required is a - # per-project decision, and it has to be expressible. + # Whether `tracking` is required is a per-project decision, and must be expressible. @test_broken ExperimentalAPI.test_surface(Shown; require_tracking=true) isa ExperimentalAPI.Audit end @testset "CI can fail a PR that increases the number of marks" begin - # The ratchet. `skip` already only shrinks; the mark count should be able to as well. + # The ratchet, in the shape `skip` already has. @test_broken ExperimentalAPI.test_surface(Shown; max_marks=0) isa ExperimentalAPI.Audit end @testset "a mark older than N releases is reported" begin - # `since` exists so that "experimental" cannot quietly become permanent. Nothing reads it yet. + # `since` exists so a mark cannot quietly become permanent. Nothing reads it yet. @test_broken ExperimentalAPI.stale_since(Shown, v"0.9.0") isa AbstractVector end diff --git a/test/spec/test_spec_lifecycle.jl b/test/spec/test_spec_lifecycle.jl index 0e13a6d..cee3b9d 100644 --- a/test/spec/test_spec_lifecycle.jl +++ b/test/spec/test_spec_lifecycle.jl @@ -1,20 +1,8 @@ -# The mark has an exit, and something has to say when it may be taken. +# The mark's exit — when it may be removed — and entry points that are not a single function. # -# From the design letter on General#166832: -# -# 「この `@experimental` を安全に外していく、というのを中間ゴールに据えた開発が可能になります」 -# -# That is the part that makes this a work item rather than a permanent label. A mark that can only -# ever be added is a decoration; a mark with a defined exit is a plan. Nothing in the rest of this -# directory covers the exit, so it is here. -# -# The other half is end-to-end analysis. The letter's reading of Lean is that `sorry` lets the -# whole development be checked with the unproven proposition still in it — -# -# 「`sorry`がついた真偽不明の命題として e2e でコードの解析を実行できる」 -# -# — and `#print axioms` answers for any declaration, not only for one you hand it. The analogue -# here is an entry point that is a MODULE or a script, not just a function with argument types. +# Scope: a mark that can only ever be added is a decoration. The exit is what makes it a work +# item. The entry point has to be a module or a script, as `#print axioms` answers for any +# declaration and not only for one it is handed. using ExperimentalAPI: ExperimentalAPI, @experimental, audit, experimental, mark using Test @@ -25,8 +13,7 @@ using ExperimentalAPI public verified_now, still_unverified, tracked_but_unresolved, settled, consumer, entry -# A mark whose reason has been discharged: the reference value now exists and the test suite -# exercises it. This is the one that should be removable. +# Reason discharged: reference value exists, suite exercises it. This one is removable. @experimental( "no reference value yet", since = v"0.1.0", @@ -41,11 +28,8 @@ public verified_now, still_unverified, tracked_but_unresolved, settled, consumer still_unverified(β::Float64) = β * 1.0000001 ) -# A third mark, and the reason it is here: without it, `verified_now` and `still_unverified` -# differ ONLY in whether `tracking` is set, so `ready_to_promote(m, n) = mark(m,n).tracking !== -# nothing` — a rule with no relationship at all to "has the reason been discharged" — satisfies -# both assertions below. This one HAS a tracking link and is still not ready, which breaks that -# shortcut. The fixture has to vary on the axis under test, not on a neighbouring one. +# Without a third mark, the two above differ only in whether `tracking` is set, and a rule keyed +# on that alone would satisfy every assertion below. This one has a link and is still not ready. @experimental( "reference value exists but disagrees with the literature at the third digit", since = v"0.1.0", @@ -78,23 +62,21 @@ end # ── end-to-end: an entry point that is not a single function ───────────────────────────────── @testset "a whole module can be the entry point" begin - # `#print axioms` answers for any declaration; the analogue is "does anything this package - # exposes reach unvalidated code". Asking function-by-function does not scale to a package. + # Function-by-function does not scale to a package. @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(Lifecycle)) === :depends end @testset "the module-level answer names which public entry points are affected" begin - # "Something in here is experimental" is not actionable for a package with 310 public names. + # "Something in here is experimental" is not actionable at package scale. @test_broken :entry in [e.name for e in ExperimentalAPI.reach(Lifecycle).affected_entries] - # …and `settled` reaches nothing marked, so it must NOT be listed. Without this, a walk that - # reports every public name whenever the module contains any mark at all passes. + # Control: `settled` reaches nothing marked and must not be listed. @test_broken :settled ∉ [e.name for e in ExperimentalAPI.reach(Lifecycle).affected_entries] end @testset "a module with nothing marked comes back clean" begin - # The negative control for the two above. + # Control for the two above. @eval module CleanModule "Settled." f(x) = x @@ -104,12 +86,8 @@ end end @testset "a script can be the entry point" begin - # The shape a researcher actually has: not a package, a file that produces a figure. - # - # Two traps avoided here. `tempname()` returns a path and creates NO file, so reading it - # throws `SystemError` forever regardless of the implementation — the same defect fixed one - # commit earlier for `stamp`. And `isa Any` is true of every Julia value, so it would have - # reported Unexpected Pass for a no-op returning `nothing`. + # The shape a researcher has: a file that produces a figure, not a package. The file must be + # written — `tempname()` alone throws regardless of the implementation. path = tempname() write(path, "1 + 1\n") @test isfile(path) @@ -119,22 +97,18 @@ end # ── the exit: what licenses removing a mark ────────────────────────────────────────────────── @testset "the tool says a mark is ready to be removed, and why" begin - # Not "is this marked" but "may this stop being marked". The evidence has to be nameable: - # the reason discharged, the definition exercised, a reference value present. + # Not "is this marked" but "may this stop being marked", with nameable evidence. @test_broken ExperimentalAPI.ready_to_promote(Lifecycle, :verified_now) === true end @testset "a mark whose reason still stands is NOT reported ready" begin - # Without this, a checker that says "ready" for everything passes the test above. All three - # definitions in the fixture are exercised by this suite, so coverage cannot be the whole - # criterion — which is the point. + # Control: rejects a checker that says "ready" for everything. All three are exercised by + # this suite, so coverage cannot be the whole criterion. @test_broken ExperimentalAPI.ready_to_promote(Lifecycle, :still_unverified) === false end @testset "having a tracking link is not the same as being ready" begin - # `tracked_but_unresolved` carries a tracking link and is still not ready. Without this, - # `ready_to_promote(m, n) = mark(m, n).tracking !== nothing` — which has nothing to do with - # the criteria the comments name — satisfies both testsets above. + # Control: rejects a rule keyed on `tracking` alone. @test mark(Lifecycle, :tracked_but_unresolved).tracking !== nothing @test mark(Lifecycle, :verified_now).tracking !== nothing @test mark(Lifecycle, :still_unverified).tracking === nothing @@ -143,11 +117,8 @@ end end @testset "removing a mark is reported as not breaking" begin - # `compare` already treats experimental → stable as non-breaking for names. The exit needs it - # stated in the direction a person asks the question: I am about to delete this line, is that - # a release event? - # Already implemented — promoted from @test_broken after the suite reported Unexpected Pass, - # which is the mechanism this directory exists for. + # Stated in the direction a person asks it: I am about to delete this line, is that a + # release event? @test !ExperimentalAPI.isbreaking( ExperimentalAPI.compare( Dict( @@ -160,9 +131,8 @@ end end @testset "…but DELETING it outright still is" begin - # The negative control. Promoting a mark and deleting the name are both "the mark is gone" to - # a careless reading, and only one of them is safe. Without this the test above would pass for - # an `isbreaking` that always answers false. + # Control: promoting a mark and deleting the name both read as "the mark is gone", and only + # one is safe. d = ExperimentalAPI.compare( Dict("stable" => ["settled", "verified_now"], "experimental" => Dict()), Dict("stable" => ["settled"], "experimental" => Dict()), @@ -172,9 +142,7 @@ end end @testset "removing the mark flips its callers, and only its callers" begin - # The propagation half of the exit. After `verified_now` is promoted, `consumer` becomes - # clean; `entry` does not, because it still reaches `still_unverified`. A tool that flips - # everything, or nothing, fails one of these. + # `consumer` becomes clean; `entry` does not, since it still reaches `still_unverified`. @test_broken ExperimentalAPI.verdict( ExperimentalAPI.reach(Lifecycle.consumer, Tuple{Float64}; ignore=[:verified_now]) ) === :clean @@ -184,38 +152,30 @@ end end @testset "how long a mark has been standing is answerable" begin - # `since` is recorded and nothing reads it. "Experimental" that never expires is just a label, - # and the letter's framing — an intermediate goal — needs the clock to be visible. + # `since` is recorded and nothing reads it; a mark that never expires is just a label. @test mark(Lifecycle, :still_unverified).since == v"0.1.0" - # Not `isa Any` — that is true of every value, including `nothing` from a stub. Assert the - # number the caller actually needs: eight minor releases have passed since v0.1.0. + # The number the caller needs, not `isa Any` — which is true of `nothing` from a stub. @test_broken ExperimentalAPI.age(Lifecycle, :still_unverified, v"0.9.0") == 8 end @testset "the number of marks can only go down under a ratchet" begin - # The mechanism that makes "an intermediate goal" real rather than aspirational, and the same - # shape as `test_surface`'s skip list, which already only shrinks. - # - # `isa Audit` will NOT do: `test_surface` is documented to return the audit on the normal - # return path whether the testset passed or not, so an implementation that accepts - # `max_marks` and ignores it satisfies that. Assert what the cap actually does. + # Same shape as `test_surface`'s skip list. Not `isa Audit`: that comes back whether the cap + # was honoured or ignored, so assert what the cap does. @test length(experimental(Lifecycle)) == 3 @test_broken ExperimentalAPI.exceeds_mark_cap(Lifecycle, 1) === true @test_broken ExperimentalAPI.exceeds_mark_cap(Lifecycle, 5) === false end @testset "a mark removed while callers still depend on it is caught" begin - # The failure mode of removing one carelessly: the line is deleted because the author looked - # at the definition, not at who reaches it. That is what propagation is for, read backwards. + # Propagation read backwards: the line gets deleted because the author looked at the + # definition, not at who reaches it. @test_broken :consumer in ExperimentalAPI.dependents(Lifecycle, :verified_now) - # `settled` calls nothing marked, so it is nobody's dependent. Without this, a `dependents` - # that returns every public name passes. + # Control: rejects a `dependents` that returns every public name. @test_broken :settled ∉ ExperimentalAPI.dependents(Lifecycle, :verified_now) end @testset "the exit works at method granularity too" begin - # `test_spec_dispatch.jl` argues that a name-keyed mark over-claims when a name has several - # methods. The exit has the same problem read backwards: promoting one method of a name must - # not promote its siblings. Neither file tests the intersection, so it is stated here. + # The intersection neither this file nor `test_spec_dispatch.jl` covers: promoting one + # method must not promote its siblings. @test_broken ExperimentalAPI.ready_to_promote isa Function end diff --git a/test/spec/test_spec_profile.jl b/test/spec/test_spec_profile.jl index 0cc8494..7ddc9f4 100644 --- a/test/spec/test_spec_profile.jl +++ b/test/spec/test_spec_profile.jl @@ -1,14 +1,8 @@ -# Profiling: what a real run actually went through. +# What a real run went through: which marked definitions it entered, how often, and how much of +# the run was spent inside them. # -# This is the half of the intent a reviewer cannot answer with "put it in the docstring". After a -# twelve-hour DMRG run the question is not "is this function experimental" but "did the number I -# am about to put in a paper come out of code nobody has validated, and how much of it". -# -# Everything here is `@test_broken`. The cheap route was measured on 2026-09-03, **Julia -# 1.12.2**, and does not -# work: `Profile.fetch` frames for inlined callees carry no `MethodInstance`, so a sampling -# profiler attributes ZERO samples to marked methods — and small functions, which is most of what -# gets marked, are exactly the ones that get inlined. +# Scope: two layers. Presence is detected by default and must cost nothing; counts, call sites +# and paths are opt-in. The measurements that put the boundary there are in `README.md`. using ExperimentalAPI: ExperimentalAPI, @experimental using Profile: Profile @@ -51,58 +45,27 @@ end # module Sim const M = Sim.Model(0.5) -# ── the default layer: you find out without asking ─────────────────────────────────────────── -# -# The goal is a tool that says WHERE experimental code was used, and a user who learns they used -# it **without opting in**. Those are two layers, and which work goes in which is measured rather -# than chosen. 10M calls of a realistic numeric body, `sqrt(abs(sin(x)*cos(x) + exp(-|x|/1e6)))`, -# Julia 1.12.2, minimum of 7-9 trials: -# -# | emitted into the body | 1 thread | 8 threads | counts correctly? | -# |-----------------------------|----------|-----------|-------------------| -# | nothing | 1.00x | 1.00x | - | -# | set-once flag, read-mostly | 1.03x | 0.985x | yes | -# | counter, plain shared `Ref` | 1.03x | 3.76x | **no** | -# | counter, global atomic | 1.17x | 4.87x | yes | -# | counter, per-thread atomic | 1.12x | 2.79x | yes | -# | `@warn` guarded, fires once | 5.65x | - | yes | -# | `@warn maxlog=1` | 59.57x | - | yes | -# -# Two things decided the split. The plain counter is not merely slow in parallel — it recorded -# 95,406,048 of 160,000,000 calls, losing 40% to races, so it is *wrong* as well. And a flag -# written once and only read afterwards never dirties the cache line again, which is why it is -# free at eight threads while every counting scheme is not. -# -# So: **presence is detected by default and costs nothing; counts, call sites and paths are -# opt-in.** The `@warn` rows are why the notice is a summary at exit rather than a warning at the -# call: the cost is the logging call sitting in the body, not the warning being printed, and a -# guard that makes it fire only once does not recover it. +# ── the default layer: no opt-in ───────────────────────────────────────────────────────────── @testset "a run reports what it entered without being asked to record" begin - # The default layer. No `record(...)` wrapper and no flag to set: the user ran their - # calculation and can find out afterwards. Sim.driver(M, 10) @test_broken :energy in [h.name for h in ExperimentalAPI.entered()] end @testset "the default answer is presence, and does not pretend to be counts" begin - # A count field that is always 1, or always the number of marks, would be worse than absent: - # it reads as a measurement. The default layer knows "yes, at least once" and must say only - # that. + # Scope: "at least once". A count field here would read as a measurement it did not make. Sim.driver(M, 10) @test_broken ExperimentalAPI.entered()[1].count === nothing end @testset "a definition that was never entered is absent from the default answer too" begin - # The control. Without it, an `entered()` that lists every mark in every loaded module passes. + # Control: separates observed from enumerated. Sim.driver(M, 10) @test_broken :cold ∉ [h.name for h in ExperimentalAPI.entered()] end -# The summary is a property of a process on its way out, so it has to be observed from outside -# one. Asserting that an `atexit` handler is registered in THIS process would pass for a handler -# that prints nothing, and asserting on `stdout` alone would pass trivially if the notice went to -# `stderr` — which stream a notice lands on is exactly what must not be assumed here. +# Observed from a child process: an `atexit` handler registered in this one would pass for a +# handler that prints nothing, and stdout alone would pass if the notice went to stderr. module ChildRun const WARMED = Ref(false) @@ -115,8 +78,7 @@ end output(code) -> String Everything a child `julia -e code` wrote, stdout and stderr merged. The first call warms the -precompilation cache with a throwaway child, so a `Precompiling …` banner is never mistaken for -the summary under test. +precompilation cache, so a `Precompiling …` banner is never mistaken for the summary under test. """ function output(code::AbstractString) if !WARMED[] @@ -132,6 +94,7 @@ function output(code::AbstractString) return String(take!(io)) end +# The two scripts differ only in the final call. const ENTERS = """ using ExperimentalAPI module Child @@ -154,31 +117,24 @@ end end # module ChildRun @testset "the summary is printed at process exit" begin - # The whole point of the default layer: the user did not ask, and still finds out. @test_broken occursin("energy", ChildRun.output(ChildRun.ENTERS)) end @testset "a process that loaded a mark but never entered it stays silent" begin - # The control, and the thing that decides whether this package is tolerable as a dependency. - # The two child scripts differ ONLY in the final call, so a summary keyed on "this module has - # marks" rather than on "this run entered one" fails here and passes above. + # Control: rejects a summary keyed on "this module has marks". @test isempty(strip(ChildRun.output(ChildRun.LOADS_ONLY))) end @testset "the summary carries the reason, not just the name" begin - # A name tells the user which line to look at; the reason is what tells them whether the - # number they are about to publish is affected. The reason is the payload everywhere else in - # this package, and must not be dropped at the one place a user reads by default. @test_broken occursin("convergence not established", ExperimentalAPI.summary_text()) end @testset "the default layer can be turned off" begin - # A package that cannot be quietened gets vendored around. The switch has to be readable - # before `using` returns, so it is an environment variable rather than a function call. + # Scope: readable before `using` returns, so an environment variable rather than a call. @test_broken ExperimentalAPI.detecting() === true end -# ── the basic question ─────────────────────────────────────────────────────────────────────── +# ── the opt-in layer: the basic question ───────────────────────────────────────────────────── @testset "a run reports which marked definitions it entered" begin @test_broken :energy in @@ -190,8 +146,7 @@ end end @testset "a marked definition the run never entered is absent, not zero" begin - # A recorder that lists every mark in the module passes the two tests above. This is the - # control that separates "observed" from "enumerated". + # Control: separates observed from enumerated. @test_broken :cold ∉ [h.name for h in ExperimentalAPI.record(() -> Sim.driver(M, 10))] end @@ -200,15 +155,13 @@ end end @testset "a record distinguishes 'touched nothing' from 'recording was off'" begin - # Both would otherwise be an empty vector, and they mean opposite things: one is a clean - # result, the other is a measurement that never happened. + # Both are an empty vector otherwise, and they mean opposite things. @test_broken ExperimentalAPI.record(() -> sum(1:10)).enabled === true end # ── granularity ────────────────────────────────────────────────────────────────────────────── @testset "attribution is to a method, not to a name" begin - # `QAtlas.fetch` has 570 methods — see `test_spec_foreign.jl`. "the run touched fetch" is unusable. @test_broken first(ExperimentalAPI.record(() -> Sim.driver(M, 10))).method isa Method end @@ -219,8 +172,7 @@ end end @testset "the call site that reached the mark is recorded" begin - # `energy` was entered from `inner`, which was entered from `driver`. Knowing only that a - # mark was hit does not tell you which part of your own code to distrust. + # Scope: which part of the caller's own code to distrust, not just that a mark was hit. @test_broken !isempty(first(ExperimentalAPI.record(() -> Sim.driver(M, 10))).callers) end @@ -231,8 +183,6 @@ end # ── proportion, not just presence ──────────────────────────────────────────────────────────── @testset "the record says what fraction of the run was inside experimental code" begin - # "It touched experimental code" and "97% of the run was inside it" are different verdicts - # about the same result, and only the second decides whether the number is usable. @test_broken 0.0 < ExperimentalAPI.experimental_fraction( ExperimentalAPI.record(() -> Sim.driver(M, 100_000)) @@ -241,31 +191,25 @@ end end @testset "inclusive and exclusive time are distinguished" begin - # A marked wrapper that spends all its time in settled code is not the same risk as a marked - # kernel that does the arithmetic itself. + # Scope: a marked wrapper over settled code is not a marked kernel. @test_broken let h = first(ExperimentalAPI.record(() -> Sim.driver(M, 1000))) h.inclusive >= h.exclusive end end # ── the floor: what may be emitted into the body ───────────────────────────────────────────── -# -# Folded in from what used to be `test_spec_runtime.jl`. Its other eight testsets restated this -# file's claims on a strictly smaller fixture; only these two measured anything of their own. - -# The control for the two assertions below. A predicate that has never returned `true` for -# anything is not evidence, and `!occursin(…)` is the shape most easily satisfied by an expansion -# that dropped the definition altogether. `@wrapping` does exactly what `@experimental` must never -# become, so the check can be shown to fire. Kept in a module because a macro defined at the top -# level of a test file is visible to every file after it in the same shard. + +# Controls for the two testsets below. `!occursin(…)` is satisfied by an expansion that dropped +# the definition, so each check is also run against an expansion that does the forbidden thing. +# Kept in a module because a macro at the top level of a test file leaks into the whole shard. module WrapControl -"A macro that wraps the call it is given — the shape the emitted flag must stay inside." +"Wraps the call it is given — the shape the emitted flag must stay inside." macro wrapping(_reason, def) return esc(Expr(:function, def.args[1], Expr(:block, :(COUNTS[] += 1), def.args[2]))) end -"A macro that puts a logging call in the body — measured at 5.65x guarded, 59.6x unguarded." +"Puts a logging call in the body." macro logging(reason, def) return esc( Expr( @@ -300,91 +244,53 @@ end end # module WrapControl @testset "today the expansion is untouched — which is what has to change" begin - # This file used to require that `@experimental` never wrap the call at all. That requirement - # is WITHDRAWN: the default layer cannot detect presence without emitting something into the - # body. What replaces it is not weaker, it is narrower and measured — see the table at the - # top. The emitted statement must be read-mostly, and must not drag the logging machinery in - # with it. - # - # Compared at the AST, not as a string: the expansion contains `Expr(:escape, …)` and the - # printed forms differ even when the bodies are identical. + # Scope: the default layer needs one statement in the body, so this equality must break. + # Compared at the AST: the expansion carries `Expr(:escape, …)` and prints differently. bare = WrapControl.method_bodies(@macroexpand f(x) = x * 2) marked = WrapControl.method_bodies(@macroexpand @experimental "why" f(x) = x * 2) @test length(bare) == 1 - @test marked == bare # the definition is emitted untouched - # …and the comparison can fail, which is the half that makes the line above worth anything. + @test marked == bare wrapped = WrapControl.method_bodies( @macroexpand WrapControl.@wrapping "why" f(x) = x * 2 ) @test wrapped != bare - # The target: exactly one statement more than the bare body, and no more. @test_broken length(marked[1].args) == length(bare[1].args) + 1 @test Sim.driver(M, 3) ≈ 3 * (0.5 * 1.0000001 + exp(-0.5)) end @testset "whatever is emitted, it is not a logging call" begin - # The hard line, and the one measurable today rather than after the fact. `@warn` in the body - # costs 59.6x unconditionally and **5.65x even when guarded so that it fires once** — the - # guard does not help, because what stops the definition inlining is the call being there at - # all. The summary at exit is the alternative that costs nothing. + # The bound on what the flag may cost: a logging call in the body stops the definition + # inlining whether or not it ever fires. emitted = string(@macroexpand @experimental "why" f(x) = x * 2) @test !occursin("CoreLogging", emitted) - # The positive control: the same predicate against an expansion that does log. Without it, - # `!occursin(…)` is satisfied by any expansion whatsoever, including an empty one. @test occursin( "CoreLogging", string(@macroexpand WrapControl.@logging "why" f(x) = x * 2) ) end -# There is still no wall-clock assertion in this file, and that is a finding rather than a gap. -# Four routes were measured on 2026-09-03 and each failed for its own reason: -# -# * A ratio `a < 5b + 1e-3` whose two arms did different work — `Sim.energy(Sim.Model(x))`, a -# struct construction per iteration, against a bare call. The 5x margin was spent on the -# fixture; it failed at a = 11.8 ms against b = 2.1 ms. -# * The same ratio with the arms made identical. Over eight trials on an idle machine the ratio -# ran 0.79 to 3.03, so the threshold sat inside the noise. -# * `@allocated` equality. Deterministic, but measuring the wrong thing: 0 on Julia 1.12 and 16 -# on 1.11 for the SAME code, and a counter wrapper `push!`ing into a warmed vector allocates -# zero too — so it cannot see the regression it was written to catch. The positive control is -# what showed that. -# * A ratio against a fixed threshold, now that a flag WILL be emitted. Ruled out in advance: -# the numbers in the table at the top come from a throwaway benchmark on an idle machine, and -# the same thresholds on a shared CI runner across three operating systems would be a flake -# generator. What CI can check is the SHAPE of the expansion; what only a benchmark can check -# is the cost, and that belongs in a benchmark. -# -# The claim is exact at the expansion and only ever approximate at run time, so it is checked -# where it is exact. +# No wall-clock assertion here by decision, not omission: the figures separating these designs +# come from an idle machine, and the same thresholds on a shared CI runner would be a flake +# generator. CI checks the shape of the expansion; cost belongs in a benchmark. # ── mechanism constraints ──────────────────────────────────────────────────────────────────── @testset "recording survives inlining" begin - # The measured reason the sampling route failed. `Sim.energy` is a one-line function; the - # optimiser will inline it. Any mechanism that only works on `@noinline` code is not a - # mechanism for this problem. + # Scope: marked definitions are usually small, so a mechanism needing `@noinline` is no + # mechanism. This is why the sampling route was rejected. @test_broken ExperimentalAPI.record(() -> Sim.driver(M, 100))[1].count == 100 end @testset "detection is on by default; counting is not" begin - # Inverted from what this file first required. "Recording off by default" was the safe answer - # only while presence detection was assumed to cost what counting costs; measured, they - # differ by a factor of four in parallel, and only one of them is affordable by default. @test_broken ExperimentalAPI.detecting() === true @test_broken ExperimentalAPI.recording() === false end @testset "the default layer's cost is stated, and it is the flag's cost" begin - # Not `>= 0`, which every number satisfies. The measured figure for a read-mostly flag is - # 0.985x-1.03x, so a default layer reporting overhead above a few percent has stopped being - # the thing that was measured — most likely by having become a counter. + # Not `>= 0`, which every number satisfies. Above a few percent it has become a counter. @test_broken ExperimentalAPI.overhead_when_detecting() < 0.10 end @testset "the opt-in layer's overhead is measured and reported, not discovered" begin - # A tool that slows a twelve-hour run by 4x will not be used on a twelve-hour run — and 4x is - # the measured figure for counting at eight threads, not a hypothetical. Whatever it costs, - # the number has to come back with the record. @test_broken ExperimentalAPI.record(() -> Sim.driver(M, 1000)).overhead isa Real end @@ -395,7 +301,6 @@ end end @testset "an exception inside the recorded block still yields a record" begin - # A run that crashed halfway is exactly when you want to know what it went through. @test_broken ExperimentalAPI.record(() -> error("boom"); rethrow=false) isa AbstractVector end @@ -403,18 +308,13 @@ end # ── concurrency and distribution ───────────────────────────────────────────────────────────── @testset "the suite runs with more than one thread" begin - # `Threads.@threads for _ in 1:8` executes its body 8 times whatever `nthreads()` is, so the - # test below would pass for a recorder that is not thread-safe at all if CI ran single - # threaded. `.github/workflows/CI.yml` sets `JULIA_NUM_THREADS` for this reason; if that ever - # regresses, this fails instead of the concurrency claim silently becoming untestable. + # `Threads.@threads for _ in 1:8` runs its body 8 times whatever `nthreads()` is, so without + # this the concurrency claims below would pass single-threaded. Set in `CI.yml`. @test Threads.nthreads() > 1 end @testset "hits from every thread are attributed" begin - # HPC code is threaded. A recorder that only sees the main thread reports a fraction of the - # truth and calls it the truth — and so does one that races: measured on 8 threads, a plain - # `Ref{Int}` incremented per call recorded 95,406,048 of 160,000,000 entries. It lost 40% and - # cost 3.76x for the privilege, which is why counting is atomic and opt-in. + # Scope: a racing counter is as wrong as a main-thread-only one, and reports a number. threaded() = Threads.@threads for _ in 1:8 Sim.driver(M, 100) end @@ -422,19 +322,15 @@ end end @testset "per-thread storage is sized by maxthreadid, not nthreads" begin - # Measured 2026-09-03 while benchmarking the counter with `-t 8`: `threadid()` returned 9, - # because the interactive pool is counted separately from the default one. A recorder that - # allocates an `nthreads()`-sized vector throws `BoundsError` on the first hit arriving from - # an interactive task — which is any hit from the REPL. + # The interactive pool is counted separately, so `threadid()` exceeds `nthreads()` — an + # `nthreads()`-sized vector throws on the first hit from a REPL task. @test Threads.maxthreadid() >= Threads.nthreads() @test_broken ExperimentalAPI.record(() -> Sim.driver(M, 10)).slots >= Threads.maxthreadid() end @testset "a merged record is still a record" begin - # `merge_records(...) isa AbstractVector` is the only constraint today, so a `record()` you - # can ask `.overhead` of, merged with another, may legally come back as a plain `Vector` you - # cannot. Losing the type across a verb's own merge is avoidable. + # `isa AbstractVector` would let the merge return a plain `Vector` with no `.enabled`. @test_broken hasproperty( ExperimentalAPI.merge_records([ExperimentalAPI.record(() -> Sim.driver(M, 10))]), :enabled, @@ -442,15 +338,13 @@ end end @testset "records from separate processes merge into one" begin - # The same shape as TestShards' shard records: a distributed sweep produces one record per - # worker, and the provenance statement is about the whole run. @test_broken ExperimentalAPI.merge_records([ ExperimentalAPI.record(() -> Sim.driver(M, 10)) for _ in 1:2 ]) isa AbstractVector end @testset "merging is associative and order-independent" begin - # Workers finish in arbitrary order; the provenance must not depend on that. + # Workers finish in arbitrary order; provenance must not depend on that. @test_broken let a = ExperimentalAPI.record(() -> Sim.driver(M, 10)), b = ExperimentalAPI.record(() -> Sim.sweep(M, 10)) @@ -461,20 +355,18 @@ end # ── coexistence with the profiler people already use ───────────────────────────────────────── @testset "recording does not disturb Profile" begin - # Nobody will adopt a provenance tool that breaks their performance workflow. @test_broken ExperimentalAPI.record(() -> Sim.driver(M, 10); with_profile=true) isa AbstractVector end @testset "an existing Profile buffer can be attributed after the fact" begin - # If someone already profiled a run, they should not have to run it again for twelve hours. + # Scope: a twelve-hour run already profiled must not have to be run again. @test_broken ExperimentalAPI.attribute(Profile.fetch()) isa AbstractVector end # ── the output is evidence, not a printout ─────────────────────────────────────────────────── @testset "a record is serialisable" begin - # The point is to put it next to a figure in a paper. A pretty-printed table is not evidence. @test_broken ExperimentalAPI.write_record( tempname(), ExperimentalAPI.record(() -> Sim.driver(M, 10)) ) isa AbstractString @@ -489,13 +381,12 @@ end end @testset "a record names the versions it was taken against" begin - # `energy` being experimental in v0.3 says nothing about v0.9. A provenance record that does - # not pin the version is not provenance. + # `energy` being experimental in v0.3 says nothing about v0.9. @test_broken ExperimentalAPI.record(() -> Sim.driver(M, 10)).versions isa AbstractDict end @testset "the reason is carried into the record" begin - # The record has to be readable a year later by someone who never saw the source. + # Scope: readable a year later by someone who never saw the source. @test_broken occursin( "convergence", first(ExperimentalAPI.record(() -> Sim.driver(M, 10))).reason ) @@ -504,11 +395,10 @@ end # ── using it as a gate ─────────────────────────────────────────────────────────────────────── @testset "a run can be asserted to have touched nothing experimental" begin - # The publishable-result gate: this figure was produced without entering unvalidated code. @test_broken ExperimentalAPI.assert_clean(() -> 1 + 1) end @testset "the assertion fails, naming the mark, when the run is not clean" begin - # A gate that cannot be shown to fire has not been shown to be a gate. + # Control: a gate that cannot be shown to fire is not a gate. @test_broken !ExperimentalAPI.assert_clean(() -> Sim.driver(M, 10); throw=false) end diff --git a/test/spec/test_spec_propagate.jl b/test/spec/test_spec_propagate.jl index 1c2f39d..798a74d 100644 --- a/test/spec/test_spec_propagate.jl +++ b/test/spec/test_spec_propagate.jl @@ -1,32 +1,17 @@ -# Propagation: a caller that never names a marked thing still depends on it. +# A caller that never names a marked thing still depends on it. # -# The model is Lean's `sorry`. Measured 2026-09-03 with Lean 4.33.1: -# -# P.lean:1:8: warning: declaration uses `sorry` -# 'unproven' depends on axioms: [sorryAx] -# 'downstream' depends on axioms: [sorryAx] <- never wrote `sorry` itself -# 'honest' does not depend on any axioms -# -# Julia cannot match that exactly, and the difference is the most important thing in this file. -# Lean's kernel has a closed dependency graph of proof terms; Julia's call graph is not closed. -# So the answer is not a Bool. It is three-valued: +# Modelled on Lean's `sorry`, but Julia's call graph is not closed, so the answer is three-valued: # # :depends a marked definition is reachable # :clean the whole call graph was resolved and nothing marked is in it # :unknown some call site could not be resolved — the honest non-answer # -# Collapsing `:unknown` into `:clean` is the one failure this file exists to prevent. A tool that -# reports "no experimental dependency" about a call graph it could not see has not made a weaker -# claim, it has made a false one. -# -# Feasibility was measured on 2026-09-03, **Julia 1.12.2**, with a custom -# `Core.Compiler.AbstractInterpreter` hooking `abstract_call_method`. Inference runs before -# inlining, so the call graph is intact there; post-processing `code_typed(...; optimize=true)` -# sees only `mul_float`/`add_float` and finds nothing. +# Scope: collapsing `:unknown` into `:clean` is the one failure this file exists to prevent. It is +# not a weaker claim, it is a false one. # -# The Julia version is load-bearing and is stated for the same reason `Lean 4.33.1` is above: -# `Core.Compiler` is internal and carries no stability guarantee across releases. This repository's -# CI spans 1.11 and 1.12, and the measurement was taken on one of them. +# The mechanism is a `Core.Compiler.AbstractInterpreter` hooking `abstract_call_method`, because +# inference runs before inlining; `code_typed(...; optimize=true)` sees only `mul_float` and finds +# nothing. using ExperimentalAPI: ExperimentalAPI, @experimental, experimental using Test @@ -65,11 +50,11 @@ mid_good(x::Float64) = solid(x) + 1.0 top_bad(x::Float64) = mid_bad(x) * 3 top_good(x::Float64) = mid_good(x) * 3 -# a function passed as a value. Julia specialises on `typeof(f)`, so inference resolves this. +# passed as a value — Julia specialises on `typeof(f)`, so this resolves apply(f, x::Float64) = f(x) top_arg(x::Float64) = apply(unstable, x) -# `@nospecialize` — measured to STILL resolve +# `@nospecialize` — still resolves top_nospec(@nospecialize(f), x::Float64) = f(x) # genuinely unresolvable: the callee is a value chosen at run time @@ -84,9 +69,8 @@ top_table(i::Int, x::Float64) = TABLE[i](x) # termination top_recursive(n::Int, x::Float64) = n <= 0 ? unstable(x) : top_recursive(n - 1, x) -# A chain deep enough that a depth limit MUST truncate before reaching the mark. `top_recursive` -# calls `unstable` in its own body, so it is visible at depth 1 whatever `maxdepth` says — a -# depth test written against it would pass for an implementation that ignores the keyword. +# Deep enough that a depth limit must truncate. `top_recursive` calls `unstable` in its own body, +# so a depth test written against that one would pass for an implementation ignoring the keyword. deep_5(x::Float64) = unstable(x) deep_4(x::Float64) = deep_5(x) deep_3(x::Float64) = deep_4(x) @@ -119,27 +103,21 @@ end # ── the type the answer lives in ───────────────────────────────────────────────────────────── # -# `Mark` and `Audit` are both pinned nominally somewhere in this directory (`isa Mark`, -# `isa Audit`). The propagation result is not: before these tests it was pinned purely by field -# name, so a `NamedTuple` with `.reached`/`.unresolved` satisfied every assertion in four files -# and `verdict` could be duck-typed on `hasproperty`. +# `Mark` and `Audit` are pinned nominally elsewhere in this directory. Without the same here, a +# `NamedTuple` with the right field names satisfies every assertion in four files. @testset "the result has a type, not just field names" begin @test_broken ExperimentalAPI.reach(Chain.top_bad, ENTRY) isa ExperimentalAPI.Reach end @testset "verdict is derived, never stored" begin - # `isbreaking(d::Diff)` is a pure function over `d.removed_stable`/`d.demoted`, never a cached - # field — which is why a `Diff` cannot claim "not breaking" while carrying a removal. Same - # rule here: if `verdict` were a stored field, `:clean` with a non-empty `.unresolved` would - # become representable, and that is the one state this whole file exists to forbid. + # A stored `verdict` makes `:clean` with a non-empty `.unresolved` representable, which is + # the one state this file forbids. Same rule as `isbreaking(d::Diff)`. @test_broken !hasproperty(ExperimentalAPI.reach(Chain.top_bad, ENTRY), :verdict) end @testset "a boolean gate exists alongside the three-valued answer" begin - # Every existing verdict in this package is a named predicate — `isbreaking`, - # `isexperimental`, `isdocumented` — never a comparison the caller writes out. The spec - # hand-writes `verdict(...) === :clean` and friends 26 times, which is the smell. + # Every other verdict here is a named predicate, never a comparison the caller writes out. @test_broken ExperimentalAPI.isclean(ExperimentalAPI.reach(Chain.top_good, ENTRY)) === true @test_broken ExperimentalAPI.isclean(ExperimentalAPI.reach(Chain.top_bad, ENTRY)) === @@ -147,9 +125,8 @@ end end @testset ":unknown absorbs when results are combined" begin - # `reach(Module)` has to fold the verdicts of every public entry into one answer, so the - # algebra has to exist. It is stated here rather than discovered: a module with one - # `:unknown` entry is not clean, whatever the others say, and folding is order-independent. + # `reach(Module)` folds every public entry into one answer, so the algebra must exist: one + # `:unknown` is not clean whatever the others say, and folding is order-independent. @test_broken ExperimentalAPI.combine(:clean, :unknown) === :unknown @test_broken ExperimentalAPI.combine(:depends, :unknown) === :depends @test_broken ExperimentalAPI.combine(:clean, :depends) === :depends @@ -168,14 +145,14 @@ end end @testset "an equally deep caller with nothing marked is reported clean" begin - # Without this the previous test passes for a tool that always says `:depends`. + # Control: rejects a tool that always says `:depends`. @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(Chain.top_good, ENTRY)) === :clean @test_broken isempty(ExperimentalAPI.reach(Chain.top_good, ENTRY).reached) end @testset "a function passed as a value is still followed" begin - # Measured: Julia specialises on `typeof(f)`, so this resolves. It is NOT a dynamic hole. + # Specialisation on `typeof(f)` resolves this; it is not a dynamic hole. @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(Chain.top_arg, ENTRY)) === :depends end @@ -189,8 +166,7 @@ end # ── the honest non-answer ──────────────────────────────────────────────────────────────────── @testset "an abstract-typed callee field is :unknown, NOT :clean" begin - # `Holder.f::Function` can hold `unstable`. Reporting `:clean` here would be a lie, and it is - # exactly what the prototype did before the third value existed. + # `Holder.f::Function` can hold `unstable`, so `:clean` here is a lie. @test_broken ExperimentalAPI.verdict( ExperimentalAPI.reach(Chain.top_field, Tuple{Chain.Holder,Float64}) ) === :unknown @@ -209,7 +185,7 @@ end end @testset "every unresolved site says where it is" begin - # "cannot tell" is only actionable if the user can go and look. + # "cannot tell" is actionable only if the user can go and look. @test_broken all( u -> hasproperty(u, :file) && hasproperty(u, :line), ExperimentalAPI.reach(Chain.top_field, Tuple{Chain.Holder,Float64}).unresolved, @@ -217,13 +193,12 @@ end end @testset "a depth limit reports :unknown rather than :clean" begin - # `deep_1` is five hops from the mark, so `maxdepth=2` must truncate. Asserting `:unknown` - # rather than `!== :clean` is what separates "the limit produced the honest non-answer" from - # "the limit was silently ignored and the mark was found anyway". + # `:unknown` rather than `!== :clean`: the latter cannot separate "the limit truncated" from + # "the limit was ignored and the mark was found anyway". @test_broken ExperimentalAPI.verdict( ExperimentalAPI.reach(Chain.deep_1, ENTRY; maxdepth=2) ) === :unknown - # …and the same entry point WITHOUT the limit finds it, so the fixture can disagree. + # …and without the limit it is found, so the fixture can disagree. @test_broken ExperimentalAPI.verdict(ExperimentalAPI.reach(Chain.deep_1, ENTRY)) === :depends end @@ -245,9 +220,8 @@ end # ── things that are not calls ──────────────────────────────────────────────────────────────── @testset "a marked const is seen where it is used" begin - # A const is not a call site, so the call-graph walk cannot find it. Either the analysis - # reads globals out of the IR as well, or this case has to be declared out of scope in the - # documentation. What it must not do is report `:clean`. + # A const is not a call site. Either the analysis reads globals out of the IR, or the case is + # declared out of scope — what it must not do is report `:clean`. @test_broken ExperimentalAPI.verdict( ExperimentalAPI.reach(Chain.top_uses_const, ENTRY) ) !== :clean @@ -260,25 +234,22 @@ end end @testset "marking a module marks what it contains" begin - # Or it does not, and that is stated. Either way it is a decision, not an omission. + # Or it does not, stated. Either way a decision, not an omission. @test_broken hasproperty(ExperimentalAPI.reach(Chain.top_bad, ENTRY), :through_modules) end # ── across packages ────────────────────────────────────────────────────────────────────────── @testset "a mark in a dependency propagates into the dependent" begin - # The QAtlas case: a downstream analysis calls `fetch`, which is marked in QAtlas. - # Requires the fixture package, so it is only pinned as an API shape here. + # Needs the fixture package, so only the API shape is pinned here. @test_broken ExperimentalAPI.reach isa Function end # ── cost ───────────────────────────────────────────────────────────────────────────────────── @testset "the macro emits nothing but the definition and one push" begin - # The previous version of this test filtered `methods(_mark!)` for a name containing "reach". - # `Method.name` is the GENERIC FUNCTION's name — always `:_mark!`, never derived from what the - # body calls — so the filter was unconditionally empty and the assertion could not fail even - # if `_mark!` called `reach` directly. Look at the expansion instead. + # Look at the expansion: `Method.name` is the generic function's name, so filtering methods + # by what their bodies call is unconditionally empty. emitted = string(@macroexpand @experimental "why" f(x) = x) @test occursin("_mark!", emitted) # the one load-time effect @test occursin("__EXPERIMENTAL_API_MARKS__", emitted) diff --git a/test/spec/test_spec_verify.jl b/test/spec/test_spec_verify.jl index 42e6f0a..27958b3 100644 --- a/test/spec/test_spec_verify.jl +++ b/test/spec/test_spec_verify.jl @@ -1,12 +1,7 @@ -# "How well is this experimental thing actually verified?" +# How well a marked definition is exercised by the tests. # -# This is the half of the intent that is cheap. A mark carries `file` and `line`; Julia's -# `--code-coverage` writes per-line execution counts. Joining them answers, with no new -# machinery: which marked definitions does the test suite never execute? -# -# It matters because the failure mode being guarded against is not a crash. It is a number that -# comes back and looks fine. A marked method with zero coverage is the worst case: unverified -# code, shipped, and never even run by its own suite. +# Scope: joining the mark's `file`/`line` against `--code-coverage` counts. No new machinery, and +# it answers the worst case — unverified code that its own suite never runs. using ExperimentalAPI: ExperimentalAPI, @experimental, experimental using Test @@ -33,8 +28,8 @@ end end # module Covered -# The suite that "verifies" the module. Deliberately partial — a fixture that exercised -# everything could not tell a working coverage join from one that always reports 100%. +# Deliberately partial: a fully exercised fixture cannot tell a working join from one that +# always reports 100%. @testset "the fixture is exercised only partly, on purpose" begin @test Covered.exercised(1) == 2 @test Covered.half_exercised(1) == 1 @@ -46,7 +41,7 @@ end for mk in experimental(Covered) @test isfile(String(mk.file)) @test mk.line > 0 - # The recorded line is the declaration, and the definition starts at or after it. + @test occursin("@experimental", readlines(String(mk.file))[mk.line]) end end @@ -56,7 +51,7 @@ end end @testset "a marked definition that IS covered is not reported" begin - # Without this, a checker that reports everything would pass the previous test. + # Control: rejects a checker that reports everything. @test_broken :exercised ∉ [mk.name for mk in ExperimentalAPI.unverified(Covered)] end @@ -65,21 +60,18 @@ end end @testset "coverage is absent, not zero, when the run had none enabled" begin - # Julia writes no `.cov` files without `--code-coverage`. Reporting 0% then would call every - # marked definition unverified on every ordinary test run — a false alarm that would get the - # check switched off. + # No `.cov` files without `--code-coverage`; reporting 0% then flags everything on every + # ordinary run. @test_broken ExperimentalAPI.coverage(Covered, :exercised) === missing || ExperimentalAPI.coverage(Covered, :exercised) isa Real end @testset "a mark whose line no longer matches its definition is reported stale" begin - # Marks record file:line at macro-expansion. An edit above the definition moves the code but - # not a previously written coverage file, and joining them then attributes the wrong lines. + # An edit above the definition moves the code but not an existing coverage file. @test_broken ExperimentalAPI.stale_marks(Covered) isa AbstractVector end @testset "the verification report is data, not a printout" begin - # Same convention as `audit`: the trust number comes back on the normal return path so a - # release script can read it, rather than being printed for a human to eyeball. + # Same convention as `audit`: the number comes back on the normal return path. @test_broken ExperimentalAPI.verification(Covered) isa AbstractVector end diff --git a/test/test_spec_table.jl b/test/test_spec_table.jl index 5b55736..ba7f568 100644 --- a/test/test_spec_table.jl +++ b/test/test_spec_table.jl @@ -1,9 +1,6 @@ -# The table in `test/spec/README.md` is generated, and this is what keeps it that way. +# The table in `test/spec/README.md` is generated; this is what keeps it that way. # -# The first version of that table was written by hand, and it drifted inside the very change that -# argues prose drifts and tests do not: two spec files landed after it was typed, so it said -# "nine files" when there were eleven and every count in it was wrong. Generating it is only half -# a fix — a generator nobody runs drifts exactly as fast. This is the other half. +# Scope: generating it is only half a fix — a generator nobody runs drifts exactly as fast. using Test @@ -13,8 +10,8 @@ const _SPEC_README = joinpath(@__DIR__, "spec", "README.md") const _BEGIN = "" const _END = "" -# Git checks the README out with CRLF on Windows, so the comparison below is between line -# endings unless they are normalised — measured, as a red `julia 1.12 — windows-latest`. +# Git checks the README out with CRLF on Windows, so without this the comparison below is +# between line endings. _lf(s::AbstractString) = replace(s, "\r\n" => "\n") @testset "the spec table is generated, not typed" begin @@ -22,8 +19,7 @@ _lf(s::AbstractString) = replace(s, "\r\n" => "\n") @test occursin(_BEGIN, md) @test occursin(_END, md) block = strip(split(split(md, _BEGIN)[2], _END)[1]) - # The failure message has to say what to do, because "a table is out of date" is not a - # defect anybody can act on without being told where the table comes from. + # "a table is out of date" is not actionable without saying where the table comes from. if block != _lf(SpecSummary.table()) @info "test/spec/README.md is stale — regenerate with `julia --project=test test/spec/summary.jl`" end @@ -31,18 +27,16 @@ _lf(s::AbstractString) = replace(s, "\r\n" => "\n") end @testset "no spec file is missing from the table" begin - # `spec_files()` reads the directory, and `table()` errors on a file with no entry, so this - # asserts the directory is non-empty and the generator ran over all of it. Without the first - # assertion a `readdir` that returned nothing would satisfy the second vacuously. + # The non-emptiness assertion is not redundant: a `readdir` returning nothing satisfies the + # set equality vacuously. files = SpecSummary.spec_files() @test length(files) >= 10 @test Set(files) == Set(keys(SpecSummary.CONCERNS)) end @testset "no spec file is missing from runtests.jl" begin - # The other silent-omission surface: a spec file can exist, be listed in the table, and never - # run, because `runtests.jl` includes them by name. Then it contributes to the published - # count while asserting nothing. + # `runtests.jl` includes spec files by name, so one can be listed in the table, never run, + # and still contribute to the published count. driver = read(joinpath(@__DIR__, "runtests.jl"), String) missing_from_driver = filter( f -> !occursin("spec/$f", driver), SpecSummary.spec_files() @@ -51,13 +45,8 @@ end end @testset "the count is behaviours, and a loop cannot inflate it" begin - # The property that makes the published measure worth publishing: `test_spec_declare.jl` - # runs 91 assertions from 36 assertion lines because several sit inside - # `for mk in experimental(Declared)`. Adding a fixture mark adds passing assertions and no - # coverage. A leaf `@testset` is something a person wrote. - # - # Pinned against a synthetic file rather than a real one, so that editing a spec file does - # not silently change what this asserts. + # The property that makes the measure worth publishing: a loop cannot inflate it. Pinned + # against a synthetic file so that editing a spec file does not change what this asserts. path = joinpath(mktempdir(), "test_spec_synthetic.jl") write( path,