From 23b4e6eb999056e6bc9a14b6969e78c0d1a1add0 Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Sat, 5 Sep 2026 08:21:14 +0000 Subject: [PATCH 1/2] docs: the front page's example runs, and the pages are executed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The review's own finding, still live: `docs/src/index.md` opened with @experimental "…" energy(m::Model) = m.β * correction(m) `Model` and `correction` are defined nowhere. That is the defect the registration review named for the README — it was fixed there and not here, one file over, where a reader looking at the documentation site meets it first. `test/test_readme.jl` covered the README's first block and nothing else. It now executes every self-contained block under `docs/src` — twenty were shipped unexecuted, and two of them did not run. Illustrative blocks (`MyPackage`, `Archeion`) and transcripts (`julia>`, `pkg>`) are skipped, and a control asserts at least five blocks survive that filter, so the loop cannot quietly skip everything. `declaring.md` quoted an error message the macro does not produce: it showed `` `Base.sum(x::Int)` ``, and `_signame` never sees the argument types — the real message says `` `Base.sum` ``. Corrected. `doctest = true` in `makedocs`, with one real `jldoctest` in `observing.md`. Only one, and the page says why: `Entry` prints its module, and Documenter's sandbox does not print as `Main`, so a doctest of the displayed form would show a line no reader sees at their own REPL. The fields it is read for are module-independent, so those are the doctest. Verified by changing one character of the expected output and watching the docs build fail. Co-Authored-By: Claude Opus 5 --- docs/make.jl | 4 +++ docs/src/declaring.md | 2 +- docs/src/index.md | 4 ++- docs/src/observing.md | 25 ++++++++++++++ test/test_readme.jl | 76 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 2 deletions(-) diff --git a/docs/make.jl b/docs/make.jl index 7bab856..b947016 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -14,6 +14,10 @@ makedocs(; edit_link="main", ), modules=[ExperimentalAPI], + # A `jldoctest` whose output stops matching fails the build. The message in `declaring.md` + # had drifted from the one the macro throws — it named the signature the macro never sees — + # which is what this catches. + doctest=true, # `:public` is the same claim this package makes about its users: a name on the public # surface without a docstring fails the build. ExperimentalAPI's own release layer is # declared @experimental AND documented, so it satisfies both. diff --git a/docs/src/declaring.md b/docs/src/declaring.md index 2d84ac4..3baaf2a 100644 --- a/docs/src/declaring.md +++ b/docs/src/declaring.md @@ -68,7 +68,7 @@ Anything else is **refused with a message naming the alternative**, never guesse ```julia julia> @experimental "why" Base.sum(x::Int) = x -ERROR: @experimental: `Base.sum(x::Int)` defines a name owned by another module, +ERROR: ArgumentError: @experimental: `Base.sum` defines a name owned by another module, which is not part of this module's public surface ``` diff --git a/docs/src/index.md b/docs/src/index.md index 68ce011..cce61a2 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -14,7 +14,9 @@ a machine so the answer arrives without anyone remembering to ask for it. ```julia using ExperimentalAPI -@experimental "convergence not established below β ≈ 0.1" energy(m::Model) = m.β * correction(m) +@experimental "convergence not established below β ≈ 0.1" energy(β) = β * 1.0000001 + +energy(0.5) ``` ```console diff --git a/docs/src/observing.md b/docs/src/observing.md index 075232b..37b3f02 100644 --- a/docs/src/observing.md +++ b/docs/src/observing.md @@ -1,5 +1,8 @@ ```@meta CurrentModule = ExperimentalAPI +DocTestSetup = quote + using ExperimentalAPI +end ``` # Observing @@ -47,6 +50,28 @@ julia> ExperimentalAPI.entered() Entry(Main.energy, "convergence not established below β ≈ 0.1") ``` +The display above is a transcript rather than a doctest on purpose: `Entry` prints its module, and +Documenter evaluates doctests in a sandbox whose module does not print as `Main`, so a doctest here +would show a line no reader ever sees at their own REPL. The fields it is read for do not depend on +where it ran, so those are checked: + +```jldoctest +julia> using ExperimentalAPI + +julia> @experimental "convergence not established below β ≈ 0.1" energy(β) = β * 1.0000001 + +julia> energy(0.5); + +julia> only(ExperimentalAPI.entered()).name +:energy + +julia> only(ExperimentalAPI.entered()).reason +"convergence not established below β ≈ 0.1" + +julia> only(ExperimentalAPI.entered()).count === nothing +true +``` + A marked definition the run never entered is **absent**, not reported with a count of zero — the difference between "observed" and "enumerated". diff --git a/test/test_readme.jl b/test/test_readme.jl index 7e749a7..fb7602a 100644 --- a/test/test_readme.jl +++ b/test/test_readme.jl @@ -131,3 +131,79 @@ end lines = split(blocks[1][2], "\n") @test count(l -> endswith(rstrip(l), "\\"), lines) == 1 end + +# ── the documentation pages ────────────────────────────────────────────────────────────────── +# +# Scope: `docs/src` gets the same treatment the README does. Twenty julia blocks were shipped +# unexecuted, and two of them did not run — including the front page's, which is the defect the +# registry review named for the README and which was fixed there and not here. + +const _DOCS = joinpath(@__DIR__, "..", "docs", "src") + +"Every fenced julia block under `docs/src`, as (page, index, text). Any fence width." +function docs_blocks() + out = Tuple{String,Int,String}[] + for f in sort(readdir(_DOCS; join=true)) + endswith(f, ".md") || continue + for (i, m) in enumerate(eachmatch(r"`{3,}julia\r?\n(.*?)`{3,}"s, read(f, String))) + push!(out, (basename(f), i, m.captures[1])) + end + end + return out +end + +# A block is illustrative if it stands in for a package the reader supplies, and a transcript if +# it shows a REPL session — `pkg>` as much as `julia>`, which is why the install block is here. +function _is_illustrative(b) + return occursin("MyPackage", b) || + occursin("MyPkg", b) || + occursin("Archeion", b) || + occursin("…", b) +end +_is_transcript(b) = occursin("julia>", b) || occursin("pkg>", b) + +@testset "every macro the documentation teaches exists" begin + blocks = docs_blocks() + @test !isempty(blocks) + @test "index.md" in [p for (p, _, _) in blocks] + foreign = Set([ + Symbol("@info"), Symbol("@test"), Symbol("@testset"), Symbol("@deprecate") + ]) + named = Set{Symbol}() + for (_, _, b) in blocks, m in eachmatch(r"@[a-zA-Z_][a-zA-Z0-9_]*", b) + push!(named, Symbol(m.match)) + end + @test Symbol("@experimental") in named # non-vacuity, anchored to content + missing_macros = sort!([ + m for m in collect(named) if m ∉ foreign && !isdefined(ExperimentalAPI, m) + ]) + @test missing_macros == Symbol[] +end + +@testset "every self-contained documentation example runs" begin + # Not "the page parses": `index.md` parsed perfectly and raised `UndefVarError: Model` on the + # first line, because the example named a type the reader was supposed to have. + failures = String[] + for (page, i, b) in docs_blocks() + (_is_illustrative(b) || _is_transcript(b)) && continue + m = Module(Symbol("DocsBlock_", replace(page, "." => "_"), "_", i)) + try + Core.eval(m, :(using ExperimentalAPI)) + Core.eval(m, Meta.parseall(b; filename=page)) + catch e + err = e isa LoadError ? e.error : e + push!(failures, "$page[$i]: " * first(sprint(showerror, err), 60)) + end + end + @test failures == String[] +end + +@testset "…and something is actually being run" begin + # Control: the loop above skips illustrative and transcript blocks, so it could quietly skip + # everything. At least the front page and one `declaring.md` block have to survive the filter. + runnable = [ + (p, i) for (p, i, b) in docs_blocks() if !_is_illustrative(b) && !_is_transcript(b) + ] + @test length(runnable) >= 5 + @test ("index.md", 1) in runnable +end From 6be4638ebe5d0eeeb250bbe917d4685f2f35ec1c Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Sat, 5 Sep 2026 08:43:13 +0000 Subject: [PATCH 2/2] fix: the documentation's examples run, and two bugs they exposed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #16, #17, #18, #19. Running every self-contained block under `docs/src` turned up two defects in shipped code, not just in prose. `record` swallowed the caller's exception (#16). On the default path it called `Base.rethrow(err)` after the `try`/`catch`, which is legal only inside a `catch`; outside one it raises its own `ErrorException` and the original is lost. The existing assertion — `@test_throws ErrorException` — passed the whole time, because the wrong exception is an `ErrorException` too. The new test pins the message and controls against the old one. `stamp(path) do … end` had no method (#17). `do` puts the function first; the methods took the path first. `record` already takes the function first, so the two verbs disagreed as well. The documentation called public-but-unexported names unqualified in 29 places (#18). Only `@experimental` is exported, so `record(...)` after `using ExperimentalAPI` is `UndefVarError` — and `api.md` states the qualification convention the pages were breaking. `docs/src/index.md` still opened with the example the registration review said does not run (#19): `energy(m::Model) = m.β * correction(m)`, with neither defined. It was fixed in the README and not here. It is now self-contained AND an `@example` block, so Documenter runs it during the build: breaking it again fails with `failed to run @example block in docs/src/index.md`. `test/test_readme.jl` executes every self-contained docs block; illustrative ones are skipped through an explicit list of the placeholders the documentation asks the reader to supply, so a new placeholder fails the test by name rather than being silently skipped. `doctest = true` in `makedocs`, with one `jldoctest` in `observing.md`. Only one, and the page says why: the error block would carry `LoadError` and an absolute-path stacktrace, and the `Entry` display prints its module, which in Documenter's sandbox is not `Main` — a doctest of it would show a line no reader sees at their own REPL. Suite: 990 passed. Co-Authored-By: Claude Opus 5 --- docs/src/adopting.md | 2 +- docs/src/analysing.md | 14 +++++++------- docs/src/checking.md | 12 ++++++------ docs/src/declaring.md | 4 ++-- docs/src/index.md | 2 +- docs/src/observing.md | 6 +++--- docs/src/releases.md | 6 +++--- src/record.jl | 5 ++++- src/release.jl | 6 ++++++ test/spec/README.md | 6 +++--- test/spec/test_spec_integration.jl | 13 +++++++++++++ test/spec/test_spec_profile.jl | 17 +++++++++++++++++ test/test_readme.jl | 26 +++++++++++++++++++------- 13 files changed, 85 insertions(+), 34 deletions(-) diff --git a/docs/src/adopting.md b/docs/src/adopting.md index 88a1268..cf67d21 100644 --- a/docs/src/adopting.md +++ b/docs/src/adopting.md @@ -12,7 +12,7 @@ and no idea which of them anybody meant. ```julia julia> using MyPackage, ExperimentalAPI -julia> audit(MyPackage) +julia> ExperimentalAPI.audit(MyPackage) ``` Two numbers matter. `undocumented` is the backlog. `dangling` should be zero on day one, because diff --git a/docs/src/analysing.md b/docs/src/analysing.md index 8711889..3f7c79c 100644 --- a/docs/src/analysing.md +++ b/docs/src/analysing.md @@ -8,14 +8,14 @@ CurrentModule = ExperimentalAPI *could* do — before running it, and including through code that never names the marked thing. ```julia -julia> r = reach(analyse, Tuple{Model,Float64}); +julia> r = ExperimentalAPI.reach(analyse, Tuple{Model,Float64}); -julia> verdict(r) +julia> ExperimentalAPI.verdict(r) :depends julia> r.reached 1-element Vector{ExperimentalAPI.Reached}: - Reached(MyModel.energy via analyse → sweep → inner → energy) + ExperimentalAPI.Reached(MyModel.energy via analyse → sweep → inner → energy) ``` The model is Lean's `sorry`: a proof that uses one is not a proof, however many layers down it @@ -79,8 +79,8 @@ A more specific unmarked method shadowing a marked one is resolved as what actua ## Whole modules, and scripts ```julia -r = reach(MyPackage) -verdict(r) # one answer for the package +r = ExperimentalAPI.reach(MyPackage) +ExperimentalAPI.verdict(r) # one answer for the package r.affected_entries # …and which public entry points are not clean ``` @@ -95,8 +95,8 @@ evaluated in a scratch module, because the analysis has to resolve the names the ## The exit, read backwards ```julia -dependents(MyPackage, :energy) # who reaches it -verdict(reach(MyPackage; ignore = [:energy])) # what removing the mark would change +ExperimentalAPI.dependents(MyPackage, :energy) # who reaches it +ExperimentalAPI.verdict(ExperimentalAPI.reach(MyPackage; ignore = [:energy])) # what removing the mark would change ``` `ignore` answers "what would removing this mark change?" without removing it. [`dependents`](@ref) diff --git a/docs/src/checking.md b/docs/src/checking.md index 0232161..e91f289 100644 --- a/docs/src/checking.md +++ b/docs/src/checking.md @@ -24,7 +24,7 @@ A mark says the shape is unsettled. That is never a reason to say nothing about does, so a marked name with no prose is a finding exactly as an unmarked one is. ```julia -julia> audit(Archeion) +julia> ExperimentalAPI.audit(Archeion) Public surface of Archeion — 36 names documented 33 experimental 0 @@ -111,12 +111,12 @@ see exactly which names it would have to argue about. Empty means the two agree. ## The method-level half ```julia -julia> audit(Downstream).contributed_methods +julia> ExperimentalAPI.audit(Downstream).contributed_methods 4-element Vector{Method}: fetch_value(::Ising, ::Energy) … ⋮ -julia> unaccounted_methods(Downstream) # neither a docstring nor a mark +julia> ExperimentalAPI.unaccounted_methods(Downstream) # neither a docstring nor a mark 2-element Vector{Method}: ⋮ ``` @@ -139,11 +139,11 @@ The mark records where it was written, and `--code-coverage` records a count per two answers the worst case a marked definition can be in: ```julia -julia> unverified(MyPackage) # marked AND never executed by the suite +julia> ExperimentalAPI.unverified(MyPackage) # marked AND never executed by the suite 1-element Vector{ExperimentalAPI.Mark}: - Mark(MyPackage.never_called, "shipped without ever being called") + ExperimentalAPI.Mark(MyPackage.never_called, "shipped without ever being called") -julia> coverage(MyPackage, :half_exercised) +julia> ExperimentalAPI.coverage(MyPackage, :half_exercised) 0.6 ``` diff --git a/docs/src/declaring.md b/docs/src/declaring.md index 3baaf2a..b1bd796 100644 --- a/docs/src/declaring.md +++ b/docs/src/declaring.md @@ -38,7 +38,7 @@ odd; they cannot see that it is odd *because* a refactor upstream has not landed is required, and the mark carries it everywhere the name goes: ```julia -julia> mark(Archeion, :ingest) +julia> ExperimentalAPI.mark(Archeion, :ingest) Archeion.ingest — experimental reason: signature will be wrapped once the write-back refactor settles since: v0.1.4 @@ -117,7 +117,7 @@ end ``` ```julia -julia> audit(M).dangling +julia> ExperimentalAPI.audit(M).dangling 1-element Vector{Symbol}: :helper ``` diff --git a/docs/src/index.md b/docs/src/index.md index cce61a2..3da377b 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -11,7 +11,7 @@ had not validated. `@experimental` is that mark, written at the definition site where the author is, and readable by a machine so the answer arrives without anyone remembering to ask for it. -```julia +```@example frontpage using ExperimentalAPI @experimental "convergence not established below β ≈ 0.1" energy(β) = β * 1.0000001 diff --git a/docs/src/observing.md b/docs/src/observing.md index 37b3f02..589bfd9 100644 --- a/docs/src/observing.md +++ b/docs/src/observing.md @@ -47,7 +47,7 @@ Three properties, each of them a decision: ```julia julia> ExperimentalAPI.entered() 1-element Vector{ExperimentalAPI.Entry}: - Entry(Main.energy, "convergence not established below β ≈ 0.1") + ExperimentalAPI.Entry(Main.energy, "convergence not established below β ≈ 0.1") ``` The display above is a transcript rather than a doctest on purpose: `Entry` prints its module, and @@ -117,7 +117,7 @@ table above. ## Recording: counts, paths and time ```julia -r = record() do +r = ExperimentalAPI.record() do simulate(model; steps = 10_000) end ``` @@ -177,7 +177,7 @@ make. [`assert_clean`](@ref) turns a record into a refusal: ```julia -assert_clean() do +ExperimentalAPI.assert_clean() do publish(compute(model)) end ``` diff --git a/docs/src/releases.md b/docs/src/releases.md index e0cc47e..3a9742f 100644 --- a/docs/src/releases.md +++ b/docs/src/releases.md @@ -28,8 +28,8 @@ tracking = "https://github.com/org/MyPackage.jl/issues/12" On the next release, compare: ```julia -d = compare(read_snapshot("api.toml"), MyPackage) -isbreaking(d) && error("breaking: $(d.removed_stable) removed, $(d.demoted) demoted") +d = ExperimentalAPI.compare(ExperimentalAPI.read_snapshot("api.toml"), MyPackage) +ExperimentalAPI.isbreaking(d) && error("breaking: $(d.removed_stable) removed, $(d.demoted) demoted") ``` ## What counts as breaking @@ -115,7 +115,7 @@ ratchet, in the shape `test_surface`'s `skip` already has. ## Provenance next to the result ```julia -stamp("figures/energy_sweep.provenance.toml") do +ExperimentalAPI.stamp("figures/energy_sweep.provenance.toml") do sweep(model; βs = 0.05:0.05:2.0) end ``` diff --git a/src/record.jl b/src/record.jl index 28dc4e8..9859a28 100644 --- a/src/record.jl +++ b/src/record.jl @@ -241,7 +241,10 @@ function record( end end end - err === nothing || rethrow && Base.rethrow(err) + # `Base.rethrow(err)` is legal only inside a `catch`; here it raises + # "rethrow(exc) not allowed outside a catch block" and the caller never sees their own + # exception. `throw` gives a fresh backtrace, which is the price of building the record first. + err === nothing || rethrow && throw(err) hits = Hit[] for p in ps diff --git a/src/release.jl b/src/release.jl index c63934e..ae2a3b9 100644 --- a/src/release.jl +++ b/src/release.jl @@ -327,6 +327,12 @@ function stamp(path::AbstractString, f) return stamp(path, record(f)) end +# `stamp(path) do … end` puts the function first, which is the form the documentation teaches and +# the order `record` already takes. Typed `::Function` so `stamp(path, value)` stays unambiguous. +function stamp(f::Function, path::AbstractString) + return stamp(path, f) +end + function stamp(path::AbstractString, r::Record) d = Dict{String,Any}( "generated" => string(Dates_now()), diff --git a/test/spec/README.md b/test/spec/README.md index 0cddebb..c517726 100644 --- a/test/spec/README.md +++ b/test/spec/README.md @@ -50,12 +50,12 @@ that is entirely `@test_broken` is a claim written down, not a check being run. | `test_spec_docstring.jl` | 9 | 9 | 0 | a mark and a docstring are different accounts and must coexist | | `test_spec_foreign.jl` | 13 | 13 | 0 | marking a method on somebody else's generic — the `QAtlas.fetch` case | | `test_spec_forms.jl` | 24 | 24 | 0 | the definition forms a real package hits on its second afternoon | -| `test_spec_integration.jl` | 18 | 18 | 0 | where the mark has to surface: docs, Aqua, releases, provenance, CI | +| `test_spec_integration.jl` | 19 | 19 | 0 | where the mark has to surface: docs, Aqua, releases, provenance, CI | | `test_spec_lifecycle.jl` | 15 | 15 | 0 | the mark's EXIT, and an entry point that is a module rather than a function | -| `test_spec_profile.jl` | 41 | 41 | 0 | what a real run went through, how often, and how much of it | +| `test_spec_profile.jl` | 42 | 42 | 0 | what a real run went through, how often, and how much of it | | `test_spec_propagate.jl` | 20 | 20 | 0 | a caller that never names a marked thing still depends on it | | `test_spec_verify.jl` | 9 | 9 | 0 | how well is a marked thing exercised by the tests | -| **10 files** | **176** | **176** | **0** | | +| **10 files** | **178** | **178** | **0** | | The table is generated and pinned by `test/test_spec_table.jl`, which fails if it goes stale — diff --git a/test/spec/test_spec_integration.jl b/test/spec/test_spec_integration.jl index dbd95ba..acf8ba5 100644 --- a/test/spec/test_spec_integration.jl +++ b/test/spec/test_spec_integration.jl @@ -291,3 +291,16 @@ end @test isempty(ExperimentalAPI.stale_since(Shown, v"0.2.0")) @test ExperimentalAPI.age(Shown, :provisional, v"0.9.0") == 7 end + +@testset "stamp takes the do-block form the documentation teaches" begin + # `stamp(path) do … end` passes the function FIRST. Only `stamp(::AbstractString, ::Any)` + # existed, so the documented form raised `MethodError` — the docs taught a call that had no + # method. Typed `::Function` on the new one so `stamp(path, value)` stays unambiguous. + path = ExperimentalAPI.stamp(tempname()) do + 1 + 1 + end + @test isfile(path) + @test occursin("julia", read(path, String)) + # …and the two-argument form still resolves the way it did. + @test ExperimentalAPI.stamp(tempname(), () -> 1 + 1) isa AbstractString +end diff --git a/test/spec/test_spec_profile.jl b/test/spec/test_spec_profile.jl index fc8cb95..114864f 100644 --- a/test/spec/test_spec_profile.jl +++ b/test/spec/test_spec_profile.jl @@ -352,6 +352,23 @@ end @test_throws ErrorException ExperimentalAPI.record(() -> error("boom")) end +@testset "…and the default propagates the caller's own exception" begin + # `rethrow = true` is the default, and the assertion above it — `@test_throws + # ErrorException` — passed for the wrong reason. `Base.rethrow(err)` outside a `catch` raises + # `ErrorException("rethrow(exc) not allowed outside a catch block")`, which satisfies a check + # on the exception TYPE while the caller never saw their own error. Pin the message. + e = try + ExperimentalAPI.record(() -> error("the caller's own message")) + nothing + catch err + err + end + @test e isa ErrorException + @test occursin("the caller's own message", sprint(showerror, e)) + # Control: the message that used to come out instead. + @test !occursin("not allowed outside a catch block", sprint(showerror, e)) +end + # ── concurrency and distribution ───────────────────────────────────────────────────────────── @testset "the suite runs with more than one thread" begin diff --git a/test/test_readme.jl b/test/test_readme.jl index fb7602a..a5cfd5e 100644 --- a/test/test_readme.jl +++ b/test/test_readme.jl @@ -154,12 +154,21 @@ end # A block is illustrative if it stands in for a package the reader supplies, and a transcript if # it shows a REPL session — `pkg>` as much as `julia>`, which is why the install block is here. -function _is_illustrative(b) - return occursin("MyPackage", b) || - occursin("MyPkg", b) || - occursin("Archeion", b) || - occursin("…", b) -end +# Names the documentation asks the reader to supply. Adding one is a deliberate act: a block that +# needs a new placeholder fails here first, naming the identifier, rather than being skipped. +const _PLACEHOLDERS = [ + "MyPackage", + "MyPkg", + "Archeion", + "…", + "simulate", + "publish", + "compute", + "sweep", + "model", +] + +_is_illustrative(b) = any(p -> occursin(p, b), _PLACEHOLDERS) _is_transcript(b) = occursin("julia>", b) || occursin("pkg>", b) @testset "every macro the documentation teaches exists" begin @@ -205,5 +214,8 @@ end (p, i) for (p, i, b) in docs_blocks() if !_is_illustrative(b) && !_is_transcript(b) ] @test length(runnable) >= 5 - @test ("index.md", 1) in runnable + # The front page's example is absent from that list because it is an `@example` block — + # Documenter runs it during the docs build, which is the stronger guard. Assert that, so + # turning it back into an inert ```julia fence is caught here. + @test occursin("```@example", read(joinpath(_DOCS, "index.md"), String)) end