diff --git a/docs/development/regression-harness.md b/docs/development/regression-harness.md index 65ad514c4..bb5217f92 100644 --- a/docs/development/regression-harness.md +++ b/docs/development/regression-harness.md @@ -96,3 +96,48 @@ regress --cases solovev_n1 --ref-range develop~10..develop - `--force` — re-run even if cached - `--verbose` — print GPEC subprocess output - `--no-instantiate` — skip `Pkg.instantiate()` (faster if deps are already resolved) +- `--no-pin-manifest` — let each ref resolve its own package set (see below) +- `--allow-env-mismatch` — reuse cached results produced in a different environment +- `--fail-on-change` — exit non-zero when any tracked quantity changed + +## Making source code the only variable + +`Manifest.toml` is untracked, so a worktree checked out at an old commit used to resolve whatever +package versions were newest at run time. Machine-epsilon differences in library math then get +amplified by the adaptive ODE step controller and by ill-conditioned near-resonant diagnostics +into double-digit-percent "regressions" that no source change caused. + +Two mechanisms prevent that: + +**The working tree's Manifest is pinned into every worktree** before `Pkg.instantiate()`, so all +refs in a comparison run against one package set. `--no-pin-manifest` opts out (and says so +loudly). If a commit declares a direct dependency the pinned Manifest lacks, `Pkg.instantiate()` +refuses to run: that ref is recorded as a failed run whose error suggests `--no-pin-manifest` to +let it resolve its own package set. + +**Every run records the environment that produced it** — Julia version, host, resolved Manifest +hash, Julia and BLAS thread counts. The cache still holds a single result per +`(commit, case)`, so a re-run replaces the stored one rather than keeping a result per +environment; what the fingerprint adds is that a cached result whose environment differs from +the current one is re-run instead of silently reused. `--allow-env-mismatch` skips that check +and reuses whatever is cached, whatever produced it. Every report prints the environment of +each ref: + +``` +Ref 1: develop @ a0cad260 (2026-08-12) + env: julia 1.11.6, arm64-apple-darwin24.0.0, manifest 7e5c34ad (pinned), 1 thread/8 BLAS +``` + +When two compared runs did not share an environment, the report says so before the table rather +than leaving you to infer it from the numbers. + +Results cached before environment fingerprinting existed carry no environment and are therefore +re-run once — those are exactly the entries whose provenance cannot be established. + +Thread counts are recorded but **not** forced: the harness does not silently change how your runs +execute. If the two refs in a comparison ran under different thread counts, the report flags it. + +## Exit status + +- `0` — every run completed (and, with `--fail-on-change`, nothing changed) +- `1` — a run failed, or a quantity changed under `--fail-on-change` diff --git a/docs/src/developer_notes.md b/docs/src/developer_notes.md index f13ac28ed..885a3de3c 100644 --- a/docs/src/developer_notes.md +++ b/docs/src/developer_notes.md @@ -220,3 +220,7 @@ regress --cases solovev_n1 --ref-range develop~10..develop - `--force` — re-run even if cached - `--verbose` — print GPEC subprocess output - `--no-instantiate` — skip `Pkg.instantiate()` (faster if deps are already resolved) + +The full flag list, the environment-pinning behaviour that keeps source code the only variable in +a comparison, and the exit-status contract are documented in +[`docs/development/regression-harness.md`](https://github.com/OpenFUSIONToolkit/GPEC/blob/develop/docs/development/regression-harness.md). diff --git a/regression-harness/regress.jl b/regression-harness/regress.jl index 3b5cd61d5..3afce5122 100644 --- a/regression-harness/regress.jl +++ b/regression-harness/regress.jl @@ -8,6 +8,7 @@ const DEFAULT_DB_PATH = joinpath(HARNESS_DIR, ".regress_cache.sqlite") const CASES_DIR = joinpath(HARNESS_DIR, "cases") include("src/types.jl") +include("src/env.jl") include("src/config.jl") include("src/database.jl") include("src/utils.jl") @@ -26,6 +27,9 @@ function parse_args(args) db_path = nothing verbose = false no_instantiate = false + no_pin_manifest = false + allow_env_mismatch = false + fail_on_change = false help = false i = 1 @@ -46,6 +50,15 @@ function parse_args(args) elseif arg == "--no-instantiate" no_instantiate = true i += 1 + elseif arg == "--no-pin-manifest" + no_pin_manifest = true + i += 1 + elseif arg == "--allow-env-mismatch" + allow_env_mismatch = true + i += 1 + elseif arg == "--fail-on-change" + fail_on_change = true + i += 1 elseif arg == "--cases" && i < length(args) cases = split(args[i+1], ",") |> collect .|> strip i += 2 @@ -69,7 +82,8 @@ function parse_args(args) end end - return CLIOptions(cases, refs, ref_range, force, list_cases, show_qty, show_case, db_path, verbose, no_instantiate, help) + return CLIOptions(cases, refs, ref_range, force, list_cases, show_qty, show_case, db_path, verbose, + no_instantiate, no_pin_manifest, allow_env_mismatch, fail_on_change, help) end const HELP_TEXT = """ @@ -93,8 +107,19 @@ Options: --db path Override database path --verbose Print subprocess output --no-instantiate Skip Pkg.instantiate() in subprocess + --no-pin-manifest Let each ref resolve its own package set (default: pin the working + tree's Manifest.toml into every worktree so source code is the only + variable in a comparison) + --allow-env-mismatch Reuse cached results produced in a different environment instead of + re-running them + --fail-on-change Exit non-zero if any tracked quantity changed (for CI use; a failed + run always exits non-zero regardless) --help Print this help message +Exit status: + 0 all runs completed (and, with --fail-on-change, nothing changed) + 1 a run failed, or a quantity changed under --fail-on-change + Examples: # Compare two refs julia --project=regression-harness regression-harness/regress.jl \\ @@ -170,6 +195,25 @@ function main(args=ARGS) error("No commits resolved from the given refs") end + warn_stale_refs(resolved_refs, REPO_ROOT) + + # Pin every ref to the working tree's package set unless asked not to, so that a + # comparison varies source code alone. + manifest_path = joinpath(REPO_ROOT, "Manifest.toml") + pin_manifest = if opts.no_pin_manifest + @warn "Manifest pinning disabled — refs may resolve different package sets, and differences below may not be caused by source changes" + nothing + elseif !isfile(manifest_path) + @warn "No Manifest.toml in the working tree; cannot pin the package set. Run Pkg.instantiate() first." + nothing + else + manifest_path + end + expected_key = opts.allow_env_mismatch ? nothing : expected_env_key(pin_manifest) + + n_failed = 0 + n_changed = 0 + # Run each case at each commit for case_spec in case_specs println("\n", "="^64) @@ -179,18 +223,28 @@ function main(args=ARGS) for ref in resolved_refs run_commit(db, ref.commit_hash, ref.name, case_spec, REPO_ROOT; force=opts.force, verbose=opts.verbose, - no_instantiate=opts.no_instantiate) + no_instantiate=opts.no_instantiate, + pin_manifest=pin_manifest, expected_key=expected_key) end # Report - if length(resolved_refs) == 1 - report_multi_ref(db, case_spec, resolved_refs) - elseif length(resolved_refs) == 2 + summary = if length(resolved_refs) == 2 report_two_ref_comparison(db, case_spec, resolved_refs[1], resolved_refs[2]) else report_multi_ref(db, case_spec, resolved_refs) end + n_failed += summary.n_failed + n_changed += summary.n_changed + end + + if n_failed > 0 + @error "$n_failed run(s) failed — see the reports above" + exit(1) + end + if opts.fail_on_change && n_changed > 0 + @error "$n_changed quantity/quantities changed (--fail-on-change)" + exit(1) end finally close_database(db) diff --git a/regression-harness/src/database.jl b/regression-harness/src/database.jl index 18e744c7f..c9a06b303 100644 --- a/regression-harness/src/database.jl +++ b/regression-harness/src/database.jl @@ -14,6 +14,13 @@ CREATE TABLE IF NOT EXISTS runs ( runtime_s REAL, success INTEGER NOT NULL DEFAULT 1, error_msg TEXT, + env_key TEXT, + julia_version TEXT, + os_arch TEXT, + manifest_sha TEXT, + nthreads INTEGER, + blas_threads INTEGER, + pinned INTEGER, UNIQUE(commit_hash, case_name) ); @@ -35,6 +42,14 @@ CREATE INDEX IF NOT EXISTS idx_runs_case ON runs(case_name); CREATE INDEX IF NOT EXISTS idx_quantities_run ON quantities(run_id); """ +""" +Value of a possibly-absent SQLite column, falling back to `default`. + +SQLite.jl returns `missing` for NULL while Julia's `something` only skips `nothing`, so columns +added by a later schema migration (NULL on every pre-existing row) need both cases handled. +""" +_column(x, default) = (x === nothing || x === missing) ? default : x + """Materialize SQLite query results as a Vector of NamedTuples.""" function query_rows(db::SQLite.DB, sql::String, params=()) result = DBInterface.execute(db, sql, params) @@ -44,6 +59,29 @@ function query_rows(db::SQLite.DB, sql::String, params=()) return [NamedTuple{keys(ct)}(Tuple(col[i] for col in values(ct))) for i in 1:nrows] end +""" +Environment columns added to `runs` after the original schema shipped. Databases created before +fingerprinting keep their rows, with NULL in these columns — such rows never match a computed +`env_key`, so they are re-run rather than silently trusted. +""" +const ENV_COLUMNS = [ + ("env_key", "TEXT"), ("julia_version", "TEXT"), ("os_arch", "TEXT"), + ("manifest_sha", "TEXT"), ("nthreads", "INTEGER"), ("blas_threads", "INTEGER"), + ("pinned", "INTEGER") +] + +"""Add any `runs` columns missing from a database created by an earlier harness version.""" +function migrate_schema!(db::SQLite.DB) + existing = Set(String[]) + for row in query_rows(db, "PRAGMA table_info(runs)") + push!(existing, String(something(row.name, ""))) + end + for (col, sqltype) in ENV_COLUMNS + col in existing && continue + DBInterface.execute(db, "ALTER TABLE runs ADD COLUMN $col $sqltype") + end +end + function open_database(path::String)::SQLite.DB db = SQLite.DB(path) DBInterface.execute(db, "PRAGMA journal_mode=WAL") @@ -53,6 +91,7 @@ function open_database(path::String)::SQLite.DB isempty(s) && continue DBInterface.execute(db, s) end + migrate_schema!(db) return db end @@ -60,12 +99,39 @@ function close_database(db::SQLite.DB) SQLite.close(db) end -function is_cached(db::SQLite.DB, commit_hash::String, case_name::String)::Bool - rows = query_rows(db, "SELECT id FROM runs WHERE commit_hash = ? AND case_name = ? AND success = 1", - (commit_hash, case_name)) +""" +Is there a usable cached result for this (commit, case)? + +With `expected_key` supplied, a cached run only counts when it was produced in the same +environment. Rows predating fingerprinting hold NULL and therefore never match — the cache +entries most likely to be misleading are exactly the ones that get re-run. +""" +function is_cached(db::SQLite.DB, commit_hash::String, case_name::String; + expected_key::Union{String,Nothing}=nothing)::Bool + if expected_key === nothing + rows = query_rows(db, "SELECT id FROM runs WHERE commit_hash = ? AND case_name = ? AND success = 1", + (commit_hash, case_name)) + return !isempty(rows) + end + rows = query_rows(db, + "SELECT id FROM runs WHERE commit_hash = ? AND case_name = ? AND success = 1 AND env_key = ?", + (commit_hash, case_name, expected_key)) return !isempty(rows) end +""" +Environment key stored for a cached run, or `nothing` when the run is absent or predates +fingerprinting. Used to explain *why* a cached result was rejected. +""" +function cached_env_key(db::SQLite.DB, commit_hash::String, case_name::String)::Union{String,Nothing} + rows = query_rows(db, "SELECT env_key FROM runs WHERE commit_hash = ? AND case_name = ?", + (commit_hash, case_name)) + isempty(rows) && return nothing + key = _column(first(rows).env_key, nothing) + key === nothing && return nothing + return String(key) +end + function delete_cached(db::SQLite.DB, commit_hash::String, case_name::String) # ON DELETE CASCADE handles quantities cleanup automatically DBInterface.execute(db, "DELETE FROM runs WHERE commit_hash = ? AND case_name = ?", @@ -76,7 +142,8 @@ function store_run(db::SQLite.DB, commit_hash::AbstractString, commit_short::Abs commit_date::AbstractString, commit_msg::AbstractString, case_name::AbstractString, runtime_s::Float64, extracted::Vector{ExtractedQuantity}; - success::Bool=true, error_msg::AbstractString="") + success::Bool=true, error_msg::AbstractString="", + fingerprint::EnvFingerprint=UNKNOWN_ENV) ran_at = Dates.format(Dates.now(), "yyyy-mm-ddTHH:MM:SS") SQLite.transaction(db) do @@ -84,10 +151,14 @@ function store_run(db::SQLite.DB, commit_hash::AbstractString, commit_short::Abs DBInterface.execute(db, """INSERT INTO runs - (commit_hash, commit_short, commit_date, commit_msg, case_name, ran_at, runtime_s, success, error_msg) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", + (commit_hash, commit_short, commit_date, commit_msg, case_name, ran_at, runtime_s, success, error_msg, + env_key, julia_version, os_arch, manifest_sha, nthreads, blas_threads, pinned) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (String(commit_hash), String(commit_short), String(commit_date), String(commit_msg), - String(case_name), ran_at, runtime_s, success ? 1 : 0, String(error_msg))) + String(case_name), ran_at, runtime_s, success ? 1 : 0, String(error_msg), + env_key(fingerprint), fingerprint.julia_version, fingerprint.os_arch, + fingerprint.manifest_sha, fingerprint.nthreads, fingerprint.blas_threads, + fingerprint.pinned ? 1 : 0)) run_id = SQLite.last_insert_rowid(db) @@ -147,11 +218,20 @@ Get run info for a (commit, case) pair. Returns NamedTuple or nothing. """ function get_run_info(db::SQLite.DB, commit_hash::String, case_name::String) rows = query_rows(db, - """SELECT commit_short, commit_date, commit_msg, runtime_s, success, error_msg + """SELECT commit_short, commit_date, commit_msg, runtime_s, success, error_msg, + julia_version, os_arch, manifest_sha, nthreads, blas_threads, pinned FROM runs WHERE commit_hash = ? AND case_name = ?""", (commit_hash, case_name)) isempty(rows) && return nothing row = first(rows) + fingerprint = EnvFingerprint( + String(_column(row.julia_version, "")), + String(_column(row.os_arch, "")), + String(_column(row.manifest_sha, "")), + Int(_column(row.nthreads, -1)), + Int(_column(row.blas_threads, -1)), + _column(row.pinned, 0) == 1 + ) return ( commit_short = something(row.commit_short, ""), commit_date = something(row.commit_date, ""), @@ -159,6 +239,7 @@ function get_run_info(db::SQLite.DB, commit_hash::String, case_name::String) runtime_s = something(row.runtime_s, 0.0), success = coalesce(row.success, 0) == 1, error_msg = something(row.error_msg, ""), + fingerprint = fingerprint, ) end diff --git a/regression-harness/src/env.jl b/regression-harness/src/env.jl new file mode 100644 index 000000000..20718c4cb --- /dev/null +++ b/regression-harness/src/env.jl @@ -0,0 +1,118 @@ +""" +Environment fingerprinting. + +A cached regression result is only comparable to a fresh one if both were produced by the same +Julia, on the same machine, against the same package set. Because `Manifest.toml` is untracked, +a worktree checkout resolves whatever is newest at run time, so two runs of the *same source* +can differ by a package set — and machine-epsilon differences in library math are amplified by +the adaptive ODE controller into large apparent regressions. The fingerprint below makes that +confound visible (and, by default, invalidating) instead of silent. +""" + +""" +Identity of the environment a run was produced in. + +## Fields + + - `julia_version::String` — version of the `julia` that ran GPEC + - `os_arch::String` — `Sys.MACHINE` of the running host + - `manifest_sha::String` — SHA-256 of the `Manifest.toml` the run resolved against ("" if absent) + - `nthreads::Int` — `Threads.nthreads()` in the run (-1 if unknown) + - `blas_threads::Int` — `BLAS.get_num_threads()` in the run (-1 if unknown) + - `pinned::Bool` — whether the harness copied its own Manifest into the run's project +""" +struct EnvFingerprint + julia_version::String + os_arch::String + manifest_sha::String + nthreads::Int + blas_threads::Int + pinned::Bool +end + +const UNKNOWN_ENV = EnvFingerprint("", "", "", -1, -1, false) + +""" +Cache key for an environment. + +Deliberately built from only the three fields that are knowable *before* a run: the Julia +version, the host, and the package set the run will be pinned to. Thread counts are recorded and +reported but not keyed, because the harness does not force them (see `--threads` in the CLI) and +so cannot predict them ahead of a run. + +When the Manifest is not pinned, the package set is unknowable in advance and the key records +`unpinned`; the resolved Manifest hash is still stored per run for display and mismatch warnings. +""" +function env_key(julia_version::AbstractString, os_arch::AbstractString, manifest_mode::AbstractString)::String + return bytes2hex(SHA.sha256("$(julia_version)|$(os_arch)|$(manifest_mode)"))[1:16] +end + +env_key(fp::EnvFingerprint) = env_key(fp.julia_version, fp.os_arch, fp.pinned ? fp.manifest_sha : "unpinned") + +"""SHA-256 of a file, or "" when it does not exist.""" +function file_sha256(path::AbstractString)::String + isfile(path) || return "" + return bytes2hex(SHA.sha256(read(path))) +end + +""" +Version string of the `julia` the harness will launch for subprocess runs. + +This is not necessarily the Julia running the harness itself, so it is probed rather than read +from `VERSION`. +""" +function subprocess_julia_version()::String + try + out = strip(read(`julia --version`, String)) + m = match(r"(\d+\.\d+\.\d+\S*)", out) + return m === nothing ? out : m.captures[1] + catch + return "unknown" + end +end + +""" +The environment key that runs launched *now* will carry. + +`manifest_path` is the Manifest the harness will pin into each worktree; pass `nothing` when +pinning is disabled. +""" +function expected_env_key(manifest_path::Union{String,Nothing})::String + mode = manifest_path === nothing ? "unpinned" : file_sha256(manifest_path) + return env_key(subprocess_julia_version(), string(Sys.MACHINE), mode) +end + +""" +Parse the `key=value` run-info file written by a run subprocess. + +Returns `(runtime_s, fingerprint)`. A missing or malformed file yields `NaN` and `UNKNOWN_ENV` +rather than throwing, so a run that produced output but no metadata still reports its numbers. +""" +function read_runinfo(path::String, pinned::Bool) + isfile(path) || return (NaN, UNKNOWN_ENV) + fields = Dict{String,String}() + for line in eachline(path) + parts = split(line, '='; limit=2) + length(parts) == 2 && (fields[strip(parts[1])] = strip(parts[2])) + end + getf = (k, d) -> get(fields, k, d) + runtime_s = tryparse(Float64, getf("runtime_s", "")) + fp = EnvFingerprint( + getf("julia_version", ""), + getf("os_arch", ""), + getf("manifest_sha", ""), + something(tryparse(Int, getf("nthreads", "")), -1), + something(tryparse(Int, getf("blas_threads", "")), -1), + pinned + ) + return (something(runtime_s, NaN), fp) +end + +"""One-line human-readable summary of an environment, for report headers.""" +function describe_env(fp::EnvFingerprint)::String + isempty(fp.julia_version) && return "environment unknown (cached before fingerprinting)" + mani = isempty(fp.manifest_sha) ? "no Manifest" : "manifest " * fp.manifest_sha[1:min(8, end)] + pin = fp.pinned ? "pinned" : "unpinned" + threads = "$(fp.nthreads) thread$(fp.nthreads == 1 ? "" : "s")/$(fp.blas_threads) BLAS" + return "julia $(fp.julia_version), $(fp.os_arch), $mani ($pin), $threads" +end diff --git a/regression-harness/src/reporter.jl b/regression-harness/src/reporter.jl index c0c57758f..4fdecfbe3 100644 --- a/regression-harness/src/reporter.jl +++ b/regression-harness/src/reporter.jl @@ -47,6 +47,31 @@ function format_value(q::NamedTuple)::String end end +""" +Print a banner when two compared runs did not share an environment. + +Source code is only the sole variable when Julia, host, package set and thread counts all match. +Anything else here means part of the reported difference may be the environment, not the code. +""" +function _warn_env_difference(fp1::EnvFingerprint, fp2::EnvFingerprint) + (fp1 === UNKNOWN_ENV || fp2 === UNKNOWN_ENV) && return + (isempty(fp1.julia_version) || isempty(fp2.julia_version)) && return + differences = String[] + fp1.julia_version != fp2.julia_version && push!(differences, "julia $(fp1.julia_version) vs $(fp2.julia_version)") + fp1.os_arch != fp2.os_arch && push!(differences, "host $(fp1.os_arch) vs $(fp2.os_arch)") + fp1.manifest_sha != fp2.manifest_sha && push!(differences, "different package sets (Manifest hashes differ)") + fp1.nthreads != fp2.nthreads && push!(differences, "$(fp1.nthreads) vs $(fp2.nthreads) Julia threads") + fp1.blas_threads != fp2.blas_threads && push!(differences, "$(fp1.blas_threads) vs $(fp2.blas_threads) BLAS threads") + isempty(differences) && return + println() + println("!! ENVIRONMENTS DIFFER — source code is not the only variable in this comparison:") + for d in differences + println(" - $d") + end + println(" Differences below may be environment artifacts. Re-run with --force to rebuild") + println(" both refs in the current environment.") +end + """ Format a diff value for display. """ @@ -87,11 +112,11 @@ function report_two_ref_comparison(db::SQLite.DB, case_spec::CaseSpec, if info1 === nothing println("ERROR: No results for ref 1 ($(ref1.name))") - return + return (n_ok=0, n_changed=0, n_missing=0, n_failed=1) end if info2 === nothing println("ERROR: No results for ref 2 ($(ref2.name))") - return + return (n_ok=0, n_changed=0, n_missing=0, n_failed=1) end failed1 = !info1.success @@ -182,7 +207,10 @@ function report_two_ref_comparison(db::SQLite.DB, case_spec::CaseSpec, tag1 = failed1 ? " (FAILED)" : "" tag2 = failed2 ? " (FAILED)" : "" println("Ref 1: $(ref1.name) @ $(info1.commit_short) ($date1)$tag1") + println(" env: $(describe_env(info1.fingerprint))") println("Ref 2: $(ref2.name) @ $(info2.commit_short) ($date2)$tag2") + println(" env: $(describe_env(info2.fingerprint))") + _warn_env_difference(info1.fingerprint, info2.fingerprint) if failed1 println(" Ref 1 error: $(_short_err(info1.error_msg))") end @@ -204,6 +232,8 @@ function report_two_ref_comparison(db::SQLite.DB, case_spec::CaseSpec, n_missing > 0 && push!(parts, "$n_missing missing/N/A") println("Summary: ", join(parts, ", ")) println() + return (n_ok=n_ok, n_changed=n_changed, n_missing=n_missing, + n_failed=count(identity, (failed1, failed2))) end """ @@ -327,6 +357,7 @@ function report_multi_ref(db::SQLite.DB, case_spec::CaseSpec, date_str = length(info.commit_date) >= 10 ? info.commit_date[1:10] : info.commit_date status_str = info.success ? "" : " (FAILED)" println("Ref $(i): $(ref.name) @ $(info.commit_short) ($date_str)$status_str") + println(" env: $(describe_env(info.fingerprint))") if !info.success println(" Ref $(i) error: $(_short_err(info.error_msg))") end @@ -334,6 +365,10 @@ function report_multi_ref(db::SQLite.DB, case_spec::CaseSpec, println("Ref $(i): $(ref.name) (no data)") end end + if length(refs) >= 2 + last_two = filter(!isnothing, run_infos[(end - 1):end]) + length(last_two) == 2 && _warn_env_difference(last_two[1].fingerprint, last_two[2].fingerprint) + end println("-"^total_w) _print_row(header, widths) @@ -351,6 +386,8 @@ function report_multi_ref(db::SQLite.DB, case_spec::CaseSpec, println("Summary (last vs prev): ", join(parts, ", ")) end println() + return (n_ok=n_ok, n_changed=n_changed, n_missing=n_missing, + n_failed=count(identity, failed_mask)) end """ diff --git a/regression-harness/src/runner.jl b/regression-harness/src/runner.jl index ecb7b760f..d8a1a5d2b 100644 --- a/regression-harness/src/runner.jl +++ b/regression-harness/src/runner.jl @@ -2,13 +2,26 @@ Runner: orchestrates checking out commits, running GPEC, and extracting results. """ -"""Read the GPEC-only runtime from the timing file written by the subprocess.""" -function _read_timing_file(path::String)::Float64 - if isfile(path) - return parse(Float64, strip(read(path, String))) +""" +Epilogue appended to every subprocess script: records the GPEC-only runtime *and* the +environment that produced it (Julia, host, resolved package set, thread counts) as `key=value` +lines. Measured inside the subprocess, after `Pkg.instantiate()` has resolved the environment, +so it describes what actually ran rather than what the harness intended to run. +""" +const RUNINFO_EPILOGUE = """ +using SHA +using LinearAlgebra: BLAS +let manifest = joinpath(dirname(Base.active_project()), "Manifest.toml") + open(ARGS[2], "w") do f + println(f, "runtime_s=", elapsed) + println(f, "julia_version=", string(VERSION)) + println(f, "os_arch=", Sys.MACHINE) + println(f, "manifest_sha=", isfile(manifest) ? bytes2hex(SHA.sha256(read(manifest))) : "") + println(f, "nthreads=", Threads.nthreads()) + println(f, "blas_threads=", BLAS.get_num_threads()) end - return NaN end +""" """ Materialize the directory GPEC will actually run in. @@ -47,16 +60,13 @@ using GeneralizedPerturbedEquilibrium t_start = time() GeneralizedPerturbedEquilibrium.main([ARGS[1]]) elapsed = time() - t_start -# Write GPEC-only runtime to a file the harness reads back -open(ARGS[2], "w") do f - println(f, elapsed) -end +%RUNINFO% """ # Self-contained computation for the GGJ inner-layer reference benchmark. # Runs the Galerkin solver on the Glasser & Wang 2020 Eq. 55 parameter set # at γ = 1 + i and writes (Δ_odd, Δ_even) into a small HDF5 file at ARGS[1]. -# Runtime is written to ARGS[2] for parity with the GPEC runner. +# Runtime and environment are written to ARGS[2] by the shared run-info epilogue. # `solve_inner` returns the named-field `InnerLayerResponse`; the historical # delta_odd / delta_even slots map to interchange / tearing respectively, which # preserves the numeric identity of each tracked quantity. @@ -77,14 +87,13 @@ h5open(ARGS[1], "w") do fid fid["ggj/delta_even_real"] = real(Δ.tearing) fid["ggj/delta_even_imag"] = imag(Δ.tearing) end -open(ARGS[2], "w") do f - println(f, elapsed) -end +%RUNINFO% """ # GGJ rotated-ray backend at Q = 500i on the q=4 benchmark surface — a regime beyond the # :galerkin backend. Builds the physical rate γ = 500i·Q₀ so inner_Q lands exactly on the -# imaginary axis at 500i, then writes the parity matching data. Runtime to ARGS[2] as usual. +# imaginary axis at 500i, then writes the parity matching data. Runtime and environment are +# written to ARGS[2] by the shared run-info epilogue. # As in the galerkin template, the delta_odd / delta_even slots map to interchange / tearing. const COMPUTED_GGJ_RAY_SCRIPT_TEMPLATE = """ using Pkg @@ -103,9 +112,7 @@ h5open(ARGS[1], "w") do fid fid["ggj/delta_even_real"] = real(Δ.tearing) fid["ggj/delta_even_imag"] = imag(Δ.tearing) end -open(ARGS[2], "w") do f - println(f, elapsed) -end +%RUNINFO% """ # Self-contained separatrix-finder regression (PR #296). Loads a fixed-boundary EFIT whose @@ -113,7 +120,7 @@ end # LCFS the coil-vacuum flux turns back above the boundary value before the grid edge, so the old # bracketed-Brent separatrix finder could not bracket psi=sibry and equilibrium setup threw. Calls # setup_equilibrium directly and writes the leading equilibrium scalars: errors on the buggy code, -# passes with the Newton finder. Runtime is written to ARGS[2] for parity with the GPEC runner. +# passes with the Newton finder. Runtime and environment are written to ARGS[2] by the run-info epilogue. const COMPUTED_SEPARATRIX_SCRIPT_TEMPLATE = """ using Pkg %INSTANTIATE% @@ -133,33 +140,50 @@ h5open(ARGS[1], "w") do fid fid["equil/betat"] = pe.params.betat fid["equil/betan"] = pe.params.betan end -open(ARGS[2], "w") do f - println(f, elapsed) -end +%RUNINFO% """ +""" +Expand a subprocess script template: the optional `Pkg.instantiate()` call and the run-info +epilogue that records runtime and environment. +""" +function _render_script(template::String; no_instantiate::Bool)::String + rendered = replace(template, "%INSTANTIATE%" => (no_instantiate ? "" : "Pkg.instantiate()")) + return replace(rendered, "%RUNINFO%" => RUNINFO_EPILOGUE) +end + """ Run GPEC for a single commit/ref and case. Dispatches to run_local for the working tree or run_at_commit for a git ref. + +`pin_manifest` is the path to a resolved `Manifest.toml` copied into each worktree so every ref +runs against the same package set; `nothing` disables pinning. `expected_key` is the environment +key a fresh run will carry — a cached run whose key differs is re-run rather than reused. """ function run_commit(db::SQLite.DB, commit_hash::String, ref_name::String, case_spec::CaseSpec, repo_root::String; force::Bool=false, verbose::Bool=false, - no_instantiate::Bool=false) + no_instantiate::Bool=false, + pin_manifest::Union{String,Nothing}=nothing, + expected_key::Union{String,Nothing}=nothing) if case_spec.kind == "computed" if commit_hash == LOCAL_REF return run_computed_local(db, case_spec, repo_root; - verbose=verbose, no_instantiate=no_instantiate) + verbose=verbose, no_instantiate=no_instantiate, + pin_manifest=pin_manifest) end return run_computed_at_commit(db, commit_hash, ref_name, case_spec, repo_root; - force=force, verbose=verbose, no_instantiate=no_instantiate) + force=force, verbose=verbose, no_instantiate=no_instantiate, + pin_manifest=pin_manifest, expected_key=expected_key) end if commit_hash == LOCAL_REF return run_local(db, case_spec, repo_root; - force=force, verbose=verbose, no_instantiate=no_instantiate) + force=force, verbose=verbose, no_instantiate=no_instantiate, + pin_manifest=pin_manifest) end return run_at_commit(db, commit_hash, ref_name, case_spec, repo_root; - force=force, verbose=verbose, no_instantiate=no_instantiate) + force=force, verbose=verbose, no_instantiate=no_instantiate, + pin_manifest=pin_manifest, expected_key=expected_key) end """ @@ -185,50 +209,63 @@ so callers can handle store_failed_run uniformly. """ function _execute_computed(case_spec::CaseSpec, project_root::String; verbose::Bool, no_instantiate::Bool, - stderr_buf::IO) - instantiate_line = no_instantiate ? "" : "Pkg.instantiate()" - script_content = replace(_computed_script_template(case_spec), - "%INSTANTIATE%" => instantiate_line) + stderr_buf::IO, pin_manifest::Union{String,Nothing}=nothing) + script_content = _render_script(_computed_script_template(case_spec); no_instantiate=no_instantiate) tmpscript = tempname() * ".jl" h5path = tempname() * ".h5" - timingfile = tempname() * ".timing" + runinfo_file = tempname() * ".runinfo" try write(tmpscript, script_content) if verbose - run(pipeline(`julia --project=$project_root $tmpscript $h5path $timingfile`)) + run(pipeline(`julia --project=$project_root $tmpscript $h5path $runinfo_file`)) else - run(pipeline(`julia --project=$project_root $tmpscript $h5path $timingfile`, + run(pipeline(`julia --project=$project_root $tmpscript $h5path $runinfo_file`, stdout=devnull, stderr=stderr_buf)) end - runtime_s = _read_timing_file(timingfile) + runtime_s, fingerprint = read_runinfo(runinfo_file, pin_manifest !== nothing) + isempty(fingerprint.julia_version) && error("subprocess wrote no run-info metadata — does the script template end with %RUNINFO%?") + _warn_pin_broken(pin_manifest, fingerprint, case_spec.name) if !isfile(h5path) error("Computed case '$(case_spec.name)' produced no output h5") end extracted = extract_quantities(h5path, case_spec.quantities, runtime_s) - return extracted, runtime_s + return extracted, runtime_s, fingerprint finally rm(tmpscript; force=true) rm(h5path; force=true) - rm(timingfile; force=true) + rm(runinfo_file; force=true) end end +""" +Add a remedy hint when a run failed because the pinned Manifest is missing a direct dependency +declared at the checked-out commit — the one incompatibility `Pkg.instantiate()` refuses to run under. +Detects on the full error text, since tail-keeping truncation can drop Pkg's ERROR line from `short_err`. +""" +function _hint_pin_incompatible(full_err::AbstractString, short_err::AbstractString)::String + occursin("is a direct dependency, but does not appear in the manifest", full_err) || return String(short_err) + return "Pinned Manifest lacks a direct dependency declared at this commit — re-run with --no-pin-manifest " * + "to let this ref resolve its own package set.\n" * short_err +end + """ Run a kind="computed" case against the working tree. """ function run_computed_local(db::SQLite.DB, case_spec::CaseSpec, repo_root::String; - verbose::Bool=false, no_instantiate::Bool=false) + verbose::Bool=false, no_instantiate::Bool=false, + pin_manifest::Union{String,Nothing}=nothing) delete_cached(db, LOCAL_REF, case_spec.name) date = Dates.format(Dates.now(), "yyyy-mm-ddTHH:MM:SS") @info "Running: $(case_spec.name) @ local (working tree, computed)" stderr_buf = IOBuffer() try - extracted, runtime_s = _execute_computed(case_spec, repo_root; - verbose=verbose, - no_instantiate=no_instantiate, - stderr_buf=stderr_buf) + extracted, runtime_s, fingerprint = _execute_computed(case_spec, repo_root; + verbose=verbose, + no_instantiate=no_instantiate, + stderr_buf=stderr_buf, + pin_manifest=pin_manifest) store_run(db, LOCAL_REF, "local", date, "working tree", case_spec.name, - runtime_s, extracted) + runtime_s, extracted; fingerprint=fingerprint) @info " Completed in $(round(runtime_s, digits=3))s — $(length(extracted)) quantities extracted" catch e err_msg = if e isa ProcessFailedException @@ -240,7 +277,7 @@ function run_computed_local(db::SQLite.DB, case_spec::CaseSpec, repo_root::Strin # Keep the tail of the message: Julia errors usually appear at the end # of the subprocess output, not the start (Pkg.instantiate output dominates the head). # `last` is unicode-safe and won't split a multibyte char like `err_msg[end-N:end]` could. - err_msg_short = length(err_msg) > 2000 ? "..." * last(err_msg, 2000) : err_msg + err_msg_short = _hint_pin_incompatible(err_msg, length(err_msg) > 2000 ? "..." * last(err_msg, 2000) : err_msg) @warn "Run failed (local computed): $(first(err_msg_short, 200))" store_failed_run(db, LOCAL_REF, "local", date, "working tree", case_spec.name, err_msg_short) @@ -253,14 +290,17 @@ Run a kind="computed" case at a specific git commit via worktree. function run_computed_at_commit(db::SQLite.DB, commit_hash::String, ref_name::String, case_spec::CaseSpec, repo_root::String; force::Bool=false, verbose::Bool=false, - no_instantiate::Bool=false) - if !force && is_cached(db, commit_hash, case_spec.name) + no_instantiate::Bool=false, + pin_manifest::Union{String,Nothing}=nothing, + expected_key::Union{String,Nothing}=nothing) + if !force && is_cached(db, commit_hash, case_spec.name; expected_key=expected_key) info = get_run_info(db, commit_hash, case_spec.name) if info !== nothing @info "Cached: $(case_spec.name) @ $(info.commit_short) ($(info.commit_date))" return end end + _warn_env_invalidated(db, commit_hash, case_spec.name, expected_key, force) if force delete_cached(db, commit_hash, case_spec.name) end @@ -272,13 +312,14 @@ function run_computed_at_commit(db::SQLite.DB, commit_hash::String, ref_name::St worktree_path = nothing stderr_buf = IOBuffer() try - worktree_path = create_worktree(commit_hash, repo_root) - extracted, runtime_s = _execute_computed(case_spec, worktree_path; - verbose=verbose, - no_instantiate=no_instantiate, - stderr_buf=stderr_buf) + worktree_path = create_worktree(commit_hash, repo_root; pin_manifest_from=pin_manifest) + extracted, runtime_s, fingerprint = _execute_computed(case_spec, worktree_path; + verbose=verbose, + no_instantiate=no_instantiate, + stderr_buf=stderr_buf, + pin_manifest=pin_manifest) store_run(db, commit_hash, commit_info.short, commit_info.date, - commit_info.msg, case_spec.name, runtime_s, extracted) + commit_info.msg, case_spec.name, runtime_s, extracted; fingerprint=fingerprint) @info " Completed in $(round(runtime_s, digits=3))s — $(length(extracted)) quantities extracted" catch e err_msg = if e isa ProcessFailedException @@ -290,7 +331,7 @@ function run_computed_at_commit(db::SQLite.DB, commit_hash::String, ref_name::St # Keep the tail of the message: Julia errors usually appear at the end # of the subprocess output, not the start (Pkg.instantiate output dominates the head). # `last` is unicode-safe and won't split a multibyte char like `err_msg[end-N:end]` could. - err_msg_short = length(err_msg) > 2000 ? "..." * last(err_msg, 2000) : err_msg + err_msg_short = _hint_pin_incompatible(err_msg, length(err_msg) > 2000 ? "..." * last(err_msg, 2000) : err_msg) @warn "Run failed (computed) for $(commit_info.short): $(first(err_msg_short, 200))" store_failed_run(db, commit_hash, commit_info.short, commit_info.date, commit_info.msg, case_spec.name, err_msg_short) @@ -301,13 +342,50 @@ function run_computed_at_commit(db::SQLite.DB, commit_hash::String, ref_name::St end end +""" +Warn if a pinned package set did not survive `Pkg.instantiate()`. + +Contingency insurance rather than a description of current behavior: on Julia 1.11/1.12, +instantiate never rewrites an out-of-sync pinned Manifest — it warns and proceeds, or errors when +the commit declares a direct dependency the Manifest lacks (which fails the run loudly). Should a +future Pkg re-resolve in place instead, this catches the pin silently not holding. +""" +function _warn_pin_broken(pin_manifest::Union{String,Nothing}, fp::EnvFingerprint, label::AbstractString) + pin_manifest === nothing && return + isempty(fp.manifest_sha) && return + pinned_sha = file_sha256(pin_manifest) + (isempty(pinned_sha) || pinned_sha == fp.manifest_sha) && return + @warn "Pinned Manifest was re-resolved for $label — its package set differs from the working tree" pinned = pinned_sha[1:8] resolved = fp.manifest_sha[1:8] +end + +""" +Explain a cache miss caused by the environment rather than by absence. + +A cached run exists for this (commit, case) but was produced under a different Julia, host, or +package set — the exact situation that used to be silently reused and reported as a physics +regression. Says so, once, before re-running. +""" +function _warn_env_invalidated(db::SQLite.DB, commit_hash::String, case_name::String, + expected_key::Union{String,Nothing}, force::Bool) + (force || expected_key === nothing) && return + stored = cached_env_key(db, commit_hash, case_name) + stored == expected_key && return + if stored === nothing + is_cached(db, commit_hash, case_name) || return + @warn "Cached result for $case_name predates environment fingerprinting — re-running" + else + @warn "Cached result for $case_name was produced in a different environment — re-running" stored_key = stored current_key = expected_key + end +end + """ Run GPEC in the current working tree (uncommitted changes included). Always re-runs (local results are never cached since the working tree is mutable). """ function run_local(db::SQLite.DB, case_spec::CaseSpec, repo_root::String; force::Bool=false, verbose::Bool=false, - no_instantiate::Bool=false) + no_instantiate::Bool=false, + pin_manifest::Union{String,Nothing}=nothing) # Always delete previous local results and re-run delete_cached(db, LOCAL_REF, case_spec.name) @@ -323,7 +401,7 @@ function run_local(db::SQLite.DB, case_spec::CaseSpec, repo_root::String; end tmpscript = nothing - timingfile = nothing + runinfo_file = nothing rundir = nothing rundir_is_temp = false stderr_buf = IOBuffer() @@ -331,19 +409,20 @@ function run_local(db::SQLite.DB, case_spec::CaseSpec, repo_root::String; try (rundir, rundir_is_temp) = _materialize_rundir(example_path, case_spec.overrides) - instantiate_line = no_instantiate ? "" : "Pkg.instantiate()" - script_content = replace(RUNNER_SCRIPT_TEMPLATE, "%INSTANTIATE%" => instantiate_line) + script_content = _render_script(RUNNER_SCRIPT_TEMPLATE; no_instantiate=no_instantiate) tmpscript = tempname() * ".jl" - timingfile = tempname() * ".timing" + runinfo_file = tempname() * ".runinfo" write(tmpscript, script_content) if verbose - run(pipeline(`julia --project=$repo_root $tmpscript $rundir $timingfile`)) + run(pipeline(`julia --project=$repo_root $tmpscript $rundir $runinfo_file`)) else - run(pipeline(`julia --project=$repo_root $tmpscript $rundir $timingfile`, + run(pipeline(`julia --project=$repo_root $tmpscript $rundir $runinfo_file`, stdout=devnull, stderr=stderr_buf)) end - runtime_s = _read_timing_file(timingfile) + runtime_s, fingerprint = read_runinfo(runinfo_file, pin_manifest !== nothing) + isempty(fingerprint.julia_version) && error("subprocess wrote no run-info metadata — does the script template end with %RUNINFO%?") + _warn_pin_broken(pin_manifest, fingerprint, case_spec.name) h5path = joinpath(rundir, "gpec.h5") if !isfile(h5path) @@ -355,7 +434,7 @@ function run_local(db::SQLite.DB, case_spec::CaseSpec, repo_root::String; extracted = extract_quantities(h5path, case_spec.quantities, runtime_s) store_run(db, LOCAL_REF, "local", date, "working tree", case_spec.name, - runtime_s, extracted) + runtime_s, extracted; fingerprint=fingerprint) @info " Completed in $(round(runtime_s, digits=1))s — $(length(extracted)) quantities extracted" @@ -369,7 +448,7 @@ function run_local(db::SQLite.DB, case_spec::CaseSpec, repo_root::String; # Keep the tail of the message: Julia errors usually appear at the end # of the subprocess output, not the start (Pkg.instantiate output dominates the head). # `last` is unicode-safe and won't split a multibyte char like `err_msg[end-N:end]` could. - err_msg_short = length(err_msg) > 2000 ? "..." * last(err_msg, 2000) : err_msg + err_msg_short = _hint_pin_incompatible(err_msg, length(err_msg) > 2000 ? "..." * last(err_msg, 2000) : err_msg) @warn "Run failed (local): $(first(err_msg_short, 200))" store_failed_run(db, LOCAL_REF, "local", date, "working tree", case_spec.name, err_msg_short) @@ -377,8 +456,8 @@ function run_local(db::SQLite.DB, case_spec::CaseSpec, repo_root::String; if tmpscript !== nothing rm(tmpscript; force=true) end - if timingfile !== nothing - rm(timingfile; force=true) + if runinfo_file !== nothing + rm(runinfo_file; force=true) end if rundir_is_temp && rundir !== nothing rm(dirname(rundir); recursive=true, force=true) @@ -388,20 +467,23 @@ end """ Run GPEC for a specific git commit via worktree. Stores results in the database. -Skips if already cached (unless force=true). +Skips if already cached in the same environment (unless force=true). """ function run_at_commit(db::SQLite.DB, commit_hash::String, ref_name::String, case_spec::CaseSpec, repo_root::String; force::Bool=false, verbose::Bool=false, - no_instantiate::Bool=false) + no_instantiate::Bool=false, + pin_manifest::Union{String,Nothing}=nothing, + expected_key::Union{String,Nothing}=nothing) # Check cache - if !force && is_cached(db, commit_hash, case_spec.name) + if !force && is_cached(db, commit_hash, case_spec.name; expected_key=expected_key) info = get_run_info(db, commit_hash, case_spec.name) if info !== nothing @info "Cached: $(case_spec.name) @ $(info.commit_short) ($(info.commit_date))" return end end + _warn_env_invalidated(db, commit_hash, case_spec.name, expected_key, force) # Delete existing cached data if force re-running if force @@ -415,14 +497,14 @@ function run_at_commit(db::SQLite.DB, commit_hash::String, ref_name::String, worktree_path = nothing tmpscript = nothing - timingfile = nothing + runinfo_file = nothing rundir = nothing rundir_is_temp = false stderr_buf = IOBuffer() try - # Create worktree - worktree_path = create_worktree(commit_hash, repo_root) + # Create worktree, pinning the package set when the caller supplied a Manifest + worktree_path = create_worktree(commit_hash, repo_root; pin_manifest_from=pin_manifest) # Check example directory exists in this commit example_path = joinpath(worktree_path, case_spec.example_dir) @@ -437,22 +519,23 @@ function run_at_commit(db::SQLite.DB, commit_hash::String, ref_name::String, (rundir, rundir_is_temp) = _materialize_rundir(example_path, case_spec.overrides) # Write temp runner script - instantiate_line = no_instantiate ? "" : "Pkg.instantiate()" - script_content = replace(RUNNER_SCRIPT_TEMPLATE, "%INSTANTIATE%" => instantiate_line) + script_content = _render_script(RUNNER_SCRIPT_TEMPLATE; no_instantiate=no_instantiate) tmpscript = tempname() * ".jl" - timingfile = tempname() * ".timing" + runinfo_file = tempname() * ".runinfo" write(tmpscript, script_content) # Run GPEC in subprocess project_root = worktree_path if verbose - run(pipeline(`julia --project=$project_root $tmpscript $rundir $timingfile`)) + run(pipeline(`julia --project=$project_root $tmpscript $rundir $runinfo_file`)) else - run(pipeline(`julia --project=$project_root $tmpscript $rundir $timingfile`, + run(pipeline(`julia --project=$project_root $tmpscript $rundir $runinfo_file`, stdout=devnull, stderr=stderr_buf)) end - runtime_s = _read_timing_file(timingfile) + runtime_s, fingerprint = read_runinfo(runinfo_file, pin_manifest !== nothing) + isempty(fingerprint.julia_version) && error("subprocess wrote no run-info metadata — does the script template end with %RUNINFO%?") + _warn_pin_broken(pin_manifest, fingerprint, commit_info.short) # Check for gpec.h5 h5path = joinpath(rundir, "gpec.h5") @@ -469,7 +552,7 @@ function run_at_commit(db::SQLite.DB, commit_hash::String, ref_name::String, # Store in database store_run(db, commit_hash, commit_info.short, commit_info.date, - commit_info.msg, case_spec.name, runtime_s, extracted) + commit_info.msg, case_spec.name, runtime_s, extracted; fingerprint=fingerprint) @info " Completed in $(round(runtime_s, digits=1))s — $(length(extracted)) quantities extracted" @@ -484,7 +567,7 @@ function run_at_commit(db::SQLite.DB, commit_hash::String, ref_name::String, # Keep the tail of the message: Julia errors usually appear at the end # of the subprocess output, not the start (Pkg.instantiate output dominates the head). # `last` is unicode-safe and won't split a multibyte char like `err_msg[end-N:end]` could. - err_msg_short = length(err_msg) > 2000 ? "..." * last(err_msg, 2000) : err_msg + err_msg_short = _hint_pin_incompatible(err_msg, length(err_msg) > 2000 ? "..." * last(err_msg, 2000) : err_msg) @warn "Run failed for $(commit_info.short): $(first(err_msg_short, 200))" store_failed_run(db, commit_hash, commit_info.short, commit_info.date, commit_info.msg, case_spec.name, err_msg_short) @@ -493,8 +576,8 @@ function run_at_commit(db::SQLite.DB, commit_hash::String, ref_name::String, if tmpscript !== nothing rm(tmpscript; force=true) end - if timingfile !== nothing - rm(timingfile; force=true) + if runinfo_file !== nothing + rm(runinfo_file; force=true) end if rundir_is_temp && rundir !== nothing rm(dirname(rundir); recursive=true, force=true) diff --git a/regression-harness/src/types.jl b/regression-harness/src/types.jl index 0d0a3e638..9417c5303 100644 --- a/regression-harness/src/types.jl +++ b/regression-harness/src/types.jl @@ -48,6 +48,11 @@ end """ Parsed CLI options. + +`no_pin_manifest` disables copying the working tree's resolved `Manifest.toml` into each +worktree (pinning is on by default, so that two refs differ only by source code). +`allow_env_mismatch` lets a cached result from a different environment be reused instead of +re-run. `fail_on_change` turns any changed quantity into a non-zero exit status, for CI use. """ struct CLIOptions cases::Vector{String} @@ -60,5 +65,8 @@ struct CLIOptions db_path::Union{String,Nothing} verbose::Bool no_instantiate::Bool + no_pin_manifest::Bool + allow_env_mismatch::Bool + fail_on_change::Bool help::Bool end diff --git a/regression-harness/src/utils.jl b/regression-harness/src/utils.jl index a7b6d8373..01afbd40a 100644 --- a/regression-harness/src/utils.jl +++ b/regression-harness/src/utils.jl @@ -68,10 +68,69 @@ function get_commit_info(commit_hash::String, repo_root::String) end """ -Create a temporary git worktree for a commit. -Returns the worktree path. +How far a ref lags the remote branch it tracks. + +Returns `(upstream, behind)` when both the ref and an upstream can be resolved, otherwise +`nothing` (detached SHAs, tags, and local-only branches have no meaningful upstream). A ref that +is behind its upstream is a stale baseline: the harness would happily benchmark against +weeks-old code without saying so. +""" +function upstream_lag(ref::String, repo_root::String) + ref == LOCAL_REF && return nothing + # `origin/` is only a meaningful fallback for an actual local branch name. Without this + # guard, `HEAD` matches the always-present symbolic ref `origin/HEAD` (→ origin/develop) and + # every feature branch gets reported as a stale copy of develop. + is_branch = success(`git -C $repo_root rev-parse --verify --quiet refs/heads/$ref`) + candidates = is_branch ? ("$(ref)@{upstream}", "origin/$(ref)") : ("$(ref)@{upstream}",) + upstream = nothing + for candidate in candidates + try + resolved = strip(read(`git -C $repo_root rev-parse --abbrev-ref --verify --quiet $candidate`, String)) + if !isempty(resolved) + upstream = resolved + break + end + catch + continue + end + end + upstream === nothing && return nothing + try + behind = parse(Int, strip(read(`git -C $repo_root rev-list --count $(ref)..$(upstream)`, String))) + return (upstream=upstream, behind=behind) + catch + return nothing + end +end + +""" +Print a banner for every resolved ref that lags its remote tracking branch. + +`resolve_ref` takes the *local* branch pointer, so a local `develop` that has not been fetched +in weeks silently becomes the baseline. Nothing else in the report says the baseline is old. +""" +function warn_stale_refs(refs::Vector{ResolvedRef}, repo_root::String) + for ref in refs + lag = upstream_lag(ref.name, repo_root) + (lag === nothing || lag.behind == 0) && continue + println() + println("!! STALE BASELINE: '$(ref.name)' is $(lag.behind) commit(s) behind $(lag.upstream).") + println(" This comparison is against out-of-date code. Update it with:") + println(" git fetch && git checkout $(ref.name) && git merge --ff-only $(lag.upstream)") + println() + end +end + """ -function create_worktree(commit_hash::String, repo_root::String)::String +Create a temporary git worktree for a commit. Returns the worktree path. + +`pin_manifest_from` copies an already-resolved `Manifest.toml` into the worktree so that the +subprocess `Pkg.instantiate()` reproduces that exact package set instead of resolving whatever is +newest. Without it, two refs are compared across two different package sets and library-level +differences surface as physics regressions. +""" +function create_worktree(commit_hash::String, repo_root::String; + pin_manifest_from::Union{String,Nothing}=nothing)::String short = commit_hash[1:min(8, length(commit_hash))] worktree_path = tempname() * "_gpec_$(short)" try @@ -79,6 +138,9 @@ function create_worktree(commit_hash::String, repo_root::String)::String catch e error("Failed to create worktree for $short: $e") end + if pin_manifest_from !== nothing && isfile(pin_manifest_from) + cp(pin_manifest_from, joinpath(worktree_path, "Manifest.toml"); force=true) + end return worktree_path end