From e44bd96c968ccb7e1cc6c6c8a4778918a197139e Mon Sep 17 00:00:00 2001
From: d-burg <daniel.burgess@columbia.edu>
Date: Wed, 12 Aug 2026 16:43:58 -0400
Subject: [PATCH 1/3] REGRESSION - IMPROVEMENT - Pin the package set and
 fingerprint run environments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The harness cached results keyed only on (commit, case), while Manifest.toml is
untracked and every worktree resolved its own package set at run time. Two runs of
the same source could therefore differ by a package set, and machine-epsilon
differences in library math are amplified by the adaptive ODE controller and by
ill-conditioned near-resonant diagnostics into double-digit-percent "regressions"
that no source change caused. Resolves the confound reported in issue #338, plus the
stale-baseline failure mode raised in its comments.

Three mechanisms:

- The working tree's resolved Manifest.toml is copied into every worktree before
  Pkg.instantiate(), so all refs in a comparison run against one package set.
  --no-pin-manifest opts out loudly. If a pinned set proves incompatible with an old
  commit's Project.toml, instantiate re-resolves it and the harness reports that the
  pin did not hold rather than reverting silently.

- Every run records the environment that produced it (Julia version, host, resolved
  Manifest hash, Julia and BLAS thread counts), measured inside the subprocess after
  instantiate so it describes what ran rather than what was intended. Results cache
  under an environment key; a cached result from a different environment is re-run,
  and reports print each ref's environment and flag differences between them. Rows
  cached before fingerprinting carry no environment and are re-run once.

- A ref that lags its remote tracking branch gets a stale-baseline banner with the
  commands to update it, so a months-old local develop can no longer be benchmarked
  against silently.

Thread counts are recorded but deliberately not forced: forcing them would change how
every run executes and would hide the parallel-vs-serial reproducibility question
rather than exposing it.

Exit status is now meaningful — non-zero when a run fails, and with --fail-on-change
when any tracked quantity moved — so the harness can gate CI. Pass/fail against
committed golden values is a separate, later change.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 docs/development/regression-harness.md |  41 +++++
 docs/src/developer_notes.md            |   4 +
 regression-harness/regress.jl          |  64 ++++++-
 regression-harness/src/database.jl     |  97 ++++++++++-
 regression-harness/src/env.jl          | 118 +++++++++++++
 regression-harness/src/reporter.jl     |  37 +++++
 regression-harness/src/runner.jl       | 220 ++++++++++++++++---------
 regression-harness/src/types.jl        |   8 +
 regression-harness/src/utils.jl        |  68 +++++++-
 9 files changed, 565 insertions(+), 92 deletions(-)
 create mode 100644 regression-harness/src/env.jl

diff --git a/docs/development/regression-harness.md b/docs/development/regression-harness.md
index 65ad514c4..b91c1c6d6 100644
--- a/docs/development/regression-harness.md
+++ b/docs/development/regression-harness.md
@@ -96,3 +96,44 @@ regress --cases solovev_n1 --ref-range develop~10..develop
 - `--force` — re-run even if cached
 - `--verbose` — print GPEC subprocess output
 - `--no-instantiate` — skip `Pkg.instantiate()` (faster if deps are already resolved)
+- `--no-pin-manifest` — let each ref resolve its own package set (see below)
+- `--allow-env-mismatch` — reuse cached results produced in a different environment
+- `--fail-on-change` — exit non-zero when any tracked quantity changed
+
+## Making source code the only variable
+
+`Manifest.toml` is untracked, so a worktree checked out at an old commit used to resolve whatever
+package versions were newest at run time. Machine-epsilon differences in library math then get
+amplified by the adaptive ODE step controller and by ill-conditioned near-resonant diagnostics
+into double-digit-percent "regressions" that no source change caused.
+
+Two mechanisms prevent that:
+
+**The working tree's Manifest is pinned into every worktree** before `Pkg.instantiate()`, so all
+refs in a comparison run against one package set. `--no-pin-manifest` opts out (and says so
+loudly). If a pinned package set turns out to be incompatible with an old commit's
+`Project.toml`, instantiate re-resolves it and the harness warns that the pin did not hold.
+
+**Every run records the environment that produced it** — Julia version, host, resolved Manifest
+hash, Julia and BLAS thread counts — and results are cached under that environment. A cached
+result from a different environment is re-run instead of silently reused, and every report prints
+the environment of each ref:
+
+```
+Ref 1: develop  @ a0cad260 (2026-08-12)
+       env: julia 1.11.6, arm64-apple-darwin24.0.0, manifest 7e5c34ad (pinned), 1 thread/8 BLAS
+```
+
+When two compared runs did not share an environment, the report says so before the table rather
+than leaving you to infer it from the numbers.
+
+Results cached before environment fingerprinting existed carry no environment and are therefore
+re-run once — those are exactly the entries whose provenance cannot be established.
+
+Thread counts are recorded but **not** forced: the harness does not silently change how your runs
+execute. If the two refs in a comparison ran under different thread counts, the report flags it.
+
+## Exit status
+
+- `0` — every run completed (and, with `--fail-on-change`, nothing changed)
+- `1` — a run failed, or a quantity changed under `--fail-on-change`
diff --git a/docs/src/developer_notes.md b/docs/src/developer_notes.md
index f13ac28ed..4e146a6c3 100644
--- a/docs/src/developer_notes.md
+++ b/docs/src/developer_notes.md
@@ -220,3 +220,7 @@ regress --cases solovev_n1 --ref-range develop~10..develop
 - `--force` — re-run even if cached
 - `--verbose` — print GPEC subprocess output
 - `--no-instantiate` — skip `Pkg.instantiate()` (faster if deps are already resolved)
+
+The full flag list, the environment-pinning behaviour that keeps source code the only variable in
+a comparison, and the exit-status contract are documented in
+[`docs/development/regression-harness.md`](../../development/regression-harness.md).
diff --git a/regression-harness/regress.jl b/regression-harness/regress.jl
index 3b5cd61d5..3afce5122 100644
--- a/regression-harness/regress.jl
+++ b/regression-harness/regress.jl
@@ -8,6 +8,7 @@ const DEFAULT_DB_PATH = joinpath(HARNESS_DIR, ".regress_cache.sqlite")
 const CASES_DIR = joinpath(HARNESS_DIR, "cases")
 
 include("src/types.jl")
+include("src/env.jl")
 include("src/config.jl")
 include("src/database.jl")
 include("src/utils.jl")
@@ -26,6 +27,9 @@ function parse_args(args)
     db_path = nothing
     verbose = false
     no_instantiate = false
+    no_pin_manifest = false
+    allow_env_mismatch = false
+    fail_on_change = false
     help = false
 
     i = 1
@@ -46,6 +50,15 @@ function parse_args(args)
         elseif arg == "--no-instantiate"
             no_instantiate = true
             i += 1
+        elseif arg == "--no-pin-manifest"
+            no_pin_manifest = true
+            i += 1
+        elseif arg == "--allow-env-mismatch"
+            allow_env_mismatch = true
+            i += 1
+        elseif arg == "--fail-on-change"
+            fail_on_change = true
+            i += 1
         elseif arg == "--cases" && i < length(args)
             cases = split(args[i+1], ",") |> collect .|> strip
             i += 2
@@ -69,7 +82,8 @@ function parse_args(args)
         end
     end
 
-    return CLIOptions(cases, refs, ref_range, force, list_cases, show_qty, show_case, db_path, verbose, no_instantiate, help)
+    return CLIOptions(cases, refs, ref_range, force, list_cases, show_qty, show_case, db_path, verbose,
+        no_instantiate, no_pin_manifest, allow_env_mismatch, fail_on_change, help)
 end
 
 const HELP_TEXT = """
@@ -93,8 +107,19 @@ Options:
     --db path              Override database path
     --verbose              Print subprocess output
     --no-instantiate       Skip Pkg.instantiate() in subprocess
+    --no-pin-manifest      Let each ref resolve its own package set (default: pin the working
+                           tree's Manifest.toml into every worktree so source code is the only
+                           variable in a comparison)
+    --allow-env-mismatch   Reuse cached results produced in a different environment instead of
+                           re-running them
+    --fail-on-change       Exit non-zero if any tracked quantity changed (for CI use; a failed
+                           run always exits non-zero regardless)
     --help                 Print this help message
 
+Exit status:
+    0  all runs completed (and, with --fail-on-change, nothing changed)
+    1  a run failed, or a quantity changed under --fail-on-change
+
 Examples:
     # Compare two refs
     julia --project=regression-harness regression-harness/regress.jl \\
@@ -170,6 +195,25 @@ function main(args=ARGS)
             error("No commits resolved from the given refs")
         end
 
+        warn_stale_refs(resolved_refs, REPO_ROOT)
+
+        # Pin every ref to the working tree's package set unless asked not to, so that a
+        # comparison varies source code alone.
+        manifest_path = joinpath(REPO_ROOT, "Manifest.toml")
+        pin_manifest = if opts.no_pin_manifest
+            @warn "Manifest pinning disabled — refs may resolve different package sets, and differences below may not be caused by source changes"
+            nothing
+        elseif !isfile(manifest_path)
+            @warn "No Manifest.toml in the working tree; cannot pin the package set. Run Pkg.instantiate() first."
+            nothing
+        else
+            manifest_path
+        end
+        expected_key = opts.allow_env_mismatch ? nothing : expected_env_key(pin_manifest)
+
+        n_failed = 0
+        n_changed = 0
+
         # Run each case at each commit
         for case_spec in case_specs
             println("\n", "="^64)
@@ -179,18 +223,28 @@ function main(args=ARGS)
             for ref in resolved_refs
                 run_commit(db, ref.commit_hash, ref.name, case_spec, REPO_ROOT;
                            force=opts.force, verbose=opts.verbose,
-                           no_instantiate=opts.no_instantiate)
+                           no_instantiate=opts.no_instantiate,
+                           pin_manifest=pin_manifest, expected_key=expected_key)
             end
 
             # Report
-            if length(resolved_refs) == 1
-                report_multi_ref(db, case_spec, resolved_refs)
-            elseif length(resolved_refs) == 2
+            summary = if length(resolved_refs) == 2
                 report_two_ref_comparison(db, case_spec,
                     resolved_refs[1], resolved_refs[2])
             else
                 report_multi_ref(db, case_spec, resolved_refs)
             end
+            n_failed += summary.n_failed
+            n_changed += summary.n_changed
+        end
+
+        if n_failed > 0
+            @error "$n_failed run(s) failed — see the reports above"
+            exit(1)
+        end
+        if opts.fail_on_change && n_changed > 0
+            @error "$n_changed quantity/quantities changed (--fail-on-change)"
+            exit(1)
         end
     finally
         close_database(db)
diff --git a/regression-harness/src/database.jl b/regression-harness/src/database.jl
index 18e744c7f..c9a06b303 100644
--- a/regression-harness/src/database.jl
+++ b/regression-harness/src/database.jl
@@ -14,6 +14,13 @@ CREATE TABLE IF NOT EXISTS runs (
     runtime_s   REAL,
     success     INTEGER NOT NULL DEFAULT 1,
     error_msg   TEXT,
+    env_key       TEXT,
+    julia_version TEXT,
+    os_arch       TEXT,
+    manifest_sha  TEXT,
+    nthreads      INTEGER,
+    blas_threads  INTEGER,
+    pinned        INTEGER,
     UNIQUE(commit_hash, case_name)
 );
 
@@ -35,6 +42,14 @@ CREATE INDEX IF NOT EXISTS idx_runs_case ON runs(case_name);
 CREATE INDEX IF NOT EXISTS idx_quantities_run ON quantities(run_id);
 """
 
+"""
+Value of a possibly-absent SQLite column, falling back to `default`.
+
+SQLite.jl returns `missing` for NULL while Julia's `something` only skips `nothing`, so columns
+added by a later schema migration (NULL on every pre-existing row) need both cases handled.
+"""
+_column(x, default) = (x === nothing || x === missing) ? default : x
+
 """Materialize SQLite query results as a Vector of NamedTuples."""
 function query_rows(db::SQLite.DB, sql::String, params=())
     result = DBInterface.execute(db, sql, params)
@@ -44,6 +59,29 @@ function query_rows(db::SQLite.DB, sql::String, params=())
     return [NamedTuple{keys(ct)}(Tuple(col[i] for col in values(ct))) for i in 1:nrows]
 end
 
+"""
+Environment columns added to `runs` after the original schema shipped. Databases created before
+fingerprinting keep their rows, with NULL in these columns — such rows never match a computed
+`env_key`, so they are re-run rather than silently trusted.
+"""
+const ENV_COLUMNS = [
+    ("env_key", "TEXT"), ("julia_version", "TEXT"), ("os_arch", "TEXT"),
+    ("manifest_sha", "TEXT"), ("nthreads", "INTEGER"), ("blas_threads", "INTEGER"),
+    ("pinned", "INTEGER")
+]
+
+"""Add any `runs` columns missing from a database created by an earlier harness version."""
+function migrate_schema!(db::SQLite.DB)
+    existing = Set(String[])
+    for row in query_rows(db, "PRAGMA table_info(runs)")
+        push!(existing, String(something(row.name, "")))
+    end
+    for (col, sqltype) in ENV_COLUMNS
+        col in existing && continue
+        DBInterface.execute(db, "ALTER TABLE runs ADD COLUMN $col $sqltype")
+    end
+end
+
 function open_database(path::String)::SQLite.DB
     db = SQLite.DB(path)
     DBInterface.execute(db, "PRAGMA journal_mode=WAL")
@@ -53,6 +91,7 @@ function open_database(path::String)::SQLite.DB
         isempty(s) && continue
         DBInterface.execute(db, s)
     end
+    migrate_schema!(db)
     return db
 end
 
@@ -60,12 +99,39 @@ function close_database(db::SQLite.DB)
     SQLite.close(db)
 end
 
-function is_cached(db::SQLite.DB, commit_hash::String, case_name::String)::Bool
-    rows = query_rows(db, "SELECT id FROM runs WHERE commit_hash = ? AND case_name = ? AND success = 1",
-                      (commit_hash, case_name))
+"""
+Is there a usable cached result for this (commit, case)?
+
+With `expected_key` supplied, a cached run only counts when it was produced in the same
+environment. Rows predating fingerprinting hold NULL and therefore never match — the cache
+entries most likely to be misleading are exactly the ones that get re-run.
+"""
+function is_cached(db::SQLite.DB, commit_hash::String, case_name::String;
+                   expected_key::Union{String,Nothing}=nothing)::Bool
+    if expected_key === nothing
+        rows = query_rows(db, "SELECT id FROM runs WHERE commit_hash = ? AND case_name = ? AND success = 1",
+                          (commit_hash, case_name))
+        return !isempty(rows)
+    end
+    rows = query_rows(db,
+        "SELECT id FROM runs WHERE commit_hash = ? AND case_name = ? AND success = 1 AND env_key = ?",
+        (commit_hash, case_name, expected_key))
     return !isempty(rows)
 end
 
+"""
+Environment key stored for a cached run, or `nothing` when the run is absent or predates
+fingerprinting. Used to explain *why* a cached result was rejected.
+"""
+function cached_env_key(db::SQLite.DB, commit_hash::String, case_name::String)::Union{String,Nothing}
+    rows = query_rows(db, "SELECT env_key FROM runs WHERE commit_hash = ? AND case_name = ?",
+                      (commit_hash, case_name))
+    isempty(rows) && return nothing
+    key = _column(first(rows).env_key, nothing)
+    key === nothing && return nothing
+    return String(key)
+end
+
 function delete_cached(db::SQLite.DB, commit_hash::String, case_name::String)
     # ON DELETE CASCADE handles quantities cleanup automatically
     DBInterface.execute(db, "DELETE FROM runs WHERE commit_hash = ? AND case_name = ?",
@@ -76,7 +142,8 @@ function store_run(db::SQLite.DB, commit_hash::AbstractString, commit_short::Abs
                    commit_date::AbstractString, commit_msg::AbstractString,
                    case_name::AbstractString, runtime_s::Float64,
                    extracted::Vector{ExtractedQuantity};
-                   success::Bool=true, error_msg::AbstractString="")
+                   success::Bool=true, error_msg::AbstractString="",
+                   fingerprint::EnvFingerprint=UNKNOWN_ENV)
     ran_at = Dates.format(Dates.now(), "yyyy-mm-ddTHH:MM:SS")
 
     SQLite.transaction(db) do
@@ -84,10 +151,14 @@ function store_run(db::SQLite.DB, commit_hash::AbstractString, commit_short::Abs
 
         DBInterface.execute(db,
             """INSERT INTO runs
-               (commit_hash, commit_short, commit_date, commit_msg, case_name, ran_at, runtime_s, success, error_msg)
-               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+               (commit_hash, commit_short, commit_date, commit_msg, case_name, ran_at, runtime_s, success, error_msg,
+                env_key, julia_version, os_arch, manifest_sha, nthreads, blas_threads, pinned)
+               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
             (String(commit_hash), String(commit_short), String(commit_date), String(commit_msg),
-             String(case_name), ran_at, runtime_s, success ? 1 : 0, String(error_msg)))
+             String(case_name), ran_at, runtime_s, success ? 1 : 0, String(error_msg),
+             env_key(fingerprint), fingerprint.julia_version, fingerprint.os_arch,
+             fingerprint.manifest_sha, fingerprint.nthreads, fingerprint.blas_threads,
+             fingerprint.pinned ? 1 : 0))
 
         run_id = SQLite.last_insert_rowid(db)
 
@@ -147,11 +218,20 @@ Get run info for a (commit, case) pair. Returns NamedTuple or nothing.
 """
 function get_run_info(db::SQLite.DB, commit_hash::String, case_name::String)
     rows = query_rows(db,
-        """SELECT commit_short, commit_date, commit_msg, runtime_s, success, error_msg
+        """SELECT commit_short, commit_date, commit_msg, runtime_s, success, error_msg,
+                  julia_version, os_arch, manifest_sha, nthreads, blas_threads, pinned
            FROM runs WHERE commit_hash = ? AND case_name = ?""",
         (commit_hash, case_name))
     isempty(rows) && return nothing
     row = first(rows)
+    fingerprint = EnvFingerprint(
+        String(_column(row.julia_version, "")),
+        String(_column(row.os_arch, "")),
+        String(_column(row.manifest_sha, "")),
+        Int(_column(row.nthreads, -1)),
+        Int(_column(row.blas_threads, -1)),
+        _column(row.pinned, 0) == 1
+    )
     return (
         commit_short = something(row.commit_short, ""),
         commit_date = something(row.commit_date, ""),
@@ -159,6 +239,7 @@ function get_run_info(db::SQLite.DB, commit_hash::String, case_name::String)
         runtime_s = something(row.runtime_s, 0.0),
         success = coalesce(row.success, 0) == 1,
         error_msg = something(row.error_msg, ""),
+        fingerprint = fingerprint,
     )
 end
 
diff --git a/regression-harness/src/env.jl b/regression-harness/src/env.jl
new file mode 100644
index 000000000..20718c4cb
--- /dev/null
+++ b/regression-harness/src/env.jl
@@ -0,0 +1,118 @@
+"""
+Environment fingerprinting.
+
+A cached regression result is only comparable to a fresh one if both were produced by the same
+Julia, on the same machine, against the same package set. Because `Manifest.toml` is untracked,
+a worktree checkout resolves whatever is newest at run time, so two runs of the *same source*
+can differ by a package set — and machine-epsilon differences in library math are amplified by
+the adaptive ODE controller into large apparent regressions. The fingerprint below makes that
+confound visible (and, by default, invalidating) instead of silent.
+"""
+
+"""
+Identity of the environment a run was produced in.
+
+## Fields
+
+  - `julia_version::String` — version of the `julia` that ran GPEC
+  - `os_arch::String` — `Sys.MACHINE` of the running host
+  - `manifest_sha::String` — SHA-256 of the `Manifest.toml` the run resolved against ("" if absent)
+  - `nthreads::Int` — `Threads.nthreads()` in the run (-1 if unknown)
+  - `blas_threads::Int` — `BLAS.get_num_threads()` in the run (-1 if unknown)
+  - `pinned::Bool` — whether the harness copied its own Manifest into the run's project
+"""
+struct EnvFingerprint
+    julia_version::String
+    os_arch::String
+    manifest_sha::String
+    nthreads::Int
+    blas_threads::Int
+    pinned::Bool
+end
+
+const UNKNOWN_ENV = EnvFingerprint("", "", "", -1, -1, false)
+
+"""
+Cache key for an environment.
+
+Deliberately built from only the three fields that are knowable *before* a run: the Julia
+version, the host, and the package set the run will be pinned to. Thread counts are recorded and
+reported but not keyed, because the harness does not force them (see `--threads` in the CLI) and
+so cannot predict them ahead of a run.
+
+When the Manifest is not pinned, the package set is unknowable in advance and the key records
+`unpinned`; the resolved Manifest hash is still stored per run for display and mismatch warnings.
+"""
+function env_key(julia_version::AbstractString, os_arch::AbstractString, manifest_mode::AbstractString)::String
+    return bytes2hex(SHA.sha256("$(julia_version)|$(os_arch)|$(manifest_mode)"))[1:16]
+end
+
+env_key(fp::EnvFingerprint) = env_key(fp.julia_version, fp.os_arch, fp.pinned ? fp.manifest_sha : "unpinned")
+
+"""SHA-256 of a file, or "" when it does not exist."""
+function file_sha256(path::AbstractString)::String
+    isfile(path) || return ""
+    return bytes2hex(SHA.sha256(read(path)))
+end
+
+"""
+Version string of the `julia` the harness will launch for subprocess runs.
+
+This is not necessarily the Julia running the harness itself, so it is probed rather than read
+from `VERSION`.
+"""
+function subprocess_julia_version()::String
+    try
+        out = strip(read(`julia --version`, String))
+        m = match(r"(\d+\.\d+\.\d+\S*)", out)
+        return m === nothing ? out : m.captures[1]
+    catch
+        return "unknown"
+    end
+end
+
+"""
+The environment key that runs launched *now* will carry.
+
+`manifest_path` is the Manifest the harness will pin into each worktree; pass `nothing` when
+pinning is disabled.
+"""
+function expected_env_key(manifest_path::Union{String,Nothing})::String
+    mode = manifest_path === nothing ? "unpinned" : file_sha256(manifest_path)
+    return env_key(subprocess_julia_version(), string(Sys.MACHINE), mode)
+end
+
+"""
+Parse the `key=value` run-info file written by a run subprocess.
+
+Returns `(runtime_s, fingerprint)`. A missing or malformed file yields `NaN` and `UNKNOWN_ENV`
+rather than throwing, so a run that produced output but no metadata still reports its numbers.
+"""
+function read_runinfo(path::String, pinned::Bool)
+    isfile(path) || return (NaN, UNKNOWN_ENV)
+    fields = Dict{String,String}()
+    for line in eachline(path)
+        parts = split(line, '='; limit=2)
+        length(parts) == 2 && (fields[strip(parts[1])] = strip(parts[2]))
+    end
+    getf = (k, d) -> get(fields, k, d)
+    runtime_s = tryparse(Float64, getf("runtime_s", ""))
+    fp = EnvFingerprint(
+        getf("julia_version", ""),
+        getf("os_arch", ""),
+        getf("manifest_sha", ""),
+        something(tryparse(Int, getf("nthreads", "")), -1),
+        something(tryparse(Int, getf("blas_threads", "")), -1),
+        pinned
+    )
+    return (something(runtime_s, NaN), fp)
+end
+
+"""One-line human-readable summary of an environment, for report headers."""
+function describe_env(fp::EnvFingerprint)::String
+    isempty(fp.julia_version) && return "environment unknown (cached before fingerprinting)"
+    mani = isempty(fp.manifest_sha) ? "no Manifest" : "manifest " * fp.manifest_sha[1:min(8, end)]
+    pin = fp.pinned ? "pinned" : "unpinned"
+    threads = "$(fp.nthreads) thread$(fp.nthreads == 1 ? "" : "s")/$(fp.blas_threads) BLAS"
+    return "julia $(fp.julia_version), $(fp.os_arch), $mani ($pin), $threads"
+end
diff --git a/regression-harness/src/reporter.jl b/regression-harness/src/reporter.jl
index c0c57758f..2bc9ec507 100644
--- a/regression-harness/src/reporter.jl
+++ b/regression-harness/src/reporter.jl
@@ -47,6 +47,31 @@ function format_value(q::NamedTuple)::String
     end
 end
 
+"""
+Print a banner when two compared runs did not share an environment.
+
+Source code is only the sole variable when Julia, host, package set and thread counts all match.
+Anything else here means part of the reported difference may be the environment, not the code.
+"""
+function _warn_env_difference(fp1::EnvFingerprint, fp2::EnvFingerprint)
+    (fp1 === UNKNOWN_ENV || fp2 === UNKNOWN_ENV) && return
+    (isempty(fp1.julia_version) || isempty(fp2.julia_version)) && return
+    differences = String[]
+    fp1.julia_version != fp2.julia_version && push!(differences, "julia $(fp1.julia_version) vs $(fp2.julia_version)")
+    fp1.os_arch != fp2.os_arch && push!(differences, "host $(fp1.os_arch) vs $(fp2.os_arch)")
+    fp1.manifest_sha != fp2.manifest_sha && push!(differences, "different package sets (Manifest hashes differ)")
+    fp1.nthreads != fp2.nthreads && push!(differences, "$(fp1.nthreads) vs $(fp2.nthreads) Julia threads")
+    fp1.blas_threads != fp2.blas_threads && push!(differences, "$(fp1.blas_threads) vs $(fp2.blas_threads) BLAS threads")
+    isempty(differences) && return
+    println()
+    println("!! ENVIRONMENTS DIFFER — source code is not the only variable in this comparison:")
+    for d in differences
+        println("     - $d")
+    end
+    println("   Differences below may be environment artifacts. Re-run with --force to rebuild")
+    println("   both refs in the current environment.")
+end
+
 """
 Format a diff value for display.
 """
@@ -182,7 +207,10 @@ function report_two_ref_comparison(db::SQLite.DB, case_spec::CaseSpec,
     tag1 = failed1 ? " (FAILED)" : ""
     tag2 = failed2 ? " (FAILED)" : ""
     println("Ref 1: $(ref1.name)  @ $(info1.commit_short) ($date1)$tag1")
+    println("       env: $(describe_env(info1.fingerprint))")
     println("Ref 2: $(ref2.name)  @ $(info2.commit_short) ($date2)$tag2")
+    println("       env: $(describe_env(info2.fingerprint))")
+    _warn_env_difference(info1.fingerprint, info2.fingerprint)
     if failed1
         println("  Ref 1 error: $(_short_err(info1.error_msg))")
     end
@@ -204,6 +232,8 @@ function report_two_ref_comparison(db::SQLite.DB, case_spec::CaseSpec,
     n_missing > 0 && push!(parts, "$n_missing missing/N/A")
     println("Summary: ", join(parts, ", "))
     println()
+    return (n_ok=n_ok, n_changed=n_changed, n_missing=n_missing,
+            n_failed=count(identity, (failed1, failed2)))
 end
 
 """
@@ -327,6 +357,7 @@ function report_multi_ref(db::SQLite.DB, case_spec::CaseSpec,
             date_str = length(info.commit_date) >= 10 ? info.commit_date[1:10] : info.commit_date
             status_str = info.success ? "" : " (FAILED)"
             println("Ref $(i): $(ref.name)  @ $(info.commit_short) ($date_str)$status_str")
+            println("       env: $(describe_env(info.fingerprint))")
             if !info.success
                 println("  Ref $(i) error: $(_short_err(info.error_msg))")
             end
@@ -334,6 +365,10 @@ function report_multi_ref(db::SQLite.DB, case_spec::CaseSpec,
             println("Ref $(i): $(ref.name)  (no data)")
         end
     end
+    if length(refs) >= 2
+        last_two = filter(!isnothing, run_infos[(end - 1):end])
+        length(last_two) == 2 && _warn_env_difference(last_two[1].fingerprint, last_two[2].fingerprint)
+    end
     println("-"^total_w)
 
     _print_row(header, widths)
@@ -351,6 +386,8 @@ function report_multi_ref(db::SQLite.DB, case_spec::CaseSpec,
         println("Summary (last vs prev): ", join(parts, ", "))
     end
     println()
+    return (n_ok=n_ok, n_changed=n_changed, n_missing=n_missing,
+            n_failed=count(identity, failed_mask))
 end
 
 """
diff --git a/regression-harness/src/runner.jl b/regression-harness/src/runner.jl
index ecb7b760f..550ea9e04 100644
--- a/regression-harness/src/runner.jl
+++ b/regression-harness/src/runner.jl
@@ -2,13 +2,26 @@
 Runner: orchestrates checking out commits, running GPEC, and extracting results.
 """
 
-"""Read the GPEC-only runtime from the timing file written by the subprocess."""
-function _read_timing_file(path::String)::Float64
-    if isfile(path)
-        return parse(Float64, strip(read(path, String)))
+"""
+Epilogue appended to every subprocess script: records the GPEC-only runtime *and* the
+environment that produced it (Julia, host, resolved package set, thread counts) as `key=value`
+lines. Measured inside the subprocess, after `Pkg.instantiate()` has resolved the environment,
+so it describes what actually ran rather than what the harness intended to run.
+"""
+const RUNINFO_EPILOGUE = """
+using SHA
+using LinearAlgebra: BLAS
+let manifest = joinpath(dirname(Base.active_project()), "Manifest.toml")
+    open(ARGS[2], "w") do f
+        println(f, "runtime_s=", elapsed)
+        println(f, "julia_version=", string(VERSION))
+        println(f, "os_arch=", Sys.MACHINE)
+        println(f, "manifest_sha=", isfile(manifest) ? bytes2hex(SHA.sha256(read(manifest))) : "")
+        println(f, "nthreads=", Threads.nthreads())
+        println(f, "blas_threads=", BLAS.get_num_threads())
     end
-    return NaN
 end
+"""
 
 """
 Materialize the directory GPEC will actually run in.
@@ -47,16 +60,13 @@ using GeneralizedPerturbedEquilibrium
 t_start = time()
 GeneralizedPerturbedEquilibrium.main([ARGS[1]])
 elapsed = time() - t_start
-# Write GPEC-only runtime to a file the harness reads back
-open(ARGS[2], "w") do f
-    println(f, elapsed)
-end
+%RUNINFO%
 """
 
 # Self-contained computation for the GGJ inner-layer reference benchmark.
 # Runs the Galerkin solver on the Glasser & Wang 2020 Eq. 55 parameter set
 # at γ = 1 + i and writes (Δ_odd, Δ_even) into a small HDF5 file at ARGS[1].
-# Runtime is written to ARGS[2] for parity with the GPEC runner.
+# Runtime and environment are written to ARGS[2] by the shared run-info epilogue.
 # `solve_inner` returns the named-field `InnerLayerResponse`; the historical
 # delta_odd / delta_even slots map to interchange / tearing respectively, which
 # preserves the numeric identity of each tracked quantity.
@@ -77,14 +87,13 @@ h5open(ARGS[1], "w") do fid
     fid["ggj/delta_even_real"] = real(Δ.tearing)
     fid["ggj/delta_even_imag"] = imag(Δ.tearing)
 end
-open(ARGS[2], "w") do f
-    println(f, elapsed)
-end
+%RUNINFO%
 """
 
 # GGJ rotated-ray backend at Q = 500i on the q=4 benchmark surface — a regime beyond the
 # :galerkin backend. Builds the physical rate γ = 500i·Q₀ so inner_Q lands exactly on the
-# imaginary axis at 500i, then writes the parity matching data. Runtime to ARGS[2] as usual.
+# imaginary axis at 500i, then writes the parity matching data. Runtime and environment are
+# written to ARGS[2] by the shared run-info epilogue.
 # As in the galerkin template, the delta_odd / delta_even slots map to interchange / tearing.
 const COMPUTED_GGJ_RAY_SCRIPT_TEMPLATE = """
 using Pkg
@@ -103,9 +112,7 @@ h5open(ARGS[1], "w") do fid
     fid["ggj/delta_even_real"] = real(Δ.tearing)
     fid["ggj/delta_even_imag"] = imag(Δ.tearing)
 end
-open(ARGS[2], "w") do f
-    println(f, elapsed)
-end
+%RUNINFO%
 """
 
 # Self-contained separatrix-finder regression (PR #296). Loads a fixed-boundary EFIT whose
@@ -113,7 +120,7 @@ end
 # LCFS the coil-vacuum flux turns back above the boundary value before the grid edge, so the old
 # bracketed-Brent separatrix finder could not bracket psi=sibry and equilibrium setup threw. Calls
 # setup_equilibrium directly and writes the leading equilibrium scalars: errors on the buggy code,
-# passes with the Newton finder. Runtime is written to ARGS[2] for parity with the GPEC runner.
+# passes with the Newton finder. Runtime and environment are written to ARGS[2] by the run-info epilogue.
 const COMPUTED_SEPARATRIX_SCRIPT_TEMPLATE = """
 using Pkg
 %INSTANTIATE%
@@ -133,33 +140,50 @@ h5open(ARGS[1], "w") do fid
     fid["equil/betat"] = pe.params.betat
     fid["equil/betan"] = pe.params.betan
 end
-open(ARGS[2], "w") do f
-    println(f, elapsed)
-end
+%RUNINFO%
+"""
+
 """
+Expand a subprocess script template: the optional `Pkg.instantiate()` call and the run-info
+epilogue that records runtime and environment.
+"""
+function _render_script(template::String; no_instantiate::Bool)::String
+    rendered = replace(template, "%INSTANTIATE%" => (no_instantiate ? "" : "Pkg.instantiate()"))
+    return replace(rendered, "%RUNINFO%" => RUNINFO_EPILOGUE)
+end
 
 """
 Run GPEC for a single commit/ref and case. Dispatches to run_local for
 the working tree or run_at_commit for a git ref.
+
+`pin_manifest` is the path to a resolved `Manifest.toml` copied into each worktree so every ref
+runs against the same package set; `nothing` disables pinning. `expected_key` is the environment
+key a fresh run will carry — a cached run whose key differs is re-run rather than reused.
 """
 function run_commit(db::SQLite.DB, commit_hash::String, ref_name::String,
                     case_spec::CaseSpec, repo_root::String;
                     force::Bool=false, verbose::Bool=false,
-                    no_instantiate::Bool=false)
+                    no_instantiate::Bool=false,
+                    pin_manifest::Union{String,Nothing}=nothing,
+                    expected_key::Union{String,Nothing}=nothing)
     if case_spec.kind == "computed"
         if commit_hash == LOCAL_REF
             return run_computed_local(db, case_spec, repo_root;
-                                      verbose=verbose, no_instantiate=no_instantiate)
+                                      verbose=verbose, no_instantiate=no_instantiate,
+                                      pin_manifest=pin_manifest)
         end
         return run_computed_at_commit(db, commit_hash, ref_name, case_spec, repo_root;
-                                      force=force, verbose=verbose, no_instantiate=no_instantiate)
+                                      force=force, verbose=verbose, no_instantiate=no_instantiate,
+                                      pin_manifest=pin_manifest, expected_key=expected_key)
     end
     if commit_hash == LOCAL_REF
         return run_local(db, case_spec, repo_root;
-                         force=force, verbose=verbose, no_instantiate=no_instantiate)
+                         force=force, verbose=verbose, no_instantiate=no_instantiate,
+                         pin_manifest=pin_manifest)
     end
     return run_at_commit(db, commit_hash, ref_name, case_spec, repo_root;
-                         force=force, verbose=verbose, no_instantiate=no_instantiate)
+                         force=force, verbose=verbose, no_instantiate=no_instantiate,
+                         pin_manifest=pin_manifest, expected_key=expected_key)
 end
 
 """
@@ -185,31 +209,30 @@ so callers can handle store_failed_run uniformly.
 """
 function _execute_computed(case_spec::CaseSpec, project_root::String;
                            verbose::Bool, no_instantiate::Bool,
-                           stderr_buf::IO)
-    instantiate_line = no_instantiate ? "" : "Pkg.instantiate()"
-    script_content = replace(_computed_script_template(case_spec),
-                             "%INSTANTIATE%" => instantiate_line)
+                           stderr_buf::IO, pin_manifest::Union{String,Nothing}=nothing)
+    script_content = _render_script(_computed_script_template(case_spec); no_instantiate=no_instantiate)
     tmpscript = tempname() * ".jl"
     h5path = tempname() * ".h5"
-    timingfile = tempname() * ".timing"
+    runinfo_file = tempname() * ".runinfo"
     try
         write(tmpscript, script_content)
         if verbose
-            run(pipeline(`julia --project=$project_root $tmpscript $h5path $timingfile`))
+            run(pipeline(`julia --project=$project_root $tmpscript $h5path $runinfo_file`))
         else
-            run(pipeline(`julia --project=$project_root $tmpscript $h5path $timingfile`,
+            run(pipeline(`julia --project=$project_root $tmpscript $h5path $runinfo_file`,
                          stdout=devnull, stderr=stderr_buf))
         end
-        runtime_s = _read_timing_file(timingfile)
+        runtime_s, fingerprint = read_runinfo(runinfo_file, pin_manifest !== nothing)
+        _warn_pin_broken(pin_manifest, fingerprint, case_spec.name)
         if !isfile(h5path)
             error("Computed case '$(case_spec.name)' produced no output h5")
         end
         extracted = extract_quantities(h5path, case_spec.quantities, runtime_s)
-        return extracted, runtime_s
+        return extracted, runtime_s, fingerprint
     finally
         rm(tmpscript; force=true)
         rm(h5path; force=true)
-        rm(timingfile; force=true)
+        rm(runinfo_file; force=true)
     end
 end
 
@@ -217,18 +240,20 @@ end
 Run a kind="computed" case against the working tree.
 """
 function run_computed_local(db::SQLite.DB, case_spec::CaseSpec, repo_root::String;
-                            verbose::Bool=false, no_instantiate::Bool=false)
+                            verbose::Bool=false, no_instantiate::Bool=false,
+                            pin_manifest::Union{String,Nothing}=nothing)
     delete_cached(db, LOCAL_REF, case_spec.name)
     date = Dates.format(Dates.now(), "yyyy-mm-ddTHH:MM:SS")
     @info "Running: $(case_spec.name) @ local (working tree, computed)"
     stderr_buf = IOBuffer()
     try
-        extracted, runtime_s = _execute_computed(case_spec, repo_root;
-                                                 verbose=verbose,
-                                                 no_instantiate=no_instantiate,
-                                                 stderr_buf=stderr_buf)
+        extracted, runtime_s, fingerprint = _execute_computed(case_spec, repo_root;
+                                                              verbose=verbose,
+                                                              no_instantiate=no_instantiate,
+                                                              stderr_buf=stderr_buf,
+                                                              pin_manifest=pin_manifest)
         store_run(db, LOCAL_REF, "local", date, "working tree", case_spec.name,
-                  runtime_s, extracted)
+                  runtime_s, extracted; fingerprint=fingerprint)
         @info "  Completed in $(round(runtime_s, digits=3))s — $(length(extracted)) quantities extracted"
     catch e
         err_msg = if e isa ProcessFailedException
@@ -253,14 +278,17 @@ Run a kind="computed" case at a specific git commit via worktree.
 function run_computed_at_commit(db::SQLite.DB, commit_hash::String, ref_name::String,
                                 case_spec::CaseSpec, repo_root::String;
                                 force::Bool=false, verbose::Bool=false,
-                                no_instantiate::Bool=false)
-    if !force && is_cached(db, commit_hash, case_spec.name)
+                                no_instantiate::Bool=false,
+                                pin_manifest::Union{String,Nothing}=nothing,
+                                expected_key::Union{String,Nothing}=nothing)
+    if !force && is_cached(db, commit_hash, case_spec.name; expected_key=expected_key)
         info = get_run_info(db, commit_hash, case_spec.name)
         if info !== nothing
             @info "Cached: $(case_spec.name) @ $(info.commit_short) ($(info.commit_date))"
             return
         end
     end
+    _warn_env_invalidated(db, commit_hash, case_spec.name, expected_key, force)
     if force
         delete_cached(db, commit_hash, case_spec.name)
     end
@@ -272,13 +300,14 @@ function run_computed_at_commit(db::SQLite.DB, commit_hash::String, ref_name::St
     worktree_path = nothing
     stderr_buf = IOBuffer()
     try
-        worktree_path = create_worktree(commit_hash, repo_root)
-        extracted, runtime_s = _execute_computed(case_spec, worktree_path;
-                                                 verbose=verbose,
-                                                 no_instantiate=no_instantiate,
-                                                 stderr_buf=stderr_buf)
+        worktree_path = create_worktree(commit_hash, repo_root; pin_manifest_from=pin_manifest)
+        extracted, runtime_s, fingerprint = _execute_computed(case_spec, worktree_path;
+                                                              verbose=verbose,
+                                                              no_instantiate=no_instantiate,
+                                                              stderr_buf=stderr_buf,
+                                                              pin_manifest=pin_manifest)
         store_run(db, commit_hash, commit_info.short, commit_info.date,
-                  commit_info.msg, case_spec.name, runtime_s, extracted)
+                  commit_info.msg, case_spec.name, runtime_s, extracted; fingerprint=fingerprint)
         @info "  Completed in $(round(runtime_s, digits=3))s — $(length(extracted)) quantities extracted"
     catch e
         err_msg = if e isa ProcessFailedException
@@ -301,13 +330,49 @@ function run_computed_at_commit(db::SQLite.DB, commit_hash::String, ref_name::St
     end
 end
 
+"""
+Warn when a pinned package set did not survive `Pkg.instantiate()`.
+
+The harness copies its Manifest into a worktree, but if that package set is incompatible with the
+checked-out commit's `Project.toml`, instantiate re-resolves and the pin silently stops holding —
+which is exactly the confound pinning exists to remove.
+"""
+function _warn_pin_broken(pin_manifest::Union{String,Nothing}, fp::EnvFingerprint, label::AbstractString)
+    pin_manifest === nothing && return
+    isempty(fp.manifest_sha) && return
+    pinned_sha = file_sha256(pin_manifest)
+    (isempty(pinned_sha) || pinned_sha == fp.manifest_sha) && return
+    @warn "Pinned Manifest was re-resolved for $label — its package set differs from the working tree" pinned = pinned_sha[1:8] resolved = fp.manifest_sha[1:8]
+end
+
+"""
+Explain a cache miss caused by the environment rather than by absence.
+
+A cached run exists for this (commit, case) but was produced under a different Julia, host, or
+package set — the exact situation that used to be silently reused and reported as a physics
+regression. Says so, once, before re-running.
+"""
+function _warn_env_invalidated(db::SQLite.DB, commit_hash::String, case_name::String,
+                               expected_key::Union{String,Nothing}, force::Bool)
+    (force || expected_key === nothing) && return
+    stored = cached_env_key(db, commit_hash, case_name)
+    stored == expected_key && return
+    if stored === nothing
+        is_cached(db, commit_hash, case_name) || return
+        @warn "Cached result for $case_name predates environment fingerprinting — re-running"
+    else
+        @warn "Cached result for $case_name was produced in a different environment — re-running" stored_key = stored current_key = expected_key
+    end
+end
+
 """
 Run GPEC in the current working tree (uncommitted changes included).
 Always re-runs (local results are never cached since the working tree is mutable).
 """
 function run_local(db::SQLite.DB, case_spec::CaseSpec, repo_root::String;
                    force::Bool=false, verbose::Bool=false,
-                   no_instantiate::Bool=false)
+                   no_instantiate::Bool=false,
+                   pin_manifest::Union{String,Nothing}=nothing)
     # Always delete previous local results and re-run
     delete_cached(db, LOCAL_REF, case_spec.name)
 
@@ -323,7 +388,7 @@ function run_local(db::SQLite.DB, case_spec::CaseSpec, repo_root::String;
     end
 
     tmpscript = nothing
-    timingfile = nothing
+    runinfo_file = nothing
     rundir = nothing
     rundir_is_temp = false
     stderr_buf = IOBuffer()
@@ -331,19 +396,19 @@ function run_local(db::SQLite.DB, case_spec::CaseSpec, repo_root::String;
     try
         (rundir, rundir_is_temp) = _materialize_rundir(example_path, case_spec.overrides)
 
-        instantiate_line = no_instantiate ? "" : "Pkg.instantiate()"
-        script_content = replace(RUNNER_SCRIPT_TEMPLATE, "%INSTANTIATE%" => instantiate_line)
+        script_content = _render_script(RUNNER_SCRIPT_TEMPLATE; no_instantiate=no_instantiate)
         tmpscript = tempname() * ".jl"
-        timingfile = tempname() * ".timing"
+        runinfo_file = tempname() * ".runinfo"
         write(tmpscript, script_content)
 
         if verbose
-            run(pipeline(`julia --project=$repo_root $tmpscript $rundir $timingfile`))
+            run(pipeline(`julia --project=$repo_root $tmpscript $rundir $runinfo_file`))
         else
-            run(pipeline(`julia --project=$repo_root $tmpscript $rundir $timingfile`,
+            run(pipeline(`julia --project=$repo_root $tmpscript $rundir $runinfo_file`,
                          stdout=devnull, stderr=stderr_buf))
         end
-        runtime_s = _read_timing_file(timingfile)
+        runtime_s, fingerprint = read_runinfo(runinfo_file, pin_manifest !== nothing)
+        _warn_pin_broken(pin_manifest, fingerprint, case_spec.name)
 
         h5path = joinpath(rundir, "gpec.h5")
         if !isfile(h5path)
@@ -355,7 +420,7 @@ function run_local(db::SQLite.DB, case_spec::CaseSpec, repo_root::String;
 
         extracted = extract_quantities(h5path, case_spec.quantities, runtime_s)
         store_run(db, LOCAL_REF, "local", date, "working tree", case_spec.name,
-                  runtime_s, extracted)
+                  runtime_s, extracted; fingerprint=fingerprint)
 
         @info "  Completed in $(round(runtime_s, digits=1))s — $(length(extracted)) quantities extracted"
 
@@ -377,8 +442,8 @@ function run_local(db::SQLite.DB, case_spec::CaseSpec, repo_root::String;
         if tmpscript !== nothing
             rm(tmpscript; force=true)
         end
-        if timingfile !== nothing
-            rm(timingfile; force=true)
+        if runinfo_file !== nothing
+            rm(runinfo_file; force=true)
         end
         if rundir_is_temp && rundir !== nothing
             rm(dirname(rundir); recursive=true, force=true)
@@ -388,20 +453,23 @@ end
 
 """
 Run GPEC for a specific git commit via worktree. Stores results in the database.
-Skips if already cached (unless force=true).
+Skips if already cached in the same environment (unless force=true).
 """
 function run_at_commit(db::SQLite.DB, commit_hash::String, ref_name::String,
                        case_spec::CaseSpec, repo_root::String;
                        force::Bool=false, verbose::Bool=false,
-                       no_instantiate::Bool=false)
+                       no_instantiate::Bool=false,
+                       pin_manifest::Union{String,Nothing}=nothing,
+                       expected_key::Union{String,Nothing}=nothing)
     # Check cache
-    if !force && is_cached(db, commit_hash, case_spec.name)
+    if !force && is_cached(db, commit_hash, case_spec.name; expected_key=expected_key)
         info = get_run_info(db, commit_hash, case_spec.name)
         if info !== nothing
             @info "Cached: $(case_spec.name) @ $(info.commit_short) ($(info.commit_date))"
             return
         end
     end
+    _warn_env_invalidated(db, commit_hash, case_spec.name, expected_key, force)
 
     # Delete existing cached data if force re-running
     if force
@@ -415,14 +483,14 @@ function run_at_commit(db::SQLite.DB, commit_hash::String, ref_name::String,
 
     worktree_path = nothing
     tmpscript = nothing
-    timingfile = nothing
+    runinfo_file = nothing
     rundir = nothing
     rundir_is_temp = false
     stderr_buf = IOBuffer()
 
     try
-        # Create worktree
-        worktree_path = create_worktree(commit_hash, repo_root)
+        # Create worktree, pinning the package set when the caller supplied a Manifest
+        worktree_path = create_worktree(commit_hash, repo_root; pin_manifest_from=pin_manifest)
 
         # Check example directory exists in this commit
         example_path = joinpath(worktree_path, case_spec.example_dir)
@@ -437,22 +505,22 @@ function run_at_commit(db::SQLite.DB, commit_hash::String, ref_name::String,
         (rundir, rundir_is_temp) = _materialize_rundir(example_path, case_spec.overrides)
 
         # Write temp runner script
-        instantiate_line = no_instantiate ? "" : "Pkg.instantiate()"
-        script_content = replace(RUNNER_SCRIPT_TEMPLATE, "%INSTANTIATE%" => instantiate_line)
+        script_content = _render_script(RUNNER_SCRIPT_TEMPLATE; no_instantiate=no_instantiate)
         tmpscript = tempname() * ".jl"
-        timingfile = tempname() * ".timing"
+        runinfo_file = tempname() * ".runinfo"
         write(tmpscript, script_content)
 
         # Run GPEC in subprocess
         project_root = worktree_path
 
         if verbose
-            run(pipeline(`julia --project=$project_root $tmpscript $rundir $timingfile`))
+            run(pipeline(`julia --project=$project_root $tmpscript $rundir $runinfo_file`))
         else
-            run(pipeline(`julia --project=$project_root $tmpscript $rundir $timingfile`,
+            run(pipeline(`julia --project=$project_root $tmpscript $rundir $runinfo_file`,
                          stdout=devnull, stderr=stderr_buf))
         end
-        runtime_s = _read_timing_file(timingfile)
+        runtime_s, fingerprint = read_runinfo(runinfo_file, pin_manifest !== nothing)
+        _warn_pin_broken(pin_manifest, fingerprint, commit_info.short)
 
         # Check for gpec.h5
         h5path = joinpath(rundir, "gpec.h5")
@@ -469,7 +537,7 @@ function run_at_commit(db::SQLite.DB, commit_hash::String, ref_name::String,
 
         # Store in database
         store_run(db, commit_hash, commit_info.short, commit_info.date,
-                  commit_info.msg, case_spec.name, runtime_s, extracted)
+                  commit_info.msg, case_spec.name, runtime_s, extracted; fingerprint=fingerprint)
 
         @info "  Completed in $(round(runtime_s, digits=1))s — $(length(extracted)) quantities extracted"
 
@@ -493,8 +561,8 @@ function run_at_commit(db::SQLite.DB, commit_hash::String, ref_name::String,
         if tmpscript !== nothing
             rm(tmpscript; force=true)
         end
-        if timingfile !== nothing
-            rm(timingfile; force=true)
+        if runinfo_file !== nothing
+            rm(runinfo_file; force=true)
         end
         if rundir_is_temp && rundir !== nothing
             rm(dirname(rundir); recursive=true, force=true)
diff --git a/regression-harness/src/types.jl b/regression-harness/src/types.jl
index 0d0a3e638..9417c5303 100644
--- a/regression-harness/src/types.jl
+++ b/regression-harness/src/types.jl
@@ -48,6 +48,11 @@ end
 
 """
 Parsed CLI options.
+
+`no_pin_manifest` disables copying the working tree's resolved `Manifest.toml` into each
+worktree (pinning is on by default, so that two refs differ only by source code).
+`allow_env_mismatch` lets a cached result from a different environment be reused instead of
+re-run. `fail_on_change` turns any changed quantity into a non-zero exit status, for CI use.
 """
 struct CLIOptions
     cases::Vector{String}
@@ -60,5 +65,8 @@ struct CLIOptions
     db_path::Union{String,Nothing}
     verbose::Bool
     no_instantiate::Bool
+    no_pin_manifest::Bool
+    allow_env_mismatch::Bool
+    fail_on_change::Bool
     help::Bool
 end
diff --git a/regression-harness/src/utils.jl b/regression-harness/src/utils.jl
index a7b6d8373..01afbd40a 100644
--- a/regression-harness/src/utils.jl
+++ b/regression-harness/src/utils.jl
@@ -68,10 +68,69 @@ function get_commit_info(commit_hash::String, repo_root::String)
 end
 
 """
-Create a temporary git worktree for a commit.
-Returns the worktree path.
+How far a ref lags the remote branch it tracks.
+
+Returns `(upstream, behind)` when both the ref and an upstream can be resolved, otherwise
+`nothing` (detached SHAs, tags, and local-only branches have no meaningful upstream). A ref that
+is behind its upstream is a stale baseline: the harness would happily benchmark against
+weeks-old code without saying so.
+"""
+function upstream_lag(ref::String, repo_root::String)
+    ref == LOCAL_REF && return nothing
+    # `origin/<ref>` is only a meaningful fallback for an actual local branch name. Without this
+    # guard, `HEAD` matches the always-present symbolic ref `origin/HEAD` (→ origin/develop) and
+    # every feature branch gets reported as a stale copy of develop.
+    is_branch = success(`git -C $repo_root rev-parse --verify --quiet refs/heads/$ref`)
+    candidates = is_branch ? ("$(ref)@{upstream}", "origin/$(ref)") : ("$(ref)@{upstream}",)
+    upstream = nothing
+    for candidate in candidates
+        try
+            resolved = strip(read(`git -C $repo_root rev-parse --abbrev-ref --verify --quiet $candidate`, String))
+            if !isempty(resolved)
+                upstream = resolved
+                break
+            end
+        catch
+            continue
+        end
+    end
+    upstream === nothing && return nothing
+    try
+        behind = parse(Int, strip(read(`git -C $repo_root rev-list --count $(ref)..$(upstream)`, String)))
+        return (upstream=upstream, behind=behind)
+    catch
+        return nothing
+    end
+end
+
+"""
+Print a banner for every resolved ref that lags its remote tracking branch.
+
+`resolve_ref` takes the *local* branch pointer, so a local `develop` that has not been fetched
+in weeks silently becomes the baseline. Nothing else in the report says the baseline is old.
+"""
+function warn_stale_refs(refs::Vector{ResolvedRef}, repo_root::String)
+    for ref in refs
+        lag = upstream_lag(ref.name, repo_root)
+        (lag === nothing || lag.behind == 0) && continue
+        println()
+        println("!! STALE BASELINE: '$(ref.name)' is $(lag.behind) commit(s) behind $(lag.upstream).")
+        println("   This comparison is against out-of-date code. Update it with:")
+        println("     git fetch && git checkout $(ref.name) && git merge --ff-only $(lag.upstream)")
+        println()
+    end
+end
+
 """
-function create_worktree(commit_hash::String, repo_root::String)::String
+Create a temporary git worktree for a commit. Returns the worktree path.
+
+`pin_manifest_from` copies an already-resolved `Manifest.toml` into the worktree so that the
+subprocess `Pkg.instantiate()` reproduces that exact package set instead of resolving whatever is
+newest. Without it, two refs are compared across two different package sets and library-level
+differences surface as physics regressions.
+"""
+function create_worktree(commit_hash::String, repo_root::String;
+                         pin_manifest_from::Union{String,Nothing}=nothing)::String
     short = commit_hash[1:min(8, length(commit_hash))]
     worktree_path = tempname() * "_gpec_$(short)"
     try
@@ -79,6 +138,9 @@ function create_worktree(commit_hash::String, repo_root::String)::String
     catch e
         error("Failed to create worktree for $short: $e")
     end
+    if pin_manifest_from !== nothing && isfile(pin_manifest_from)
+        cp(pin_manifest_from, joinpath(worktree_path, "Manifest.toml"); force=true)
+    end
     return worktree_path
 end
 

From 8d26ec90f19f16a0a5f889c2496ecc4baa8c587b Mon Sep 17 00:00:00 2001
From: d-burg <daniel.burgess@columbia.edu>
Date: Wed, 12 Aug 2026 23:41:00 -0400
Subject: [PATCH 2/3] REGRESSION - BUG FIX - Use an absolute link and describe
 the cache precisely
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The cross-reference added to developer_notes.md pointed outside docs/src/, which
Documenter rejects as an invalid local link, failing the docs build. Use the full
GitHub URL, matching how the file already links to DOC_STANDARD.md.

Also corrects the harness documentation: the schema keeps one row per
(commit, case), so a re-run replaces the stored result rather than caching one
result per environment. What the fingerprint adds is invalidation — a cached
result from a different environment is re-run instead of reused — and
--allow-env-mismatch reuses whatever is cached regardless of its provenance.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 docs/development/regression-harness.md | 9 ++++++---
 docs/src/developer_notes.md            | 2 +-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/docs/development/regression-harness.md b/docs/development/regression-harness.md
index b91c1c6d6..aeb13ad9b 100644
--- a/docs/development/regression-harness.md
+++ b/docs/development/regression-harness.md
@@ -115,9 +115,12 @@ loudly). If a pinned package set turns out to be incompatible with an old commit
 `Project.toml`, instantiate re-resolves it and the harness warns that the pin did not hold.
 
 **Every run records the environment that produced it** — Julia version, host, resolved Manifest
-hash, Julia and BLAS thread counts — and results are cached under that environment. A cached
-result from a different environment is re-run instead of silently reused, and every report prints
-the environment of each ref:
+hash, Julia and BLAS thread counts. The cache still holds a single result per
+`(commit, case)`, so a re-run replaces the stored one rather than keeping a result per
+environment; what the fingerprint adds is that a cached result whose environment differs from
+the current one is re-run instead of silently reused. `--allow-env-mismatch` skips that check
+and reuses whatever is cached, whatever produced it. Every report prints the environment of
+each ref:
 
 ```
 Ref 1: develop  @ a0cad260 (2026-08-12)
diff --git a/docs/src/developer_notes.md b/docs/src/developer_notes.md
index 4e146a6c3..885a3de3c 100644
--- a/docs/src/developer_notes.md
+++ b/docs/src/developer_notes.md
@@ -223,4 +223,4 @@ regress --cases solovev_n1 --ref-range develop~10..develop
 
 The full flag list, the environment-pinning behaviour that keeps source code the only variable in
 a comparison, and the exit-status contract are documented in
-[`docs/development/regression-harness.md`](../../development/regression-harness.md).
+[`docs/development/regression-harness.md`](https://github.com/OpenFUSIONToolkit/GPEC/blob/develop/docs/development/regression-harness.md).

From 52db9d2e82beb82f924c891142543b1a5e32627f Mon Sep 17 00:00:00 2001
From: Matthew Pharr <m.pharr@protonmail.com>
Date: Fri, 14 Aug 2026 12:09:33 -0400
Subject: [PATCH 3/3] Regression - Minor - claude review items

---
 docs/development/regression-harness.md |  5 +++--
 regression-harness/src/reporter.jl     |  4 ++--
 regression-harness/src/runner.jl       | 31 +++++++++++++++++++-------
 3 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/docs/development/regression-harness.md b/docs/development/regression-harness.md
index aeb13ad9b..bb5217f92 100644
--- a/docs/development/regression-harness.md
+++ b/docs/development/regression-harness.md
@@ -111,8 +111,9 @@ Two mechanisms prevent that:
 
 **The working tree's Manifest is pinned into every worktree** before `Pkg.instantiate()`, so all
 refs in a comparison run against one package set. `--no-pin-manifest` opts out (and says so
-loudly). If a pinned package set turns out to be incompatible with an old commit's
-`Project.toml`, instantiate re-resolves it and the harness warns that the pin did not hold.
+loudly). If a commit declares a direct dependency the pinned Manifest lacks, `Pkg.instantiate()`
+refuses to run: that ref is recorded as a failed run whose error suggests `--no-pin-manifest` to
+let it resolve its own package set.
 
 **Every run records the environment that produced it** — Julia version, host, resolved Manifest
 hash, Julia and BLAS thread counts. The cache still holds a single result per
diff --git a/regression-harness/src/reporter.jl b/regression-harness/src/reporter.jl
index 2bc9ec507..4fdecfbe3 100644
--- a/regression-harness/src/reporter.jl
+++ b/regression-harness/src/reporter.jl
@@ -112,11 +112,11 @@ function report_two_ref_comparison(db::SQLite.DB, case_spec::CaseSpec,
 
     if info1 === nothing
         println("ERROR: No results for ref 1 ($(ref1.name))")
-        return
+        return (n_ok=0, n_changed=0, n_missing=0, n_failed=1)
     end
     if info2 === nothing
         println("ERROR: No results for ref 2 ($(ref2.name))")
-        return
+        return (n_ok=0, n_changed=0, n_missing=0, n_failed=1)
     end
 
     failed1 = !info1.success
diff --git a/regression-harness/src/runner.jl b/regression-harness/src/runner.jl
index 550ea9e04..d8a1a5d2b 100644
--- a/regression-harness/src/runner.jl
+++ b/regression-harness/src/runner.jl
@@ -223,6 +223,7 @@ function _execute_computed(case_spec::CaseSpec, project_root::String;
                          stdout=devnull, stderr=stderr_buf))
         end
         runtime_s, fingerprint = read_runinfo(runinfo_file, pin_manifest !== nothing)
+        isempty(fingerprint.julia_version) && error("subprocess wrote no run-info metadata — does the script template end with %RUNINFO%?")
         _warn_pin_broken(pin_manifest, fingerprint, case_spec.name)
         if !isfile(h5path)
             error("Computed case '$(case_spec.name)' produced no output h5")
@@ -236,6 +237,17 @@ function _execute_computed(case_spec::CaseSpec, project_root::String;
     end
 end
 
+"""
+Add a remedy hint when a run failed because the pinned Manifest is missing a direct dependency
+declared at the checked-out commit — the one incompatibility `Pkg.instantiate()` refuses to run under.
+Detects on the full error text, since tail-keeping truncation can drop Pkg's ERROR line from `short_err`.
+"""
+function _hint_pin_incompatible(full_err::AbstractString, short_err::AbstractString)::String
+    occursin("is a direct dependency, but does not appear in the manifest", full_err) || return String(short_err)
+    return "Pinned Manifest lacks a direct dependency declared at this commit — re-run with --no-pin-manifest " *
+           "to let this ref resolve its own package set.\n" * short_err
+end
+
 """
 Run a kind="computed" case against the working tree.
 """
@@ -265,7 +277,7 @@ function run_computed_local(db::SQLite.DB, case_spec::CaseSpec, repo_root::Strin
         # Keep the tail of the message: Julia errors usually appear at the end
         # of the subprocess output, not the start (Pkg.instantiate output dominates the head).
         # `last` is unicode-safe and won't split a multibyte char like `err_msg[end-N:end]` could.
-        err_msg_short = length(err_msg) > 2000 ? "..." * last(err_msg, 2000) : err_msg
+        err_msg_short = _hint_pin_incompatible(err_msg, length(err_msg) > 2000 ? "..." * last(err_msg, 2000) : err_msg)
         @warn "Run failed (local computed): $(first(err_msg_short, 200))"
         store_failed_run(db, LOCAL_REF, "local", date, "working tree", case_spec.name,
                          err_msg_short)
@@ -319,7 +331,7 @@ function run_computed_at_commit(db::SQLite.DB, commit_hash::String, ref_name::St
         # Keep the tail of the message: Julia errors usually appear at the end
         # of the subprocess output, not the start (Pkg.instantiate output dominates the head).
         # `last` is unicode-safe and won't split a multibyte char like `err_msg[end-N:end]` could.
-        err_msg_short = length(err_msg) > 2000 ? "..." * last(err_msg, 2000) : err_msg
+        err_msg_short = _hint_pin_incompatible(err_msg, length(err_msg) > 2000 ? "..." * last(err_msg, 2000) : err_msg)
         @warn "Run failed (computed) for $(commit_info.short): $(first(err_msg_short, 200))"
         store_failed_run(db, commit_hash, commit_info.short, commit_info.date,
                          commit_info.msg, case_spec.name, err_msg_short)
@@ -331,11 +343,12 @@ function run_computed_at_commit(db::SQLite.DB, commit_hash::String, ref_name::St
 end
 
 """
-Warn when a pinned package set did not survive `Pkg.instantiate()`.
+Warn if a pinned package set did not survive `Pkg.instantiate()`.
 
-The harness copies its Manifest into a worktree, but if that package set is incompatible with the
-checked-out commit's `Project.toml`, instantiate re-resolves and the pin silently stops holding —
-which is exactly the confound pinning exists to remove.
+Contingency insurance rather than a description of current behavior: on Julia 1.11/1.12,
+instantiate never rewrites an out-of-sync pinned Manifest — it warns and proceeds, or errors when
+the commit declares a direct dependency the Manifest lacks (which fails the run loudly). Should a
+future Pkg re-resolve in place instead, this catches the pin silently not holding.
 """
 function _warn_pin_broken(pin_manifest::Union{String,Nothing}, fp::EnvFingerprint, label::AbstractString)
     pin_manifest === nothing && return
@@ -408,6 +421,7 @@ function run_local(db::SQLite.DB, case_spec::CaseSpec, repo_root::String;
                          stdout=devnull, stderr=stderr_buf))
         end
         runtime_s, fingerprint = read_runinfo(runinfo_file, pin_manifest !== nothing)
+        isempty(fingerprint.julia_version) && error("subprocess wrote no run-info metadata — does the script template end with %RUNINFO%?")
         _warn_pin_broken(pin_manifest, fingerprint, case_spec.name)
 
         h5path = joinpath(rundir, "gpec.h5")
@@ -434,7 +448,7 @@ function run_local(db::SQLite.DB, case_spec::CaseSpec, repo_root::String;
         # Keep the tail of the message: Julia errors usually appear at the end
         # of the subprocess output, not the start (Pkg.instantiate output dominates the head).
         # `last` is unicode-safe and won't split a multibyte char like `err_msg[end-N:end]` could.
-        err_msg_short = length(err_msg) > 2000 ? "..." * last(err_msg, 2000) : err_msg
+        err_msg_short = _hint_pin_incompatible(err_msg, length(err_msg) > 2000 ? "..." * last(err_msg, 2000) : err_msg)
         @warn "Run failed (local): $(first(err_msg_short, 200))"
         store_failed_run(db, LOCAL_REF, "local", date, "working tree", case_spec.name,
                          err_msg_short)
@@ -520,6 +534,7 @@ function run_at_commit(db::SQLite.DB, commit_hash::String, ref_name::String,
                          stdout=devnull, stderr=stderr_buf))
         end
         runtime_s, fingerprint = read_runinfo(runinfo_file, pin_manifest !== nothing)
+        isempty(fingerprint.julia_version) && error("subprocess wrote no run-info metadata — does the script template end with %RUNINFO%?")
         _warn_pin_broken(pin_manifest, fingerprint, commit_info.short)
 
         # Check for gpec.h5
@@ -552,7 +567,7 @@ function run_at_commit(db::SQLite.DB, commit_hash::String, ref_name::String,
         # Keep the tail of the message: Julia errors usually appear at the end
         # of the subprocess output, not the start (Pkg.instantiate output dominates the head).
         # `last` is unicode-safe and won't split a multibyte char like `err_msg[end-N:end]` could.
-        err_msg_short = length(err_msg) > 2000 ? "..." * last(err_msg, 2000) : err_msg
+        err_msg_short = _hint_pin_incompatible(err_msg, length(err_msg) > 2000 ? "..." * last(err_msg, 2000) : err_msg)
         @warn "Run failed for $(commit_info.short): $(first(err_msg_short, 200))"
         store_failed_run(db, commit_hash, commit_info.short, commit_info.date,
                          commit_info.msg, case_spec.name, err_msg_short)