From ca3a5b182264315eb8ccf7a2d7c21d346d99e5c8 Mon Sep 17 00:00:00 2001 From: mattral Date: Thu, 8 Oct 2026 14:45:23 +0900 Subject: [PATCH] docs: pre-registration Amendment 1 (power, underpowered reading) Made before any run-2 data exists. Adds a Rule B power table from scripts/power_rule_b.py (Gaussian copula, 300 sims/cell, seed 0), fixes that a Rule B failure with rho_gap >= 0.80 reads "underpowered to settle it", adds the non-exchangeable-conditions limitation, and pre-registers blind-mode testing of the analysis script on real footage. Co-Authored-By: Claude Opus 5.5 --- docs/v1_run2_preregistration.md | 75 ++++++++++++++++++++- scripts/power_rule_b.py | 113 ++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+), 2 deletions(-) create mode 100755 scripts/power_rule_b.py diff --git a/docs/v1_run2_preregistration.md b/docs/v1_run2_preregistration.md index 76ee181..4944dfc 100644 --- a/docs/v1_run2_preregistration.md +++ b/docs/v1_run2_preregistration.md @@ -5,6 +5,26 @@ recording exists**. Its git commit is its identity. Nothing below may be changed after run-2 data is recorded; any departure during the run is reported in the results as a deviation from this document, not edited in. +### Amendments + +**Amendment 1 — 2026-10-08, made before any run-2 data existed.** The +original text is in git history (merge `c153902`). Amended passages are marked +*[A1]*. + +- **Why:** the original lacked a power analysis for Rule B, and its §8.4 read + every Rule B failure as "no claim of added value", including failures where + the test simply had little chance to pass. Interpreting those afterwards + would look like an excuse; fixing the reading now does not. +- **§8.4:** a Rule B failure with ρ_gap ≥ 0.80 now reads *"underpowered to + settle it"*, not *"no added value"*. The original row was: "pass | fail | + The gap score tracks degradation, but not demonstrably better than counting + dropouts. No claim of added value." +- **§13:** adds a Rule B power table (with its assumptions, so it can be + re-run) and a limitation: the 24 conditions are 4 factors × 6 ordered + severities, not 24 exchangeable draws. +- **§12:** adds the testing protocol for the analysis script before run-2 + data exists (synthetic data, and run 1's footage in blind mode). + Run 1 (`v1_first_run_results.md`) was null and underpowered: the conditions barely made MediaPipe fail, take-to-take noise was comparable to the between-condition signal, and spec 5.2 normalization was missing. Run 2 is @@ -233,7 +253,8 @@ valid resamples is reported. For reference, in the pilot ρ_base was **+0.56** over these 24 conditions. With 24 conditions, passing Rule B is expected to need ρ_gap well above that. This is -stated in advance so a near-miss is not later read as a success. +stated in advance so a near-miss is not later read as a success. *[A1]* The +estimated power is in §13: about 17–51% at ρ_gap = 0.80 and 54–98% at 0.90. ### 8.3 Secondary analyses (descriptive; no decision attached) @@ -255,9 +276,13 @@ stated in advance so a near-miss is not later read as a success. | Rule A | Rule B | What will be said | |---|---|---| | pass | pass | On one subject and one camera, across 24 pre-registered software-degraded conditions, the gap score ranked MediaPipe landmark degradation, and did so better than counting dropouts. Not "validated" in general. | -| pass | fail | The gap score tracks degradation, but not demonstrably better than counting dropouts. No claim of added value. | +| pass | fail, **ρ_gap ≥ 0.80** *[A1]* | The gap score tracks degradation. Whether it adds value over counting dropouts is **not settled: the test was underpowered** to detect an advantage (§13, power ≈ 17–51% at ρ_gap = 0.80). This is **not** evidence of no added value. | +| pass | fail, **ρ_gap < 0.80** *[A1]* | The gap score tracks degradation, but not demonstrably better than counting dropouts. No claim of added value. | | fail | — | The gap score did not track landmark degradation across these conditions. Reported as a negative result. | +*[A1]* ρ_gap here is the point estimate of §8.1. The 0.80 threshold is fixed +now, before data, from the §13 power table; it is not moved afterwards. + None of these outcomes is generalised beyond one person, one camera and simulated degradations of real footage. @@ -310,6 +335,13 @@ load on the machine; run 2 is therefore cached and run overnight. - The run-2 analysis script, which implements exactly §5–§8 including caching, is committed **before any run-2 recording is processed**. Its commit hash and `study_settings.json` are recorded with the results. +- *[A1]* **Testing before run-2 data exists.** The analysis script + (`scripts/run_v1_run2.py`) is tested end to end in CI on synthetic videos, + and on run 1's footage in **blind mode** (`--blind`). Blind mode runs every + stage (extraction, caching, fitting, gap scores, both rules, secondary + analyses) but writes only a health report and discards every score and + correlation unread. A non-blind run on real footage, which would preview the + result for these exact conditions, is not made before run 2. - Recordings stay on the recording machine (gitignored; spec 12.16). Results are published as numbers only. @@ -330,3 +362,42 @@ load on the machine; run 2 is therefore cached and run overnight. orders severities within each factor correctly. §8.3 item 5 reports this. - 24 conditions limit the precision of every correlation, and of Δ most of all. +- *[A1]* **The 24 conditions are not exchangeable draws.** They are 4 factors + × 6 ordered severities. The bootstrap resamples them as if they were + exchangeable, ignoring that severities within a factor are related, so every + CI is **narrower than the dependence structure warrants**, Δ's most of all. + A Rule A or B pass should be read with that in mind. + +### Rule B power *[A1]* (an estimate) + +Probability that Rule B passes (95% paired-bootstrap CI of Δ above 0), for +n = 24 and ρ_base = 0.56, by the true ρ_gap and the unknown correlation +between the gap score and the dropout baseline: + +| ρ_gap | r(gap, base) = 0.2 | 0.4 | 0.6 | 0.8 | range | +|---|---:|---:|---:|---:|---:| +| 0.70 | 0.08 | 0.11 | 0.10 | 0.21 | 0.08–0.21 | +| 0.80 | 0.17 | 0.25 | 0.33 | 0.51 | 0.17–0.51 | +| 0.85 | 0.34 | 0.41 | 0.50 | 0.76 | 0.34–0.76 | +| 0.90 | 0.54 | 0.57 | 0.77 | 0.98 | 0.54–0.98 | +| 0.95 | infeasible | 0.88 | 0.97 | infeasible | 0.88–0.97 | + +Assumptions, so the numbers can be reproduced with +`python scripts/power_rule_b.py --sims 300` (seed 0): + +- The 24 conditions are exchangeable draws from a **Gaussian copula** over + (landmark error, gap score, baseline). Spearman targets are converted to + latent Pearson correlations as r = 2·sin(π·ρ/6). +- **300 simulated studies per cell**, each analysed exactly as pre-registered + (paired percentile bootstrap, 10,000 resamples, 95%). The Monte Carlo + error is about ±3 percentage points per cell. +- Scores are continuous, with no ties. Real dropout ties heavily at 0%, so the + baseline will have coarser resolution than modelled. +- Cells whose correlation matrix is not positive definite are infeasible. +- The real conditions violate the exchangeability assumption (see the + limitation above). + +An independent simulation by the author under the same copula model gave +similar ranges (0.70: 9–16%; 0.80: 29–47%; 0.85: 48–70%; 0.90: 75–86%; 0.95: +94–98%). The differences come from the assumed range of r(gap, base) and from +Monte Carlo error. diff --git a/scripts/power_rule_b.py b/scripts/power_rule_b.py new file mode 100755 index 0000000..d4bf02a --- /dev/null +++ b/scripts/power_rule_b.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Power of run 2's Rule B (docs/v1_run2_preregistration.md, Amendment 1). + +Rule B passes when the 95% paired-bootstrap CI of + Δ = Spearman(gap, truth) − Spearman(baseline, truth) +has its lower bound above 0, across n conditions. This estimates how often +that happens for a given true ρ_gap, with ρ_base fixed at the pilot's value. + +Model (an assumption, stated so the numbers are reproducible): +- The n conditions are exchangeable draws from a Gaussian copula over + (truth, gap, baseline). Target Spearman correlations are converted to the + latent Pearson correlations with r = 2·sin(π·ρ_s / 6). +- Scores are continuous (no ties). Real dropout has many ties at 0%, so this + is optimistic about the baseline's resolution. +- Each simulated study is analysed exactly as pre-registered: percentile + bootstrap over conditions, paired (the same resample for both + correlations), 10,000 resamples, 95%; resamples where a correlation is + undefined are discarded. +- corr(gap, baseline) is unknown in advance, so power is reported across a + grid of values; cells whose correlation matrix is not positive definite + are skipped. + + python scripts/power_rule_b.py --sims 300 +""" + +from __future__ import annotations + +import argparse +import json +import sys + +import numpy as np +from scipy.stats import rankdata + + +def spearman_to_pearson(rho_s: float) -> float: + return 2.0 * np.sin(np.pi * rho_s / 6.0) + + +def rowwise_spearman(a: np.ndarray, b: np.ndarray) -> np.ndarray: + """Spearman per row of two (B, n) arrays (average ranks for ties).""" + ra = rankdata(a, axis=1) + rb = rankdata(b, axis=1) + ra -= ra.mean(axis=1, keepdims=True) + rb -= rb.mean(axis=1, keepdims=True) + num = (ra * rb).sum(axis=1) + den = np.sqrt((ra * ra).sum(axis=1) * (rb * rb).sum(axis=1)) + with np.errstate(invalid="ignore", divide="ignore"): + return num / den + + +def rule_b_passes(truth, gap, base, n_boot: int, rng) -> bool: + n = truth.shape[0] + idx = rng.integers(0, n, size=(n_boot, n)) + t = truth[idx] + d = rowwise_spearman(gap[idx], t) - rowwise_spearman(base[idx], t) + d = d[np.isfinite(d)] + return d.size > 0 and np.percentile(d, 2.5) > 0 + + +def power(n: int, rho_gap: float, rho_base: float, rho_gap_base: float, sims: int, n_boot: int, seed: int): + r = np.array([ + [1.0, spearman_to_pearson(rho_gap), spearman_to_pearson(rho_base)], + [spearman_to_pearson(rho_gap), 1.0, spearman_to_pearson(rho_gap_base)], + [spearman_to_pearson(rho_base), spearman_to_pearson(rho_gap_base), 1.0], + ]) + if np.linalg.eigvalsh(r).min() <= 1e-9: + return None # infeasible combination + rng = np.random.default_rng(seed) + chol = np.linalg.cholesky(r) + hits = 0 + for _ in range(sims): + z = rng.standard_normal((n, 3)) @ chol.T + hits += rule_b_passes(z[:, 0], z[:, 1], z[:, 2], n_boot, rng) + return hits / sims + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--n", type=int, default=24) + ap.add_argument("--rho-base", type=float, default=0.56) + ap.add_argument("--rho-gap", type=float, nargs="*", default=[0.70, 0.80, 0.85, 0.90, 0.95]) + ap.add_argument("--rho-gap-base", type=float, nargs="*", default=[0.2, 0.4, 0.6, 0.8]) + ap.add_argument("--sims", type=int, default=300) + ap.add_argument("--boot", type=int, default=10_000) + ap.add_argument("--seed", type=int, default=0) + ap.add_argument("--json", action="store_true", help="print the table as JSON") + args = ap.parse_args() + + table = {} + for i, rg in enumerate(args.rho_gap): + for j, rgb in enumerate(args.rho_gap_base): + p = power(args.n, rg, args.rho_base, rgb, args.sims, args.boot, args.seed + 1000 * i + j) + table[(rg, rgb)] = p + + if args.json: + print(json.dumps([{"rho_gap": k[0], "rho_gap_base": k[1], "power": v} for k, v in table.items()])) + return 0 + + head = "".join(f" r(gap,base)={c:<4}" for c in args.rho_gap_base) + print(f"Rule B power, n={args.n}, rho_base={args.rho_base}, {args.sims} sims/cell, {args.boot} resamples") + print(f"rho_gap {head} range") + for rg in args.rho_gap: + vals = [table[(rg, c)] for c in args.rho_gap_base] + cells = "".join(f" {'infeasible':>16}" if v is None else f" {v:>16.2f}" for v in vals) + ok = [v for v in vals if v is not None] + rng_txt = f"{min(ok):.2f}-{max(ok):.2f}" if ok else "-" + print(f"{rg:<7} {cells} {rng_txt}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())