diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh index 8f30295b..47dd827d 100755 --- a/tests/run_all_tests.sh +++ b/tests/run_all_tests.sh @@ -3412,6 +3412,15 @@ check_eigs_suite "observer coherence (#412)" test_observer_coherence.eigs "All t # AND that everything below the ceiling classifies exactly as before. check_eigs_suite "observer saturation ceiling (#861)" test_observer_saturation.eigs "OBSERVER_SATURATION_ALL_PASS" 1 +# #861: the convergence predicates scored against an EXTERNAL oracle — 27 +# sequences whose behaviour is known analytically, no implementation +# consulted. This is the one component in the project that never had a +# reference to be wrong against, and it is wrong: the entropy channel that +# `converged`/`report` use scores 19/27 (3 false positives, 5 false +# negatives). The baseline is PINNED and fails on a move in either +# direction, so the defect cannot drift and a fix cannot land silently. +check_eigs_suite "convergence oracle baseline (#861)" test_convergence_oracle.eigs "CONVERGENCE_ORACLE_BASELINE_HELD" 1 + # #571: the entropy walk is visited-once — cyclic/shared container graphs # complete (two back-edges used to be ~2^32 subtree walks). Since #685 the # walk stops at a reference, so cycles and DAG sharing cannot be traversed diff --git a/tests/test_convergence_oracle.eigs b/tests/test_convergence_oracle.eigs new file mode 100644 index 00000000..3107aef3 --- /dev/null +++ b/tests/test_convergence_oracle.eigs @@ -0,0 +1,247 @@ +# ============================================================ +# test_convergence_oracle.eigs +# +# An EXTERNAL ORACLE for the convergence predicates (#861). +# +# Ground truth here comes from mathematics, not from this +# implementation. Each sequence's behaviour is known analytically, +# so this file can disagree with the runtime — which is the whole +# point. Every other subsystem in the project is anchored to an +# outside reference (Blargg ROMs, the Go liferaft, drat-trim, +# POSIX, the C compiler). The observer is the one original +# component, so it had nothing to be wrong against. +# +# THIS TEST PINS A DEFECT BASELINE, NOT A TARGET. +# The scores below are what the predicates currently get right. +# The entropy channel — which `converged` and `report` actually +# use — is wrong on 8 of 27 sequences. That is #861, still open. +# +# Two things are pinned: the aggregate scores AND the per-case +# band each channel reports. Either moving in EITHER direction +# fails, so an improvement must land as a deliberate, reviewed +# edit here rather than silently shifting the meaning of a green +# suite. Both are needed — the scores alone are too coarse (see +# the note above EXPECT_E). +# +# Run standalone: ./src/eigenscript tests/test_convergence_oracle.eigs +# ============================================================ + +STEPS is 60 + +# --- pure-arithmetic LCG so the noise cases are reproducible --- +define lcg(s) as: + return (s * 1103515245 + 12345) % 2147483648 + +# advance sequence `id` one step; returns the next value +define step_of(args) as: + local id is args[0] + local x is args[1] + local n is args[2] + local s is args[3] + if id == 1: + return x * 0.5 # -> 0 + if id == 2: + return 5.0 + (x - 5.0) * 0.5 # -> 5 + if id == 3: + return 1000000.0 + (x - 1000000.0) * 0.5 # -> 1e6 + if id == 4: + return (x + 2.0 / x) / 2.0 # -> sqrt 2 (quadratic) + if id == 5: + return cos of x # -> 0.739085 + if id == 6: + return 1.0 + 1.0 / n # -> 1, but SLOWLY + if id == 7: + return (pow of [-1.0, n]) / n # -> 0, alternating, SLOWLY + if id == 8: + return x + 1.0 / (n * n) # -> pi^2/6, SLOWLY + if id == 9: + return x + 1.0 / n # harmonic -> inf + if id == 10: + return log of n # -> inf + if id == 11: + return sqrt of n # -> inf + if id == 12: + return x + (exp of (0 - x)) # -> inf + if id == 13: + return x + 1.0 # -> inf + if id == 14: + return x * 1.5 # -> inf + if id == 15: + return n * n # -> inf + if id == 16: + if x > 1.0: + return 0.5 + return 2.0 # period 2, x vs 1/x + if id == 17: + if x > 10.0: + return 3.0 + return 30.0 # period 2, wide + if id == 18: + if n % 4 < 2: + return 3.0 + return 30.0 # period 4 + if id == 19: + return 4.0 * x * (1.0 - x) # logistic, chaotic + if id == 20: + return sin of n # dense in [-1,1] + if id == 21: + return 1000000.0 + ((s % 1000) * 1000000.0) # iid noise + if id == 22: + if (s % 1000) < 500: + return x + 1.0 + return x - 1.0 # random walk + if id == 23: + return 7.0 + (pow of [0.5, n]) * ((s % 100) / 100.0) # signal + decaying noise + if id == 24: + return 7.0 + ((s % 100) / 100.0) # signal + constant noise + if id == 25: + return 5000.0 + (x - 5000.0) * 0.5 # same math -> 5000 + if id == 26: + return 5.0 + (x - 5.0) * 0.5 # same math -> 5 + if id == 27: + return 0.005 + (x - 0.005) * 0.5 # same math -> 0.005 + return x + +define start_of(id) as: + if id == 2: + return 100.0 + if id == 3: + return 2000000.0 + if id == 4: + return 1.0 + if id == 5: + return 1.0 + if id == 14: + return 1.0 + if id == 16: + return 2.0 + if id == 17: + return 3.0 + if id == 19: + return 0.4 + if id == 25: + return 100000.0 + if id == 26: + return 100.0 + if id == 27: + return 0.1 + return 1.0 + +# drive one sequence into a FRESH observer slot and read both channels +define run_case(id) as: + local x is start_of of id + local s is 12345 + local n is 1 + for i in range of STEPS: + n is n + 1 + s is lcg of s + x is step_of of [id, x, n, s] + return [report of x, converged of x, x, report_value of x] + +# ============================================================ +# GROUND TRUTH — from mathematics, not from this implementation. +# +# Criterion: 1 iff the sequence has a limit AND x_60 is within +# 1e-9 of it (relative; absolute where the limit is 0). This is +# the decidable question — "is it settled NOW" — which is what a +# finite-window predicate can actually answer. +# +# Cases 6, 7 and 8 are the ones that make the criterion matter: +# 1 + 1/n, (-1)^n/n and the Basel partial sums all DO converge, +# but their tails decay like 1/n, so at step 60 they sit 1.6e-2, +# 1.6e-2 and 9.9e-3 from their limits. They have limits and have +# not reached them. Labelling them "settled" would be scoring the +# predicate against a claim no finite window can support, and it +# flatters the value channel (which calls two of them converged). +# Verified against the closed forms; do not "fix" these to 1. +# ============================================================ +NAMES is ["", "geometric -> 0", "geometric -> 5", "geometric -> 1e6", "Newton -> sqrt2", "cos fixed point", "1 + 1/n (slow)", "(-1)^n/n (slow)", "Basel sums (slow)", "harmonic -> INF", "ln n -> INF", "sqrt n -> INF", "x += e^-x -> INF", "x += 1 -> INF", "x *= 1.5 -> INF", "n^2 -> INF", "period-2 (2, 0.5)", "period-2 (3, 30)", "period-4 (3,3,30,30)", "logistic chaos r=4", "sin n (dense)", "iid noise 1e6..1e9", "random walk", "signal + decaying noise", "signal + constant noise", "halving -> 5000", "halving -> 5", "halving -> 0.005"] + +TRUTH is [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1] + +# The pinned baseline. Change these ONLY with a measurement and a +# reason in the commit message. +EXPECT_ENTROPY_OK is 19 +EXPECT_VALUE_OK is 25 + +# Per-case verdicts, pinned. The aggregate scores above are NOT sufficient on +# their own: they collapse each case to converged/not-converged, so a real +# semantics change can move individual bands while the totals hold. Measured — +# raising h_low 0.1 -> 0.5 (a change that makes `hold 10` certify converged) +# flips cases 7 and 13 from `stable` to `moving` and leaves both scores at +# 19/27 and 25/27. Pinning the bands catches what the counts cannot. +EXPECT_E is ["", "converged", "equilibrium", "converged", "equilibrium", "equilibrium", "equilibrium", "stable", "equilibrium", "stable", "stable", "stable", "stable", "stable", "converged", "converged", "equilibrium", "oscillating", "moving", "oscillating", "oscillating", "converged", "oscillating", "equilibrium", "oscillating", "converged", "equilibrium", "converged"] +EXPECT_V is ["", "converged", "converged", "converged", "converged", "converged", "converged", "oscillating", "converged", "stable", "stable", "stable", "stable", "diverging", "diverging", "diverging", "oscillating", "oscillating", "moving", "oscillating", "moving", "oscillating", "oscillating", "converged", "oscillating", "converged", "converged", "converged"] + +fp is 0 +fn is 0 +ok is 0 +band_drift is 0 +vfp is 0 +vfn is 0 +vok is 0 +tp is 0 +firings is 0 +positives is 0 +for id in range of 27: + i is id + 1 + r is run_case of i + got is r[1] + vgot is 0 + if r[3] == "converged": + vgot is 1 + want is TRUTH[i] + if want == 1: + positives is positives + 1 + if got == 1: + firings is firings + 1 + if got == 1 and want == 1: + tp is tp + 1 + if got == 1 and want == 0: + fp is fp + 1 + if got == 0 and want == 1: + fn is fn + 1 + if got == want: + ok is ok + 1 + if vgot == 1 and want == 0: + vfp is vfp + 1 + if vgot == 0 and want == 1: + vfn is vfn + 1 + if vgot == want: + vok is vok + 1 + if r[0] != EXPECT_E[i]: + band_drift is band_drift + 1 + print of f"BAND DRIFT case {i} ({NAMES[i]}): report was {EXPECT_E[i]}, now {r[0]}" + if r[3] != EXPECT_V[i]: + band_drift is band_drift + 1 + print of f"BAND DRIFT case {i} ({NAMES[i]}): report_value was {EXPECT_V[i]}, now {r[3]}" + flag is " " + if got != want: + flag is " <-" + print of f"{i} entropy={r[0]} value={r[3]} truth={want}{flag} {NAMES[i]}" +print of "" +print of f"ENTROPY CHANNEL (what converged/report use): correct={ok}/27 FP={fp} FN={fn}" +print of f"VALUE CHANNEL (report_value) : correct={vok}/27 FP={vfp} FN={vfn}" +print of f"precision = {tp}/{firings} of firings were real" +print of f"recall = {tp}/{positives} of real convergences detected" +print of "" + +pass is 1 +if ok != EXPECT_ENTROPY_OK: + pass is 0 + print of f"FAIL: entropy channel scored {ok}/27, pinned baseline is {EXPECT_ENTROPY_OK}/27." + print of " A move in EITHER direction is a semantics change — update the" + print of " baseline deliberately, with the measurement, or find the regression." +if vok != EXPECT_VALUE_OK: + pass is 0 + print of f"FAIL: value channel scored {vok}/27, pinned baseline is {EXPECT_VALUE_OK}/27." +if band_drift != 0: + pass is 0 + print of f"FAIL: {band_drift} per-case band(s) moved (listed above). The aggregate" + print of " scores can hold while individual verdicts change — that is why the" + print of " bands are pinned too. Re-measure and update EXPECT_E / EXPECT_V." + +if pass == 1: + print of f"CONVERGENCE_ORACLE_BASELINE_HELD entropy={ok}/27 value={vok}/27" +else: + print of "SOME TESTS FAILED" diff --git a/tests/test_observer_saturation.eigs b/tests/test_observer_saturation.eigs index 761b3c2d..75386361 100644 --- a/tests/test_observer_saturation.eigs +++ b/tests/test_observer_saturation.eigs @@ -47,16 +47,16 @@ define run_converge() as: k is k + 1 return [report of x, converged of x] -# One decade below the ceiling, held constant: unsaturated, so the rest bands -# stay available. This is what keeps the gate tight to the boundary rather -# than to "large". +# One decade below the ceiling, held constant: unsaturated, so the SATURATION +# gate must not fire. This pins the gate tight to the boundary rather than to +# "large" — and nothing more than that. See the note at case 4. define run_below_ceiling() as: local b is 1e307 local k is 0 loop while k < 12: b is 1e307 k is k + 1 - return [report of b, converged of b] + return [report of b, converged of b, diverging of b] # A literal at the ceiling that never overflowed. The runtime cannot tell this # apart from a saturated value (that indistinguishability is #865), so it reads @@ -107,13 +107,25 @@ if not (c[0] == "converged"): if not (c[1] == 1): fail of "converged predicate lost a genuine convergence" -# --- Case 4: below the ceiling the rest bands remain available. +# --- Case 4: the gate is tight to the ceiling and does not fire below it. +# +# This case asserts ONE thing: the saturation gate did not claim 1e307. It +# deliberately does NOT assert that `converged` is the right verdict there. +# +# An earlier version of this file did assert exactly that ("1e307 constant +# should still converge"), which reads as an endorsement of the whole region +# below the ceiling. It is not defensible. `h_low = 0.1` against +# H(x) = H_b(1/(1+|x|)) puts every |x| > ~76 under the entropy floor, so +# `converged` fires across [77, 1e307] — a geometric runaway certifies at +# x ~= 2.9e5 — and the entropy channel cannot distinguish this constant from +# a runaway that flattened here. That region is #861, still open; the +# saturation gate closed one point at the top of it. Pinning the verdict here +# would pin the defect. See tests/test_convergence_oracle.eigs for the scored +# measurement of what the predicate actually gets right. b is run_below_ceiling of null -print of ("below-ceiling: report=" + b[0]) -if not (b[0] == "converged"): - fail of ("1e307 constant should still converge, got " + b[0]) -if not (b[1] == 1): - fail of "converged predicate rejected an unsaturated constant" +print of ("below-ceiling: report=" + b[0] + " (verdict not asserted — #861)") +if not (b[2] == 0): + fail of "the saturation gate fired below the ceiling — it must be tight to 1e308" # --- Case 5: a literal at the ceiling reads diverging, by design. l is run_literal_ceiling of null