From c74980a3ea89474d4f0234ccda35068b7c36738e Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Fri, 31 Jul 2026 13:51:40 +0200 Subject: [PATCH 1/3] fix(loadmodel): stop the outlier filter locking out a new load level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A rejected sample updates neither MAE nor Samples, so the band max(MAE × 10, 200) could never grow in response to being persistently wrong. With the filter arming after 50 samples — 51 minutes at the 60 s cadence — a model calibrated on one quiet hour rejected the real house permanently. Measured on a clean NewModel(10000): after one overnight hour at 400 W: samples=60 MAE=57.0 band=570.5 then four hours at 5 kW: accepted=0 rejected=240 prediction for that hour: 1794 W (truth 5000 W) after a full week at 5 kW: 1794 W — MAE still 57.0 100% rejection, permanently, because MAE only moves on accepted samples and nothing is accepted. The failure is silent and reads as a forecast problem from outside: a small reported error next to an arbitrarily large real one. Three changes: - Hard bound at 3 × PeakW, always on. Above the site's rated draw a reading is a fault, not a household. Absolute and derived from configured hardware, so it holds from the first sample and cannot be talked down by a model that has mislearned — this is what keeps the first day safe now that the soft filter arms later. - The soft filter arms after a day rather than an hour, so its band spans a night-and-day cycle instead of whichever hour followed a restart. - Ten consecutive same-direction rejections widen the band by exactly enough to admit the residual; the ordinary EMA takes over from there. Accepting the one sample and resetting was not enough — the band stayed narrow, the next nine were rejected too, and a day at a new level only reached 1650 of 3000 W. After: the 3-minute 6 kW spike is still rejected in full, a sustained shift is picked up within ten minutes, and every trained bucket tracks it exactly. Refs #739 Co-Authored-By: Claude Opus 5 --- .changeset/loadmodel-outlier-lockout.md | 33 ++++++++++ go/internal/loadmodel/model.go | 80 +++++++++++++++++++++++- go/internal/loadmodel/model_test.go | 82 +++++++++++++++++++++++++ 3 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 .changeset/loadmodel-outlier-lockout.md diff --git a/.changeset/loadmodel-outlier-lockout.md b/.changeset/loadmodel-outlier-lockout.md new file mode 100644 index 00000000..01662663 --- /dev/null +++ b/.changeset/loadmodel-outlier-lockout.md @@ -0,0 +1,33 @@ +--- +"ftw": patch +--- + +Load model: the outlier filter could lock the model out of learning a load +level it had not seen before, permanently. A rejected sample updates +neither `MAE` nor `Samples`, so the band `max(MAE × 10, 200)` never grew in +response to being persistently wrong; combined with the filter arming after +50 samples — 51 minutes at the 60 s cadence — a model calibrated on one +quiet hour rejected the real house forever. + +Measured on a clean model: an hour at 400 W left MAE at 57 W and the band +at 570 W. A following week at 5 kW was rejected in full, 100% of samples, +and the prediction never moved off 1794 W. The failure is silent and looks +like a bad forecast from the outside, because the model reports a small +error while being arbitrarily wrong. + +Three changes: + +- A hard bound at `3 × PeakW`, always on. A sample above the site's rated + draw is a measurement fault, not a household. It is absolute — derived + from configured hardware, never from what the model has learned — so it + holds from the first sample and cannot be widened by a model that has + mislearned. +- The soft MAE filter now arms after a day of samples rather than an hour, + so its band reflects a full night-and-day cycle instead of whichever + arbitrary hour followed the last restart. +- Ten consecutive same-direction rejections widen the band by exactly + enough to admit the residual, after which the ordinary EMA takes over. + Spikes are short and alternate in sign; a real level shift is sustained. + +A three-minute 6 kW spike is still rejected in full. A sustained shift to a +new level is picked up within ten minutes and tracked exactly. diff --git a/go/internal/loadmodel/model.go b/go/internal/loadmodel/model.go index dc4d1f8d..4b46adfd 100644 --- a/go/internal/loadmodel/model.go +++ b/go/internal/loadmodel/model.go @@ -63,6 +63,29 @@ const HeatingMinDeltaT = 3.0 // home. Clamp prevents one anomalous sample from blowing up the fit. const HeatingCoefMaxW = 1500.0 +// implausibleLoadFactor bounds a single sample against the site's rated +// power. Above this the reading is a fault, not consumption. Same factor +// Predict clamps its output with, so training and prediction agree on +// what counts as physically possible. +const implausibleLoadFactor = 3.0 + +// outlierArmSamples is how much history the outlier filter needs before +// it starts rejecting anything. The old threshold of 50 armed the filter +// after 51 minutes at the 60 s sample cadence — calibrating a whole +// house's plausible-residual band on one arbitrary hour, usually a quiet +// one, because that is when a restart is least disruptive. A day of +// samples spans at least one full night-and-day cycle, so MAE reflects +// the range the house actually moves through. +const outlierArmSamples = 24 * 60 + +// outlierLevelShiftRun is how many consecutive same-direction rejections +// mean "the level moved" rather than "a spike". At the 60 s cadence this +// is ten minutes of the model being wrong the same way before it concedes +// and starts learning again. Short enough that a genuine shift is picked +// up within the hour, long enough that an oven or a car starting to +// charge is still filtered as the transient it is. +const outlierLevelShiftRun = 10 + // Profile selects which learned occupancy profile is used for training // and prediction. type Profile string @@ -104,6 +127,12 @@ type Model struct { MAE float64 `json:"mae"` Alpha float64 `json:"alpha"` // EMA coefficient for bucket updates PriorScale float64 `json:"prior_scale,omitempty"` + + // RejectRun counts consecutive same-direction outlier rejections. + // It is the model's only way to notice that it is not filtering noise + // but refusing reality — see the outlier filter in Update. + RejectRun int `json:"reject_run,omitempty"` + RejectRunPositive bool `json:"reject_run_positive,omitempty"` } // typicalPrior returns an approximate W load for a given hour-of-week @@ -273,11 +302,58 @@ func (m *Model) Update(t time.Time, actualLoadW, tempC float64) (updated bool) { } } + // Hard sanity bound, always on. A residual this far above the site's + // rated draw is a measurement fault, not a household. It is deliberately + // absolute — derived from configured hardware, never from what the model + // has learned — so unlike the MAE band below it cannot be talked down by + // a model that has mislearned, and it protects the first day before the + // soft filter arms. Mirrors the ceiling Predict already applies. + if m.PeakW > 0 && actualLoadW > implausibleLoadFactor*m.PeakW { + return false + } + // Outlier filter: once we have some history, reject 10× MAE residuals. - if m.Samples > 50 { + // + // Two guards keep this from locking the model out of a load level it + // has not seen before. A rejected sample updates neither MAE nor + // Samples, so without them the band can never grow in response to + // being persistently wrong, and a model calibrated on one quiet hour + // rejects the real house forever. Measured before the fix: an hour of + // 400 W overnight load gave MAE 57 W and a 570 W band; a subsequent + // week at 5 kW was rejected in full, 100% of samples, and the + // prediction never moved off 1794 W. + if m.Samples > outlierArmSamples { band := math.Max(m.MAE*10, 200) if math.Abs(err) > band { - return false + // A run of same-direction rejections is not noise — it is the + // house telling us the level moved. Spikes are short and + // alternate in sign; a real shift is sustained. Let the run + // through so the band can re-fit, and keep the filter's actual + // job (rejecting the isolated spike) intact. + sameDirection := (err > 0) == (m.RejectRunPositive) + if m.RejectRun > 0 && sameDirection { + m.RejectRun++ + } else { + m.RejectRun = 1 + m.RejectRunPositive = err > 0 + } + if m.RejectRun < outlierLevelShiftRun { + return false + } + // The run is long enough to be real. Widening the band by + // exactly enough to admit this residual is what lets the model + // re-fit: simply accepting the one sample and resetting the run + // leaves the band untouched, so the next nine are rejected too + // and the model crawls in at one sample in ten. Measured that + // way, a day at a new 3 kW level only reached 1650 W. + // + // max() means the band never shrinks here, and the ordinary EMA + // below takes over from this point — so the widening is a floor + // set by observation, not a permanent loosening. + m.MAE = math.Max(m.MAE, math.Abs(err)/10) + m.RejectRun = 0 + } else { + m.RejectRun = 0 } } diff --git a/go/internal/loadmodel/model_test.go b/go/internal/loadmodel/model_test.go index 3eff2396..38609578 100644 --- a/go/internal/loadmodel/model_test.go +++ b/go/internal/loadmodel/model_test.go @@ -329,3 +329,85 @@ func TestHeatingFitWaitsForBucketTrust(t *testing.T) { t.Errorf("untrusted bucket must not drive heating fit, coef = %.0f", m.HeatingW_per_degC) } } + +// A model calibrated on one quiet hour used to reject the real house +// forever: a rejected sample updates neither MAE nor Samples, so the band +// could never grow in response to being persistently wrong. Measured +// before the fix — an hour at 400 W gave a 570 W band, and a subsequent +// week at 5 kW was rejected in full, with the prediction stuck at 1794 W. +func TestSustainedLevelShiftIsLearned(t *testing.T) { + m := NewModel(10000) + start := time.Date(2026, 7, 1, 2, 0, 0, 0, time.UTC) + + // Two quiet days, enough to arm the outlier filter on a narrow band. + n := 0 + for ; n < 2*24*60; n++ { + m.Update(start.Add(time.Duration(n)*time.Minute), 400, HeatingReferenceC) + } + if band := math.Max(m.MAE*10, 200); band > 600 { + t.Fatalf("expected a narrow band after quiet training, got %.0f", band) + } + + // The house moves to 3 kW and stays there for three days. + trainedFrom := n + for i := 0; i < 3*24*60; i++ { + m.Update(start.Add(time.Duration(n)*time.Minute), 3000, HeatingReferenceC) + n++ + } + + // Every hour-of-week bucket the run covered must have followed. + for i := trainedFrom; i < n; i += 60 { + got := m.Predict(start.Add(time.Duration(i)*time.Minute), HeatingReferenceC) + if math.Abs(got-3000) > 300 { + t.Fatalf("bucket at minute %d predicts %.0f W, want ~3000 W", i, got) + } + } +} + +// The level-shift concession must not cost us the filter's actual job. +func TestShortSpikeIsStillRejected(t *testing.T) { + m := NewModel(10000) + start := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC) + n := 0 + for ; n < 2*24*60; n++ { + m.Update(start.Add(time.Duration(n)*time.Minute), 400, HeatingReferenceC) + } + + accepted := 0 + for i := 0; i < outlierLevelShiftRun-1; i++ { + if m.Update(start.Add(time.Duration(n)*time.Minute), 6000, HeatingReferenceC) { + accepted++ + } + n++ + } + if accepted != 0 { + t.Errorf("a spike shorter than the level-shift run was accepted %d times", accepted) + } + + // Returning to normal must reset the run, so two separate spikes never + // add up to a level shift. + m.Update(start.Add(time.Duration(n)*time.Minute), 400, HeatingReferenceC) + n++ + if m.RejectRun != 0 { + t.Errorf("reject run = %d after a normal sample, want 0", m.RejectRun) + } +} + +// The hard bound is absolute and derived from configured hardware, so it +// holds from the very first sample — before the MAE band arms — and cannot +// be widened by a mislearned model. +func TestImplausibleLoadRejectedBeforeFilterArms(t *testing.T) { + m := NewModel(4000) + t0 := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + if m.Update(t0, 50000, HeatingReferenceC) { + t.Error("50 kW on a 4 kW site should never be accepted") + } + if m.Samples != 0 { + t.Errorf("rejected sample must not count, samples = %d", m.Samples) + } + // Just under the bound still trains — the guard is for faults, not for + // houses that occasionally draw hard. + if !m.Update(t0, 3*4000-1, HeatingReferenceC) { + t.Error("a load just under the bound should be accepted") + } +} From 6244f77df816ad8a5f1fce33610b58cd77934b6b Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Fri, 31 Jul 2026 14:37:00 +0200 Subject: [PATCH 2/3] fix(loadmodel): anchor outlier rejection to the fuse, not to learned MAE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MAE band rejected the upper half of the real load distribution, and could lock the model out of a level it had not seen before, permanently. A rejected sample updates neither MAE nor Samples, so the band max(MAE × 10, 200) never grew in response to being persistently wrong. Measured on a clean NewModel(10000): after one overnight hour at 400 W: samples=60 MAE=57.0 band=570.5 then four hours at 5 kW: accepted=0 rejected=240 prediction for that hour: 1794 W (truth 5000 W) after a full week at 5 kW: 1794 W — MAE still 57.0 The first fix attempt kept the band and let a run of same-direction rejections widen it. That worked, but it was propping up the wrong instrument. Household load is multimodal — a few hundred watts of baseline, then 11 kW when the sauna, oven and car overlap — and nothing about a residual's size separates "unusual but real" from "wrong". A band fitted to the quiet hours always excludes the busy ones, so the filter's normal operation, not just its failure mode, was discarding real consumption. It was also redundant. Telemetry already runs a Kalman filter per signal and this model reads the smoothed values, so short-term noise is handled one layer down, where it belongs. What remains is the rejection physics licenses: a house cannot draw more than its main fuse passes. The ceiling is fuse capacity × 1.25, threaded in from configuration rather than derived from loadPeakW — that one is a tunable proxy for "typical peak", this is physics, and deriving the second from the first would silently move a safety bound whenever somebody retuned the proxy. No fuse configured disables the check rather than inventing a limit. A sustained shift is now learned directly. A one-minute spike still trains, because it is real, and the bucket EMA damps it to a tenth of the gap. Refs #739 Co-Authored-By: Claude Opus 5 --- .changeset/loadmodel-outlier-lockout.md | 49 +++++----- go/cmd/ftw/main.go | 10 +- go/internal/api/api_loadmodel_test.go | 4 +- go/internal/calendar/e2e_test.go | 2 +- go/internal/loadmodel/model.go | 109 ++++++--------------- go/internal/loadmodel/model_test.go | 122 +++++++++++++++--------- go/internal/loadmodel/service.go | 11 ++- go/internal/loadmodel/service_test.go | 16 ++-- 8 files changed, 161 insertions(+), 162 deletions(-) diff --git a/.changeset/loadmodel-outlier-lockout.md b/.changeset/loadmodel-outlier-lockout.md index 01662663..f4c95218 100644 --- a/.changeset/loadmodel-outlier-lockout.md +++ b/.changeset/loadmodel-outlier-lockout.md @@ -2,32 +2,33 @@ "ftw": patch --- -Load model: the outlier filter could lock the model out of learning a load -level it had not seen before, permanently. A rejected sample updates -neither `MAE` nor `Samples`, so the band `max(MAE × 10, 200)` never grew in -response to being persistently wrong; combined with the filter arming after -50 samples — 51 minutes at the 60 s cadence — a model calibrated on one -quiet hour rejected the real house forever. +Load model: replace the MAE-band outlier filter with a physical ceiling +taken from the main fuse. The band rejected the upper half of the real +load distribution, and could lock the model out of a level it had not seen +before, permanently. +A rejected sample updates neither `MAE` nor `Samples`, so the band +`max(MAE × 10, 200)` never grew in response to being persistently wrong. Measured on a clean model: an hour at 400 W left MAE at 57 W and the band -at 570 W. A following week at 5 kW was rejected in full, 100% of samples, -and the prediction never moved off 1794 W. The failure is silent and looks -like a bad forecast from the outside, because the model reports a small -error while being arbitrarily wrong. +at 570 W; a following week at 5 kW was rejected in full, 100% of samples, +and the prediction never moved off 1794 W. -Three changes: +The band was the wrong instrument, not merely mistuned. Household load is +multimodal — a few hundred watts of baseline, then 11 kW when the sauna, +oven and car overlap — and nothing about a residual's size separates +"unusual but real" from "wrong". A band fitted to the quiet hours always +excludes the busy ones. Short-term measurement noise is already handled a +layer down, by the Kalman filter in telemetry whose smoothed output this +model reads, so the second filter was both harmful and redundant. -- A hard bound at `3 × PeakW`, always on. A sample above the site's rated - draw is a measurement fault, not a household. It is absolute — derived - from configured hardware, never from what the model has learned — so it - holds from the first sample and cannot be widened by a model that has - mislearned. -- The soft MAE filter now arms after a day of samples rather than an hour, - so its band reflects a full night-and-day cycle instead of whichever - arbitrary hour followed the last restart. -- Ten consecutive same-direction rejections widen the band by exactly - enough to admit the residual, after which the ordinary EMA takes over. - Spikes are short and alternate in sign; a real level shift is sustained. +What remains is the rejection physics licenses: a house cannot draw more +than its main fuse passes. The ceiling is `fuse capacity × 1.25`, passed in +from configuration and never derived from what the model has learned, so it +holds from the first sample and cannot be talked down by a model that has +mislearned. With no fuse configured the check disables itself rather than +inventing a limit. -A three-minute 6 kW spike is still rejected in full. A sustained shift to a -new level is picked up within ten minutes and tracked exactly. +A sustained shift to a new load level is now learned directly. A one-minute +spike still trains — it is real consumption — and the bucket EMA damps it +to a tenth of the gap, which is what keeps an hour from being defined by +its loudest minute. diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 509f7946..6b989e5e 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -1030,7 +1030,15 @@ func main() { if loadPeakW <= 0 { loadPeakW = 5000 } - loadSvc = loadmodel.NewService(st, tel, cfg.SiteMeterDriver(), loadPeakW) + // Training ceiling: the main fuse is the one hard limit on what a house + // can actually draw, so a sample above it is a measurement fault rather + // than an unusual hour. Passed separately from loadPeakW — that one is a + // tunable proxy for "typical peak", this one is physics, and deriving + // the second from the first would silently move a safety bound whenever + // somebody retuned the proxy. Zero when no fuse is configured, which + // disables the check rather than inventing a limit. + loadMaxPlausibleW := cfg.Fuse.MaxPowerW() * loadmodel.PlausibleLoadHeadroom + loadSvc = loadmodel.NewService(st, tel, cfg.SiteMeterDriver(), loadPeakW, loadMaxPlausibleW) // SeedHeatingCoef — operator config is a cold-start prior. Once the // load model has accumulated samples in production, its // telemetry-fit HeatingW_per_degC survives restart and the config diff --git a/go/internal/api/api_loadmodel_test.go b/go/internal/api/api_loadmodel_test.go index ca42ffac..1573406c 100644 --- a/go/internal/api/api_loadmodel_test.go +++ b/go/internal/api/api_loadmodel_test.go @@ -12,7 +12,7 @@ import ( ) func TestHandleLoadModelProfileSwitch(t *testing.T) { - lm := loadmodel.NewService(nil, telemetry.NewStore(), "site", 4000) + lm := loadmodel.NewService(nil, telemetry.NewStore(), "site", 4000, 17250) srv := New(&Deps{LoadModel: lm}) req := httptest.NewRequest(http.MethodPost, "/api/loadmodel/profile", strings.NewReader(`{"profile":"away"}`)) @@ -51,7 +51,7 @@ func TestHandleLoadModelProfileSwitch(t *testing.T) { } func TestHandleLoadModelProfileRejectsUnknown(t *testing.T) { - lm := loadmodel.NewService(nil, telemetry.NewStore(), "site", 4000) + lm := loadmodel.NewService(nil, telemetry.NewStore(), "site", 4000, 17250) srv := New(&Deps{LoadModel: lm}) req := httptest.NewRequest(http.MethodPost, "/api/loadmodel/profile", strings.NewReader(`{"profile":"vacation"}`)) diff --git a/go/internal/calendar/e2e_test.go b/go/internal/calendar/e2e_test.go index 452a077e..82955ac0 100644 --- a/go/internal/calendar/e2e_test.go +++ b/go/internal/calendar/e2e_test.go @@ -24,7 +24,7 @@ func TestEndToEndRealCollaborators(t *testing.T) { } defer st.Close() - loadSvc := loadmodel.NewService(st, nil, "", 5000) + loadSvc := loadmodel.NewService(st, nil, "", 5000, 17250) lpMgr := loadpoint.NewManager() lpMgr.Load([]loadpoint.Config{{ID: "garage", VehicleCapacityWh: 60000}}) diff --git a/go/internal/loadmodel/model.go b/go/internal/loadmodel/model.go index 4b46adfd..135cd73e 100644 --- a/go/internal/loadmodel/model.go +++ b/go/internal/loadmodel/model.go @@ -63,28 +63,12 @@ const HeatingMinDeltaT = 3.0 // home. Clamp prevents one anomalous sample from blowing up the fit. const HeatingCoefMaxW = 1500.0 -// implausibleLoadFactor bounds a single sample against the site's rated -// power. Above this the reading is a fault, not consumption. Same factor -// Predict clamps its output with, so training and prediction agree on -// what counts as physically possible. -const implausibleLoadFactor = 3.0 - -// outlierArmSamples is how much history the outlier filter needs before -// it starts rejecting anything. The old threshold of 50 armed the filter -// after 51 minutes at the 60 s sample cadence — calibrating a whole -// house's plausible-residual band on one arbitrary hour, usually a quiet -// one, because that is when a restart is least disruptive. A day of -// samples spans at least one full night-and-day cycle, so MAE reflects -// the range the house actually moves through. -const outlierArmSamples = 24 * 60 - -// outlierLevelShiftRun is how many consecutive same-direction rejections -// mean "the level moved" rather than "a spike". At the 60 s cadence this -// is ten minutes of the model being wrong the same way before it concedes -// and starts learning again. Short enough that a genuine shift is picked -// up within the hour, long enough that an oven or a car starting to -// charge is still filtered as the transient it is. -const outlierLevelShiftRun = 10 +// PlausibleLoadHeadroom multiplies the main fuse capacity to get the +// point past which a reading must be a fault. Above 1.0 because a fuse +// tolerates brief overload and a meter can overshoot a step; well below +// anything a real house sustains, so a genuine 11 kW hour on a 25 A +// service is nowhere near it. +const PlausibleLoadHeadroom = 1.25 // Profile selects which learned occupancy profile is used for training // and prediction. @@ -128,11 +112,12 @@ type Model struct { Alpha float64 `json:"alpha"` // EMA coefficient for bucket updates PriorScale float64 `json:"prior_scale,omitempty"` - // RejectRun counts consecutive same-direction outlier rejections. - // It is the model's only way to notice that it is not filtering noise - // but refusing reality — see the outlier filter in Update. - RejectRun int `json:"reject_run,omitempty"` - RejectRunPositive bool `json:"reject_run_positive,omitempty"` + // MaxPlausibleW is the physical ceiling a sample must fall under to be + // trained on: main fuse capacity plus headroom. Derived from configured + // hardware and never from what the model has learned, so it cannot be + // talked down by a model that has mislearned. 0 disables the check — + // the state a site with no fuse configuration is in. + MaxPlausibleW float64 `json:"max_plausible_w,omitempty"` } // typicalPrior returns an approximate W load for a given hour-of-week @@ -302,59 +287,25 @@ func (m *Model) Update(t time.Time, actualLoadW, tempC float64) (updated bool) { } } - // Hard sanity bound, always on. A residual this far above the site's - // rated draw is a measurement fault, not a household. It is deliberately - // absolute — derived from configured hardware, never from what the model - // has learned — so unlike the MAE band below it cannot be talked down by - // a model that has mislearned, and it protects the first day before the - // soft filter arms. Mirrors the ceiling Predict already applies. - if m.PeakW > 0 && actualLoadW > implausibleLoadFactor*m.PeakW { - return false - } - - // Outlier filter: once we have some history, reject 10× MAE residuals. + // Physical bound — the only sample filter this model needs. // - // Two guards keep this from locking the model out of a load level it - // has not seen before. A rejected sample updates neither MAE nor - // Samples, so without them the band can never grow in response to - // being persistently wrong, and a model calibrated on one quiet hour - // rejects the real house forever. Measured before the fix: an hour of - // 400 W overnight load gave MAE 57 W and a 570 W band; a subsequent - // week at 5 kW was rejected in full, 100% of samples, and the - // prediction never moved off 1794 W. - if m.Samples > outlierArmSamples { - band := math.Max(m.MAE*10, 200) - if math.Abs(err) > band { - // A run of same-direction rejections is not noise — it is the - // house telling us the level moved. Spikes are short and - // alternate in sign; a real shift is sustained. Let the run - // through so the band can re-fit, and keep the filter's actual - // job (rejecting the isolated spike) intact. - sameDirection := (err > 0) == (m.RejectRunPositive) - if m.RejectRun > 0 && sameDirection { - m.RejectRun++ - } else { - m.RejectRun = 1 - m.RejectRunPositive = err > 0 - } - if m.RejectRun < outlierLevelShiftRun { - return false - } - // The run is long enough to be real. Widening the band by - // exactly enough to admit this residual is what lets the model - // re-fit: simply accepting the one sample and resetting the run - // leaves the band untouched, so the next nine are rejected too - // and the model crawls in at one sample in ten. Measured that - // way, a day at a new 3 kW level only reached 1650 W. - // - // max() means the band never shrinks here, and the ordinary EMA - // below takes over from this point — so the widening is a floor - // set by observation, not a permanent loosening. - m.MAE = math.Max(m.MAE, math.Abs(err)/10) - m.RejectRun = 0 - } else { - m.RejectRun = 0 - } + // A household's real load is strongly multimodal: a few hundred watts + // of baseline for most of the day, then 11 kW when the sauna, oven and + // car overlap. Both are true readings. Nothing about a residual's size + // distinguishes "unusual but real" from "wrong", so the only defensible + // rejection is the one physics licenses: a house cannot draw more than + // its main fuse passes. + // + // Short-term noise is already handled a layer down — telemetry runs a + // Kalman filter per signal, and this model reads the smoothed values. + // Filtering again here, against a band derived from what the model has + // already learned, rejected the upper half of the real distribution: an + // hour of 400 W overnight load produced a 570 W band, after which a + // week at 5 kW was rejected in full and the prediction never moved off + // 1794 W. That is not a corner case; a band fitted to the quiet hours + // always excludes the busy ones. + if m.MaxPlausibleW > 0 && actualLoadW > m.MaxPlausibleW { + return false } // Bucket update: exact running mean for the first 10 samples (crisp diff --git a/go/internal/loadmodel/model_test.go b/go/internal/loadmodel/model_test.go index 38609578..2c397712 100644 --- a/go/internal/loadmodel/model_test.go +++ b/go/internal/loadmodel/model_test.go @@ -125,8 +125,15 @@ func TestRejectsNegativeLoad(t *testing.T) { } } +// Outlier rejection is now anchored to the site's fuse rather than to a +// band fitted from the model's own history. The behaviour this test +// protects is unchanged — a 50 kW reading must not move the bucket — but +// it needs the physical ceiling set, which production wires from the fuse +// configuration. The band version of this check also rejected genuine +// household peaks; see TestFullHouseholdRangeIsTrained. func TestRejectsOutliers(t *testing.T) { m := NewModel(4000) + m.MaxPlausibleW = 17250 start := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) for i := 0; i < 200; i++ { m.Update(start.Add(time.Duration(i)*time.Minute), 1500, HeatingReferenceC) @@ -330,32 +337,35 @@ func TestHeatingFitWaitsForBucketTrust(t *testing.T) { } } -// A model calibrated on one quiet hour used to reject the real house -// forever: a rejected sample updates neither MAE nor Samples, so the band -// could never grow in response to being persistently wrong. Measured -// before the fix — an hour at 400 W gave a 570 W band, and a subsequent -// week at 5 kW was rejected in full, with the prediction stuck at 1794 W. +// The MAE-band outlier filter used to reject the real house permanently. +// A rejected sample updates neither MAE nor Samples, so the band could +// never grow in response to being persistently wrong: one quiet hour at +// 400 W produced a 570 W band, after which a week at 5 kW was rejected in +// full and the prediction never moved off 1794 W. +// +// The band was the wrong instrument. Household load is multimodal — a few +// hundred watts of baseline, then 11 kW when the sauna, oven and car +// overlap — and nothing about a residual's size separates "unusual but +// real" from "wrong". Short-term noise is already handled by the Kalman +// filter in telemetry, one layer down. func TestSustainedLevelShiftIsLearned(t *testing.T) { m := NewModel(10000) + m.MaxPlausibleW = 17000 start := time.Date(2026, 7, 1, 2, 0, 0, 0, time.UTC) - // Two quiet days, enough to arm the outlier filter on a narrow band. + // Two quiet days — under the old filter this is what armed a band so + // narrow that the house could never be learned. n := 0 for ; n < 2*24*60; n++ { m.Update(start.Add(time.Duration(n)*time.Minute), 400, HeatingReferenceC) } - if band := math.Max(m.MAE*10, 200); band > 600 { - t.Fatalf("expected a narrow band after quiet training, got %.0f", band) - } - // The house moves to 3 kW and stays there for three days. trainedFrom := n for i := 0; i < 3*24*60; i++ { m.Update(start.Add(time.Duration(n)*time.Minute), 3000, HeatingReferenceC) n++ } - // Every hour-of-week bucket the run covered must have followed. for i := trainedFrom; i < n; i += 60 { got := m.Predict(start.Add(time.Duration(i)*time.Minute), HeatingReferenceC) if math.Abs(got-3000) > 300 { @@ -364,50 +374,76 @@ func TestSustainedLevelShiftIsLearned(t *testing.T) { } } -// The level-shift concession must not cost us the filter's actual job. -func TestShortSpikeIsStillRejected(t *testing.T) { - m := NewModel(10000) - start := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC) - n := 0 - for ; n < 2*24*60; n++ { - m.Update(start.Add(time.Duration(n)*time.Minute), 400, HeatingReferenceC) - } +// A house that goes from near-idle to 11 kW is doing something ordinary, +// not reporting a fault. That whole range has to reach the model. +func TestFullHouseholdRangeIsTrained(t *testing.T) { + m := NewModel(8000) + m.MaxPlausibleW = 17250 // 25 A × 3 × 230 V + start := time.Date(2026, 7, 1, 3, 0, 0, 0, time.UTC) - accepted := 0 - for i := 0; i < outlierLevelShiftRun-1; i++ { - if m.Update(start.Add(time.Duration(n)*time.Minute), 6000, HeatingReferenceC) { - accepted++ + for i, load := range []float64{200, 11000, 300, 9500, 250, 11000} { + if !m.Update(start.Add(time.Duration(i)*time.Minute), load, HeatingReferenceC) { + t.Errorf("%.0f W is a normal household reading and was rejected", load) } - n++ - } - if accepted != 0 { - t.Errorf("a spike shorter than the level-shift run was accepted %d times", accepted) } - - // Returning to normal must reset the run, so two separate spikes never - // add up to a level shift. - m.Update(start.Add(time.Duration(n)*time.Minute), 400, HeatingReferenceC) - n++ - if m.RejectRun != 0 { - t.Errorf("reject run = %d after a normal sample, want 0", m.RejectRun) + if m.Samples != 6 { + t.Errorf("samples = %d, want 6", m.Samples) } } -// The hard bound is absolute and derived from configured hardware, so it -// holds from the very first sample — before the MAE band arms — and cannot -// be widened by a mislearned model. -func TestImplausibleLoadRejectedBeforeFilterArms(t *testing.T) { +// Rejection is licensed by physics alone: a house cannot draw more than +// its main fuse passes. Because the bound comes from configured hardware +// rather than from what the model has learned, it holds from the first +// sample and a mislearned model cannot talk it down. +func TestImplausibleLoadRejected(t *testing.T) { m := NewModel(4000) + m.MaxPlausibleW = 11000 * PlausibleLoadHeadroom // 16 A service t0 := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + if m.Update(t0, 50000, HeatingReferenceC) { - t.Error("50 kW on a 4 kW site should never be accepted") + t.Error("50 kW past a 16 A service should never be accepted") } if m.Samples != 0 { t.Errorf("rejected sample must not count, samples = %d", m.Samples) } - // Just under the bound still trains — the guard is for faults, not for - // houses that occasionally draw hard. - if !m.Update(t0, 3*4000-1, HeatingReferenceC) { - t.Error("a load just under the bound should be accepted") + // Right at the service limit is high but real — a fuse passes its + // rating, so this has to train. + if !m.Update(t0, 11000, HeatingReferenceC) { + t.Error("a load at the service limit should be accepted") + } +} + +// No fuse configured means no defensible ceiling, so the check disables +// itself rather than inventing one. +func TestNoFuseConfiguredTrainsEverything(t *testing.T) { + m := NewModel(4000) + t0 := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + if !m.Update(t0, 50000, HeatingReferenceC) { + t.Error("with MaxPlausibleW unset the model should not reject") + } +} + +// A one-minute spike is real and trains, but the bucket EMA is what keeps +// it from dominating the hour — no separate rejection needed. +func TestSpikeIsDampedByTheBucketEMA(t *testing.T) { + m := NewModel(8000) + m.MaxPlausibleW = 17250 + start := time.Date(2026, 7, 1, 4, 0, 0, 0, time.UTC) + + // Settle the bucket at 400 W, past the exact-running-mean phase. + for i := 0; i < 40; i++ { + m.Update(start.Add(time.Duration(i)*time.Minute), 400, HeatingReferenceC) + } + before := m.Predict(start, HeatingReferenceC) + m.Update(start.Add(41*time.Minute), 11000, HeatingReferenceC) + after := m.Predict(start, HeatingReferenceC) + + moved := after - before + if moved <= 0 { + t.Error("a real 11 kW reading should move the estimate up") + } + // alpha = 0.1, so one sample carries a tenth of the gap and no more. + if moved > 0.15*(11000-400) { + t.Errorf("one spike moved the hour by %.0f W — EMA is not damping it", moved) } } diff --git a/go/internal/loadmodel/service.go b/go/internal/loadmodel/service.go index d489de4d..5fe9b3f7 100644 --- a/go/internal/loadmodel/service.go +++ b/go/internal/loadmodel/service.go @@ -61,7 +61,7 @@ type Service struct { } // NewService constructs + restores from state if present. -func NewService(st *state.Store, tel *telemetry.Store, siteMeter string, peakW float64) *Service { +func NewService(st *state.Store, tel *telemetry.Store, siteMeter string, peakW, maxPlausibleW float64) *Service { s := &Service{ Store: st, Tele: tel, @@ -75,6 +75,7 @@ func NewService(st *state.Store, tel *telemetry.Store, siteMeter string, peakW f } for _, profile := range Profiles() { s.models[profile] = newProfileModel(peakW, profile) + s.models[profile].MaxPlausibleW = maxPlausibleW } if st != nil { loadedProfiles := make(map[Profile]bool) @@ -85,7 +86,7 @@ func NewService(st *state.Store, tel *telemetry.Store, siteMeter string, peakW f } for _, profile := range Profiles() { if js, ok := st.LoadConfig(stateKey(profile)); ok && js != "" { - if m, ok := restoreModel(js, peakW, profile); ok { + if m, ok := restoreModel(js, peakW, maxPlausibleW, profile); ok { s.models[profile] = m loadedProfiles[profile] = true slog.Info("loadmodel restored", @@ -99,6 +100,7 @@ func NewService(st *state.Store, tel *telemetry.Store, siteMeter string, peakW f if err := json.Unmarshal([]byte(js), &m); err == nil && m.Alpha > 0 { if !loadedProfiles[ProfileHome] { m.PeakW = peakW // config may have changed + m.MaxPlausibleW = maxPlausibleW if m.PriorScale <= 0 { m.PriorScale = 1 } @@ -125,12 +127,13 @@ func (s *Service) SetSiteMeter(name string) { s.mu.Unlock() } -func restoreModel(js string, peakW float64, profile Profile) (*Model, bool) { +func restoreModel(js string, peakW, maxPlausibleW float64, profile Profile) (*Model, bool) { var m Model if err := json.Unmarshal([]byte(js), &m); err != nil || m.Alpha <= 0 { return nil, false } - m.PeakW = peakW // config may have changed + m.PeakW = peakW // config may have changed + m.MaxPlausibleW = maxPlausibleW // ditto — fuse size is editable if m.PriorScale <= 0 { m.PriorScale = newProfileModel(peakW, profile).PriorScale } diff --git a/go/internal/loadmodel/service_test.go b/go/internal/loadmodel/service_test.go index ca9ba5bb..9b6e2664 100644 --- a/go/internal/loadmodel/service_test.go +++ b/go/internal/loadmodel/service_test.go @@ -11,7 +11,7 @@ import ( ) func TestResetPreservesHeatingCoefficient(t *testing.T) { - s := NewService(nil, telemetry.NewStore(), "site", 4000) + s := NewService(nil, telemetry.NewStore(), "site", 4000, 17250) s.SetHeatingCoef(275) s.Reset() @@ -28,7 +28,7 @@ func TestResetPreservesHeatingCoefficient(t *testing.T) { // startup wiring in cmd/ftw/main.go uses SeedHeatingCoef // precisely because operator config is a *prior*, not an override. func TestSeedHeatingCoefDoesNotOverwriteLearnedValue(t *testing.T) { - s := NewService(nil, telemetry.NewStore(), "site", 4000) + s := NewService(nil, telemetry.NewStore(), "site", 4000, 17250) // Simulate a model that has been adapting in production. s.mu.Lock() m := s.activeModelLocked() @@ -44,7 +44,7 @@ func TestSeedHeatingCoefDoesNotOverwriteLearnedValue(t *testing.T) { } func TestSeedHeatingCoefAppliesOnColdStart(t *testing.T) { - s := NewService(nil, telemetry.NewStore(), "site", 4000) + s := NewService(nil, telemetry.NewStore(), "site", 4000, 17250) // Fresh model — no samples observed yet. s.SeedHeatingCoef(300) @@ -59,7 +59,7 @@ func TestProfileSwitchTrainsOnlyActiveProfile(t *testing.T) { tel.Update("site", telemetry.DerMeter, 1000, nil, nil) tel.RecordDriverSuccess("site") - s := NewService(nil, tel, "site", 4000) + s := NewService(nil, tel, "site", 4000, 17250) now := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) s.sampleAt(now) @@ -89,7 +89,7 @@ func TestProfileAndModelsPersist(t *testing.T) { } defer st.Close() - s := NewService(st, telemetry.NewStore(), "site", 4000) + s := NewService(st, telemetry.NewStore(), "site", 4000, 17250) if err := s.SetProfile(ProfileAway); err != nil { t.Fatalf("set profile: %v", err) } @@ -103,7 +103,7 @@ func TestProfileAndModelsPersist(t *testing.T) { t.Fatalf("persist: %v", err) } - restored := NewService(st, telemetry.NewStore(), "site", 4000) + restored := NewService(st, telemetry.NewStore(), "site", 4000, 17250) if got := restored.Profile(); got != ProfileAway { t.Fatalf("restored profile = %q, want %q", got, ProfileAway) } @@ -116,7 +116,7 @@ func TestSampleRequiresOnlineSiteMeter(t *testing.T) { tel := telemetry.NewStore() tel.Update("site", telemetry.DerMeter, 1000, nil, nil) - s := NewService(nil, tel, "site", 4000) + s := NewService(nil, tel, "site", 4000, 17250) s.sampleAt(time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC)) if got := s.Model().Samples; got != 0 { @@ -137,7 +137,7 @@ func TestSampleUsesOnlyOnlineDERsAndSubtractsEV(t *testing.T) { tel.Update("charger", telemetry.DerEV, 300, nil, nil) tel.RecordDriverSuccess("charger") - s := NewService(nil, tel, "site", 4000) + s := NewService(nil, tel, "site", 4000, 17250) now := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) s.sampleAt(now) From 7389102ac5173d7c3e061ab4c15ff407d0c14827 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Fri, 31 Jul 2026 14:43:46 +0200 Subject: [PATCH 3/3] fix(loadmodel): reject faults before the heating fit, not after MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The physical bound ran after the online heating fit, so a cold-weather meter fault applied its residual to HeatingW_per_degC and only then got rejected — with Update still reporting that no update was applied. A run of faults could walk the coefficient toward its ceiling, inflating every cold-weather prediction from readings the model had ostensibly discarded. Measured with the bound in its old position: 40 rejected 50 kW samples at 0 °C moved the coefficient from 0 to 919 W/°C. At an 18 °C delta that is 16.5 kW of invented heating load. A fault must touch nothing on its way out — not the buckets, not MAE, not the heating coefficient — so the bound is now the first thing Update does after the negative-load guard. The comment about fitting heating before the outlier filter explained why a stale coefficient needs to be able to recover from the adaptive band; it never applied to an absolute physical limit, and that band is gone anyway. The regression test keeps every sample inside one hour-of-week bucket. The first version let the faults land in the next hour, where the fit is gated on bucket trust rather than on the bound — it passed against the buggy ordering and proved nothing. Codex P2 on PR #742. Co-Authored-By: Claude Opus 5 --- go/internal/loadmodel/model.go | 43 +++++++++++----------- go/internal/loadmodel/model_test.go | 56 +++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 21 deletions(-) diff --git a/go/internal/loadmodel/model.go b/go/internal/loadmodel/model.go index 135cd73e..faf5ac99 100644 --- a/go/internal/loadmodel/model.go +++ b/go/internal/loadmodel/model.go @@ -254,6 +254,28 @@ func (m *Model) Update(t time.Time, actualLoadW, tempC float64) (updated bool) { if actualLoadW < 0 { return false } + + // Physical bound — the only sample filter this model needs, and the + // first thing it does. A house cannot draw more than its main fuse + // passes, so a reading above it is a fault and must touch nothing: + // not the buckets, not MAE, and not the heating coefficient. Running + // it after the heating fit let one cold-weather meter fault move + // HeatingW_per_degC while Update still reported no update applied, + // and repeated faults could walk the coefficient to its ceiling. + // + // A household's real load is strongly multimodal: a few hundred watts + // of baseline for most of the day, then 11 kW when the sauna, oven and + // car overlap. Both are true readings. Nothing about a residual's size + // distinguishes "unusual but real" from "wrong", which is why this is + // the only rejection left — see the git history for the MAE band that + // used to sit below, and what it cost. + // + // Short-term noise is handled a layer down: telemetry runs a Kalman + // filter per signal and this model reads the smoothed values. + if m.MaxPlausibleW > 0 && actualLoadW > m.MaxPlausibleW { + return false + } + idx := HourOfWeek(t) b := &m.Bucket[idx] predicted := m.Predict(t, tempC) @@ -287,27 +309,6 @@ func (m *Model) Update(t time.Time, actualLoadW, tempC float64) (updated bool) { } } - // Physical bound — the only sample filter this model needs. - // - // A household's real load is strongly multimodal: a few hundred watts - // of baseline for most of the day, then 11 kW when the sauna, oven and - // car overlap. Both are true readings. Nothing about a residual's size - // distinguishes "unusual but real" from "wrong", so the only defensible - // rejection is the one physics licenses: a house cannot draw more than - // its main fuse passes. - // - // Short-term noise is already handled a layer down — telemetry runs a - // Kalman filter per signal, and this model reads the smoothed values. - // Filtering again here, against a band derived from what the model has - // already learned, rejected the upper half of the real distribution: an - // hour of 400 W overnight load produced a 570 W band, after which a - // week at 5 kW was rejected in full and the prediction never moved off - // 1794 W. That is not a corner case; a band fitted to the quiet hours - // always excludes the busy ones. - if m.MaxPlausibleW > 0 && actualLoadW > m.MaxPlausibleW { - return false - } - // Bucket update: exact running mean for the first 10 samples (crisp // early convergence), EMA after (smooth drift as the home evolves). // Subtract the current heating-gain estimate so the bucket learns diff --git a/go/internal/loadmodel/model_test.go b/go/internal/loadmodel/model_test.go index 2c397712..b39c0d67 100644 --- a/go/internal/loadmodel/model_test.go +++ b/go/internal/loadmodel/model_test.go @@ -447,3 +447,59 @@ func TestSpikeIsDampedByTheBucketEMA(t *testing.T) { t.Errorf("one spike moved the hour by %.0f W — EMA is not damping it", moved) } } + +// A fault must touch nothing on its way out — not the buckets, not MAE, +// and not the heating coefficient. The physical bound used to run after +// the online heating fit, so a cold-weather meter fault moved +// HeatingW_per_degC while Update still reported no update applied, and a +// run of faults could walk the coefficient to its ceiling. Codex P2 on +// PR #742. +// +// Every sample here stays inside one hour-of-week bucket: the heating fit +// is gated on that bucket's trust, so a fault landing in a fresh bucket +// would be filtered by the gate rather than by the bound, and the test +// would pass either way without proving anything. +func TestImplausibleLoadDoesNotMoveHeatingCoefficient(t *testing.T) { + m := NewModel(4000) + m.MaxPlausibleW = 11000 + start := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + const coldC = 0.0 // well past HeatingMinDeltaT + + // Warm the bucket past MinTrustSamples so the fit is ungated, and hold + // the load below the learned mean so the residual — and therefore the + // coefficient — is driven somewhere a fault could visibly move it. + for i := 0; i < 20; i++ { + m.Update(start.Add(time.Duration(i)*time.Minute), 1200, coldC) + } + idx := HourOfWeek(start) + if m.Bucket[idx].Samples < MinTrustSamples { + t.Fatalf("warmup left the bucket untrusted (%d samples) — the fit would be gated, not bounded", + m.Bucket[idx].Samples) + } + + before := m.HeatingW_per_degC + beforeMAE := m.MAE + beforeSamples := m.Samples + + // Faults in the SAME bucket, so only the bound can stop them. + for i := 20; i < 60; i++ { + at := start.Add(time.Duration(i) * time.Minute) + if HourOfWeek(at) != idx { + t.Fatalf("sample %d escaped the bucket under test", i) + } + if m.Update(at, 50000, coldC) { + t.Fatal("an implausible reading was accepted") + } + } + + if m.HeatingW_per_degC != before { + t.Errorf("heating coefficient moved on rejected faults: %.4f → %.4f", + before, m.HeatingW_per_degC) + } + if m.MAE != beforeMAE { + t.Errorf("MAE moved on rejected faults: %.1f → %.1f", beforeMAE, m.MAE) + } + if m.Samples != beforeSamples { + t.Errorf("sample count moved on rejected faults: %d → %d", beforeSamples, m.Samples) + } +}