diff --git a/.changeset/loadmodel-outlier-lockout.md b/.changeset/loadmodel-outlier-lockout.md new file mode 100644 index 00000000..f4c95218 --- /dev/null +++ b/.changeset/loadmodel-outlier-lockout.md @@ -0,0 +1,34 @@ +--- +"ftw": patch +--- + +Load model: replace the MAE-band outlier filter with a physical ceiling +taken from the main fuse. The band rejected the upper half of the real +load distribution, and could lock the model out of a level it had not seen +before, permanently. + +A rejected sample updates neither `MAE` nor `Samples`, so the band +`max(MAE × 10, 200)` never grew in response to being persistently wrong. +Measured on a clean model: an hour at 400 W left MAE at 57 W and the band +at 570 W; a following week at 5 kW was rejected in full, 100% of samples, +and the prediction never moved off 1794 W. + +The band was the wrong instrument, not merely mistuned. Household load is +multimodal — a few hundred watts of baseline, then 11 kW when the sauna, +oven and car overlap — and nothing about a residual's size separates +"unusual but real" from "wrong". A band fitted to the quiet hours always +excludes the busy ones. Short-term measurement noise is already handled a +layer down, by the Kalman filter in telemetry whose smoothed output this +model reads, so the second filter was both harmful and redundant. + +What remains is the rejection physics licenses: a house cannot draw more +than its main fuse passes. The ceiling is `fuse capacity × 1.25`, passed in +from configuration and never derived from what the model has learned, so it +holds from the first sample and cannot be talked down by a model that has +mislearned. With no fuse configured the check disables itself rather than +inventing a limit. + +A sustained shift to a new load level is now learned directly. A one-minute +spike still trains — it is real consumption — and the bucket EMA damps it +to a tenth of the gap, which is what keeps an hour from being defined by +its loudest minute. diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 509f7946..6b989e5e 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -1030,7 +1030,15 @@ func main() { if loadPeakW <= 0 { loadPeakW = 5000 } - loadSvc = loadmodel.NewService(st, tel, cfg.SiteMeterDriver(), loadPeakW) + // Training ceiling: the main fuse is the one hard limit on what a house + // can actually draw, so a sample above it is a measurement fault rather + // than an unusual hour. Passed separately from loadPeakW — that one is a + // tunable proxy for "typical peak", this one is physics, and deriving + // the second from the first would silently move a safety bound whenever + // somebody retuned the proxy. Zero when no fuse is configured, which + // disables the check rather than inventing a limit. + loadMaxPlausibleW := cfg.Fuse.MaxPowerW() * loadmodel.PlausibleLoadHeadroom + loadSvc = loadmodel.NewService(st, tel, cfg.SiteMeterDriver(), loadPeakW, loadMaxPlausibleW) // SeedHeatingCoef — operator config is a cold-start prior. Once the // load model has accumulated samples in production, its // telemetry-fit HeatingW_per_degC survives restart and the config diff --git a/go/internal/api/api_loadmodel_test.go b/go/internal/api/api_loadmodel_test.go index ca42ffac..1573406c 100644 --- a/go/internal/api/api_loadmodel_test.go +++ b/go/internal/api/api_loadmodel_test.go @@ -12,7 +12,7 @@ import ( ) func TestHandleLoadModelProfileSwitch(t *testing.T) { - lm := loadmodel.NewService(nil, telemetry.NewStore(), "site", 4000) + lm := loadmodel.NewService(nil, telemetry.NewStore(), "site", 4000, 17250) srv := New(&Deps{LoadModel: lm}) req := httptest.NewRequest(http.MethodPost, "/api/loadmodel/profile", strings.NewReader(`{"profile":"away"}`)) @@ -51,7 +51,7 @@ func TestHandleLoadModelProfileSwitch(t *testing.T) { } func TestHandleLoadModelProfileRejectsUnknown(t *testing.T) { - lm := loadmodel.NewService(nil, telemetry.NewStore(), "site", 4000) + lm := loadmodel.NewService(nil, telemetry.NewStore(), "site", 4000, 17250) srv := New(&Deps{LoadModel: lm}) req := httptest.NewRequest(http.MethodPost, "/api/loadmodel/profile", strings.NewReader(`{"profile":"vacation"}`)) diff --git a/go/internal/calendar/e2e_test.go b/go/internal/calendar/e2e_test.go index 452a077e..82955ac0 100644 --- a/go/internal/calendar/e2e_test.go +++ b/go/internal/calendar/e2e_test.go @@ -24,7 +24,7 @@ func TestEndToEndRealCollaborators(t *testing.T) { } defer st.Close() - loadSvc := loadmodel.NewService(st, nil, "", 5000) + loadSvc := loadmodel.NewService(st, nil, "", 5000, 17250) lpMgr := loadpoint.NewManager() lpMgr.Load([]loadpoint.Config{{ID: "garage", VehicleCapacityWh: 60000}}) diff --git a/go/internal/loadmodel/model.go b/go/internal/loadmodel/model.go index dc4d1f8d..faf5ac99 100644 --- a/go/internal/loadmodel/model.go +++ b/go/internal/loadmodel/model.go @@ -63,6 +63,13 @@ const HeatingMinDeltaT = 3.0 // home. Clamp prevents one anomalous sample from blowing up the fit. const HeatingCoefMaxW = 1500.0 +// PlausibleLoadHeadroom multiplies the main fuse capacity to get the +// point past which a reading must be a fault. Above 1.0 because a fuse +// tolerates brief overload and a meter can overshoot a step; well below +// anything a real house sustains, so a genuine 11 kW hour on a 25 A +// service is nowhere near it. +const PlausibleLoadHeadroom = 1.25 + // Profile selects which learned occupancy profile is used for training // and prediction. type Profile string @@ -104,6 +111,13 @@ type Model struct { MAE float64 `json:"mae"` Alpha float64 `json:"alpha"` // EMA coefficient for bucket updates PriorScale float64 `json:"prior_scale,omitempty"` + + // MaxPlausibleW is the physical ceiling a sample must fall under to be + // trained on: main fuse capacity plus headroom. Derived from configured + // hardware and never from what the model has learned, so it cannot be + // talked down by a model that has mislearned. 0 disables the check — + // the state a site with no fuse configuration is in. + MaxPlausibleW float64 `json:"max_plausible_w,omitempty"` } // typicalPrior returns an approximate W load for a given hour-of-week @@ -240,6 +254,28 @@ func (m *Model) Update(t time.Time, actualLoadW, tempC float64) (updated bool) { if actualLoadW < 0 { return false } + + // Physical bound — the only sample filter this model needs, and the + // first thing it does. A house cannot draw more than its main fuse + // passes, so a reading above it is a fault and must touch nothing: + // not the buckets, not MAE, and not the heating coefficient. Running + // it after the heating fit let one cold-weather meter fault move + // HeatingW_per_degC while Update still reported no update applied, + // and repeated faults could walk the coefficient to its ceiling. + // + // A household's real load is strongly multimodal: a few hundred watts + // of baseline for most of the day, then 11 kW when the sauna, oven and + // car overlap. Both are true readings. Nothing about a residual's size + // distinguishes "unusual but real" from "wrong", which is why this is + // the only rejection left — see the git history for the MAE band that + // used to sit below, and what it cost. + // + // Short-term noise is handled a layer down: telemetry runs a Kalman + // filter per signal and this model reads the smoothed values. + if m.MaxPlausibleW > 0 && actualLoadW > m.MaxPlausibleW { + return false + } + idx := HourOfWeek(t) b := &m.Bucket[idx] predicted := m.Predict(t, tempC) @@ -273,14 +309,6 @@ func (m *Model) Update(t time.Time, actualLoadW, tempC float64) (updated bool) { } } - // Outlier filter: once we have some history, reject 10× MAE residuals. - if m.Samples > 50 { - band := math.Max(m.MAE*10, 200) - if math.Abs(err) > band { - return false - } - } - // Bucket update: exact running mean for the first 10 samples (crisp // early convergence), EMA after (smooth drift as the home evolves). // Subtract the current heating-gain estimate so the bucket learns diff --git a/go/internal/loadmodel/model_test.go b/go/internal/loadmodel/model_test.go index 3eff2396..b39c0d67 100644 --- a/go/internal/loadmodel/model_test.go +++ b/go/internal/loadmodel/model_test.go @@ -125,8 +125,15 @@ func TestRejectsNegativeLoad(t *testing.T) { } } +// Outlier rejection is now anchored to the site's fuse rather than to a +// band fitted from the model's own history. The behaviour this test +// protects is unchanged — a 50 kW reading must not move the bucket — but +// it needs the physical ceiling set, which production wires from the fuse +// configuration. The band version of this check also rejected genuine +// household peaks; see TestFullHouseholdRangeIsTrained. func TestRejectsOutliers(t *testing.T) { m := NewModel(4000) + m.MaxPlausibleW = 17250 start := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) for i := 0; i < 200; i++ { m.Update(start.Add(time.Duration(i)*time.Minute), 1500, HeatingReferenceC) @@ -329,3 +336,170 @@ func TestHeatingFitWaitsForBucketTrust(t *testing.T) { t.Errorf("untrusted bucket must not drive heating fit, coef = %.0f", m.HeatingW_per_degC) } } + +// The MAE-band outlier filter used to reject the real house permanently. +// A rejected sample updates neither MAE nor Samples, so the band could +// never grow in response to being persistently wrong: one quiet hour at +// 400 W produced a 570 W band, after which a week at 5 kW was rejected in +// full and the prediction never moved off 1794 W. +// +// The band was the wrong instrument. Household load is multimodal — a few +// hundred watts of baseline, then 11 kW when the sauna, oven and car +// overlap — and nothing about a residual's size separates "unusual but +// real" from "wrong". Short-term noise is already handled by the Kalman +// filter in telemetry, one layer down. +func TestSustainedLevelShiftIsLearned(t *testing.T) { + m := NewModel(10000) + m.MaxPlausibleW = 17000 + start := time.Date(2026, 7, 1, 2, 0, 0, 0, time.UTC) + + // Two quiet days — under the old filter this is what armed a band so + // narrow that the house could never be learned. + n := 0 + for ; n < 2*24*60; n++ { + m.Update(start.Add(time.Duration(n)*time.Minute), 400, HeatingReferenceC) + } + + trainedFrom := n + for i := 0; i < 3*24*60; i++ { + m.Update(start.Add(time.Duration(n)*time.Minute), 3000, HeatingReferenceC) + n++ + } + + for i := trainedFrom; i < n; i += 60 { + got := m.Predict(start.Add(time.Duration(i)*time.Minute), HeatingReferenceC) + if math.Abs(got-3000) > 300 { + t.Fatalf("bucket at minute %d predicts %.0f W, want ~3000 W", i, got) + } + } +} + +// A house that goes from near-idle to 11 kW is doing something ordinary, +// not reporting a fault. That whole range has to reach the model. +func TestFullHouseholdRangeIsTrained(t *testing.T) { + m := NewModel(8000) + m.MaxPlausibleW = 17250 // 25 A × 3 × 230 V + start := time.Date(2026, 7, 1, 3, 0, 0, 0, time.UTC) + + for i, load := range []float64{200, 11000, 300, 9500, 250, 11000} { + if !m.Update(start.Add(time.Duration(i)*time.Minute), load, HeatingReferenceC) { + t.Errorf("%.0f W is a normal household reading and was rejected", load) + } + } + if m.Samples != 6 { + t.Errorf("samples = %d, want 6", m.Samples) + } +} + +// Rejection is licensed by physics alone: a house cannot draw more than +// its main fuse passes. Because the bound comes from configured hardware +// rather than from what the model has learned, it holds from the first +// sample and a mislearned model cannot talk it down. +func TestImplausibleLoadRejected(t *testing.T) { + m := NewModel(4000) + m.MaxPlausibleW = 11000 * PlausibleLoadHeadroom // 16 A service + t0 := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + + if m.Update(t0, 50000, HeatingReferenceC) { + t.Error("50 kW past a 16 A service should never be accepted") + } + if m.Samples != 0 { + t.Errorf("rejected sample must not count, samples = %d", m.Samples) + } + // Right at the service limit is high but real — a fuse passes its + // rating, so this has to train. + if !m.Update(t0, 11000, HeatingReferenceC) { + t.Error("a load at the service limit should be accepted") + } +} + +// No fuse configured means no defensible ceiling, so the check disables +// itself rather than inventing one. +func TestNoFuseConfiguredTrainsEverything(t *testing.T) { + m := NewModel(4000) + t0 := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + if !m.Update(t0, 50000, HeatingReferenceC) { + t.Error("with MaxPlausibleW unset the model should not reject") + } +} + +// A one-minute spike is real and trains, but the bucket EMA is what keeps +// it from dominating the hour — no separate rejection needed. +func TestSpikeIsDampedByTheBucketEMA(t *testing.T) { + m := NewModel(8000) + m.MaxPlausibleW = 17250 + start := time.Date(2026, 7, 1, 4, 0, 0, 0, time.UTC) + + // Settle the bucket at 400 W, past the exact-running-mean phase. + for i := 0; i < 40; i++ { + m.Update(start.Add(time.Duration(i)*time.Minute), 400, HeatingReferenceC) + } + before := m.Predict(start, HeatingReferenceC) + m.Update(start.Add(41*time.Minute), 11000, HeatingReferenceC) + after := m.Predict(start, HeatingReferenceC) + + moved := after - before + if moved <= 0 { + t.Error("a real 11 kW reading should move the estimate up") + } + // alpha = 0.1, so one sample carries a tenth of the gap and no more. + if moved > 0.15*(11000-400) { + t.Errorf("one spike moved the hour by %.0f W — EMA is not damping it", moved) + } +} + +// A fault must touch nothing on its way out — not the buckets, not MAE, +// and not the heating coefficient. The physical bound used to run after +// the online heating fit, so a cold-weather meter fault moved +// HeatingW_per_degC while Update still reported no update applied, and a +// run of faults could walk the coefficient to its ceiling. Codex P2 on +// PR #742. +// +// Every sample here stays inside one hour-of-week bucket: the heating fit +// is gated on that bucket's trust, so a fault landing in a fresh bucket +// would be filtered by the gate rather than by the bound, and the test +// would pass either way without proving anything. +func TestImplausibleLoadDoesNotMoveHeatingCoefficient(t *testing.T) { + m := NewModel(4000) + m.MaxPlausibleW = 11000 + start := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + const coldC = 0.0 // well past HeatingMinDeltaT + + // Warm the bucket past MinTrustSamples so the fit is ungated, and hold + // the load below the learned mean so the residual — and therefore the + // coefficient — is driven somewhere a fault could visibly move it. + for i := 0; i < 20; i++ { + m.Update(start.Add(time.Duration(i)*time.Minute), 1200, coldC) + } + idx := HourOfWeek(start) + if m.Bucket[idx].Samples < MinTrustSamples { + t.Fatalf("warmup left the bucket untrusted (%d samples) — the fit would be gated, not bounded", + m.Bucket[idx].Samples) + } + + before := m.HeatingW_per_degC + beforeMAE := m.MAE + beforeSamples := m.Samples + + // Faults in the SAME bucket, so only the bound can stop them. + for i := 20; i < 60; i++ { + at := start.Add(time.Duration(i) * time.Minute) + if HourOfWeek(at) != idx { + t.Fatalf("sample %d escaped the bucket under test", i) + } + if m.Update(at, 50000, coldC) { + t.Fatal("an implausible reading was accepted") + } + } + + if m.HeatingW_per_degC != before { + t.Errorf("heating coefficient moved on rejected faults: %.4f → %.4f", + before, m.HeatingW_per_degC) + } + if m.MAE != beforeMAE { + t.Errorf("MAE moved on rejected faults: %.1f → %.1f", beforeMAE, m.MAE) + } + if m.Samples != beforeSamples { + t.Errorf("sample count moved on rejected faults: %d → %d", beforeSamples, m.Samples) + } +} diff --git a/go/internal/loadmodel/service.go b/go/internal/loadmodel/service.go index d489de4d..5fe9b3f7 100644 --- a/go/internal/loadmodel/service.go +++ b/go/internal/loadmodel/service.go @@ -61,7 +61,7 @@ type Service struct { } // NewService constructs + restores from state if present. -func NewService(st *state.Store, tel *telemetry.Store, siteMeter string, peakW float64) *Service { +func NewService(st *state.Store, tel *telemetry.Store, siteMeter string, peakW, maxPlausibleW float64) *Service { s := &Service{ Store: st, Tele: tel, @@ -75,6 +75,7 @@ func NewService(st *state.Store, tel *telemetry.Store, siteMeter string, peakW f } for _, profile := range Profiles() { s.models[profile] = newProfileModel(peakW, profile) + s.models[profile].MaxPlausibleW = maxPlausibleW } if st != nil { loadedProfiles := make(map[Profile]bool) @@ -85,7 +86,7 @@ func NewService(st *state.Store, tel *telemetry.Store, siteMeter string, peakW f } for _, profile := range Profiles() { if js, ok := st.LoadConfig(stateKey(profile)); ok && js != "" { - if m, ok := restoreModel(js, peakW, profile); ok { + if m, ok := restoreModel(js, peakW, maxPlausibleW, profile); ok { s.models[profile] = m loadedProfiles[profile] = true slog.Info("loadmodel restored", @@ -99,6 +100,7 @@ func NewService(st *state.Store, tel *telemetry.Store, siteMeter string, peakW f if err := json.Unmarshal([]byte(js), &m); err == nil && m.Alpha > 0 { if !loadedProfiles[ProfileHome] { m.PeakW = peakW // config may have changed + m.MaxPlausibleW = maxPlausibleW if m.PriorScale <= 0 { m.PriorScale = 1 } @@ -125,12 +127,13 @@ func (s *Service) SetSiteMeter(name string) { s.mu.Unlock() } -func restoreModel(js string, peakW float64, profile Profile) (*Model, bool) { +func restoreModel(js string, peakW, maxPlausibleW float64, profile Profile) (*Model, bool) { var m Model if err := json.Unmarshal([]byte(js), &m); err != nil || m.Alpha <= 0 { return nil, false } - m.PeakW = peakW // config may have changed + m.PeakW = peakW // config may have changed + m.MaxPlausibleW = maxPlausibleW // ditto — fuse size is editable if m.PriorScale <= 0 { m.PriorScale = newProfileModel(peakW, profile).PriorScale } diff --git a/go/internal/loadmodel/service_test.go b/go/internal/loadmodel/service_test.go index ca9ba5bb..9b6e2664 100644 --- a/go/internal/loadmodel/service_test.go +++ b/go/internal/loadmodel/service_test.go @@ -11,7 +11,7 @@ import ( ) func TestResetPreservesHeatingCoefficient(t *testing.T) { - s := NewService(nil, telemetry.NewStore(), "site", 4000) + s := NewService(nil, telemetry.NewStore(), "site", 4000, 17250) s.SetHeatingCoef(275) s.Reset() @@ -28,7 +28,7 @@ func TestResetPreservesHeatingCoefficient(t *testing.T) { // startup wiring in cmd/ftw/main.go uses SeedHeatingCoef // precisely because operator config is a *prior*, not an override. func TestSeedHeatingCoefDoesNotOverwriteLearnedValue(t *testing.T) { - s := NewService(nil, telemetry.NewStore(), "site", 4000) + s := NewService(nil, telemetry.NewStore(), "site", 4000, 17250) // Simulate a model that has been adapting in production. s.mu.Lock() m := s.activeModelLocked() @@ -44,7 +44,7 @@ func TestSeedHeatingCoefDoesNotOverwriteLearnedValue(t *testing.T) { } func TestSeedHeatingCoefAppliesOnColdStart(t *testing.T) { - s := NewService(nil, telemetry.NewStore(), "site", 4000) + s := NewService(nil, telemetry.NewStore(), "site", 4000, 17250) // Fresh model — no samples observed yet. s.SeedHeatingCoef(300) @@ -59,7 +59,7 @@ func TestProfileSwitchTrainsOnlyActiveProfile(t *testing.T) { tel.Update("site", telemetry.DerMeter, 1000, nil, nil) tel.RecordDriverSuccess("site") - s := NewService(nil, tel, "site", 4000) + s := NewService(nil, tel, "site", 4000, 17250) now := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) s.sampleAt(now) @@ -89,7 +89,7 @@ func TestProfileAndModelsPersist(t *testing.T) { } defer st.Close() - s := NewService(st, telemetry.NewStore(), "site", 4000) + s := NewService(st, telemetry.NewStore(), "site", 4000, 17250) if err := s.SetProfile(ProfileAway); err != nil { t.Fatalf("set profile: %v", err) } @@ -103,7 +103,7 @@ func TestProfileAndModelsPersist(t *testing.T) { t.Fatalf("persist: %v", err) } - restored := NewService(st, telemetry.NewStore(), "site", 4000) + restored := NewService(st, telemetry.NewStore(), "site", 4000, 17250) if got := restored.Profile(); got != ProfileAway { t.Fatalf("restored profile = %q, want %q", got, ProfileAway) } @@ -116,7 +116,7 @@ func TestSampleRequiresOnlineSiteMeter(t *testing.T) { tel := telemetry.NewStore() tel.Update("site", telemetry.DerMeter, 1000, nil, nil) - s := NewService(nil, tel, "site", 4000) + s := NewService(nil, tel, "site", 4000, 17250) s.sampleAt(time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC)) if got := s.Model().Samples; got != 0 { @@ -137,7 +137,7 @@ func TestSampleUsesOnlyOnlineDERsAndSubtractsEV(t *testing.T) { tel.Update("charger", telemetry.DerEV, 300, nil, nil) tel.RecordDriverSuccess("charger") - s := NewService(nil, tel, "site", 4000) + s := NewService(nil, tel, "site", 4000, 17250) now := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) s.sampleAt(now)