diff --git a/AGENTS.md b/AGENTS.md index 5f0464b..f9284e6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -109,6 +109,7 @@ samples/fixture/ synthetic workbooks for tests/CI (committed) - `analytics/catchModel.ts`: negative-binomial counts per timed gillnet set, offset log(net-hours), natural-spline day of season (knots at the 0/25/50/75/100% quantiles), nursery and season as effect-coded fixed effects (each level vs the average; no random effects), and nine environment terms per +1 SD. Complete cases only; nurseries need ≥ 30 complete sets, and a species gets a nursery effect only where ≥ 10 of it were caught (otherwise the estimate runs off to zero or infinity). - The headline: sea temperature's raw link with catch rate (≈ −25% per °C) vanishes once day of the season is in the model. Don't describe SST as a driver of catches on other pages. +- Fit diagnostics (#125, `analytics/modelDiagnostics.ts`) reuse each fit's own rows, design matrix, counts and offsets, so they can't drift from the model. NB2 variance is μ + μ²/θ; P(0) falls back to Poisson when θ is infinite. Calibration rates are summed sharks over summed net-hours per group of sets. They are in-sample checks: not forecasting skill, and no leave-one-season-out (season is a fixed effect). Only rounded columns reach the page; keep it that way (the first draft sent 500 kB of JSON). - `glm.ts` is generic (IRLS + ML θ, Wald intervals via `contrast`); reuse it rather than writing another fitter. ### Sex ratio mixed model (#117) diff --git a/e2e/model-diagnostics.spec.ts b/e2e/model-diagnostics.spec.ts new file mode 100644 index 0000000..5182702 --- /dev/null +++ b/e2e/model-diagnostics.spec.ts @@ -0,0 +1,26 @@ +import { expect, test } from '@playwright/test'; +import AxeBuilder from '@axe-core/playwright'; + +test('model fit checks follow the catch model', async ({ page }) => { + await page.goto('/drivers'); + const summary = page.getByText('How well does the model describe the catches?', { exact: true }); + // The synthetic fixture has too few complete sets to fit the model, so there is nothing to check. + if (!(await page.getByRole('heading', { name: 'Sea temperature, step by step' }).count())) { + await expect(summary).toHaveCount(0); + return; + } + await summary.click(); + const model = page.getByRole('combobox', { name: 'Model', exact: true }); + await expect(model).toBeVisible(); + for (const theme of ['light', 'dark'] as const) { + await page.emulateMedia({ colorScheme: theme }); + const result = await new AxeBuilder({ page }).include('main').analyze(); + expect( + result.violations.filter((v) => ['serious', 'critical'].includes(v.impact ?? '')) + ).toEqual([]); + } + for (const i of ['1', '2']) { + await model.selectOption(i); + await expect(page.locator('#model-fit figure').first()).toBeVisible(); + } +}); diff --git a/src/lib/components/ModelDiagnostics.svelte b/src/lib/components/ModelDiagnostics.svelte new file mode 100644 index 0000000..b2afa7c --- /dev/null +++ b/src/lib/components/ModelDiagnostics.svelte @@ -0,0 +1,190 @@ + + +
+ How well does the model describe the catches? +

+ These checks compare the model with the same sets it was fitted to. They show where it expects + too many or too few sharks. They don't show how well it would forecast another season, and a + good fit doesn't make an effect a cause. Sets from the same night or nursery aren't independent, + so the charts carry no uncertainty bands. +

+ + {#if d} +

+ {plural(d.n, 'set')} in this model's sample. Zero-catch charts leave out groups with fewer than + {MIN_GROUP_SETS} sets; their tables keep every group. +

+
+ + ({ + x: b.expected, + y: b.observed, + g: group.name, + label: `Group ${b.bin}: ${plural(b.n, 'set')}` + }))} + groups={[{ name: group.name, color: group.color }]} + fits={[ + { + name: 'Observed = expected', + color: 'var(--ink-2)', + dashed: true, + points: [ + { x: 0, y: 0 }, + { x: maxRate, y: maxRate } + ] + } + ]} + xLabel="Expected sharks per net-hour" + yLabel="Observed sharks per net-hour" + r={5} + opacity={0.9} + /> + + {#each panels as panel (panel.name)} + {@const rows = panel.rows.filter((r) => r.n >= MIN_GROUP_SETS)} + + {#if rows.length} + r.group)} + series={[ + { name: 'Observed', color: group.color, values: rows.map((r) => r.observed) }, + { name: 'Expected', color: 'var(--muted)', values: rows.map((r) => r.expected) } + ]} + yLabel="Sets with no sharks (%)" + valueFormat={(v) => `${num(v)}%`} + /> + {:else} +

No {panel.name} has {MIN_GROUP_SETS} sets in this sample.

+ {/if} +
+ {/each} + {#each residualPanels as r (r.title)} + + ({ x, y: d.residuals.value[i], g: group.name }))} + groups={[{ name: group.name, color: group.color }]} + fits={[ + { + name: 'As expected', + color: 'var(--ink-2)', + dashed: true, + points: [ + { x: Math.min(...r.xs), y: 0 }, + { x: Math.max(...r.xs), y: 0 } + ] + } + ]} + xLabel={r.xLabel} + xLog={r.xLog} + xFormat={r.xFormat} + yLabel="Pearson residual" + /> + + {/each} +
+ {:else} +

Diagnostics are unavailable: the model needs a converged fit and a supported sample.

+ {/if} +
+ + diff --git a/src/lib/server/analytics/catchModel.ts b/src/lib/server/analytics/catchModel.ts index 3c328bb..b3ac99e 100644 --- a/src/lib/server/analytics/catchModel.ts +++ b/src/lib/server/analytics/catchModel.ts @@ -5,11 +5,13 @@ * fall through the season); the model estimates each one with the others held fixed. Still * observational: it adjusts for what is measured, nothing more. */ +import { SPECIES } from '#lib/species'; import { mean, quantile, sd } from '#lib/stats'; import type { Dataset, SetRow } from '../data'; import { effortSets, round, seasonDay, seasonDayLabel, siteName } from './common'; import { SET_VARIABLES } from './env'; import { contrast, fitGlm, naturalSpline, type GlmFit } from './glm'; +import { modelDiagnostics } from './modelDiagnostics'; /** Environmental terms, chosen to cover each family once (no near-duplicates like air vs sea). */ const TERMS = [ @@ -189,6 +191,15 @@ export function catchModel(d: Dataset) { return { n: all.rows.length, + diagnostics: [ + { m: all, name: 'All sharks', color: 'var(--series-3)' }, + { m: cm, name: 'Blacktip reef sharks', color: SPECIES.Cm.color }, + { m: na, name: 'Sicklefin lemon sharks', color: SPECIES.Na.color } + ].map(({ m, name, color }) => ({ + name, + color, + data: modelDiagnostics(m.fit, m.rows, m.X, m.y, m.offset, (id) => siteName(d, id)) + })), dropped: effortSets(d).filter((s) => s.cpue != null).length - all.rows.length, knots: knots.map((k) => seasonDayLabel(Math.round(k))), terms: vars.map((v, j) => ({ diff --git a/src/lib/server/analytics/modelDiagnostics.test.ts b/src/lib/server/analytics/modelDiagnostics.test.ts new file mode 100644 index 0000000..cfa704d --- /dev/null +++ b/src/lib/server/analytics/modelDiagnostics.test.ts @@ -0,0 +1,44 @@ +import { expect, it } from 'vitest'; +import type { SetRow } from '../data'; +import type { GlmFit } from './glm'; +import { modelDiagnostics, zeroProbability } from './modelDiagnostics'; + +it('uses the NB2 zero probability, including the stable Poisson limit', () => { + expect(zeroProbability(2, 1)).toBeCloseTo(1 / 3); + expect(zeroProbability(2, Infinity)).toBeCloseTo(Math.exp(-2)); + expect(zeroProbability(2, 1e12)).toBeCloseTo(Math.exp(-2)); +}); + +const rows = Array.from({ length: 20 }, (_, i) => ({ + date: '2020-10-01', + siteId: i < 10 ? 'b' : 'a', + season: '2020-2021', + durationMin: i % 2 ? 120 : 60 +})) as SetRow[]; +const fit = { beta: [Math.log(2)], theta: 1, n: 20, converged: true } as GlmFit; +const X = rows.map(() => [1]), + y = rows.map((_, i) => (i % 2 ? 4 : 0)), + offset = rows.map((s) => Math.log(s.durationMin! / 60)); + +it('uses duration offsets and the fitted variance without refiltering the sample', () => { + const d = modelDiagnostics(fit, rows, X, y, offset, (id) => `Nursery ${id.toUpperCase()}`)!; + // μ = 2 per net-hour × 1 h; NB2 variance 2 + 2²/1 = 6. + expect(d.residuals.value[0]).toBeCloseTo(-2 / Math.sqrt(6)); + expect(d.residuals.expected[1]).toBeCloseTo(4); + expect(d.residuals.day).toHaveLength(20); + // Groups are sorted by id and labelled; shares are percentages. + expect(d.sites.map((s) => s.group)).toEqual(['Nursery A', 'Nursery B']); + expect(d.sites[0].observed).toBe(50); + expect(d.sites[0].expected).toBeCloseTo((100 * (1 / 3 + 1 / 5)) / 2, 1); + expect(d.bins.reduce((n, b) => n + b.n, 0)).toBe(20); + // Unequal durations: summed sharks over summed net-hours stays at the true rate of 2. + expect(d.bins.every((b) => Math.abs(b.expected - 2) < 1e-9)).toBe(true); +}); + +it('handles the Poisson limit and refuses failed or mismatched fits', () => { + const p = modelDiagnostics({ ...fit, theta: Infinity }, rows, X, y, offset)!; + expect(p.residuals.value[0]).toBeCloseTo(-2 / Math.sqrt(2)); + expect(modelDiagnostics(null, rows, X, y, offset)).toBeNull(); + expect(modelDiagnostics({ ...fit, converged: false }, rows, X, y, offset)).toBeNull(); + expect(modelDiagnostics(fit, rows.slice(1), X, y, offset)).toBeNull(); +}); diff --git a/src/lib/server/analytics/modelDiagnostics.ts b/src/lib/server/analytics/modelDiagnostics.ts new file mode 100644 index 0000000..f0c87bb --- /dev/null +++ b/src/lib/server/analytics/modelDiagnostics.ts @@ -0,0 +1,94 @@ +/** + * In-sample fit checks for the catch model (#125): calibration, zero-catch shares and Pearson + * residuals, from the exact rows, design matrix, counts and offsets of each fit. Only what the + * charts draw goes to the page, rounded, so /drivers stays light (#76). + */ +import type { SetRow } from '../data'; +import type { GlmFit } from './glm'; +import { round, seasonDay } from './common'; + +/** Calibration groups: sets sorted by expected count, split into this many near-equal bins. */ +export const CALIBRATION_BINS = 10; + +/** NB2 P(Y = 0) = (θ / (θ + μ))^θ, with the Poisson limit e^−μ as θ → ∞. */ +export function zeroProbability(mu: number, theta: number) { + return Number.isFinite(theta) ? Math.exp(-theta * Math.log1p(mu / theta)) : Math.exp(-mu); +} + +/** Same X, response and offsets as the fit; all diagnostics are in-sample. */ +export function modelDiagnostics( + fit: GlmFit | null, + rows: SetRow[], + X: number[][], + y: number[], + offsets: number[], + siteLabel: (id: string) => string = (id) => id +) { + if ( + !fit?.converged || + !rows.length || + rows.length !== fit.n || + X.length !== rows.length || + y.length !== rows.length || + offsets.length !== rows.length + ) + return null; + const points = rows.map((s, i) => { + const mu = Math.exp( + Math.min(30, X[i].reduce((sum, v, j) => sum + v * fit.beta[j], 0) + offsets[i]) + ); + return { + expected: mu, + observed: y[i], + hours: Math.exp(offsets[i]), + residual: + (y[i] - mu) / Math.sqrt(mu + (Number.isFinite(fit.theta) ? (mu * mu) / fit.theta : 0)), + zero: zeroProbability(mu, fit.theta), + day: seasonDay(s.date!), + site: s.siteId!, + season: s.season + }; + }); + if (points.some((p) => !Number.isFinite(p.residual) || !Number.isFinite(p.hours))) return null; + const sorted = [...points].sort((a, b) => a.expected - b.expected); + const nBins = Math.min(CALIBRATION_BINS, points.length); + const bins = Array.from({ length: nBins }, (_, i) => { + const ps = sorted.slice( + Math.floor((i * points.length) / nBins), + Math.floor(((i + 1) * points.length) / nBins) + ); + const hours = ps.reduce((n, p) => n + p.hours, 0); + return { + bin: i + 1, + n: ps.length, + min: round(ps[0].expected, 2)!, + max: round(ps.at(-1)!.expected, 2)!, + hours: round(hours, 1)!, + // Rates are summed sharks over summed net-hours, never a mean of per-set rates. + expected: round(ps.reduce((n, p) => n + p.expected, 0) / hours, 3)!, + observed: round(ps.reduce((n, p) => n + p.observed, 0) / hours, 3)! + }; + }); + const zero = (key: 'site' | 'season', label: (v: string) => string) => + [...new Set(points.map((p) => p[key]))].sort().map((group) => { + const ps = points.filter((p) => p[key] === group); + return { + group: label(group), + n: ps.length, + observed: round((100 * ps.filter((p) => p.observed === 0).length) / ps.length, 1)!, + expected: round((100 * ps.reduce((n, p) => n + p.zero, 0)) / ps.length, 1)! + }; + }); + return { + n: points.length, + bins, + sites: zero('site', siteLabel), + seasons: zero('season', (s) => s), + // Columns rather than objects: a few thousand points per model. + residuals: { + expected: points.map((p) => Number(p.expected.toPrecision(3))), + day: points.map((p) => p.day), + value: points.map((p) => round(p.residual, 2)!) + } + }; +} diff --git a/src/lib/server/analytics/realDiagnostics.test.ts b/src/lib/server/analytics/realDiagnostics.test.ts new file mode 100644 index 0000000..5d72b1e --- /dev/null +++ b/src/lib/server/analytics/realDiagnostics.test.ts @@ -0,0 +1,21 @@ +import { expect, it } from 'vitest'; +import { dataset } from '../data'; +import { catchModel } from './catchModel'; + +it.skipIf(!process.env.REAL_DATA_DIR)( + 'real catch diagnostics reconcile to the fitted samples', + () => { + process.env.DATA_DIR = process.env.REAL_DATA_DIR; + process.env.DATABASE_PATH = ':memory:'; + const model = catchModel(dataset()); + expect(model).not.toBeNull(); + const diagnostics = model!.diagnostics[0].data; + expect(diagnostics).not.toBeNull(); + expect(diagnostics!.n).toBe(model!.n); + expect(diagnostics!.bins.reduce((n, b) => n + b.n, 0)).toBe(model!.n); + expect(diagnostics!.residuals.value.every(Number.isFinite)).toBe(true); + for (const g of model!.diagnostics) + if (g.data) expect(g.data.residuals.value).toHaveLength(g.data.n); + }, + 60000 +); diff --git a/src/routes/drivers/+page.svelte b/src/routes/drivers/+page.svelte index fd63444..0411af4 100644 --- a/src/routes/drivers/+page.svelte +++ b/src/routes/drivers/+page.svelte @@ -1,6 +1,7 @@