From 26c9608642288327855de750fadd39ca2fcb6ae6 Mon Sep 17 00:00:00 2001 From: luen Date: Wed, 4 Nov 2026 14:11:52 +1000 Subject: [PATCH 1/4] Charts: add catch-model fit diagnostics (#125) --- AGENTS.md | 2 + e2e/model-diagnostics.spec.ts | 21 +++ src/lib/components/ModelDiagnostics.svelte | 139 ++++++++++++++++++ src/lib/server/analytics/catchModel.ts | 5 + .../server/analytics/modelDiagnostics.test.ts | 31 ++++ src/lib/server/analytics/modelDiagnostics.ts | 72 +++++++++ .../server/analytics/realDiagnostics.test.ts | 19 +++ src/lib/server/import/index.ts | 2 +- src/routes/drivers/+page.svelte | 8 + 9 files changed, 298 insertions(+), 1 deletion(-) create mode 100644 e2e/model-diagnostics.spec.ts create mode 100644 src/lib/components/ModelDiagnostics.svelte create mode 100644 src/lib/server/analytics/modelDiagnostics.test.ts create mode 100644 src/lib/server/analytics/modelDiagnostics.ts create mode 100644 src/lib/server/analytics/realDiagnostics.test.ts diff --git a/AGENTS.md b/AGENTS.md index ace5450..13c7335 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -110,6 +110,8 @@ samples/fixture/ synthetic workbooks for tests/CI (committed) ### School holidays (#25) +- Catch-model diagnostics (#125) use each fit's exact rows/design/offsets; NB2 variance is μ + μ²/θ. They are in-sample checks, not validation of forecasting skill. Calibration rates are ratios of summed counts to summed net-hours; zero-catch plots require 20 sets per group. + - `fetch-external.mjs --only=school` writes `external/school-holidays.csv` (first and last day off, local dates). 2013-14 to 2016-17 are transcribed in the script from the Polynesian government's calendar orders for collèges and lycées (arrêtés 0270/CM 2013, 0399, 0401 and 0403/CM 2014), archived from education.pf by the Wayback Machine; the national dataset only starts in 2017-18. From 2017-18 it is the Ministry's `fr-en-calendrier-scolaire` open data (Polynésie, Licence Ouverte 2.0, credit it): pupils only, a day counts when either level is off, `end_date` is the day classes resume. - The national data has slips (the 2023 November break "starts" on 28 September): fix them in `SCHOOL_ERRATA`. The job throws if breaks overlap, so a new typo fails the weekly refresh instead of reaching the site. - `dayType` precedence: public holiday > festive window > school holidays > weekend. The pressure page also compares school-holiday weekdays with term-time weekdays, which keeps weekends out of the contrast. diff --git a/e2e/model-diagnostics.spec.ts b/e2e/model-diagnostics.spec.ts new file mode 100644 index 0000000..de1022c --- /dev/null +++ b/e2e/model-diagnostics.spec.ts @@ -0,0 +1,21 @@ +import { expect, test } from '@playwright/test'; +import AxeBuilder from '@axe-core/playwright'; + +test('model diagnostics expose accessible fit checks and species selection', async ({ page }) => { + await page.goto('/drivers'); + await page.getByText('How well does the model describe the catches?', { exact: true }).click(); + await expect(page.getByRole('combobox', { name: 'Model', exact: true })).toBeVisible(); + for (const theme of ['light', 'dark'] as const) { + await page.emulateMedia({ colorScheme: theme }); + const result = await new AxeBuilder({ page }).include('main').analyze(); + expect( + result.violations.filter((v) => ['serious', 'critical'].includes(v.impact ?? '')) + ).toEqual([]); + } + await page.getByRole('combobox', { name: 'Model', exact: true }).selectOption('1'); + await expect( + page.getByText( + 'Diagnostics are unavailable: the model needs a converged fit and a supported sample.' + ) + ).toBeVisible(); +}); diff --git a/src/lib/components/ModelDiagnostics.svelte b/src/lib/components/ModelDiagnostics.svelte new file mode 100644 index 0000000..2d71e07 --- /dev/null +++ b/src/lib/components/ModelDiagnostics.svelte @@ -0,0 +1,139 @@ + + +
+ How well does the model describe the catches? +

+ In-sample fit checks, not forecasting skill or causal evidence. Repeated sets and nights are not + independent. No uncertainty bands are implied. Nursery and season fixed effects cannot directly + predict unseen levels. +

+ + {#if d} +

+ {d.n} sets in the fitted sample. Calibration bins contain approximately equal numbers of sets, ordered + by expected count. Zero-catch groups need at least 20 sets; all group counts remain in the tables. +

+
+ + ({ + x: b.expected, + y: b.observed, + label: `Bin ${b.bin}: ${b.n} sets` + }))} + fits={[ + { + name: '1:1', + color: 'var(--ink-2)', + dashed: true, + points: [ + { x: 0, y: 0 }, + { x: maxRate, y: maxRate } + ] + } + ]} + xLabel="Expected sharks / net-hour" + yLabel="Observed sharks / net-hour" + /> + + {#each [{ name: 'nursery', rows: d.sites }, { name: 'season', rows: d.seasons }] as panel (panel.name)} + {@const rows = panel.rows.filter((r) => r.n >= 20)} + + {#if rows.length} r.group)} + series={[ + { + name: 'Observed', + color: 'var(--series-1)', + values: rows.map((r) => r.observed * 100) + }, + { + name: 'Expected', + color: 'var(--series-2)', + values: rows.map((r) => r.expected * 100) + } + ]} + yLabel="Zero-catch sets (%)" + />{:else}

No groups with at least 20 sets.

{/if} +
+ {/each} + {#each [{ key: 'expected' as const, label: 'Expected count' }, { key: 'day' as const, label: 'Days since 1 August' }] as axis (axis.key)} + + ({ x: p[axis.key], y: p.residual }))} + fits={[ + { + name: 'Zero residual', + color: 'var(--ink-2)', + dashed: true, + points: [ + { x: Math.min(...d.points.map((p) => p[axis.key])), y: 0 }, + { x: Math.max(...d.points.map((p) => p[axis.key])), y: 0 } + ] + } + ]} + xLabel={axis.label} + yLabel="Pearson residual" + /> + + {/each} +
+ {:else}

+ Diagnostics are unavailable: the model needs a converged fit and a supported sample. +

{/if} +
+ + diff --git a/src/lib/server/analytics/catchModel.ts b/src/lib/server/analytics/catchModel.ts index 3c328bb..2e5a542 100644 --- a/src/lib/server/analytics/catchModel.ts +++ b/src/lib/server/analytics/catchModel.ts @@ -6,6 +6,7 @@ * observational: it adjusts for what is measured, nothing more. */ import { mean, quantile, sd } from '#lib/stats'; +import { modelDiagnostics } from './modelDiagnostics'; import type { Dataset, SetRow } from '../data'; import { effortSets, round, seasonDay, seasonDayLabel, siteName } from './common'; import { SET_VARIABLES } from './env'; @@ -189,6 +190,10 @@ export function catchModel(d: Dataset) { return { n: all.rows.length, + diagnostics: [all, cm, na].map((m, i) => ({ + name: ['All sharks', 'Blacktip reef sharks', 'Sicklefin lemon sharks'][i], + data: modelDiagnostics(m.fit, m.rows, m.X, m.y, m.offset) + })), dropped: effortSets(d).filter((s) => s.cpue != null).length - all.rows.length, knots: knots.map((k) => seasonDayLabel(Math.round(k))), terms: vars.map((v, j) => ({ diff --git a/src/lib/server/analytics/modelDiagnostics.test.ts b/src/lib/server/analytics/modelDiagnostics.test.ts new file mode 100644 index 0000000..9c93f8f --- /dev/null +++ b/src/lib/server/analytics/modelDiagnostics.test.ts @@ -0,0 +1,31 @@ +import { expect, it } from 'vitest'; +import type { SetRow } from '../data'; +import type { GlmFit } from './glm'; +import { modelDiagnostics, zeroProbability } from './modelDiagnostics'; + +it('uses the NB2 zero probability, including the stable Poisson limit', () => { + expect(zeroProbability(2, 1)).toBeCloseTo(1 / 3); + expect(zeroProbability(2, Infinity)).toBeCloseTo(Math.exp(-2)); + expect(zeroProbability(2, 1e12)).toBeCloseTo(Math.exp(-2)); +}); +it('uses duration offsets and the fitted variance without refiltering the sample', () => { + const rows = Array.from({ length: 20 }, (_, i) => ({ + date: '2020-10-01', + siteId: 'a', + season: '2020-2021', + durationMin: i % 2 ? 120 : 60 + })) as SetRow[]; + const fit = { beta: [Math.log(2)], theta: 1, n: 20, converged: true } as GlmFit; + const X = rows.map(() => [1]), + y = rows.map((_, i) => (i % 2 ? 4 : 0)), + offset = rows.map((s) => Math.log(s.durationMin! / 60)); + const d = modelDiagnostics(fit, rows, X, y, offset)!; + expect(d.points[0].residual).toBeCloseTo(-2 / Math.sqrt(6)); + expect(d.points[1].expected).toBeCloseTo(4); + expect(d.sites[0].observed).toBe(0.5); + expect(d.sites[0].expected).toBeCloseTo((1 / 3 + 1 / 5) / 2); + expect(d.bins.reduce((n, b) => n + b.n, 0)).toBe(20); + expect(d.bins.every((b) => Math.abs(b.expected - 2) < 1e-10)).toBe(true); + expect(modelDiagnostics({ ...fit, converged: false }, rows, X, y, offset)).toBeNull(); + expect(modelDiagnostics(fit, rows.slice(1), X, y, offset)).toBeNull(); +}); diff --git a/src/lib/server/analytics/modelDiagnostics.ts b/src/lib/server/analytics/modelDiagnostics.ts new file mode 100644 index 0000000..ff144dd --- /dev/null +++ b/src/lib/server/analytics/modelDiagnostics.ts @@ -0,0 +1,72 @@ +import type { SetRow } from '../data'; +import type { GlmFit } from './glm'; +import { seasonDay } from './common'; + +export function zeroProbability(mu: number, theta: number) { + return Number.isFinite(theta) ? Math.exp(-theta * Math.log1p(mu / theta)) : Math.exp(-mu); +} + +/** Same X, response and offsets as the fit; all diagnostics are in-sample. */ +export function modelDiagnostics( + fit: GlmFit | null, + rows: SetRow[], + X: number[][], + y: number[], + offsets: number[] +) { + if ( + !fit?.converged || + !rows.length || + rows.length !== fit.n || + X.length !== rows.length || + y.length !== rows.length || + offsets.length !== rows.length + ) + return null; + const points = rows.map((s, i) => { + const mu = Math.exp( + Math.min(30, X[i].reduce((sum, v, j) => sum + v * fit.beta[j], 0) + offsets[i]) + ); + return { + expected: mu, + observed: y[i], + hours: Math.exp(offsets[i]), + residual: + (y[i] - mu) / Math.sqrt(mu + (Number.isFinite(fit.theta) ? (mu * mu) / fit.theta : 0)), + zero: zeroProbability(mu, fit.theta), + day: seasonDay(s.date!), + site: s.siteId!, + season: s.season + }; + }); + if (points.some((p) => !Number.isFinite(p.residual) || !Number.isFinite(p.hours))) return null; + const sorted = [...points].sort((a, b) => a.expected - b.expected); + const nBins = Math.min(10, points.length); + const bins = Array.from({ length: nBins }, (_, i) => { + const ps = sorted.slice( + Math.floor((i * points.length) / nBins), + Math.floor(((i + 1) * points.length) / nBins) + ); + const hours = ps.reduce((n, p) => n + p.hours, 0); + return { + bin: i + 1, + n: ps.length, + min: ps[0].expected, + max: ps.at(-1)!.expected, + hours, + expected: ps.reduce((n, p) => n + p.expected, 0) / hours, + observed: ps.reduce((n, p) => n + p.observed, 0) / hours + }; + }); + const zero = (key: 'site' | 'season') => + [...new Set(points.map((p) => p[key]))].map((group) => { + const ps = points.filter((p) => p[key] === group); + return { + group, + n: ps.length, + observed: ps.filter((p) => p.observed === 0).length / ps.length, + expected: ps.reduce((n, p) => n + p.zero, 0) / ps.length + }; + }); + return { n: points.length, bins, sites: zero('site'), seasons: zero('season'), points }; +} diff --git a/src/lib/server/analytics/realDiagnostics.test.ts b/src/lib/server/analytics/realDiagnostics.test.ts new file mode 100644 index 0000000..66b7a53 --- /dev/null +++ b/src/lib/server/analytics/realDiagnostics.test.ts @@ -0,0 +1,19 @@ +import { expect, it } from 'vitest'; +import { dataset } from '../data'; +import { catchModel } from './catchModel'; + +it.skipIf(!process.env.REAL_DATA_DIR)( + 'real catch diagnostics reconcile to the fitted samples', + () => { + process.env.DATA_DIR = process.env.REAL_DATA_DIR; + process.env.DATABASE_PATH = ':memory:'; + const model = catchModel(dataset()); + expect(model).not.toBeNull(); + const diagnostics = model!.diagnostics[0].data; + expect(diagnostics).not.toBeNull(); + expect(diagnostics!.n).toBe(model!.n); + expect(diagnostics!.bins.reduce((n, b) => n + b.n, 0)).toBe(model!.n); + expect(diagnostics!.points.every((p) => Number.isFinite(p.residual))).toBe(true); + }, + 60000 +); diff --git a/src/lib/server/import/index.ts b/src/lib/server/import/index.ts index 3cfd073..6ce12c6 100644 --- a/src/lib/server/import/index.ts +++ b/src/lib/server/import/index.ts @@ -73,7 +73,7 @@ import { SITE_DEFS } from './sites'; import { readWorkbook } from './xlsx'; /** Bump when import logic changes so existing caches rebuild on the next start. */ -export const IMPORTER_VERSION = '8'; +export const IMPORTER_VERSION = '9'; const MIGRATIONS = import.meta.glob('/drizzle/*.sql', { query: '?raw', diff --git a/src/routes/drivers/+page.svelte b/src/routes/drivers/+page.svelte index ad88e00..7f8a411 100644 --- a/src/routes/drivers/+page.svelte +++ b/src/routes/drivers/+page.svelte @@ -1,4 +1,5 @@ -
+
How well does the model describe the catches? -

- In-sample fit checks, not forecasting skill or causal evidence. Repeated sets and nights are not - independent. No uncertainty bands are implied. Nursery and season fixed effects cannot directly - predict unseen levels. +

+ These checks compare the model with the same sets it was fitted to. They show where it expects + too many or too few sharks. They don't show how well it would forecast another season, and a + good fit doesn't make an effect a cause. Sets from the same night or nursery aren't independent, + so the charts carry no uncertainty bands.

- + {#if d} -

- {d.n} sets in the fitted sample. Calibration bins contain approximately equal numbers of sets, ordered - by expected count. Zero-catch groups need at least 20 sets; all group counts remain in the tables. +

+ {plural(d.n, 'set')} in this model's sample. Zero-catch charts leave out groups with fewer than + {MIN_GROUP_SETS} sets; their tables keep every group.

({ x: b.expected, y: b.observed, - label: `Bin ${b.bin}: ${b.n} sets` + g: group.name, + label: `Group ${b.bin}: ${plural(b.n, 'set')}` }))} + groups={[{ name: group.name, color: group.color }]} fits={[ { - name: '1:1', + name: 'Observed = expected', color: 'var(--ink-2)', dashed: true, points: [ @@ -62,78 +105,86 @@ ] } ]} - xLabel="Expected sharks / net-hour" - yLabel="Observed sharks / net-hour" + xLabel="Expected sharks per net-hour" + yLabel="Observed sharks per net-hour" + r={5} + opacity={0.9} /> - {#each [{ name: 'nursery', rows: d.sites }, { name: 'season', rows: d.seasons }] as panel (panel.name)} - {@const rows = panel.rows.filter((r) => r.n >= 20)} + {#each panels as panel (panel.name)} + {@const rows = panel.rows.filter((r) => r.n >= MIN_GROUP_SETS)} - {#if rows.length} r.group)} series={[ - { - name: 'Observed', - color: 'var(--series-1)', - values: rows.map((r) => r.observed * 100) - }, - { - name: 'Expected', - color: 'var(--series-2)', - values: rows.map((r) => r.expected * 100) - } + { name: 'Observed', color: group.color, values: rows.map((r) => r.observed) }, + { name: 'Expected', color: 'var(--muted)', values: rows.map((r) => r.expected) } ]} - yLabel="Zero-catch sets (%)" - />{:else}

No groups with at least 20 sets.

{/if} + yLabel="Sets with no sharks (%)" + valueFormat={(v) => `${num(v)}%`} + /> + {:else} +

No {panel.name} has {MIN_GROUP_SETS} sets in this sample.

+ {/if}
{/each} - {#each [{ key: 'expected' as const, label: 'Expected count' }, { key: 'day' as const, label: 'Days since 1 August' }] as axis (axis.key)} + {#each residualPanels as r (r.title)} ({ x: p[axis.key], y: p.residual }))} + points={r.xs.map((x, i) => ({ x, y: d.residuals.value[i], g: group.name }))} + groups={[{ name: group.name, color: group.color }]} fits={[ { - name: 'Zero residual', + name: 'As expected', color: 'var(--ink-2)', dashed: true, points: [ - { x: Math.min(...d.points.map((p) => p[axis.key])), y: 0 }, - { x: Math.max(...d.points.map((p) => p[axis.key])), y: 0 } + { x: Math.min(...r.xs), y: 0 }, + { x: Math.max(...r.xs), y: 0 } ] } ]} - xLabel={axis.label} + xLabel={r.xLabel} + xLog={r.xLog} + xFormat={r.xFormat} yLabel="Pearson residual" /> {/each}
- {:else}

- Diagnostics are unavailable: the model needs a converged fit and a supported sample. -

{/if} + {:else} +

Diagnostics are unavailable: the model needs a converged fit and a supported sample.

+ {/if}
diff --git a/src/lib/server/analytics/catchModel.ts b/src/lib/server/analytics/catchModel.ts index 2e5a542..b3ac99e 100644 --- a/src/lib/server/analytics/catchModel.ts +++ b/src/lib/server/analytics/catchModel.ts @@ -5,12 +5,13 @@ * fall through the season); the model estimates each one with the others held fixed. Still * observational: it adjusts for what is measured, nothing more. */ +import { SPECIES } from '#lib/species'; import { mean, quantile, sd } from '#lib/stats'; -import { modelDiagnostics } from './modelDiagnostics'; import type { Dataset, SetRow } from '../data'; import { effortSets, round, seasonDay, seasonDayLabel, siteName } from './common'; import { SET_VARIABLES } from './env'; import { contrast, fitGlm, naturalSpline, type GlmFit } from './glm'; +import { modelDiagnostics } from './modelDiagnostics'; /** Environmental terms, chosen to cover each family once (no near-duplicates like air vs sea). */ const TERMS = [ @@ -190,9 +191,14 @@ export function catchModel(d: Dataset) { return { n: all.rows.length, - diagnostics: [all, cm, na].map((m, i) => ({ - name: ['All sharks', 'Blacktip reef sharks', 'Sicklefin lemon sharks'][i], - data: modelDiagnostics(m.fit, m.rows, m.X, m.y, m.offset) + diagnostics: [ + { m: all, name: 'All sharks', color: 'var(--series-3)' }, + { m: cm, name: 'Blacktip reef sharks', color: SPECIES.Cm.color }, + { m: na, name: 'Sicklefin lemon sharks', color: SPECIES.Na.color } + ].map(({ m, name, color }) => ({ + name, + color, + data: modelDiagnostics(m.fit, m.rows, m.X, m.y, m.offset, (id) => siteName(d, id)) })), dropped: effortSets(d).filter((s) => s.cpue != null).length - all.rows.length, knots: knots.map((k) => seasonDayLabel(Math.round(k))), diff --git a/src/lib/server/analytics/modelDiagnostics.test.ts b/src/lib/server/analytics/modelDiagnostics.test.ts index 9c93f8f..cfa704d 100644 --- a/src/lib/server/analytics/modelDiagnostics.test.ts +++ b/src/lib/server/analytics/modelDiagnostics.test.ts @@ -8,24 +8,37 @@ it('uses the NB2 zero probability, including the stable Poisson limit', () => { expect(zeroProbability(2, Infinity)).toBeCloseTo(Math.exp(-2)); expect(zeroProbability(2, 1e12)).toBeCloseTo(Math.exp(-2)); }); + +const rows = Array.from({ length: 20 }, (_, i) => ({ + date: '2020-10-01', + siteId: i < 10 ? 'b' : 'a', + season: '2020-2021', + durationMin: i % 2 ? 120 : 60 +})) as SetRow[]; +const fit = { beta: [Math.log(2)], theta: 1, n: 20, converged: true } as GlmFit; +const X = rows.map(() => [1]), + y = rows.map((_, i) => (i % 2 ? 4 : 0)), + offset = rows.map((s) => Math.log(s.durationMin! / 60)); + it('uses duration offsets and the fitted variance without refiltering the sample', () => { - const rows = Array.from({ length: 20 }, (_, i) => ({ - date: '2020-10-01', - siteId: 'a', - season: '2020-2021', - durationMin: i % 2 ? 120 : 60 - })) as SetRow[]; - const fit = { beta: [Math.log(2)], theta: 1, n: 20, converged: true } as GlmFit; - const X = rows.map(() => [1]), - y = rows.map((_, i) => (i % 2 ? 4 : 0)), - offset = rows.map((s) => Math.log(s.durationMin! / 60)); - const d = modelDiagnostics(fit, rows, X, y, offset)!; - expect(d.points[0].residual).toBeCloseTo(-2 / Math.sqrt(6)); - expect(d.points[1].expected).toBeCloseTo(4); - expect(d.sites[0].observed).toBe(0.5); - expect(d.sites[0].expected).toBeCloseTo((1 / 3 + 1 / 5) / 2); + const d = modelDiagnostics(fit, rows, X, y, offset, (id) => `Nursery ${id.toUpperCase()}`)!; + // μ = 2 per net-hour × 1 h; NB2 variance 2 + 2²/1 = 6. + expect(d.residuals.value[0]).toBeCloseTo(-2 / Math.sqrt(6)); + expect(d.residuals.expected[1]).toBeCloseTo(4); + expect(d.residuals.day).toHaveLength(20); + // Groups are sorted by id and labelled; shares are percentages. + expect(d.sites.map((s) => s.group)).toEqual(['Nursery A', 'Nursery B']); + expect(d.sites[0].observed).toBe(50); + expect(d.sites[0].expected).toBeCloseTo((100 * (1 / 3 + 1 / 5)) / 2, 1); expect(d.bins.reduce((n, b) => n + b.n, 0)).toBe(20); - expect(d.bins.every((b) => Math.abs(b.expected - 2) < 1e-10)).toBe(true); + // Unequal durations: summed sharks over summed net-hours stays at the true rate of 2. + expect(d.bins.every((b) => Math.abs(b.expected - 2) < 1e-9)).toBe(true); +}); + +it('handles the Poisson limit and refuses failed or mismatched fits', () => { + const p = modelDiagnostics({ ...fit, theta: Infinity }, rows, X, y, offset)!; + expect(p.residuals.value[0]).toBeCloseTo(-2 / Math.sqrt(2)); + expect(modelDiagnostics(null, rows, X, y, offset)).toBeNull(); expect(modelDiagnostics({ ...fit, converged: false }, rows, X, y, offset)).toBeNull(); expect(modelDiagnostics(fit, rows.slice(1), X, y, offset)).toBeNull(); }); diff --git a/src/lib/server/analytics/modelDiagnostics.ts b/src/lib/server/analytics/modelDiagnostics.ts index ff144dd..f0c87bb 100644 --- a/src/lib/server/analytics/modelDiagnostics.ts +++ b/src/lib/server/analytics/modelDiagnostics.ts @@ -1,7 +1,16 @@ +/** + * In-sample fit checks for the catch model (#125): calibration, zero-catch shares and Pearson + * residuals, from the exact rows, design matrix, counts and offsets of each fit. Only what the + * charts draw goes to the page, rounded, so /drivers stays light (#76). + */ import type { SetRow } from '../data'; import type { GlmFit } from './glm'; -import { seasonDay } from './common'; +import { round, seasonDay } from './common'; +/** Calibration groups: sets sorted by expected count, split into this many near-equal bins. */ +export const CALIBRATION_BINS = 10; + +/** NB2 P(Y = 0) = (θ / (θ + μ))^θ, with the Poisson limit e^−μ as θ → ∞. */ export function zeroProbability(mu: number, theta: number) { return Number.isFinite(theta) ? Math.exp(-theta * Math.log1p(mu / theta)) : Math.exp(-mu); } @@ -12,7 +21,8 @@ export function modelDiagnostics( rows: SetRow[], X: number[][], y: number[], - offsets: number[] + offsets: number[], + siteLabel: (id: string) => string = (id) => id ) { if ( !fit?.converged || @@ -41,7 +51,7 @@ export function modelDiagnostics( }); if (points.some((p) => !Number.isFinite(p.residual) || !Number.isFinite(p.hours))) return null; const sorted = [...points].sort((a, b) => a.expected - b.expected); - const nBins = Math.min(10, points.length); + const nBins = Math.min(CALIBRATION_BINS, points.length); const bins = Array.from({ length: nBins }, (_, i) => { const ps = sorted.slice( Math.floor((i * points.length) / nBins), @@ -51,22 +61,34 @@ export function modelDiagnostics( return { bin: i + 1, n: ps.length, - min: ps[0].expected, - max: ps.at(-1)!.expected, - hours, - expected: ps.reduce((n, p) => n + p.expected, 0) / hours, - observed: ps.reduce((n, p) => n + p.observed, 0) / hours + min: round(ps[0].expected, 2)!, + max: round(ps.at(-1)!.expected, 2)!, + hours: round(hours, 1)!, + // Rates are summed sharks over summed net-hours, never a mean of per-set rates. + expected: round(ps.reduce((n, p) => n + p.expected, 0) / hours, 3)!, + observed: round(ps.reduce((n, p) => n + p.observed, 0) / hours, 3)! }; }); - const zero = (key: 'site' | 'season') => - [...new Set(points.map((p) => p[key]))].map((group) => { + const zero = (key: 'site' | 'season', label: (v: string) => string) => + [...new Set(points.map((p) => p[key]))].sort().map((group) => { const ps = points.filter((p) => p[key] === group); return { - group, + group: label(group), n: ps.length, - observed: ps.filter((p) => p.observed === 0).length / ps.length, - expected: ps.reduce((n, p) => n + p.zero, 0) / ps.length + observed: round((100 * ps.filter((p) => p.observed === 0).length) / ps.length, 1)!, + expected: round((100 * ps.reduce((n, p) => n + p.zero, 0)) / ps.length, 1)! }; }); - return { n: points.length, bins, sites: zero('site'), seasons: zero('season'), points }; + return { + n: points.length, + bins, + sites: zero('site', siteLabel), + seasons: zero('season', (s) => s), + // Columns rather than objects: a few thousand points per model. + residuals: { + expected: points.map((p) => Number(p.expected.toPrecision(3))), + day: points.map((p) => p.day), + value: points.map((p) => round(p.residual, 2)!) + } + }; } diff --git a/src/lib/server/analytics/realDiagnostics.test.ts b/src/lib/server/analytics/realDiagnostics.test.ts index 66b7a53..5d72b1e 100644 --- a/src/lib/server/analytics/realDiagnostics.test.ts +++ b/src/lib/server/analytics/realDiagnostics.test.ts @@ -13,7 +13,9 @@ it.skipIf(!process.env.REAL_DATA_DIR)( expect(diagnostics).not.toBeNull(); expect(diagnostics!.n).toBe(model!.n); expect(diagnostics!.bins.reduce((n, b) => n + b.n, 0)).toBe(model!.n); - expect(diagnostics!.points.every((p) => Number.isFinite(p.residual))).toBe(true); + expect(diagnostics!.residuals.value.every(Number.isFinite)).toBe(true); + for (const g of model!.diagnostics) + if (g.data) expect(g.data.residuals.value).toHaveLength(g.data.n); }, 60000 ); diff --git a/src/routes/drivers/+page.svelte b/src/routes/drivers/+page.svelte index 35d7ba9..65e67c2 100644 --- a/src/routes/drivers/+page.svelte +++ b/src/routes/drivers/+page.svelte @@ -1,7 +1,7 @@