From 08ec0bb9884a07e8a2023a3f0dac48fda9be9fcb Mon Sep 17 00:00:00 2001 From: luen Date: Tue, 3 Nov 2026 13:56:23 +1000 Subject: [PATCH] Drivers: negative-binomial model of catches (#54) A joint model of sharks per gillnet set: negative binomial with log(net-hours) as the offset, a natural-spline day of season, nursery and season effects (each against the average) and nine environment terms per +1 SD, fitted per species. New "All together" section on /drivers with forest plots (new ForestPlot chart), the adjusted seasonal curve, and a sea-temperature step-by-step comparison; a live finding reports how the raw SST link changes once the calendar is in the model. glm.ts holds a small generic NB/Poisson GLM (IRLS, ML theta, Wald contrasts, natural spline basis) with tests on simulated data. seasonDayLabel moves to #lib/format so pages can use it. Closes #54 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 8 + src/lib/components/charts/ForestPlot.svelte | 198 +++++++++++++++++ src/lib/format.ts | 6 + src/lib/server/analytics/catchModel.ts | 211 ++++++++++++++++++ src/lib/server/analytics/common.ts | 5 +- src/lib/server/analytics/findings.ts | 26 +++ src/lib/server/analytics/glm.test.ts | 106 +++++++++ src/lib/server/analytics/glm.ts | 231 ++++++++++++++++++++ src/lib/stats.ts | 26 ++- src/routes/drivers/+page.server.ts | 2 + src/routes/drivers/+page.svelte | 209 +++++++++++++++++- 11 files changed, 1021 insertions(+), 7 deletions(-) create mode 100644 src/lib/components/charts/ForestPlot.svelte create mode 100644 src/lib/server/analytics/catchModel.ts create mode 100644 src/lib/server/analytics/glm.test.ts create mode 100644 src/lib/server/analytics/glm.ts diff --git a/AGENTS.md b/AGENTS.md index 0873f2e..bd5e202 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,11 +43,13 @@ src/lib/server/data.ts loads the whole dataset into memory per import; memo() src/lib/server/analytics common.ts (CPUE, Fulton's K, recapture pairs…) · env.ts (per-set variables) findings.ts (live findings) · insitu.ts (logger temperature summaries) population.ts (Schnabel abundance, von Bertalanffy growth) + glm.ts (NB/Poisson GLM, splines) · catchModel.ts (joint catch model) src/lib/server/astro.ts sunset/twilight/moon (local formulas) · tides.ts (extrema, interpolation) src/lib/server/holidays.ts French Polynesia public holidays (Easter-based too) + festive window src/lib/stats.ts regression, Spearman, Welch, Wilson, binomial, histograms, box stats src/lib/components/ ChartCard, StatTile, DataTable, FindingCard, MooreaMap (SSR SVG), InteractiveMap (Leaflet, lazy; MooreaMap is its fallback), charts/* + (ForestPlot: rate ratios with intervals on a log scale) src/routes/ one folder per page (+page.server.ts computes, +page.svelte renders) scripts/ fetch-external.mjs · make-fixture.mjs · server.mjs · nightlyRelease.mjs external/ public environmental snapshots (committed) @@ -83,6 +85,12 @@ samples/fixture/ synthetic workbooks for tests/CI (committed) - ERA5 `cloud_evening` covers every set; the crew's 1–4 `cloud_cover` score exists only in 2020–22 and tracks ERA5 (score 1 ≈ 43%, 4 ≈ 94%). `moonlight()` in `analytics/cloud.ts` = illumination × sin(moon altitude) × clear-sky fraction. - Cloud vs water temperature must be compared within the same month: the cloudy wet season is also the warmest, so raw correlations are ~0 while within-month ones are strongly negative. +### Catch model (#54) + +- `analytics/catchModel.ts`: negative-binomial counts per timed gillnet set, offset log(net-hours), natural-spline day of season (knots at the 0/25/50/75/100% quantiles), nursery and season as effect-coded fixed effects (each level vs the average; no random effects), and nine environment terms per +1 SD. Complete cases only; nurseries need ≥ 30 complete sets, and a species gets a nursery effect only where ≥ 10 of it were caught (otherwise the estimate runs off to zero or infinity). +- The headline: sea temperature's raw link with catch rate (≈ −25% per °C) vanishes once day of the season is in the model. Don't describe SST as a driver of catches on other pages. +- `glm.ts` is generic (IRLS + ML θ, Wald intervals via `contrast`); reuse it rather than writing another fitter. + ### Adding a new season Drop the workbook in `data/` named like `Physioshark_Shark Log 2026-2027.xlsx` (the `YYYY-YYYY` in the file name sets the season). Check the Data quality page for unknown sites, unmatched headers ("Sheet not imported") and unit warnings, and extend aliases if needed. Never rename columns in the workbook to make the importer happy. diff --git a/src/lib/components/charts/ForestPlot.svelte b/src/lib/components/charts/ForestPlot.svelte new file mode 100644 index 0000000..9365eea --- /dev/null +++ b/src/lib/components/charts/ForestPlot.svelte @@ -0,0 +1,198 @@ + + + + + ({ label: s.name, color: s.color, kind: 'dot' as const }))} /> +
+ (tip = null)} + > + {#each xt as t (t)} + + {format(t)} + {/each} + + {#each rows as row, i (i)} + {@const cy = m.top + band * i + band / 2} + + show(e, i)} + /> + {#each series as s, k (s.name)} + {@const v = s.values[i]} + {#if v} + + + {/if} + {/each} + + {#each labels[i] as line, j (j)} + 1 ? '-0.25em' : '0.32em'} + >{line} + {/each} + {row} + + + {/each} + {#if xLabel} + {xLabel} + {/if} + + +
+ + diff --git a/src/lib/format.ts b/src/lib/format.ts index 254d1c0..ec7643a 100644 --- a/src/lib/format.ts +++ b/src/lib/format.ts @@ -53,6 +53,12 @@ export function monthYear(iso: string): string { return `${MONTHS[m - 1]} ${y}`; } +/** Day of the fieldwork season (1 Aug = day 0) → `5 Nov`. */ +export const seasonDayLabel = (day: number) => { + const d = new Date(Date.UTC(2001, 7, 1) + day * 86_400_000); + return `${d.getUTCDate()} ${MONTHS[d.getUTCMonth()]}`; +}; + /** `2013-2014` → `2013–14`. */ export function season(s: string): string { return s.length === 9 ? `${s.slice(0, 4)}–${s.slice(7)}` : s; diff --git a/src/lib/server/analytics/catchModel.ts b/src/lib/server/analytics/catchModel.ts new file mode 100644 index 0000000..3c328bb --- /dev/null +++ b/src/lib/server/analytics/catchModel.ts @@ -0,0 +1,211 @@ +/** + * Joint model of catches (#54): negative-binomial counts per gillnet set with log(net-hours) as the + * offset, a smooth for day of the season, nursery and season effects and the environment. The + * single-variable correlations on /drivers mix these together (sea temperature rises as catches + * fall through the season); the model estimates each one with the others held fixed. Still + * observational: it adjusts for what is measured, nothing more. + */ +import { mean, quantile, sd } from '#lib/stats'; +import type { Dataset, SetRow } from '../data'; +import { effortSets, round, seasonDay, seasonDayLabel, siteName } from './common'; +import { SET_VARIABLES } from './env'; +import { contrast, fitGlm, naturalSpline, type GlmFit } from './glm'; + +/** Environmental terms, chosen to cover each family once (no near-duplicates like air vs sea). */ +const TERMS = [ + 'sst', + 'moonIllum', + 'tide', + 'wind', + 'rain24h', + 'cloud', + 'waves', + 'startSunset', + 'crew' +] as const; +/** Nurseries need this many complete sets to get their own effect. */ +const MIN_SITE_SETS = 30; +const SPLINE_KNOTS = [0, 0.25, 0.5, 0.75, 1]; + +type Response = 'all' | 'Cm' | 'Na'; + +const ci = (c: { est: number; lo: number; hi: number }) => ({ + est: round(Math.exp(c.est), 3)!, + lo: round(Math.exp(c.lo), 3)!, + hi: round(Math.exp(c.hi), 3)! +}); + +/** Effect (sum-to-zero) coding: each level against the average level; the last is −Σ others. */ +const effectCode = (levels: string[], v: string) => + levels.slice(0, -1).map((l) => (v === levels[levels.length - 1] ? -1 : v === l ? 1 : 0)); + +function count(s: SetRow, r: Response): number { + const hours = s.durationMin! / 60; + const rate = r === 'all' ? s.cpue : r === 'Cm' ? s.cpueCm : s.cpueNa; + return Math.round((rate ?? 0) * hours); +} + +/** A species gets its own nursery effect only where it was caught at least this often. */ +const MIN_SITE_CATCH = 10; + +export function catchModel(d: Dataset) { + const vars = TERMS.map((k) => SET_VARIABLES.find((v) => v.key === k)!); + const complete = effortSets(d).filter( + (s) => s.cpue != null && s.date && s.siteId && vars.every((v) => v.get(s) != null) + ); + const perSite = new Map(); + for (const s of complete) perSite.set(s.siteId!, (perSite.get(s.siteId!) ?? 0) + 1); + const sites = [...perSite] + .filter(([, n]) => n >= MIN_SITE_SETS) + .map(([id]) => id) + .sort(); + const base = complete.filter((s) => sites.includes(s.siteId!)); + const seasons = [...new Set(base.map((s) => s.season))].sort(); + if (base.length < 200 || sites.length < 2 || seasons.length < 2) return null; + + // Scales and spline knots come from the full model so every species shares the same units. + const scale = vars.map((v) => { + const xs = base.map((s) => v.get(s)!); + return { mean: mean(xs)!, sd: sd(xs) || 1 }; + }); + const baseDays = base.map((s) => seasonDay(s.date!)); + const knots = SPLINE_KNOTS.map((q) => quantile(baseDays, q)!); + const spline = naturalSpline(knots); + const nSpline = knots.length - 1; + const dayGrid: number[] = []; + for (let x = Math.ceil(knots[0] / 7) * 7; x <= knots[knots.length - 1]; x += 7) dayGrid.push(x); + + // Column layout: intercept | environment (per SD) | day spline | nursery | season. + const iEnv = 1; + const iDay = iEnv + vars.length; + const fitFor = (r: Response) => { + const caught = new Map(); + for (const s of base) caught.set(s.siteId!, (caught.get(s.siteId!) ?? 0) + count(s, r)); + const own = sites.filter((id) => r === 'all' || (caught.get(id) ?? 0) >= MIN_SITE_CATCH); + const rows = base.filter((s) => own.includes(s.siteId!)); + const ownSeasons = seasons.filter((x) => rows.some((s) => s.season === x)); + const days = rows.map((s) => seasonDay(s.date!)); + const meanBasis = new Array(nSpline).fill(0); + for (const day of days) spline(day).forEach((v, j) => (meanBasis[j] += v / days.length)); + const iSite = iDay + nSpline; + const iSeason = iSite + own.length - 1; + const p = iSeason + ownSeasons.length - 1; + const X = rows.map((s, i) => [ + 1, + ...vars.map((v, j) => (v.get(s)! - scale[j].mean) / scale[j].sd), + ...spline(days[i]), + ...effectCode(own, s.siteId!), + ...effectCode(ownSeasons, s.season) + ]); + const y = rows.map((s) => count(s, r)); + const offset = rows.map((s) => Math.log(s.durationMin! / 60)); + const fit = own.length >= 2 ? fitGlm(X, y, offset) : null; + const unit = (j: number) => Array.from({ length: p }, (_, k) => (k === j ? 1 : 0)); + const level = (start: number, n: number, i: number) => + Array.from({ length: p }, (_, k) => + k < start || k >= start + n - 1 ? 0 : i === n - 1 ? -1 : k - start === i ? 1 : 0 + ); + const effect = (g: number[]) => { + const c = contrast(fit!, g); + return { ...ci(c), p: c.p }; + }; + return { + fit, + rows, + offset, + y, + X, + summary: fit && { + n: fit.n, + sharks: y.reduce((a, b) => a + b, 0), + theta: round(fit.theta, 2), + devExplained: round(1 - fit.deviance / fit.nullDeviance, 3), + env: vars.map((_, j) => effect(unit(iEnv + j))), + sites: sites.map((id) => { + const i = own.indexOf(id); + return i < 0 ? null : effect(level(iSite, own.length, i)); + }), + seasons: seasons.map((x) => { + const i = ownSeasons.indexOf(x); + return i < 0 ? null : effect(level(iSeason, ownSeasons.length, i)); + }), + // Day-of-season curve relative to the season average. + day: dayGrid.map((x) => { + const b = spline(x); + const g = Array.from({ length: p }, (_, k) => + k >= iDay && k < iDay + nSpline ? b[k - iDay] - meanBasis[k - iDay] : 0 + ); + return { x, ...ci(contrast(fit, g)) }; + }) + } + }; + }; + const all = fitFor('all'); + const cm = fitFor('Cm'); + const na = fitFor('Na'); + if (!all.summary) return null; + + // The SST question from #54: how much of the raw SST link is really the time of season? + const sstJ = vars.findIndex((v) => v.key === 'sst'); + const sstC = all.rows.map((s) => vars[sstJ].get(s)! - scale[sstJ].mean); + const days = all.rows.map((s) => seasonDay(s.date!)); + const perDegree = (fit: GlmFit | null, j: number, per = 1) => { + if (!fit) return null; + const c = contrast( + fit, + Array.from({ length: fit.beta.length }, (_, k) => (k === j ? 1 / per : 0)) + ); + return { ...ci(c), p: c.p }; + }; + const sst = [ + { + label: 'Sea temperature alone', + v: perDegree( + fitGlm( + sstC.map((x) => [1, x]), + all.y, + all.offset + ), + 1 + ) + }, + { + label: '+ day of the season', + v: perDegree( + fitGlm( + sstC.map((x, i) => [1, x, ...spline(days[i])]), + all.y, + all.offset + ), + 1 + ) + }, + { + label: '+ nursery, season and conditions', + v: perDegree(all.fit, iEnv + sstJ, scale[sstJ].sd) + } + ]; + const poisson = fitGlm(all.X, all.y, all.offset, 'poisson'); + + return { + n: all.rows.length, + dropped: effortSets(d).filter((s) => s.cpue != null).length - all.rows.length, + knots: knots.map((k) => seasonDayLabel(Math.round(k))), + terms: vars.map((v, j) => ({ + key: v.key, + label: v.label, + unit: v.unit, + sd: round(scale[j].sd, v.dp + 1)!, + dp: v.dp + })), + sites: sites.map((id) => ({ id, name: siteName(d, id), n: perSite.get(id)! })), + seasons, + days: dayGrid.map((x) => ({ x, label: seasonDayLabel(x) })), + all: all.summary, + cm: cm.summary, + na: na.summary, + sst, + aicNb: round(all.fit?.aic, 1), + aicPoisson: round(poisson?.aic, 1) + }; +} diff --git a/src/lib/server/analytics/common.ts b/src/lib/server/analytics/common.ts index b112932..594dfb2 100644 --- a/src/lib/server/analytics/common.ts +++ b/src/lib/server/analytics/common.ts @@ -42,10 +42,7 @@ export function seasonDay(date: string): number { return daysBetween(start, date); } -export const seasonDayLabel = (day: number) => { - const d = new Date(Date.UTC(2001, 7, 1) + day * 86_400_000); - return `${d.getUTCDate()} ${['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'][d.getUTCMonth()]}`; -}; +export { seasonDayLabel } from '#lib/format'; export interface BinnedMean { label: string; diff --git a/src/lib/server/analytics/findings.ts b/src/lib/server/analytics/findings.ts index 495b8d3..8838272 100644 --- a/src/lib/server/analytics/findings.ts +++ b/src/lib/server/analytics/findings.ts @@ -18,6 +18,7 @@ import { seasonDayLabel, siteName } from './common'; +import { catchModel } from './catchModel'; import { cloudWater } from './cloud'; import { dailyMaxBySite, versusSst } from './insitu'; @@ -391,6 +392,31 @@ function compute(d: Dataset): Finding[] { href: '/drivers' }); + // Sea temperature or the calendar? (#54) + const model = memo('catchModel', catchModel); + const [alone, , adjusted] = model?.sst.map((x) => x.v) ?? []; + const warming = spearman( + pairs( + eff, + (s) => (s.date ? seasonDay(s.date) : null), + (s) => s.sst + ) + ); + if (model && alone && adjusted && alone.hi < 1 && (warming?.rho ?? 0) > 0.3) { + const peak = model.all.day.reduce((a, b) => (b.est > a.est ? b : a)); + const change = (rr: number) => `${rr < 1 ? '−' : '+'}${pct(Math.abs(rr - 1))}`; + const words = (rr: number) => `${pct(Math.abs(rr - 1))} ${rr < 1 ? 'fewer' : 'more'}`; + const flat = adjusted.lo <= 1 && adjusted.hi >= 1; + out.push({ + id: 'sst-calendar', + kind: 'pattern', + title: 'Warm water, or just late in the season?', + value: `${change(alone.est)} → ${flat ? 'no link' : change(adjusted.est)}`, + text: `On its own, each extra °C of sea temperature goes with ${words(alone.est)} sharks per net-hour. Model the day of the season, nursery, season and weather together and the link ${flat ? `disappears (rate ratio ${num(adjusted.est, 2)}, 95% interval ${num(adjusted.lo, 2)}–${num(adjusted.hi, 2)}, where 1 means no change)` : `becomes ${words(adjusted.est)} per °C`}: the sea keeps warming as the season goes on, while catches fall away after peaking around ${seasonDayLabel(peak.x)}.`, + href: '/drivers#model' + }); + } + // Fun totals const totalTl = caps.reduce((a, c) => a + (c.tlMm ?? 0), 0) / 1e6; const totalKg = caps.reduce((a, c) => a + (c.massKg ?? 0), 0); diff --git a/src/lib/server/analytics/glm.test.ts b/src/lib/server/analytics/glm.test.ts new file mode 100644 index 0000000..9815f5b --- /dev/null +++ b/src/lib/server/analytics/glm.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from 'vitest'; +import { contrast, fitGlm, invertSpd, naturalSpline } from './glm'; + +// Deterministic generators so the tests are stable. +function rng(seed: number) { + let s = seed; + const u = () => (s = (s * 16807) % 2147483647) / 2147483647; + const normal = () => Math.sqrt(-2 * Math.log(u())) * Math.cos(2 * Math.PI * u()); + const gamma = (shape: number): number => { + if (shape < 1) return gamma(shape + 1) * u() ** (1 / shape); + const d = shape - 1 / 3; + const c = 1 / Math.sqrt(9 * d); + for (;;) { + const x = normal(); + const v = (1 + c * x) ** 3; + if (v > 0 && Math.log(u()) < 0.5 * x * x + d - d * v + d * Math.log(v)) return d * v; + } + }; + const poisson = (lambda: number) => { + const l = Math.exp(-lambda); + let k = 0; + let p = 1; + do { + k++; + p *= u(); + } while (p > l); + return k - 1; + }; + return { u, normal, gamma, poisson }; +} + +describe('invertSpd', () => { + it('inverts a symmetric positive-definite matrix', () => { + const a = [ + [4, 2, 0.6], + [2, 2, 0.4], + [0.6, 0.4, 1] + ]; + const inv = invertSpd(a)!; + for (let i = 0; i < 3; i++) + for (let j = 0; j < 3; j++) + expect(a[i].reduce((s, v, k) => s + v * inv[k][j], 0)).toBeCloseTo(i === j ? 1 : 0, 10); + }); + it('returns null for a singular matrix', () => { + expect( + invertSpd([ + [1, 1], + [1, 1] + ]) + ).toBeNull(); + }); +}); + +describe('fitGlm', () => { + it('intercept-only Poisson gives the pooled rate', () => { + const y = [0, 2, 1, 5, 0, 3]; + const hours = [1, 2, 1.5, 3, 0.5, 2]; + const fit = fitGlm( + y.map(() => [1]), + y, + hours.map(Math.log), + 'poisson' + )!; + expect(Math.exp(fit.beta[0])).toBeCloseTo(11 / 10, 6); + }); + + it('recovers negative-binomial coefficients and θ from simulated catches', () => { + const r = rng(42); + const beta = [-0.5, 0.3, -0.4]; + const theta = 1.5; + const X: number[][] = []; + const y: number[] = []; + const off: number[] = []; + for (let i = 0; i < 4000; i++) { + const row = [1, r.normal(), r.u() < 0.4 ? 1 : 0]; + const o = Math.log(1 + 3 * r.u()); + const mu = Math.exp(row.reduce((s, v, k) => s + v * beta[k], 0) + o); + X.push(row); + off.push(o); + y.push(r.poisson((mu * r.gamma(theta)) / theta)); + } + const fit = fitGlm(X, y, off)!; + expect(fit.converged).toBe(true); + for (let k = 0; k < 3; k++) { + const c = contrast(fit, k === 0 ? [1, 0, 0] : k === 1 ? [0, 1, 0] : [0, 0, 1]); + expect(c.lo).toBeLessThan(beta[k]); + expect(c.hi).toBeGreaterThan(beta[k]); + } + expect(fit.theta).toBeGreaterThan(1.1); + expect(fit.theta).toBeLessThan(2.1); + // Overdispersed data: the NB fits far better than a Poisson with the same terms. + const pois = fitGlm(X, y, off, 'poisson')!; + expect(fit.aic).toBeLessThan(pois.aic - 50); + }); +}); + +describe('naturalSpline', () => { + it('is linear beyond the boundary knots', () => { + const f = naturalSpline([0, 25, 50, 75, 100]); + const diff = (a: number, b: number) => f(b).map((v, i) => v - f(a)[i]); + const d1 = diff(110, 120); + const d2 = diff(120, 130); + d1.forEach((v, i) => expect(v).toBeCloseTo(d2[i], 10)); + expect(f(50)).toHaveLength(4); + }); +}); diff --git a/src/lib/server/analytics/glm.ts b/src/lib/server/analytics/glm.ts new file mode 100644 index 0000000..077ecb4 --- /dev/null +++ b/src/lib/server/analytics/glm.ts @@ -0,0 +1,231 @@ +/** + * Negative-binomial (NB2) and Poisson regression with a log link and an offset (#54): IRLS for + * the coefficients, θ by maximum likelihood, alternating until both settle. Dense and small: a + * few dozen columns and a couple of thousand rows. + */ +import { logGamma, normalP } from '#lib/stats'; + +export interface GlmFit { + beta: number[]; + /** Covariance of β (inverse Fisher information). */ + cov: number[][]; + /** NB dispersion: Var = μ + μ²/θ. Infinity for Poisson. */ + theta: number; + loglik: number; + aic: number; + deviance: number; + nullDeviance: number; + n: number; + converged: boolean; +} + +/** Inverse of a symmetric positive-definite matrix by Cholesky; null if singular. */ +export function invertSpd(a: number[][]): number[][] | null { + const n = a.length; + const l = a.map(() => new Array(n).fill(0)); + for (let i = 0; i < n; i++) + for (let j = 0; j <= i; j++) { + let s = a[i][j]; + for (let k = 0; k < j; k++) s -= l[i][k] * l[j][k]; + if (i === j) { + if (s <= 1e-12 * Math.max(1, Math.abs(a[i][i]))) return null; + l[i][i] = Math.sqrt(s); + } else l[i][j] = s / l[j][j]; + } + // inv(L), then inv(A) = inv(L)ᵀ inv(L). + const li = l.map(() => new Array(n).fill(0)); + for (let i = 0; i < n; i++) { + li[i][i] = 1 / l[i][i]; + for (let j = 0; j < i; j++) { + let s = 0; + for (let k = j; k < i; k++) s -= l[i][k] * li[k][j]; + li[i][j] = s / l[i][i]; + } + } + const out = l.map(() => new Array(n).fill(0)); + for (let i = 0; i < n; i++) + for (let j = 0; j <= i; j++) { + let s = 0; + for (let k = i; k < n; k++) s += li[k][i] * li[k][j]; + out[i][j] = s; + out[j][i] = s; + } + return out; +} + +function logLik(y: number[], mu: number[], theta: number): number { + let ll = 0; + for (let i = 0; i < y.length; i++) { + const m = Math.max(mu[i], 1e-12); + if (!Number.isFinite(theta)) ll += y[i] * Math.log(m) - m - logGamma(y[i] + 1); + else + ll += + logGamma(y[i] + theta) - + logGamma(theta) - + logGamma(y[i] + 1) + + theta * Math.log(theta / (theta + m)) + + y[i] * Math.log(m / (theta + m)); + } + return ll; +} + +function deviance(y: number[], mu: number[], theta: number): number { + let d = 0; + for (let i = 0; i < y.length; i++) { + const m = Math.max(mu[i], 1e-12); + const t = y[i] > 0 ? y[i] * Math.log(y[i] / m) : 0; + d += Number.isFinite(theta) + ? 2 * (t - (y[i] + theta) * Math.log((y[i] + theta) / (m + theta))) + : 2 * (t - (y[i] - m)); + } + return d; +} + +/** θ maximising the NB likelihood for fixed μ (golden section on log θ). */ +function bestTheta(y: number[], mu: number[]): number { + let a = Math.log(0.01); + let b = Math.log(1e4); + const g = (Math.sqrt(5) - 1) / 2; + const f = (lt: number) => -logLik(y, mu, Math.exp(lt)); + let c = b - g * (b - a); + let d = a + g * (b - a); + let fc = f(c); + let fd = f(d); + for (let i = 0; i < 80 && b - a > 1e-6; i++) { + if (fc < fd) { + b = d; + d = c; + fd = fc; + c = b - g * (b - a); + fc = f(c); + } else { + a = c; + c = d; + fc = fd; + d = a + g * (b - a); + fd = f(d); + } + } + return Math.exp((a + b) / 2); +} + +/** IRLS for β at fixed θ, starting from μ. */ +function irls(X: number[][], y: number[], offset: number[], theta: number, mu0: number[]) { + const n = y.length; + const p = X[0].length; + let mu = mu0.slice(); + let beta = new Array(p).fill(0); + let cov: number[][] | null = null; + let dev = Infinity; + let converged = false; + for (let it = 0; it < 100; it++) { + const xtwx = Array.from({ length: p }, () => new Array(p).fill(0)); + const xtwz = new Array(p).fill(0); + for (let i = 0; i < n; i++) { + const m = Math.max(mu[i], 1e-10); + const w = Number.isFinite(theta) ? m / (1 + m / theta) : m; + const z = Math.log(m) - offset[i] + (y[i] - m) / m; + const xi = X[i]; + for (let j = 0; j < p; j++) { + const wx = w * xi[j]; + xtwz[j] += wx * z; + for (let k = 0; k <= j; k++) xtwx[j][k] += wx * xi[k]; + } + } + for (let j = 0; j < p; j++) for (let k = 0; k < j; k++) xtwx[k][j] = xtwx[j][k]; + cov = invertSpd(xtwx); + if (!cov) return null; + beta = cov.map((row) => row.reduce((s, v, k) => s + v * xtwz[k], 0)); + mu = X.map((xi, i) => + Math.exp(Math.min(30, xi.reduce((s, v, k) => s + v * beta[k], 0) + offset[i])) + ); + const d = deviance(y, mu, theta); + if (Math.abs(d - dev) < 1e-8 * (Math.abs(d) + 0.1)) { + dev = d; + converged = true; + break; + } + dev = d; + } + return { beta, cov: cov!, mu, dev, converged }; +} + +/** + * Fit log(μ) = Xβ + offset. `family: 'nb'` estimates θ; 'poisson' fixes θ = ∞. X should carry its + * own intercept column. Returns null when the design is singular. + */ +export function fitGlm( + X: number[][], + y: number[], + offset: number[], + family: 'nb' | 'poisson' = 'nb' +): GlmFit | null { + const n = y.length; + if (!n || !X[0]?.length) return null; + const rate = y.reduce((a, b) => a + b, 0) / offset.reduce((a, o) => a + Math.exp(o), 0); + let mu = offset.map((o, i) => (y[i] + Math.max(rate * Math.exp(o), 0.1)) / 2); + let theta = family === 'nb' ? 1 : Infinity; + let fit = irls(X, y, offset, theta, mu); + if (!fit) return null; + let converged = fit.converged; + if (family === 'nb') { + converged = false; + for (let outer = 0; outer < 50; outer++) { + const t = bestTheta(y, fit.mu); + const next = irls(X, y, offset, t, fit.mu); + if (!next) return null; + const done = Math.abs(Math.log(t / theta)) < 1e-6 && Math.abs(next.dev - fit.dev) < 1e-6; + theta = t; + fit = next; + if (done) { + converged = next.converged; + break; + } + } + } + mu = fit.mu; + const ll = logLik(y, mu, theta); + const p = X[0].length + (family === 'nb' ? 1 : 0); + // Null deviance: intercept + offset only, same θ. + const nullMu = offset.map((o) => rate * Math.exp(o)); + return { + beta: fit.beta, + cov: fit.cov, + theta, + loglik: ll, + aic: -2 * ll + 2 * p, + deviance: fit.dev, + nullDeviance: deviance(y, nullMu, theta), + n, + converged + }; +} + +/** Wald estimate, 95% interval and p for a linear combination gᵀβ (on the log scale). */ +export function contrast(fit: GlmFit, g: number[]) { + const est = g.reduce((s, v, i) => s + v * fit.beta[i], 0); + let v = 0; + for (let i = 0; i < g.length; i++) + if (g[i]) for (let j = 0; j < g.length; j++) if (g[j]) v += g[i] * fit.cov[i][j] * g[j]; + const se = Math.sqrt(Math.max(0, v)); + return { est, se, lo: est - 1.96 * se, hi: est + 1.96 * se, p: se > 0 ? normalP(est / se) : NaN }; +} + +/** + * Natural cubic spline basis (ESL §5.2.1) with the given knots; x and the knots are scaled to the + * boundary knots so the columns stay well conditioned. Returns K − 1 columns (linear term first). + */ +export function naturalSpline(knots: number[]): (x: number) => number[] { + const lo = knots[0]; + const span = knots[knots.length - 1] - lo || 1; + const k = knots.map((v) => (v - lo) / span); + const K = k.length; + const cube = (v: number) => (v > 0 ? v * v * v : 0); + const d = (x: number, j: number) => (cube(x - k[j]) - cube(x - k[K - 1])) / (k[K - 1] - k[j]); + return (raw: number) => { + const x = (raw - lo) / span; + const out = [x]; + for (let j = 0; j < K - 2; j++) out.push(d(x, j) - d(x, K - 2)); + return out; + }; +} diff --git a/src/lib/stats.ts b/src/lib/stats.ts index 188a8eb..2844c68 100644 --- a/src/lib/stats.ts +++ b/src/lib/stats.ts @@ -64,7 +64,7 @@ export function pairs(rows: T[], fx: (r: T) => Num, fy: (r: T) => Num): [numb // --- distributions ----------------------------------------------------------------------------- -function logGamma(x: number): number { +export function logGamma(x: number): number { const c = [ 76.1800917294715, -86.5053203294168, 24.0140982408309, -1.23173957245016, 0.00120865097386618, -0.5395239384953e-5 @@ -119,6 +119,30 @@ export function incompleteBeta(x: number, a: number, b: number): number { : 1 - (bt * betacf(b, a, 1 - x)) / b; } +/** Two-sided p-value for a standard normal statistic (erfc, Numerical Recipes; error < 1.2e-7). */ +export function normalP(z: number): number { + if (!Number.isFinite(z)) return NaN; + const x = Math.abs(z) / Math.SQRT2; + const t = 1 / (1 + 0.5 * x); + const poly = + -x * x - + 1.26551223 + + t * + (1.00002368 + + t * + (0.37409196 + + t * + (0.09678418 + + t * + (-0.18628806 + + t * + (0.27886807 + + t * + (-1.13520398 + + t * (1.48851587 + t * (-0.82215223 + t * 0.17087277)))))))); + return Math.min(1, t * Math.exp(poly)); +} + /** Two-sided p-value for a t statistic with `df` degrees of freedom. */ export function tTestP(t: number, df: number): number { if (!Number.isFinite(t) || df <= 0) return NaN; diff --git a/src/routes/drivers/+page.server.ts b/src/routes/drivers/+page.server.ts index 5fb7327..38eb0b7 100644 --- a/src/routes/drivers/+page.server.ts +++ b/src/routes/drivers/+page.server.ts @@ -1,5 +1,6 @@ import { memo } from '#lib/server/data'; import { binnedMean, cpueOf, effortSets, round } from '#lib/server/analytics/common'; +import { catchModel } from '#lib/server/analytics/catchModel'; import { correlations, SET_VARIABLES } from '#lib/server/analytics/env'; import { quantile, spearman, pairs } from '#lib/stats'; import type { PageServerLoad } from './$types'; @@ -53,6 +54,7 @@ export const load: PageServerLoad = () => const zero = sets.filter((s) => !s.nTotal).length; return { + model: memo('catchModel', catchModel), n: sets.length, zero, corr: corr.map((c) => ({ diff --git a/src/routes/drivers/+page.svelte b/src/routes/drivers/+page.svelte index 85cc6a0..3296c14 100644 --- a/src/routes/drivers/+page.svelte +++ b/src/routes/drivers/+page.svelte @@ -2,12 +2,35 @@ import ChartCard from '#lib/components/ChartCard.svelte'; import DataTable from '#lib/components/DataTable.svelte'; import BarChart from '#lib/components/charts/BarChart.svelte'; + import ForestPlot from '#lib/components/charts/ForestPlot.svelte'; import Heatmap from '#lib/components/charts/Heatmap.svelte'; - import { num, pct, pValue } from '#lib/format'; + import LineChart from '#lib/components/charts/LineChart.svelte'; + import { num, pct, pValue, season, seasonDayLabel } from '#lib/format'; import { SPECIES } from '#lib/species'; import type { PageProps } from './$types'; let { data }: PageProps = $props(); + + type Effect = { est: number; lo: number; hi: number; p: number } | null; + const fv = (e: Effect | undefined) => (e ? { est: e.est, lo: e.lo, hi: e.hi } : null); + const clear = (e: Effect | undefined) => !!e && (e.hi < 1 || e.lo > 1); + const effectText = (e: Effect | undefined) => + e ? `${num(e.est, 2)} (${num(e.lo, 2)}–${num(e.hi, 2)}), ${pValue(e.p)}` : '–'; + const md = $derived(data.model); + const termLabels = $derived( + md?.terms.map( + (t) => + `${t.label} (+${num(t.sd, t.sd >= 10 ? 0 : t.sd >= 1 ? 1 : 2)}${t.unit === '%' ? '%' : t.unit ? ` ${t.unit}` : ''})` + ) ?? [] + ); + const siteOrder = $derived( + md + ? md.sites + .map((_, i) => i) + .sort((a, b) => (md.cm?.sites[b]?.est ?? 0) - (md.cm?.sites[a]?.est ?? 0)) + : [] + ); + const lastSeason = $derived(md?.seasons[md.seasons.length - 1]);
@@ -23,7 +46,7 @@

These are single-variable associations: sites, seasons and variables overlap (warm years are also El Niño years, late sets are also darker), so a link here is a lead to test, not a cause. - A multivariable model is tracked in issue #54. + The model further down estimates them together.

@@ -69,6 +92,174 @@ + {#if md} + {@const sst = md.sst} + {@const strong = md.terms.filter((_, j) => clear(md.all.env[j]))} +
+

All together: a model of catches

+

+ A negative-binomial regression of the sharks caught in each set ({num(md.n)} sets with every variable; + net-hours as the exposure), with a smooth curve for the day of the season, an effect for each + nursery and each season, and the environment. Each effect is the change in catch rate with all + the others held where they were. It can only adjust for what was measured, so it is still a guide, + not proof of cause. +

+
+ + x.label)} + series={[ + { name: 'All sharks', color: 'var(--series-1)', values: sst.map((x) => fv(x.v)) } + ]} + xLabel="Rate ratio per +1 °C" + /> + + t.label.toLowerCase()).join(', ')}. The rest are within the noise.` + : 'For all sharks together, no environmental variable clears the noise once the calendar, nursery and season are in the model.'} + table={{ + columns: [ + { key: 'label', label: 'Variable (+1 SD)' }, + { key: 'all', label: 'All sharks' }, + { key: 'cm', label: 'Blacktip' }, + { key: 'na', label: 'Lemon' } + ], + rows: termLabels.map((label, j) => ({ + label, + all: effectText(md.all.env[j]), + cm: effectText(md.cm?.env[j]), + na: effectText(md.na?.env[j]) + })) + }} + > + fv(md.cm?.env[j])) + }, + { + name: SPECIES.Na.common, + color: SPECIES.Na.color, + values: md.terms.map((_, j) => fv(md.na?.env[j])) + } + ]} + xLabel="Rate ratio (1 = no change)" + /> + + + ({ x: p.x, y: p.est })) ?? [] + }, + { + name: SPECIES.Na.common, + color: SPECIES.Na.color, + points: md.na?.day.map((p) => ({ x: p.x, y: p.est })) ?? [] + } + ]} + refs={[{ y: 1, label: 'season average' }]} + xFormat={(x) => seasonDayLabel(Math.round(x))} + yFormat={(v) => `×${num(v, 1)}`} + yLabel="Rate ratio" + /> + + + md.sites[i].name)} + series={[ + { + name: SPECIES.Cm.common, + color: SPECIES.Cm.color, + values: siteOrder.map((i) => fv(md.cm?.sites[i])) + }, + { + name: SPECIES.Na.common, + color: SPECIES.Na.color, + values: siteOrder.map((i) => fv(md.na?.sites[i])) + } + ]} + xLabel="Rate ratio vs the average nursery" + /> + + + season(x))} + series={[ + { + name: SPECIES.Cm.common, + color: SPECIES.Cm.color, + values: md.seasons.map((_, i) => fv(md.cm?.seasons[i])) + }, + { + name: SPECIES.Na.common, + color: SPECIES.Na.color, + values: md.seasons.map((_, i) => fv(md.na?.seasons[i])) + } + ]} + xLabel="Rate ratio vs the average season" + /> + +
+

About the model

+
    +
  • + Counts per set (blacktips, sicklefin lemons, and all sharks for the sea-temperature + steps) with log(net-hours) as the offset, so effects are on catch rate. +
  • +
  • + Negative binomial, because catches are clumped: the dispersion θ is {num( + md.all.theta, + 2 + )} for all sharks, and the model beats a Poisson by {num( + (md.aicPoisson ?? 0) - (md.aicNb ?? 0) + )} AIC points. +
  • +
  • + Day of the season: natural cubic spline with knots at {md.knots.join(', ')}. Nurseries + and seasons are fixed effects coded against their average. +
  • +
  • + {num(md.dropped)} timed sets are left out for a missing variable (mostly the tide) or because + their nursery has fewer than 30 complete sets. The model explains {pct( + md.all.devExplained + )} of the deviance for all sharks. +
  • +
  • Each rate ratio has a Wald 95% interval, with no correction for the many terms.
  • +
+
+
+
+ {/if} +

Catch rate across each variable's range

@@ -135,3 +326,17 @@

+ +