Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ src/lib/server/analytics common.ts (CPUE, Fulton's K, recapture pairs…) · env
diet.ts (gastric lavage: fullness, prey groups)
events.ts (cyclones and heavy-rain days vs catches)
glm.ts (NB/Poisson GLM, splines) · catchModel.ts (joint catch model)
glmm.ts (binomial mixed model, crossed random intercepts) · sex.ts
src/lib/server/astro.ts sunset/twilight/moon (local formulas) · tides.ts (extrema, interpolation)
src/lib/server/holidays.ts French Polynesia public holidays (Easter-based too), festive window,
school holidays (schoolDays) and the one-label-per-date dayType
Expand Down Expand Up @@ -101,6 +102,11 @@ samples/fixture/ synthetic workbooks for tests/CI (committed)
- The headline: sea temperature's raw link with catch rate (≈ −25% per °C) vanishes once day of the season is in the model. Don't describe SST as a driver of catches on other pages.
- `glm.ts` is generic (IRLS + ML θ, Wald intervals via `contrast`); reuse it rather than writing another fitter.

### Sex ratio mixed model (#117)

- `analytics/sex.ts`: female (1) vs male (0) per individual, counted once at its first capture, with the nursery and season of that capture as crossed random intercepts (`glmm.ts`: Laplace approximation, PIRLS as lme4's nAGQ = 0, SDs by Nelder–Mead). Tests fix one SD at 0 and halve the χ²₁ p (boundary).
- Reuse `fitBinomialGlmm` for any yes/no outcome with grouping (fed or empty, recaptured or not) instead of pooling groups; there is no R in CI, so validate new uses against simulated data as `glmm.test.ts` does.

### School holidays (#25)

- `fetch-external.mjs --only=school` writes `external/school-holidays.csv` (first and last day off, local dates). 2013-14 to 2016-17 are transcribed in the script from the Polynesian government's calendar orders for collèges and lycées (arrêtés 0270/CM 2013, 0399, 0401 and 0403/CM 2014), archived from education.pf by the Wayback Machine; the national dataset only starts in 2017-18. From 2017-18 it is the Ministry's `fr-en-calendrier-scolaire` open data (Polynésie, Licence Ouverte 2.0, credit it): pupils only, a day counts when either level is off, `end_date` is the day classes resume.
Expand Down
14 changes: 12 additions & 2 deletions src/lib/server/analytics/findings.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import { catchModel } from './catchModel';
import { cloudWater } from './cloud';
import { diet } from './diet';
import { dailyMaxBySite, versusSst } from './insitu';
import { sexModel } from './sex';

export interface Finding {
id: string;
Expand Down Expand Up @@ -106,18 +107,27 @@ function compute(d: Dataset): Finding[] {
});
}

// Sex ratio
// Sex ratio, with the mixed model's verdict on nurseries (#117)
const sexFits = memo('sexModel', sexModel);
for (const sp of ['Cm', 'Na'] as const) {
const ind = d.individuals.filter((i) => i.species === sp && i.sex);
const f = ind.filter((i) => i.sex === 'F').length;
if (ind.length > 50) {
const p = binomTestHalf(f, ind.length);
const m = sexFits.find((x) => x.species === sp)?.fit;
const top = m?.nurseries.filter((r) => r.n >= 20) ?? [];
const byNursery =
m?.nurseryP != null && m.nurseryP < 0.05 && top.length >= 2
? ` It does shift between nurseries (mixed model p = ${m.nurseryP.toFixed(3)}): from ${pct(top[0].est)} female at ${top[0].name} to ${pct(top[top.length - 1].est)} at ${top[top.length - 1].name}.`
: m?.nurseryP != null
? ` Nurseries don't differ beyond chance (mixed model p = ${m.nurseryP.toFixed(2)}).`
: '';
out.push({
id: `sex-${sp}`,
kind: 'pattern',
title: `${speciesName(sp)} sex ratio`,
value: `${pct(f / ind.length)} female`,
text: `${num(f)} females and ${num(ind.length - f)} males among individually identified ${speciesName(sp).toLowerCase()}s; ${p < 0.05 ? `a ratio unlikely to be chance (binomial p = ${p.toFixed(3)})` : `close to 1:1 (binomial p = ${p.toFixed(2)})`}.`,
text: `${num(f)} females and ${num(ind.length - f)} males among individually identified ${speciesName(sp).toLowerCase()}s; ${p < 0.05 ? `a ratio unlikely to be chance (binomial p = ${p.toFixed(3)})` : `close to 1:1 (binomial p = ${p.toFixed(2)})`}.${byNursery}`,
href: '/morphometrics#sex'
});
}
Expand Down
65 changes: 65 additions & 0 deletions src/lib/server/analytics/glmm.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
import { describe, expect, it } from 'vitest';
import { fitBinomialGlmm, testRandomEffect, type Factor } from './glmm';

/** Seeded uniform (mulberry32) and normal (Box–Muller) draws. */
function rng(seed: number) {
let a = seed;
const u = () => {
a = (a + 0x6d2b79f5) | 0;
let t = Math.imul(a ^ (a >>> 15), 1 | a);
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
const z = () => Math.sqrt(-2 * Math.log(u() || 1e-12)) * Math.cos(2 * Math.PI * u());
return { u, z };
}

/** Crossed design: `levels[f]` levels per factor, true SDs `sd[f]`, intercept b0. */
function simulate(seed: number, n: number, levels: number[], sd: number[], b0: number) {
const r = rng(seed);
const effects = levels.map((L, f) => Array.from({ length: L }, () => sd[f] * r.z()));
const factors: Factor[] = levels.map((L) => ({ levels: L, index: [] }));
const y: number[] = [];
for (let i = 0; i < n; i++) {
let eta = b0;
levels.forEach((L, f) => {
const l = Math.floor(r.u() * L);
factors[f].index.push(l);
eta += effects[f][l];
});
y.push(r.u() < 1 / (1 + Math.exp(-eta)) ? 1 : 0);
}
return { y, X: y.map(() => [1]), factors };
}

describe('binomial GLMM (#117)', () => {
it('reduces to logistic regression when the random effects are switched off', () => {
const { y, X, factors } = simulate(1, 400, [10], [1], 0.4);
const fit = fitBinomialGlmm(y, X, factors, [0])!;
const k = y.filter(Boolean).length;
expect(fit.beta[0]).toBeCloseTo(Math.log(k / (400 - k)), 6);
expect(fit.betaSe[0]).toBeCloseTo(Math.sqrt(1 / k + 1 / (400 - k)), 4);
expect(fit.loglik).toBeCloseTo(k * Math.log(k / 400) + (400 - k) * Math.log(1 - k / 400), 6);
});

it('recovers a random-intercept SD and finds it significant', () => {
const { y, X, factors } = simulate(7, 3000, [30], [1], 0.3);
const fit = fitBinomialGlmm(y, X, factors)!;
expect(fit.sd[0]).toBeGreaterThan(0.65);
expect(fit.sd[0]).toBeLessThan(1.4);
expect(Math.abs(fit.beta[0] - 0.3)).toBeLessThan(0.45);
expect(testRandomEffect(fit, y, X, factors, 0)!.p).toBeLessThan(0.001);
});

it('separates a real crossed effect from an absent one', () => {
const { y, X, factors } = simulate(11, 2500, [12, 10], [0.8, 0], -0.2);
const fit = fitBinomialGlmm(y, X, factors)!;
expect(fit.sd[0]).toBeGreaterThan(0.4);
expect(fit.sd[1]).toBeLessThan(0.25);
expect(testRandomEffect(fit, y, X, factors, 0)!.p).toBeLessThan(0.001);
expect(testRandomEffect(fit, y, X, factors, 1)!.p).toBeGreaterThan(0.05);
// Conditional modes shrink towards zero and come with standard errors.
expect(fit.re[0]).toHaveLength(12);
expect(fit.re[0].every((e) => Number.isFinite(e.se) && e.se > 0)).toBe(true);
});
});
248 changes: 248 additions & 0 deletions src/lib/server/analytics/glmm.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,248 @@
/**
* Binomial mixed model (#117): logistic regression with crossed random intercepts (nursery and
* season, say), fitted by maximum likelihood with the Laplace approximation. For given standard
* deviations the fixed effects and the spherical random effects v (u = σ·v) are found together
* by penalised IRLS, as lme4's glmer with nAGQ = 0; the standard deviations then maximise the
* approximate log-likelihood (Nelder–Mead). Random-intercept tests compare the fit with that
* standard deviation fixed at zero (likelihood ratio, halved p for the boundary).
*/
import { chiSquareP } from '#lib/stats';
import { invertSpd } from './glm';

/** A grouping factor: `index[i]` is observation i's level, 0 … levels − 1. */
export interface Factor {
levels: number;
index: number[];
}

export interface GlmmFit {
n: number;
beta: number[];
betaSe: number[];
/** Random-intercept standard deviation per factor (log-odds scale). */
sd: number[];
/** Laplace-approximate log-likelihood. */
loglik: number;
/** Conditional modes of the random effects with their standard errors, per factor and level. */
re: { est: number; se: number }[][];
converged: boolean;
}

/** Cholesky factor of a symmetric positive-definite matrix, or null. */
function chol(a: number[][]): number[][] | null {
const n = a.length;
const l = a.map(() => new Array<number>(n).fill(0));
for (let i = 0; i < n; i++)
for (let j = 0; j <= i; j++) {
let s = a[i][j];
for (let k = 0; k < j; k++) s -= l[i][k] * l[j][k];
if (i === j) {
if (s <= 1e-12 * Math.max(1, Math.abs(a[i][i]))) return null;
l[i][i] = Math.sqrt(s);
} else l[i][j] = s / l[j][j];
}
return l;
}

function solveChol(l: number[][], b: number[]): number[] {
const n = b.length;
const z = new Array<number>(n);
for (let i = 0; i < n; i++) {
let s = b[i];
for (let k = 0; k < i; k++) s -= l[i][k] * z[k];
z[i] = s / l[i][i];
}
const x = new Array<number>(n);
for (let i = n - 1; i >= 0; i--) {
let s = z[i];
for (let k = i + 1; k < n; k++) s -= l[k][i] * x[k];
x[i] = s / l[i][i];
}
return x;
}

const logDet = (l: number[][]) => 2 * l.reduce((a, row, i) => a + Math.log(row[i]), 0);

/** log p(y | η) for 0/1 outcomes, computed stably. */
const bernoulli = (y: number, eta: number) =>
y ? -Math.log1p(Math.exp(-eta)) : -Math.log1p(Math.exp(eta));

interface Mode {
beta: number[];
v: number[];
A: number[][];
loglik: number;
ok: boolean;
}

/** Penalised IRLS for fixed standard deviations: the joint mode of (β, v). */
function mode(
y: number[],
X: number[][],
factors: Factor[],
sds: number[],
start?: { beta: number[]; v: number[] }
): Mode | null {
const n = y.length;
const p = X[0].length;
const offs = factors.map((_, f) => factors.slice(0, f).reduce((a, g) => a + g.levels, 0));
const q = factors.reduce((a, g) => a + g.levels, 0);
const m = p + q;
let beta = start?.beta.slice() ?? new Array<number>(p).fill(0);
let v = start?.v.slice() ?? new Array<number>(q).fill(0);
const etaOf = (b: number[], vv: number[], i: number) => {
let e = 0;
for (let j = 0; j < p; j++) e += X[i][j] * b[j];
for (let f = 0; f < factors.length; f++) e += sds[f] * vv[offs[f] + factors[f].index[i]];
return e;
};
const penalised = (b: number[], vv: number[]) => {
let s = 0;
for (let i = 0; i < n; i++) s += bernoulli(y[i], etaOf(b, vv, i));
return s - 0.5 * vv.reduce((a, x) => a + x * x, 0);
};
let pen = penalised(beta, v);
let A: number[][] = [];
for (let iter = 0; iter < 60; iter++) {
A = Array.from({ length: m }, () => new Array<number>(m).fill(0));
const g = new Array<number>(m).fill(0);
const cols = new Array<number>(p + factors.length);
const vals = new Array<number>(p + factors.length);
for (let i = 0; i < n; i++) {
const mu = 1 / (1 + Math.exp(-etaOf(beta, v, i)));
const w = mu * (1 - mu);
const r = y[i] - mu;
for (let j = 0; j < p; j++) [cols[j], vals[j]] = [j, X[i][j]];
for (let f = 0; f < factors.length; f++)
[cols[p + f], vals[p + f]] = [p + offs[f] + factors[f].index[i], sds[f]];
for (let a = 0; a < cols.length; a++) {
g[cols[a]] += r * vals[a];
for (let b = 0; b < cols.length; b++) A[cols[a]][cols[b]] += w * vals[a] * vals[b];
}
}
for (let k = 0; k < q; k++) {
A[p + k][p + k] += 1;
g[p + k] -= v[k];
}
const l = chol(A);
if (!l) return null;
const delta = solveChol(l, g);
let step = 1;
let next = pen;
let nb = beta;
let nv = v;
for (let h = 0; h < 30; h++) {
nb = beta.map((x, j) => x + step * delta[j]);
nv = v.map((x, k) => x + step * delta[p + k]);
next = penalised(nb, nv);
if (next >= pen - 1e-10) break;
step /= 2;
}
const done = Math.max(...delta.map(Math.abs)) * step < 1e-8 || Math.abs(next - pen) < 1e-10;
[beta, v, pen] = [nb, nv, next];
if (done) break;
}
// Laplace: log p(y | mode) − ½|v|² − ½ log det(ΛZᵀWZΛ + I), the random-effect block of A.
const lv = chol(A.slice(p).map((row) => row.slice(p)));
if (!lv) return null;
return { beta, v, A, loglik: pen - 0.5 * logDet(lv), ok: true };
}

/** Nelder–Mead maximisation inside [0, hi] in every coordinate. */
function maximise(f: (x: number[]) => number, x0: number[], hi = 5): number[] {
const k = x0.length;
const clamp = (x: number[]) => x.map((v) => Math.min(hi, Math.max(0, v)));
let pts = [clamp(x0), ...x0.map((_, i) => clamp(x0.map((v, j) => (i === j ? v + 0.4 : v))))];
let vals = pts.map(f);
for (let it = 0; it < 200; it++) {
const order = vals.map((_, i) => i).sort((a, b) => vals[b] - vals[a]);
pts = order.map((i) => pts[i]);
vals = order.map((i) => vals[i]);
if (Math.abs(vals[0] - vals[k]) < 1e-7) break;
const c = x0.map((_, j) => pts.slice(0, k).reduce((a, q) => a + q[j], 0) / k);
const at = (t: number) => clamp(c.map((v, j) => v + t * (pts[k][j] - v)));
const r = at(-1);
const fr = f(r);
if (fr > vals[0]) {
const e = at(-2);
const fe = f(e);
[pts[k], vals[k]] = fe > fr ? [e, fe] : [r, fr];
} else if (fr > vals[k - 1]) [pts[k], vals[k]] = [r, fr];
else {
const ct = at(fr > vals[k] ? -0.5 : 0.5);
const fc = f(ct);
if (fc > Math.max(fr, vals[k])) [pts[k], vals[k]] = [ct, fc];
else
for (let i = 1; i <= k; i++) {
pts[i] = clamp(pts[i].map((v, j) => pts[0][j] + 0.5 * (v - pts[0][j])));
vals[i] = f(pts[i]);
}
}
}
return pts[vals.indexOf(Math.max(...vals))];
}

/**
* Fit y ~ X + (1 | factor₁) + (1 | factor₂) … for 0/1 outcomes. `fixedSd[f]` pins a standard
* deviation (0 drops that random effect, for tests).
*/
export function fitBinomialGlmm(
y: number[],
X: number[][],
factors: Factor[],
fixedSd: (number | null)[] = []
): GlmmFit | null {
if (!y.length || !X.length) return null;
const free = factors.map((_, f) => fixedSd[f] == null);
const full = (x: number[]) => {
let k = 0;
return factors.map((_, f) => (free[f] ? x[k++] : fixedSd[f]!));
};
let warm: { beta: number[]; v: number[] } | undefined;
const objective = (x: number[]) => {
const md = mode(y, X, factors, full(x), warm);
if (!md) return -Infinity;
warm = { beta: md.beta, v: md.v };
return md.loglik;
};
const x0 = factors.filter((_, f) => free[f]).map(() => 0.5);
const best = x0.length ? maximise(objective, x0) : [];
const sds = full(best);
const md = mode(y, X, factors, sds, warm);
if (!md) return null;
const inv = invertSpd(md.A);
const p = X[0].length;
let off = p;
const re = factors.map((g, f) => {
const out = Array.from({ length: g.levels }, (_, l) => ({
est: sds[f] * md.v[off - p + l],
se: inv ? sds[f] * Math.sqrt(Math.max(0, inv[off + l][off + l])) : NaN
}));
off += g.levels;
return out;
});
return {
n: y.length,
beta: md.beta,
betaSe: md.beta.map((_, j) => (inv ? Math.sqrt(Math.max(0, inv[j][j])) : NaN)),
sd: sds,
loglik: md.loglik,
re,
converged: md.ok
};
}

/** Likelihood-ratio test of one random intercept (σ = 0 on the boundary: halved χ²₁ p). */
export function testRandomEffect(
fit: GlmmFit,
y: number[],
X: number[][],
factors: Factor[],
f: number
) {
const fixed = factors.map((_, k) => (k === f ? 0 : null));
const reduced = fitBinomialGlmm(y, X, factors, fixed);
if (!reduced) return null;
const lr = Math.max(0, 2 * (fit.loglik - reduced.loglik));
return { lr, p: lr > 1e-6 ? 0.5 * chiSquareP(lr, 1) : 1 };
}
Loading
Loading