From b190185820375c659d6053f298dfc9f1bdd5c38c Mon Sep 17 00:00:00 2001 From: luen Date: Wed, 4 Nov 2026 14:13:04 +1000 Subject: [PATCH 1/2] Charts: show measurement coverage and model selection (#123) --- AGENTS.md | 2 + e2e/chart-123.spec.ts | 20 ++++ src/lib/components/DataTable.svelte | 3 +- src/lib/components/MeasurementCoverage.svelte | 46 ++++++++++ src/lib/components/ModelSelection.svelte | 41 +++++++++ src/lib/server/analytics/catchModel.ts | 18 +++- .../analytics/measurementCoverage.test.ts | 25 +++++ .../server/analytics/measurementCoverage.ts | 91 +++++++++++++++++++ src/routes/data-quality/+page.server.ts | 2 + src/routes/data-quality/+page.svelte | 2 + src/routes/drivers/+page.server.ts | 2 + src/routes/drivers/+page.svelte | 2 + 12 files changed, 248 insertions(+), 6 deletions(-) create mode 100644 e2e/chart-123.spec.ts create mode 100644 src/lib/components/MeasurementCoverage.svelte create mode 100644 src/lib/components/ModelSelection.svelte create mode 100644 src/lib/server/analytics/measurementCoverage.test.ts create mode 100644 src/lib/server/analytics/measurementCoverage.ts diff --git a/AGENTS.md b/AGENTS.md index 5f0464b..50b7843 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -107,6 +107,8 @@ samples/fixture/ synthetic workbooks for tests/CI (committed) ### Catch model (#54) +- Coverage (#123) reports recorded/eligible counts, not measurement validity. Lavage results use sampled-or-result-present captures. `selectCatchModel()` owns the shared model and selection-flow filters; sequential exclusions must reconcile without double-counting. + - `analytics/catchModel.ts`: negative-binomial counts per timed gillnet set, offset log(net-hours), natural-spline day of season (knots at the 0/25/50/75/100% quantiles), nursery and season as effect-coded fixed effects (each level vs the average; no random effects), and nine environment terms per +1 SD. Complete cases only; nurseries need ≥ 30 complete sets, and a species gets a nursery effect only where ≥ 10 of it were caught (otherwise the estimate runs off to zero or infinity). - The headline: sea temperature's raw link with catch rate (≈ −25% per °C) vanishes once day of the season is in the model. Don't describe SST as a driver of catches on other pages. - `glm.ts` is generic (IRLS + ML θ, Wald intervals via `contrast`); reuse it rather than writing another fitter. diff --git a/e2e/chart-123.spec.ts b/e2e/chart-123.spec.ts new file mode 100644 index 0000000..3af476a --- /dev/null +++ b/e2e/chart-123.spec.ts @@ -0,0 +1,20 @@ +import { expect, test } from '@playwright/test'; +import AxeBuilder from '@axe-core/playwright'; +test('new charts render on mobile and pass both-theme accessibility', async ({ page }) => { + await page.setViewportSize({ width: 375, height: 800 }); + await page.goto('/data-quality'); + await expect( + page.getByRole('heading', { name: 'Measurement availability', exact: true }) + ).toBeVisible(); + for (const colorScheme of ['light', 'dark'] as const) { + await page.emulateMedia({ colorScheme }); + const results = await new AxeBuilder({ page }).include('main').analyze(); + expect( + results.violations.filter((v) => v.impact === 'serious' || v.impact === 'critical') + ).toEqual([]); + } + expect( + await page.evaluate(() => document.documentElement.scrollWidth - innerWidth) + ).toBeLessThanOrEqual(1); + await expect(page.getByRole('button', { name: /Download CSV/ }).first()).toBeVisible(); +}); diff --git a/src/lib/components/DataTable.svelte b/src/lib/components/DataTable.svelte index 2cf96b8..5f247d0 100644 --- a/src/lib/components/DataTable.svelte +++ b/src/lib/components/DataTable.svelte @@ -24,7 +24,8 @@ : String(v); -
+ +
diff --git a/src/lib/components/MeasurementCoverage.svelte b/src/lib/components/MeasurementCoverage.svelte new file mode 100644 index 0000000..c7be1ca --- /dev/null +++ b/src/lib/components/MeasurementCoverage.svelte @@ -0,0 +1,46 @@ + + +
+

Measurement availability

+

+ Recorded values, not accuracy or lab performance. Zero counts as available. Null does not tell + us whether a measurement was omitted by protocol. Grey means no eligible records; pale blue + means 0% recorded and dark blue means 100%. Counts and denominators are in each table. +

+

+ Capture denominators are all field captures (including recaptures); set denominators are all + gillnet sets. The lavage column includes only captures marked lavaged or with a recorded result. {data.unknownLavage} + captures have unknown sampling status and no result; {data.notLavaged} are explicitly not lavaged + with no result. +

+ {#each [{ name: 'Field captures', panel: data.captures }, { name: 'Gillnet sets', panel: data.sets }] as p (p.name)} + + `${v.toFixed(1)}%`} + xTitle="Measurement" + yTitle="Season" + /> + + {/each} +
diff --git a/src/lib/components/ModelSelection.svelte b/src/lib/components/ModelSelection.svelte new file mode 100644 index 0000000..d4aa39b --- /dev/null +++ b/src/lib/components/ModelSelection.svelte @@ -0,0 +1,41 @@ + + +
+ Which sets are eligible for the catch model? +

+ Sequential filters from the same selection logic used by the model. A set missing several fields + is excluded once. Eligibility does not guarantee a converged fit: the model also needs at least + 200 base sets, two nurseries and two seasons, and each species fit needs two supported + nurseries. The all-sharks model has no species catch threshold. +

+ {#each groups as g (g.name)} + + r.stage)} + series={[ + { + name: 'Remaining sets', + color: 'var(--series-1)', + values: g.rows.map((r) => r.remaining) + } + ]} + /> + + {/each} +
diff --git a/src/lib/server/analytics/catchModel.ts b/src/lib/server/analytics/catchModel.ts index 3c328bb..f300335 100644 --- a/src/lib/server/analytics/catchModel.ts +++ b/src/lib/server/analytics/catchModel.ts @@ -48,7 +48,7 @@ function count(s: SetRow, r: Response): number { /** A species gets its own nursery effect only where it was caught at least this often. */ const MIN_SITE_CATCH = 10; -export function catchModel(d: Dataset) { +export function selectCatchModel(d: Dataset) { const vars = TERMS.map((k) => SET_VARIABLES.find((v) => v.key === k)!); const complete = effortSets(d).filter( (s) => s.cpue != null && s.date && s.siteId && vars.every((v) => v.get(s) != null) @@ -61,6 +61,17 @@ export function catchModel(d: Dataset) { .sort(); const base = complete.filter((s) => sites.includes(s.siteId!)); const seasons = [...new Set(base.map((s) => s.season))].sort(); + const selectSpecies = (r: Response) => { + const caught = new Map(); + for (const s of base) caught.set(s.siteId!, (caught.get(s.siteId!) ?? 0) + count(s, r)); + const own = sites.filter((id) => r === 'all' || (caught.get(id) ?? 0) >= MIN_SITE_CATCH); + return { own, rows: base.filter((s) => own.includes(s.siteId!)) }; + }; + return { vars, complete, perSite, sites, base, seasons, selectSpecies }; +} + +export function catchModel(d: Dataset) { + const { vars, perSite, sites, base, seasons, selectSpecies } = selectCatchModel(d); if (base.length < 200 || sites.length < 2 || seasons.length < 2) return null; // Scales and spline knots come from the full model so every species shares the same units. @@ -79,10 +90,7 @@ export function catchModel(d: Dataset) { const iEnv = 1; const iDay = iEnv + vars.length; const fitFor = (r: Response) => { - const caught = new Map(); - for (const s of base) caught.set(s.siteId!, (caught.get(s.siteId!) ?? 0) + count(s, r)); - const own = sites.filter((id) => r === 'all' || (caught.get(id) ?? 0) >= MIN_SITE_CATCH); - const rows = base.filter((s) => own.includes(s.siteId!)); + const { own, rows } = selectSpecies(r); const ownSeasons = seasons.filter((x) => rows.some((s) => s.season === x)); const days = rows.map((s) => seasonDay(s.date!)); const meanBasis = new Array(nSpline).fill(0); diff --git a/src/lib/server/analytics/measurementCoverage.test.ts b/src/lib/server/analytics/measurementCoverage.test.ts new file mode 100644 index 0000000..5ca72ea --- /dev/null +++ b/src/lib/server/analytics/measurementCoverage.test.ts @@ -0,0 +1,25 @@ +import { expect, it } from 'vitest'; +import { dataset } from '../data'; +import { coverageCell, modelSelectionFlow } from './measurementCoverage'; +it('distinguishes zero, missing and no eligible records', () => { + expect(coverageCell([0, null, 2], (v) => v != null)).toEqual({ + available: 2, + eligible: 3, + percent: 200 / 3 + }); + expect( + coverageCell( + [1], + () => true, + () => false + ) + ).toEqual({ available: 0, eligible: 0, percent: null }); +}); +it('reconciles mutually sequential exclusions for each model sample', () => { + for (const g of modelSelectionFlow(dataset())) { + expect(g.rows[0].remaining - g.rows.slice(1).reduce((n, r) => n + r.excluded, 0)).toBe( + g.rows.at(-1)!.remaining + ); + expect(g.rows.every((r) => r.excluded >= 0)).toBe(true); + } +}); diff --git a/src/lib/server/analytics/measurementCoverage.ts b/src/lib/server/analytics/measurementCoverage.ts new file mode 100644 index 0000000..f7f53aa --- /dev/null +++ b/src/lib/server/analytics/measurementCoverage.ts @@ -0,0 +1,91 @@ +import type { Dataset, Capture, SetRow } from '../data'; +import { effortSets } from './common'; +import { selectCatchModel } from './catchModel'; + +/** Availability is distinct from validity; zero is a recorded value. */ +export function coverageCell( + rows: T[], + present: (r: T) => boolean, + eligible: (r: T) => boolean = () => true +) { + const subset = rows.filter(eligible), + available = subset.filter(present).length; + return { + available, + eligible: subset.length, + percent: subset.length ? (100 * available) / subset.length : null + }; +} +export function measurementCoverage(d: Dataset) { + const captures = d.captures.filter((c) => c.kind === 'field'); + const sets = d.sets.filter((s) => s.gear === 'Gillnet'); + const panel = ( + rows: T[], + fields: { label: string; present: (r: T) => boolean; eligible?: (r: T) => boolean }[] + ) => { + const table = d.seasons.flatMap((season) => + fields.map((f) => ({ + season, + field: f.label, + ...coverageCell( + rows.filter((r) => r.season === season), + f.present, + f.eligible + ) + })) + ); + return { + fields: fields.map((f) => f.label), + seasons: d.seasons, + rows: table, + values: d.seasons.map((s) => table.filter((r) => r.season === s).map((r) => r.percent)) + }; + }; + return { + captures: panel(captures, [ + { label: 'Length', present: (c) => c.tlMm != null }, + { label: 'Mass', present: (c) => c.massKg != null }, + { label: 'Length + mass', present: (c) => c.tlMm != null && c.massKg != null }, + { label: 'Sex', present: (c) => c.sex === 'M' || c.sex === 'F' }, + { label: 'Scar', present: (c) => c.umbilical != null }, + { label: 'Capture time', present: (c) => c.captureMin != null }, + { + label: 'Lavage result', + present: (c) => c.stomach != null, + eligible: (c) => c.lavage === 1 || c.stomach != null + } + ]), + sets: panel(sets, [ + { label: 'Duration', present: (s) => s.durationMin != null }, + { label: 'Water °C', present: (s) => s.waterTemp != null }, + { label: 'pH', present: (s) => s.ph != null }, + { label: 'Oxygen', present: (s) => s.doMg != null }, + { label: 'Salinity', present: (s) => s.salinity != null }, + { label: 'SST', present: (s) => s.sst != null }, + { label: 'Rain', present: (s) => s.rain24h != null }, + { label: 'Tide', present: (s) => s.tideMidMm != null } + ]), + unknownLavage: captures.filter((c) => c.lavage == null && c.stomach == null).length, + notLavaged: captures.filter((c) => c.lavage === 0 && c.stomach == null).length + }; +} +export function modelSelectionFlow(d: Dataset) { + const s = selectCatchModel(d); + return ['all', 'Cm', 'Na'].map((response) => { + const species = s.selectSpecies(response as 'all' | 'Cm' | 'Na'); + const counts = [effortSets(d).length, s.complete.length, s.base.length, species.rows.length]; + return { + name: response, + rows: counts.map((remaining, i) => ({ + stage: [ + 'Timed gillnet sets', + 'Complete dates, nursery and environment', + 'Nurseries with ≥30 complete sets', + 'Species nursery support (≥10 catches)' + ][i], + remaining, + excluded: i ? counts[i - 1] - remaining : 0 + })) + }; + }); +} diff --git a/src/routes/data-quality/+page.server.ts b/src/routes/data-quality/+page.server.ts index c0c1ce4..40e51e8 100644 --- a/src/routes/data-quality/+page.server.ts +++ b/src/routes/data-quality/+page.server.ts @@ -1,4 +1,5 @@ import { memo } from '#lib/server/data'; +import { measurementCoverage } from '#lib/server/analytics/measurementCoverage'; import type { PageServerLoad } from './$types'; export const load: PageServerLoad = () => @@ -11,6 +12,7 @@ export const load: PageServerLoad = () => cats.set(k, c); } return { + coverage: measurementCoverage(d), summary: [...cats.values()].sort((a, b) => b.n - a.n), issues: d.issues.map((i) => ({ ...i, diff --git a/src/routes/data-quality/+page.svelte b/src/routes/data-quality/+page.svelte index 2cd3127..0673f81 100644 --- a/src/routes/data-quality/+page.svelte +++ b/src/routes/data-quality/+page.svelte @@ -1,4 +1,5 @@ -
+
diff --git a/src/lib/components/MeasurementCoverage.svelte b/src/lib/components/MeasurementCoverage.svelte index c7be1ca..407bbbb 100644 --- a/src/lib/components/MeasurementCoverage.svelte +++ b/src/lib/components/MeasurementCoverage.svelte @@ -1,33 +1,54 @@ -
-

Measurement availability

-

- Recorded values, not accuracy or lab performance. Zero counts as available. Null does not tell - us whether a measurement was omitted by protocol. Grey means no eligible records; pale blue - means 0% recorded and dark blue means 100%. Counts and denominators are in each table. +

+

What was recorded, season by season

+

+ The share of records with a value for each measurement, which shows what each comparison on this + site can draw on. It is not a check that the values are right, and it says nothing about the + lab's work. A zero counts as recorded. A blank can't tell a skipped measurement from one the + protocol didn't call for that season. A dashed cell has no records to count. Each table gives + the counts.

-

- Capture denominators are all field captures (including recaptures); set denominators are all - gillnet sets. The lavage column includes only captures marked lavaged or with a recorded result. {data.unknownLavage} - captures have unknown sampling status and no result; {data.notLavaged} are explicitly not lavaged - with no result. +

+ Lavage results count only captures marked as lavaged or with a result. Of the rest, {plural( + data.unknownLavage, + 'capture doesn’t', + 'captures don’t' + )} say whether the shark was lavaged and {plural(data.notLavaged, 'was', 'were')} marked as not lavaged.

- {#each [{ name: 'Field captures', panel: data.captures }, { name: 'Gillnet sets', panel: data.sets }] as p (p.name)} + {#each panels as p (p.title)} `${v.toFixed(1)}%`} + format={share} + cellLabels xTitle="Measurement" yTitle="Season" /> diff --git a/src/lib/components/ModelSelection.svelte b/src/lib/components/ModelSelection.svelte index d4aa39b..095aadc 100644 --- a/src/lib/components/ModelSelection.svelte +++ b/src/lib/components/ModelSelection.svelte @@ -1,41 +1,74 @@ -
- Which sets are eligible for the catch model? +
+ Which sets does the model use?

- Sequential filters from the same selection logic used by the model. A set missing several fields - is excluded once. Eligibility does not guarantee a converged fit: the model also needs at least - 200 base sets, two nurseries and two seasons, and each species fit needs two supported - nurseries. The all-sharks model has no species catch threshold. + Each bar is the number of sets left after a filter, counted by the same code the model uses. The + first three filters run in order and leave the sample for all sharks. Each species model then + starts again from that sample and drops the nurseries where that species was rarely caught. +

+ {#if incomplete} +

+ Of the {plural(incomplete, 'set')} missing a model variable, {missingText.length > 1 + ? `${missingText.slice(0, -1).join(', ')} and ${missingText.at(-1)}` + : missingText[0]}{flow.missing.length > 1 ? `; a set can lack more than one` : ''}. +

+ {/if} +

+ Being in the sample doesn't guarantee a fit: the model also needs at least {num( + flow.minModelSets + )} sets, two nurseries and two seasons. The sample for all sharks has {plural( + flow.modelSets, + 'set' + )} from {plural(flow.nurseries, 'nursery', 'nurseries')} over {plural(flow.seasons, 'season')}.

- {#each groups as g (g.name)} - - r.stage)} - series={[ - { - name: 'Remaining sets', - color: 'var(--series-1)', - values: g.rows.map((r) => r.remaining) - } - ]} - /> - - {/each} + + r.stage)} + series={[ + { name: 'Sets', color: 'var(--series-1)', values: flow.rows.map((r) => r.remaining) } + ]} + valueFormat={(v) => num(v)} + /> +
+ + diff --git a/src/lib/components/charts/Heatmap.svelte b/src/lib/components/charts/Heatmap.svelte index c790a63..40ea86d 100644 --- a/src/lib/components/charts/Heatmap.svelte +++ b/src/lib/components/charts/Heatmap.svelte @@ -49,22 +49,27 @@ const hi = $derived(domain?.[1] ?? (flat.length ? Math.max(...flat) : 1)); const absMax = $derived(Math.max(Math.abs(lo), Math.abs(hi)) || 1); - function fill(v: number | null): { color: string; opacity: number; dark: boolean } { + function fill(v: number | null): { color: string; opacity: number; ink: string } { // No data: an empty dashed cell, never a colour from the scale (the lightest steps are // close to --surface-2, in dark mode especially). - if (v == null || !Number.isFinite(v)) return { color: 'none', opacity: 1, dark: false }; + if (v == null || !Number.isFinite(v)) return { color: 'none', opacity: 1, ink: 'var(--ink)' }; if (mode === 'div') { const t = Math.max(-1, Math.min(1, v / absMax)); - if (Math.abs(t) < 0.06) return { color: 'var(--div-mid)', opacity: 1, dark: false }; + if (Math.abs(t) < 0.06) return { color: 'var(--div-mid)', opacity: 1, ink: 'var(--ink)' }; return { color: t < 0 ? 'var(--div-neg)' : 'var(--div-pos)', opacity: 0.18 + 0.82 * Math.abs(t), - dark: Math.abs(t) > 0.6 + ink: Math.abs(t) > 0.6 ? '#ffffff' : 'var(--ink)' }; } const t = hi === lo ? 1 : (v - lo) / (hi - lo); const k = Math.min(7, 1 + Math.floor(t * 6.999)); - return { color: `var(--seq-${k})`, opacity: 1, dark: k >= 5 }; + // The strong end is dark in light mode and pale in dark mode, so its text flips with it. + return { + color: `var(--seq-${k})`, + opacity: 1, + ink: k >= 5 ? 'var(--seq-strong-ink)' : 'var(--ink)' + }; } let tip = $state(null); @@ -133,7 +138,7 @@ dy="0.32em" text-anchor="middle" font-size="10" - fill={f.dark ? '#ffffff' : 'var(--ink)'} + fill={f.ink} pointer-events="none">{format(v)} {/if} diff --git a/src/lib/server/analytics/catchModel.ts b/src/lib/server/analytics/catchModel.ts index f300335..65570ff 100644 --- a/src/lib/server/analytics/catchModel.ts +++ b/src/lib/server/analytics/catchModel.ts @@ -24,7 +24,9 @@ const TERMS = [ 'crew' ] as const; /** Nurseries need this many complete sets to get their own effect. */ -const MIN_SITE_SETS = 30; +export const MIN_SITE_SETS = 30; +/** The model needs this many sets from supported nurseries (and two nurseries, two seasons). */ +export const MIN_MODEL_SETS = 200; const SPLINE_KNOTS = [0, 0.25, 0.5, 0.75, 1]; type Response = 'all' | 'Cm' | 'Na'; @@ -46,8 +48,9 @@ function count(s: SetRow, r: Response): number { } /** A species gets its own nursery effect only where it was caught at least this often. */ -const MIN_SITE_CATCH = 10; +export const MIN_SITE_CATCH = 10; +/** The sample selection behind every fit; /drivers shows it step by step (#123). */ export function selectCatchModel(d: Dataset) { const vars = TERMS.map((k) => SET_VARIABLES.find((v) => v.key === k)!); const complete = effortSets(d).filter( @@ -72,7 +75,7 @@ export function selectCatchModel(d: Dataset) { export function catchModel(d: Dataset) { const { vars, perSite, sites, base, seasons, selectSpecies } = selectCatchModel(d); - if (base.length < 200 || sites.length < 2 || seasons.length < 2) return null; + if (base.length < MIN_MODEL_SETS || sites.length < 2 || seasons.length < 2) return null; // Scales and spline knots come from the full model so every species shares the same units. const scale = vars.map((v) => { diff --git a/src/lib/server/analytics/measurementCoverage.test.ts b/src/lib/server/analytics/measurementCoverage.test.ts index 5ca72ea..69b5c50 100644 --- a/src/lib/server/analytics/measurementCoverage.test.ts +++ b/src/lib/server/analytics/measurementCoverage.test.ts @@ -1,6 +1,8 @@ import { expect, it } from 'vitest'; import { dataset } from '../data'; -import { coverageCell, modelSelectionFlow } from './measurementCoverage'; +import { catchModel } from './catchModel'; +import { coverageCell, measurementCoverage, modelSelectionFlow } from './measurementCoverage'; + it('distinguishes zero, missing and no eligible records', () => { expect(coverageCell([0, null, 2], (v) => v != null)).toEqual({ available: 2, @@ -15,11 +17,34 @@ it('distinguishes zero, missing and no eligible records', () => { ) ).toEqual({ available: 0, eligible: 0, percent: null }); }); -it('reconciles mutually sequential exclusions for each model sample', () => { - for (const g of modelSelectionFlow(dataset())) { - expect(g.rows[0].remaining - g.rows.slice(1).reduce((n, r) => n + r.excluded, 0)).toBe( - g.rows.at(-1)!.remaining - ); - expect(g.rows.every((r) => r.excluded >= 0)).toBe(true); + +it('keeps empty strata missing rather than 0%', () => { + const c = measurementCoverage(dataset()); + for (const p of [c.captures, c.sets]) { + expect(p.values).toHaveLength(p.seasons.length); + for (const r of p.rows) { + if (r.eligible === 0) expect(r.percent).toBeNull(); + else expect(r.percent).toBeCloseTo((100 * r.available) / r.eligible, 1); + } + } +}); + +it('reconciles the selection flow with the fitted samples', () => { + const d = dataset(); + const f = modelSelectionFlow(d); + const [timed, complete, base, cm, na] = f.rows; + expect(f.rows.every((r) => r.excluded >= 0)).toBe(true); + // Sequential steps: each set is left out once. + expect(timed.remaining - complete.excluded - base.excluded).toBe(base.remaining); + // Species steps start again from the all-shark sample. + for (const r of [cm, na]) expect(base.remaining - r.excluded).toBe(r.remaining); + // Missing variables overlap: together they cover at least the sets left out for them. + expect(f.missing.reduce((n, m) => n + m.n, 0)).toBeGreaterThanOrEqual(complete.excluded); + const model = catchModel(d); + if (model) { + expect(model.n).toBe(base.remaining); + expect(model.dropped).toBe(timed.remaining - base.remaining); + if (model.cm) expect(model.cm.n).toBe(cm.remaining); + if (model.na) expect(model.na.n).toBe(na.remaining); } }); diff --git a/src/lib/server/analytics/measurementCoverage.ts b/src/lib/server/analytics/measurementCoverage.ts index f7f53aa..f2b5013 100644 --- a/src/lib/server/analytics/measurementCoverage.ts +++ b/src/lib/server/analytics/measurementCoverage.ts @@ -1,6 +1,10 @@ +/** + * Measurement coverage (#123): how many records have a value for each measurement, season by + * season, and the sample selection behind the catch model. Coverage is availability, not validity. + */ import type { Dataset, Capture, SetRow } from '../data'; -import { effortSets } from './common'; -import { selectCatchModel } from './catchModel'; +import { effortSets, round } from './common'; +import { MIN_MODEL_SETS, MIN_SITE_CATCH, MIN_SITE_SETS, selectCatchModel } from './catchModel'; /** Availability is distinct from validity; zero is a recorded value. */ export function coverageCell( @@ -16,6 +20,7 @@ export function coverageCell( percent: subset.length ? (100 * available) / subset.length : null }; } + export function measurementCoverage(d: Dataset) { const captures = d.captures.filter((c) => c.kind === 'field'); const sets = d.sets.filter((s) => s.gear === 'Gillnet'); @@ -24,15 +29,14 @@ export function measurementCoverage(d: Dataset) { fields: { label: string; present: (r: T) => boolean; eligible?: (r: T) => boolean }[] ) => { const table = d.seasons.flatMap((season) => - fields.map((f) => ({ - season, - field: f.label, - ...coverageCell( + fields.map((f) => { + const c = coverageCell( rows.filter((r) => r.season === season), f.present, f.eligible - ) - })) + ); + return { season, field: f.label, ...c, percent: round(c.percent, 1) ?? null }; + }) ); return { fields: fields.map((f) => f.label), @@ -57,6 +61,8 @@ export function measurementCoverage(d: Dataset) { ]), sets: panel(sets, [ { label: 'Duration', present: (s) => s.durationMin != null }, + { label: 'Start time', present: (s) => s.startMin != null }, + { label: 'Crew size', present: (s) => s.crewSize != null }, { label: 'Water °C', present: (s) => s.waterTemp != null }, { label: 'pH', present: (s) => s.ph != null }, { label: 'Oxygen', present: (s) => s.doMg != null }, @@ -69,23 +75,68 @@ export function measurementCoverage(d: Dataset) { notLavaged: captures.filter((c) => c.lavage === 0 && c.stomach == null).length }; } + +/** + * The catch model's sample, filter by filter, from the same selectCatchModel() the fits use. The + * first three steps are sequential; each species model then drops nurseries from the all-shark + * sample, so its "left out" counts from that sample, not from the bar above. + */ export function modelSelectionFlow(d: Dataset) { const s = selectCatchModel(d); - return ['all', 'Cm', 'Na'].map((response) => { - const species = s.selectSpecies(response as 'all' | 'Cm' | 'Na'); - const counts = [effortSets(d).length, s.complete.length, s.base.length, species.rows.length]; - return { - name: response, - rows: counts.map((remaining, i) => ({ - stage: [ - 'Timed gillnet sets', - 'Complete dates, nursery and environment', - 'Nurseries with ≥30 complete sets', - 'Species nursery support (≥10 catches)' - ][i], - remaining, - excluded: i ? counts[i - 1] - remaining : 0 - })) - }; + const timed = effortSets(d).filter((x) => x.cpue != null); + const step = (stage: string, rule: string, rows: SetRow[], from: number) => ({ + stage, + rule, + remaining: rows.length, + nurseries: new Set(rows.map((x) => x.siteId).filter(Boolean)).size, + excluded: from - rows.length }); + const rows = [ + step('Timed gillnet sets', 'Gillnet sets with a duration', timed, timed.length), + step( + 'Every variable recorded', + 'Date, nursery and every model variable', + s.complete, + timed.length + ), + step( + `Nurseries with ≥ ${MIN_SITE_SETS} sets`, + `The sample for all sharks: nurseries with at least ${MIN_SITE_SETS} of those sets`, + s.base, + s.complete.length + ), + ...( + [ + ['Cm', 'Blacktip model', 'blacktips'], + ['Na', 'Sicklefin lemon model', 'sicklefin lemons'] + ] as const + ).map(([code, label, plural]) => + step( + label, + `From the all-shark sample: nurseries where at least ${MIN_SITE_CATCH} ${plural} were caught`, + s.selectSpecies(code).rows, + s.base.length + ) + ) + ]; + // A set can miss several variables, so these overlap and can add up to more than the step. + const missing = [ + { label: 'date', n: timed.filter((x) => !x.date).length }, + { label: 'nursery', n: timed.filter((x) => !x.siteId).length }, + ...s.vars.map((v) => ({ + label: v.label.toLowerCase(), + n: timed.filter((x) => v.get(x) == null).length + })) + ] + .filter((m) => m.n > 0) + .sort((a, b) => b.n - a.n); + return { + rows, + missing, + minModelSets: MIN_MODEL_SETS, + minSiteSets: MIN_SITE_SETS, + modelSets: s.base.length, + nurseries: s.sites.length, + seasons: s.seasons.length + }; } diff --git a/src/routes/data-quality/+page.svelte b/src/routes/data-quality/+page.svelte index 0673f81..30fd08e 100644 --- a/src/routes/data-quality/+page.svelte +++ b/src/routes/data-quality/+page.svelte @@ -22,7 +22,6 @@ the source sheets.

-
@@ -35,6 +34,7 @@ { key: 'sheets', label: 'Sheets / CSVs', num: true } ]} rows={data.files} + label="Imported files" />

{Object.entries(data.counts) @@ -57,10 +57,13 @@ { key: 'n', label: 'Notes', num: true } ]} rows={data.summary} + label="Notes by type" />

+ +
diff --git a/src/routes/drivers/+page.svelte b/src/routes/drivers/+page.svelte index e500e57..5265f6d 100644 --- a/src/routes/drivers/+page.svelte +++ b/src/routes/drivers/+page.svelte @@ -50,7 +50,6 @@ The model further down estimates them together.

-
+ +
+ {:else} +
+

All together: a model of catches

+

+ There aren't enough complete sets to fit the model. It needs at least {num( + data.selection.minModelSets + )} sets with every variable, from nurseries with at least {data.selection.minSiteSets} such sets + each, across two nurseries and two seasons. +

+
{/if}