diff --git a/AGENTS.md b/AGENTS.md index eab8948..f36a503 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,6 +77,11 @@ samples/fixture/ synthetic workbooks for tests/CI (committed) - Cleaning: trim 2 h at the start of each file and 2 h (EnvLogger) or 24 h (HOBO, read out back at the station) at the end; drop readings outside 15–40 °C; merge repeated downloads of one logger by timestamp. Days need ≥ 18 site-hours. Tables: `logger_files`, `logger_daily`, `logger_diel`, plus `sets.insitu_temp`. - Comparisons between nurseries only use those logged ≥ 1 year with both seasons covered (a summer-only logger looks hottest). +### Cloud cover (#104) + +- ERA5 `cloud_evening` covers every set; the crew's 1–4 `cloud_cover` score exists only in 2020–22 and tracks ERA5 (score 1 ≈ 43%, 4 ≈ 94%). `moonlight()` in `analytics/cloud.ts` = illumination × sin(moon altitude) × clear-sky fraction. +- Cloud vs water temperature must be compared within the same month: the cloudy wet season is also the warmest, so raw correlations are ~0 while within-month ones are strongly negative. + ### Adding a new season Drop the workbook in `data/` named like `Physioshark_Shark Log 2026-2027.xlsx` (the `YYYY-YYYY` in the file name sets the season). Check the Data quality page for unknown sites, unmatched headers ("Sheet not imported") and unit warnings, and extend aliases if needed. Never rename columns in the workbook to make the importer happy. diff --git a/src/lib/server/analytics/cloud.test.ts b/src/lib/server/analytics/cloud.test.ts new file mode 100644 index 0000000..2bc797b --- /dev/null +++ b/src/lib/server/analytics/cloud.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from 'vitest'; +import { dataset, type SetRow } from '../data'; +import { cloudWater, moonCloud, moonlight } from './cloud'; + +describe('cloud cover (#104)', () => { + it('scales moonlight by illumination, moon height and clear sky', () => { + const s = (moonIllum: number, moonAltMid: number, cloudEvening: number) => + ({ moonIllum, moonAltMid, cloudEvening }) as SetRow; + expect(moonlight(s(1, 90, 0))).toBeCloseTo(1, 6); + expect(moonlight(s(1, 30, 50))).toBeCloseTo(0.25, 6); + expect(moonlight(s(1, -5, 0))).toBe(0); + expect(moonlight({ ...s(1, 30, 0), cloudEvening: null })).toBeNull(); + }); + + it('runs on the synthetic fixture', () => { + const d = dataset(); + const mc = moonCloud(d); + expect(mc.groups).toHaveLength(3); + expect(mc.n).toBeGreaterThan(0); + const cw = cloudWater(d); + expect(cw.days).toBeGreaterThan(0); + for (const b of cw.bins) expect(b.n).toBeGreaterThanOrEqual(20); + }); +}); diff --git a/src/lib/server/analytics/cloud.ts b/src/lib/server/analytics/cloud.ts new file mode 100644 index 0000000..3ef6ac8 --- /dev/null +++ b/src/lib/server/analytics/cloud.ts @@ -0,0 +1,154 @@ +/** + * Cloud cover (#104): the crew's own cloud score against ERA5, cloud as a filter on moonlight, + * and cloud against the in-situ water temperature. Exploratory: correlations, not causes. + */ +import { finite, mean, pairs, se, spearman, welch } from '#lib/stats'; +import type { Dataset, SetRow } from '../data'; +import { effortSets, round } from './common'; + +/** Bright moon: at least this illuminated and this high (degrees) at mid-set. */ +const BRIGHT_ILLUM = 0.6; +const BRIGHT_ALT = 10; +/** Sky classes from ERA5 cloud cover (%). */ +export const CLEAR_MAX = 40; +export const OVERCAST_MIN = 70; + +/** + * Moonlight reaching the water at mid-set, 0–1: illuminated fraction × sine of the moon's + * altitude (0 below the horizon) × clear-sky fraction from ERA5 cloud cover. + */ +export function moonlight(s: SetRow): number | null { + if (s.moonIllum == null || s.moonAltMid == null || s.cloudEvening == null) return null; + const sinAlt = Math.max(0, Math.sin((s.moonAltMid * Math.PI) / 180)); + return s.moonIllum * sinAlt * (1 - s.cloudEvening / 100); +} + +const ci = (ys: (number | null)[]) => { + const m = mean(ys); + const e = se(ys); + return { + n: finite(ys).length, + mean: round(m), + lo: m != null && e != null ? round(Math.max(0, m - 1.96 * e)) : null, + hi: m != null && e != null ? round(m + 1.96 * e) : null + }; +}; + +/** Mean ERA5 cloud cover during the sets the crew scored 1–4 (2020–22 set logs). */ +export function crewVsEra5(d: Dataset) { + const scored = d.sets.filter((s) => s.cloudCover != null && s.cloudEvening != null); + const rows = [1, 2, 3, 4].map((score) => ({ + score, + ...ci(scored.filter((s) => Math.round(s.cloudCover!) === score).map((s) => s.cloudEvening)) + })); + const r = spearman( + pairs( + scored, + (s) => s.cloudCover, + (s) => s.cloudEvening + ) + ); + const means = rows.map((x) => x.mean).filter((m): m is number => m != null); + return { + rows, + n: scored.length, + rho: r ? round(r.rho) : null, + increasing: means.length === 4 && means.every((m, i) => i === 0 || m > means[i - 1]) + }; +} + +/** Catch rate on dark nights and on bright-moon nights under clear or overcast skies. */ +export function moonCloud(d: Dataset) { + const sets = effortSets(d).filter( + (s) => s.cpue != null && s.moonIllum != null && s.moonAltMid != null && s.cloudEvening != null + ); + const bright = sets.filter((s) => s.moonIllum! >= BRIGHT_ILLUM && s.moonAltMid! > BRIGHT_ALT); + const clear = bright.filter((s) => s.cloudEvening! < CLEAR_MAX); + const overcast = bright.filter((s) => s.cloudEvening! >= OVERCAST_MIN); + const dark = sets.filter((s) => s.moonAltMid! <= 0 || s.moonIllum! < 0.25); + const w = welch( + clear.map((s) => s.cpue), + overcast.map((s) => s.cpue) + ); + const r = spearman(pairs(sets, moonlight, (s) => s.cpue)); + return { + groups: [ + { k: 'Dark: moon down or < 25% lit', ...ci(dark.map((s) => s.cpue)) }, + { k: 'Bright moon, clear sky', ...ci(clear.map((s) => s.cpue)) }, + { k: 'Bright moon, overcast', ...ci(overcast.map((s) => s.cpue)) } + ], + p: w ? w.p : null, + rho: r ? round(r.rho) : null, + n: sets.length + }; +} + +/** + * Nursery water against daily cloud cover. Cloudy wet-season months are also the warmest, so + * each day is compared with its own month: anomalies of the island-wide daily high and swing. + */ +export function cloudWater(d: Dataset) { + const cloud = new Map(d.weather.map((w) => [w.date, w.cloud])); + const byDate = new Map(); + for (const r of d.loggerDaily) { + const e = byDate.get(r.date) ?? { max: [], min: [] }; + e.max.push(r.max); + e.min.push(r.min); + byDate.set(r.date, e); + } + const days = [...byDate] + .map(([date, e]) => ({ + date, + cloud: cloud.get(date) ?? null, + max: mean(e.max)!, + swing: mean(e.max)! - mean(e.min)! + })) + .filter( + (x): x is { date: string; cloud: number; max: number; swing: number } => x.cloud != null + ); + const byMonth = new Map(); + for (const x of days) { + const l = byMonth.get(x.date.slice(0, 7)); + if (l) l.push(x); + else byMonth.set(x.date.slice(0, 7), [x]); + } + const anom: { cloud: number; max: number; swing: number }[] = []; + // Within-month contrast of overcast vs clear days, weighted by the smaller group. + let wSum = 0; + let dMax = 0; + let dSwing = 0; + for (const l of byMonth.values()) { + if (l.length < 10) continue; + const mMax = mean(l.map((x) => x.max))!; + const mSwing = mean(l.map((x) => x.swing))!; + for (const x of l) anom.push({ cloud: x.cloud, max: x.max - mMax, swing: x.swing - mSwing }); + const c = l.filter((x) => x.cloud < CLEAR_MAX); + const o = l.filter((x) => x.cloud >= OVERCAST_MIN); + if (c.length >= 3 && o.length >= 3) { + const w = Math.min(c.length, o.length); + dMax += w * (mean(o.map((x) => x.max))! - mean(c.map((x) => x.max))!); + dSwing += w * (mean(o.map((x) => x.swing))! - mean(c.map((x) => x.swing))!); + wSum += w; + } + } + const bins = [0, 20, 40, 60, 80, 100].slice(0, -1).map((a) => { + const b = a + 20; + const inBin = anom.filter((x) => x.cloud >= a && (b === 100 ? x.cloud <= b : x.cloud < b)); + return { + x: a + 10, + label: `${a}–${b}%`, + n: inBin.length, + max: round(mean(inBin.map((x) => x.max))), + swing: round(mean(inBin.map((x) => x.swing))) + }; + }); + const rMax = spearman(anom.map((x) => [x.cloud, x.max])); + return { + bins: bins.filter((b) => b.n >= 20), + days: anom.length, + rhoMax: rMax ? round(rMax.rho) : null, + overcastVsClear: wSum + ? { max: round(dMax / wSum), swing: round(dSwing / wSum), months: wSum } + : null + }; +} diff --git a/src/lib/server/analytics/env.ts b/src/lib/server/analytics/env.ts index 66d513c..d19c651 100644 --- a/src/lib/server/analytics/env.ts +++ b/src/lib/server/analytics/env.ts @@ -1,6 +1,7 @@ /** Per-set explanatory variables used by the drivers, tides, moon and weather pages (#45). */ import { spearman, pairs } from '#lib/stats'; import type { SetRow } from '../data'; +import { moonlight } from './cloud'; import { seasonDay } from './common'; export interface SetVariable { @@ -31,6 +32,20 @@ export const SET_VARIABLES: SetVariable[] = [ { key: 'rain24h', label: 'Rain in previous 24 h', unit: 'mm', dp: 1, get: (s) => s.rain24h }, { key: 'rainSet', label: 'Rain during the set', unit: 'mm', dp: 1, get: (s) => s.rainEvening }, { key: 'cloud', label: 'Cloud cover', unit: '%', dp: 0, get: (s) => s.cloudEvening }, + { + key: 'cloudCrew', + label: 'Cloud cover noted by the crew (1–4)', + unit: '', + dp: 0, + get: (s) => s.cloudCover + }, + { + key: 'moonlight', + label: 'Moonlight reaching the water', + unit: '', + dp: 2, + get: moonlight + }, { key: 'pressure', label: 'Air pressure', unit: 'hPa', dp: 1, get: (s) => s.pressure }, { key: 'waves', label: 'Wave height (offshore)', unit: 'm', dp: 1, get: (s) => s.waveHeight }, { key: 'swell', label: 'Swell height (offshore)', unit: 'm', dp: 1, get: (s) => s.swellHeight }, diff --git a/src/lib/server/analytics/findings.ts b/src/lib/server/analytics/findings.ts index c7c2a43..495b8d3 100644 --- a/src/lib/server/analytics/findings.ts +++ b/src/lib/server/analytics/findings.ts @@ -18,6 +18,7 @@ import { seasonDayLabel, siteName } from './common'; +import { cloudWater } from './cloud'; import { dailyMaxBySite, versusSst } from './insitu'; export interface Finding { @@ -317,6 +318,18 @@ function compute(d: Dataset): Finding[] { }); } + // Cloud cover and nursery water (#104): within-month contrast, so season doesn't confound it. + const cw = cloudWater(d).overcastVsClear; + if (cw?.max != null && cw.swing != null && cw.months >= 30 && cw.max < 0) + out.push({ + id: 'cloud-cools-nursery', + kind: 'pattern', + title: 'Clouds take the edge off nursery afternoons', + value: `−${num(Math.abs(cw.max), 2)} °C`, + text: `Compared with clear days in the same month, overcast days peak ${num(Math.abs(cw.max), 2)} °C cooler on the nursery sand, and the day–night swing is ${num(Math.abs(cw.swing), 2)} °C ${cw.swing <= 0 ? 'smaller' : 'larger'}. Shallow water heats fast under full sun.`, + href: '/water-quality' + }); + // Earliest newborn const openScar = caps.filter((c) => c.umbilical === 'open' && c.date); if (openScar.length) { diff --git a/src/routes/moon/+page.server.ts b/src/routes/moon/+page.server.ts index d6b647b..4d732aa 100644 --- a/src/routes/moon/+page.server.ts +++ b/src/routes/moon/+page.server.ts @@ -1,6 +1,7 @@ import { memo } from '#lib/server/data'; import { binnedMean, cpueOf, effortSets, fieldCaptures, round } from '#lib/server/analytics/common'; import { MOON_PHASES } from '#lib/server/astro'; +import { moonCloud } from '#lib/server/analytics/cloud'; import { histogram, median, welch } from '#lib/stats'; import type { PageServerLoad } from './$types'; @@ -27,6 +28,7 @@ export const load: PageServerLoad = () => const bright = caps.filter((c) => (c.moonIllum ?? 0.5) > 0.75).map((c) => c.minAfterSunset); const w = welch(bright, dark); return { + moonCloud: moonCloud(d), phases, illum: binnedMean( sets, diff --git a/src/routes/moon/+page.svelte b/src/routes/moon/+page.svelte index 1231868..ca0b8e3 100644 --- a/src/routes/moon/+page.svelte +++ b/src/routes/moon/+page.svelte @@ -9,6 +9,7 @@ let { data }: PageProps = $props(); const t = $derived(data.timing); + const mc = $derived(data.moonCloud);
@@ -106,6 +107,25 @@ labels /> + 0.05 && mc.rho != null && Math.abs(mc.rho) < 0.1 ? ' Accounting for cloud does not reveal a moonlight effect.' : ''}`} + > + `${g.k} (${g.n})`)} + series={[ + { + name: 'Sharks / net-hour', + color: 'var(--series-7)', + values: mc.groups.map((g) => g.mean) + } + ]} + errors={mc.groups.map((g) => (g.lo != null && g.hi != null ? [g.lo, g.hi] : null))} + valueFormat={(v) => num(v, 2)} + /> + diel, dielAmp: { warm: amp(diel.warm), cool: amp(diel.cool) }, dailyMax: dailyMaxBySite(d), + cloud: cloudWater(d), coverage: coverage(d) } : null; diff --git a/src/routes/water-quality/+page.svelte b/src/routes/water-quality/+page.svelte index 8b285e8..93a7cc8 100644 --- a/src/routes/water-quality/+page.svelte +++ b/src/routes/water-quality/+page.svelte @@ -118,6 +118,39 @@ yFormat={(v) => num(v, 1)} /> + {#if ins.cloud.bins.length >= 3} + {@const oc = ins.cloud.overcastVsClear} + + ins.cloud.bins.find((b) => b.x === v)?.label ?? `${v}%`} + yFormat={(v) => num(v, 2)} + refs={[{ y: 0, label: 'month average' }]} + series={[ + { + name: 'Daily high', + color: 'var(--series-2)', + dots: true, + points: ins.cloud.bins.map((b) => ({ x: b.x, y: b.max })) + }, + { + name: 'Day–night swing', + color: 'var(--series-1)', + dots: true, + points: ins.cloud.bins.map((b) => ({ x: b.x, y: b.swing })) + } + ]} + /> + + {/if} (a: number, b: number) => `${round(a, dp)}–${round(b, dp)}${u}`; return { + crewCloud: crewVsEra5(d), byMonth, rainMonthly: [...monthly.entries()] .sort() diff --git a/src/routes/weather/+page.svelte b/src/routes/weather/+page.svelte index b280727..d89e82f 100644 --- a/src/routes/weather/+page.svelte +++ b/src/routes/weather/+page.svelte @@ -118,6 +118,31 @@ /> {/each} + {#if data.crewCloud.n >= 30} + + `Score ${r.score} (${r.n})`)} + series={[ + { + name: 'ERA5 cloud cover (%)', + color: 'var(--series-6)', + values: data.crewCloud.rows.map((r) => r.mean) + } + ]} + errors={data.crewCloud.rows.map((r) => + r.lo != null && r.hi != null ? [r.lo, r.hi] : null + )} + yLabel="%" + valueFormat={(v) => num(v, 0)} + /> + + {/if}