Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,11 @@ samples/fixture/ synthetic workbooks for tests/CI (committed)
- Cleaning: trim 2 h at the start of each file and 2 h (EnvLogger) or 24 h (HOBO, read out back at the station) at the end; drop readings outside 15–40 °C; merge repeated downloads of one logger by timestamp. Days need ≥ 18 site-hours. Tables: `logger_files`, `logger_daily`, `logger_diel`, plus `sets.insitu_temp`.
- Comparisons between nurseries only use those logged ≥ 1 year with both seasons covered (a summer-only logger looks hottest).

### Cloud cover (#104)

- ERA5 `cloud_evening` covers every set; the crew's 1–4 `cloud_cover` score exists only in 2020–22 and tracks ERA5 (score 1 ≈ 43%, 4 ≈ 94%). `moonlight()` in `analytics/cloud.ts` = illumination × sin(moon altitude) × clear-sky fraction.
- Cloud vs water temperature must be compared within the same month: the cloudy wet season is also the warmest, so raw correlations are ~0 while within-month ones are strongly negative.

### Adding a new season

Drop the workbook in `data/` named like `Physioshark_Shark Log 2026-2027.xlsx` (the `YYYY-YYYY` in the file name sets the season). Check the Data quality page for unknown sites, unmatched headers ("Sheet not imported") and unit warnings, and extend aliases if needed. Never rename columns in the workbook to make the importer happy.
Expand Down
24 changes: 24 additions & 0 deletions src/lib/server/analytics/cloud.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import { describe, expect, it } from 'vitest';
import { dataset, type SetRow } from '../data';
import { cloudWater, moonCloud, moonlight } from './cloud';

describe('cloud cover (#104)', () => {
it('scales moonlight by illumination, moon height and clear sky', () => {
const s = (moonIllum: number, moonAltMid: number, cloudEvening: number) =>
({ moonIllum, moonAltMid, cloudEvening }) as SetRow;
expect(moonlight(s(1, 90, 0))).toBeCloseTo(1, 6);
expect(moonlight(s(1, 30, 50))).toBeCloseTo(0.25, 6);
expect(moonlight(s(1, -5, 0))).toBe(0);
expect(moonlight({ ...s(1, 30, 0), cloudEvening: null })).toBeNull();
});

it('runs on the synthetic fixture', () => {
const d = dataset();
const mc = moonCloud(d);
expect(mc.groups).toHaveLength(3);
expect(mc.n).toBeGreaterThan(0);
const cw = cloudWater(d);
expect(cw.days).toBeGreaterThan(0);
for (const b of cw.bins) expect(b.n).toBeGreaterThanOrEqual(20);
});
});
154 changes: 154 additions & 0 deletions src/lib/server/analytics/cloud.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
/**
* Cloud cover (#104): the crew's own cloud score against ERA5, cloud as a filter on moonlight,
* and cloud against the in-situ water temperature. Exploratory: correlations, not causes.
*/
import { finite, mean, pairs, se, spearman, welch } from '#lib/stats';
import type { Dataset, SetRow } from '../data';
import { effortSets, round } from './common';

/** Bright moon: at least this illuminated and this high (degrees) at mid-set. */
const BRIGHT_ILLUM = 0.6;
const BRIGHT_ALT = 10;
/** Sky classes from ERA5 cloud cover (%). */
export const CLEAR_MAX = 40;
export const OVERCAST_MIN = 70;

/**
* Moonlight reaching the water at mid-set, 0–1: illuminated fraction × sine of the moon's
* altitude (0 below the horizon) × clear-sky fraction from ERA5 cloud cover.
*/
export function moonlight(s: SetRow): number | null {
if (s.moonIllum == null || s.moonAltMid == null || s.cloudEvening == null) return null;
const sinAlt = Math.max(0, Math.sin((s.moonAltMid * Math.PI) / 180));
return s.moonIllum * sinAlt * (1 - s.cloudEvening / 100);
}

const ci = (ys: (number | null)[]) => {
const m = mean(ys);
const e = se(ys);
return {
n: finite(ys).length,
mean: round(m),
lo: m != null && e != null ? round(Math.max(0, m - 1.96 * e)) : null,
hi: m != null && e != null ? round(m + 1.96 * e) : null
};
};

/** Mean ERA5 cloud cover during the sets the crew scored 1–4 (2020–22 set logs). */
export function crewVsEra5(d: Dataset) {
const scored = d.sets.filter((s) => s.cloudCover != null && s.cloudEvening != null);
const rows = [1, 2, 3, 4].map((score) => ({
score,
...ci(scored.filter((s) => Math.round(s.cloudCover!) === score).map((s) => s.cloudEvening))
}));
const r = spearman(
pairs(
scored,
(s) => s.cloudCover,
(s) => s.cloudEvening
)
);
const means = rows.map((x) => x.mean).filter((m): m is number => m != null);
return {
rows,
n: scored.length,
rho: r ? round(r.rho) : null,
increasing: means.length === 4 && means.every((m, i) => i === 0 || m > means[i - 1])
};
}

/** Catch rate on dark nights and on bright-moon nights under clear or overcast skies. */
export function moonCloud(d: Dataset) {
const sets = effortSets(d).filter(
(s) => s.cpue != null && s.moonIllum != null && s.moonAltMid != null && s.cloudEvening != null
);
const bright = sets.filter((s) => s.moonIllum! >= BRIGHT_ILLUM && s.moonAltMid! > BRIGHT_ALT);
const clear = bright.filter((s) => s.cloudEvening! < CLEAR_MAX);
const overcast = bright.filter((s) => s.cloudEvening! >= OVERCAST_MIN);
const dark = sets.filter((s) => s.moonAltMid! <= 0 || s.moonIllum! < 0.25);
const w = welch(
clear.map((s) => s.cpue),
overcast.map((s) => s.cpue)
);
const r = spearman(pairs(sets, moonlight, (s) => s.cpue));
return {
groups: [
{ k: 'Dark: moon down or < 25% lit', ...ci(dark.map((s) => s.cpue)) },
{ k: 'Bright moon, clear sky', ...ci(clear.map((s) => s.cpue)) },
{ k: 'Bright moon, overcast', ...ci(overcast.map((s) => s.cpue)) }
],
p: w ? w.p : null,
rho: r ? round(r.rho) : null,
n: sets.length
};
}

/**
* Nursery water against daily cloud cover. Cloudy wet-season months are also the warmest, so
* each day is compared with its own month: anomalies of the island-wide daily high and swing.
*/
export function cloudWater(d: Dataset) {
const cloud = new Map(d.weather.map((w) => [w.date, w.cloud]));
const byDate = new Map<string, { max: number[]; min: number[] }>();
for (const r of d.loggerDaily) {
const e = byDate.get(r.date) ?? { max: [], min: [] };
e.max.push(r.max);
e.min.push(r.min);
byDate.set(r.date, e);
}
const days = [...byDate]
.map(([date, e]) => ({
date,
cloud: cloud.get(date) ?? null,
max: mean(e.max)!,
swing: mean(e.max)! - mean(e.min)!
}))
.filter(
(x): x is { date: string; cloud: number; max: number; swing: number } => x.cloud != null
);
const byMonth = new Map<string, typeof days>();
for (const x of days) {
const l = byMonth.get(x.date.slice(0, 7));
if (l) l.push(x);
else byMonth.set(x.date.slice(0, 7), [x]);
}
const anom: { cloud: number; max: number; swing: number }[] = [];
// Within-month contrast of overcast vs clear days, weighted by the smaller group.
let wSum = 0;
let dMax = 0;
let dSwing = 0;
for (const l of byMonth.values()) {
if (l.length < 10) continue;
const mMax = mean(l.map((x) => x.max))!;
const mSwing = mean(l.map((x) => x.swing))!;
for (const x of l) anom.push({ cloud: x.cloud, max: x.max - mMax, swing: x.swing - mSwing });
const c = l.filter((x) => x.cloud < CLEAR_MAX);
const o = l.filter((x) => x.cloud >= OVERCAST_MIN);
if (c.length >= 3 && o.length >= 3) {
const w = Math.min(c.length, o.length);
dMax += w * (mean(o.map((x) => x.max))! - mean(c.map((x) => x.max))!);
dSwing += w * (mean(o.map((x) => x.swing))! - mean(c.map((x) => x.swing))!);
wSum += w;
}
}
const bins = [0, 20, 40, 60, 80, 100].slice(0, -1).map((a) => {
const b = a + 20;
const inBin = anom.filter((x) => x.cloud >= a && (b === 100 ? x.cloud <= b : x.cloud < b));
return {
x: a + 10,
label: `${a}–${b}%`,
n: inBin.length,
max: round(mean(inBin.map((x) => x.max))),
swing: round(mean(inBin.map((x) => x.swing)))
};
});
const rMax = spearman(anom.map((x) => [x.cloud, x.max]));
return {
bins: bins.filter((b) => b.n >= 20),
days: anom.length,
rhoMax: rMax ? round(rMax.rho) : null,
overcastVsClear: wSum
? { max: round(dMax / wSum), swing: round(dSwing / wSum), months: wSum }
: null
};
}
15 changes: 15 additions & 0 deletions src/lib/server/analytics/env.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/** Per-set explanatory variables used by the drivers, tides, moon and weather pages (#45). */
import { spearman, pairs } from '#lib/stats';
import type { SetRow } from '../data';
import { moonlight } from './cloud';
import { seasonDay } from './common';

export interface SetVariable {
Expand Down Expand Up @@ -31,6 +32,20 @@ export const SET_VARIABLES: SetVariable[] = [
{ key: 'rain24h', label: 'Rain in previous 24 h', unit: 'mm', dp: 1, get: (s) => s.rain24h },
{ key: 'rainSet', label: 'Rain during the set', unit: 'mm', dp: 1, get: (s) => s.rainEvening },
{ key: 'cloud', label: 'Cloud cover', unit: '%', dp: 0, get: (s) => s.cloudEvening },
{
key: 'cloudCrew',
label: 'Cloud cover noted by the crew (1–4)',
unit: '',
dp: 0,
get: (s) => s.cloudCover
},
{
key: 'moonlight',
label: 'Moonlight reaching the water',
unit: '',
dp: 2,
get: moonlight
},
{ key: 'pressure', label: 'Air pressure', unit: 'hPa', dp: 1, get: (s) => s.pressure },
{ key: 'waves', label: 'Wave height (offshore)', unit: 'm', dp: 1, get: (s) => s.waveHeight },
{ key: 'swell', label: 'Swell height (offshore)', unit: 'm', dp: 1, get: (s) => s.swellHeight },
Expand Down
13 changes: 13 additions & 0 deletions src/lib/server/analytics/findings.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {
seasonDayLabel,
siteName
} from './common';
import { cloudWater } from './cloud';
import { dailyMaxBySite, versusSst } from './insitu';

export interface Finding {
Expand Down Expand Up @@ -317,6 +318,18 @@ function compute(d: Dataset): Finding[] {
});
}

// Cloud cover and nursery water (#104): within-month contrast, so season doesn't confound it.
const cw = cloudWater(d).overcastVsClear;
if (cw?.max != null && cw.swing != null && cw.months >= 30 && cw.max < 0)
out.push({
id: 'cloud-cools-nursery',
kind: 'pattern',
title: 'Clouds take the edge off nursery afternoons',
value: `−${num(Math.abs(cw.max), 2)} °C`,
text: `Compared with clear days in the same month, overcast days peak ${num(Math.abs(cw.max), 2)} °C cooler on the nursery sand, and the day–night swing is ${num(Math.abs(cw.swing), 2)} °C ${cw.swing <= 0 ? 'smaller' : 'larger'}. Shallow water heats fast under full sun.`,
href: '/water-quality'
});

// Earliest newborn
const openScar = caps.filter((c) => c.umbilical === 'open' && c.date);
if (openScar.length) {
Expand Down
2 changes: 2 additions & 0 deletions src/routes/moon/+page.server.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { memo } from '#lib/server/data';
import { binnedMean, cpueOf, effortSets, fieldCaptures, round } from '#lib/server/analytics/common';
import { MOON_PHASES } from '#lib/server/astro';
import { moonCloud } from '#lib/server/analytics/cloud';
import { histogram, median, welch } from '#lib/stats';
import type { PageServerLoad } from './$types';

Expand All @@ -27,6 +28,7 @@ export const load: PageServerLoad = () =>
const bright = caps.filter((c) => (c.moonIllum ?? 0.5) > 0.75).map((c) => c.minAfterSunset);
const w = welch(bright, dark);
return {
moonCloud: moonCloud(d),
phases,
illum: binnedMean(
sets,
Expand Down
20 changes: 20 additions & 0 deletions src/routes/moon/+page.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

let { data }: PageProps = $props();
const t = $derived(data.timing);
const mc = $derived(data.moonCloud);
</script>

<div class="container">
Expand Down Expand Up @@ -106,6 +107,25 @@
labels
/>
</ChartCard>
<ChartCard
title="Does cloud hide the moon?"
papers={['debaere2024lunar']}
subtitle="Catch rate on dark nights and on bright-moon nights (≥ 60% lit, moon above 10° at mid-set) under a clear (< 40% cloud) or overcast (≥ 70%) sky. Whiskers: 95% confidence intervals."
note={`${mc.groups[1].mean != null && mc.groups[2].mean != null ? `Bright-moon nights averaged ${num(mc.groups[1].mean, 2)} sharks per net-hour under a clear sky and ${num(mc.groups[2].mean, 2)} under cloud (Welch ${pValue(mc.p)}). ` : ''}Moonlight reaching the water (illumination × moon height × clear sky) against catch rate: Spearman ρ = ${num(mc.rho, 2)} over ${num(mc.n)} sets.${mc.p != null && mc.p > 0.05 && mc.rho != null && Math.abs(mc.rho) < 0.1 ? ' Accounting for cloud does not reveal a moonlight effect.' : ''}`}
>
<BarChart
categories={mc.groups.map((g) => `${g.k} (${g.n})`)}
series={[
{
name: 'Sharks / net-hour',
color: 'var(--series-7)',
values: mc.groups.map((g) => g.mean)
}
]}
errors={mc.groups.map((g) => (g.lo != null && g.hi != null ? [g.lo, g.hi] : null))}
valueFormat={(v) => num(v, 2)}
/>
</ChartCard>
<ChartCard
title="Bright nights vs dark nights: when do sharks hit the net?"
papers={['debaere2024lunar']}
Expand Down
2 changes: 2 additions & 0 deletions src/routes/water-quality/+page.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
versusSst,
withGaps
} from '#lib/server/analytics/insitu';
import { cloudWater } from '#lib/server/analytics/cloud';
import { linearFit, mean, pairs } from '#lib/stats';
import type { PageServerLoad } from './$types';

Expand Down Expand Up @@ -120,6 +121,7 @@ export const load: PageServerLoad = () =>
diel,
dielAmp: { warm: amp(diel.warm), cool: amp(diel.cool) },
dailyMax: dailyMaxBySite(d),
cloud: cloudWater(d),
coverage: coverage(d)
}
: null;
Expand Down
33 changes: 33 additions & 0 deletions src/routes/water-quality/+page.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,39 @@
yFormat={(v) => num(v, 1)}
/>
</ChartCard>
{#if ins.cloud.bins.length >= 3}
{@const oc = ins.cloud.overcastVsClear}
<ChartCard
title="Clouds and the nursery's afternoon"
papers={['trujillo2025shallows']}
subtitle="Nursery daily high and day–night swing by that day's cloud cover (ERA5), relative to the average for the same month: the cloudy wet season is also the warm season"
note={oc
? `Within the same month, overcast days (≥ 70% cloud) peaked ${num(Math.abs(oc.max ?? 0), 2)} °C ${(oc.max ?? 0) <= 0 ? 'cooler' : 'warmer'} than clear days (< 40%), and the day–night swing was ${num(Math.abs(oc.swing ?? 0), 2)} °C ${(oc.swing ?? 0) <= 0 ? 'smaller' : 'larger'}. Spearman ρ between cloud and the daily high: ${num(ins.cloud.rhoMax, 2)} over ${num(ins.cloud.days)} days.`
: ''}
>
<LineChart
xLabel="Cloud cover"
yLabel="°C vs month average"
xFormat={(v) => ins.cloud.bins.find((b) => b.x === v)?.label ?? `${v}%`}
yFormat={(v) => num(v, 2)}
refs={[{ y: 0, label: 'month average' }]}
series={[
{
name: 'Daily high',
color: 'var(--series-2)',
dots: true,
points: ins.cloud.bins.map((b) => ({ x: b.x, y: b.max }))
},
{
name: 'Day–night swing',
color: 'var(--series-1)',
dots: true,
points: ins.cloud.bins.map((b) => ({ x: b.x, y: b.swing }))
}
]}
/>
</ChartCard>
{/if}
<ChartCard
title="Logger coverage"
subtitle="Each bar is one logger download, from first to last kept reading"
Expand Down
2 changes: 2 additions & 0 deletions src/routes/weather/+page.server.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { memo } from '#lib/server/data';
import { binnedMean, cpueOf, effortSets, round } from '#lib/server/analytics/common';
import { crewVsEra5 } from '#lib/server/analytics/cloud';
import { mean, sum } from '#lib/stats';
import type { PageServerLoad } from './$types';

Expand Down Expand Up @@ -50,6 +51,7 @@ export const load: PageServerLoad = () =>
(a: number, b: number) =>
`${round(a, dp)}–${round(b, dp)}${u}`;
return {
crewCloud: crewVsEra5(d),
byMonth,
rainMonthly: [...monthly.entries()]
.sort()
Expand Down
Loading
Loading