diff --git a/AGENTS.md b/AGENTS.md index abdb6be..af795e5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -73,6 +73,7 @@ samples/fixture/ synthetic workbooks for tests/CI (committed) - Species codes: `Cm` blacktip, `Na` sicklefin lemon, `Nf` tawny nurse shark. 2024-25 writes the nurse shark as `GF` / `Gf1` (`GF Log Juv`), probably from _Ginglymostoma_, the Atlantic nurse shark genus; it is read as `Nf` (confirmed by Jodie Rummer, #12). - Diet (#40): `Lavage (y/n)` / `Gastic lavage` → `captures.lavage`; `Lavage (full/empty)` / `Stomach content` → `stomach` (full | traces | empty; "no sample" is null); `Lavage (contents)` / `Stomach sample type` / 2013-14 `State of Stomach` → `stomach_contents` free text. A fullness cell that isn't a fullness word (e.g. "worm found") becomes contents. Analyses in `analytics/diet.ts` (fed = full or traces). - Individuals: union-find over tags (PIT 15-digit, `COFA 12345`, re-tags) plus recapture groups. Conflicting sex/species/shrinking length is logged, not fixed. +- The sheets' capture numbers never decide who is who. `recaptures.ts` logs where they disagree with the tags (#168): capture 2+ with nothing earlier, capture 1 whose tag was caught earlier that season, one tag on two unnumbered captures in a day. Numbers restart each season. 2013-14 writes how many times the shark was caught on every row (2, 2), not the order. - Captures link to sets by season + set number (±1 day), then date + site, then a unique set that date. - CPUE = max(set-log count, linked captures) / net-hours. 2013-14 has no set times (no CPUE). - Sites → `sites.ts` regexes (review in #13). Crew → `crew.ts` aliases plus accent/case merging in the orchestrator (#14). diff --git a/src/lib/server/import/index.ts b/src/lib/server/import/index.ts index 7f410c5..c66c23d 100644 --- a/src/lib/server/import/index.ts +++ b/src/lib/server/import/index.ts @@ -71,11 +71,12 @@ import { type RegisterRow } from './loggers'; import { buildLoggerTables } from './insitu'; +import { checkRecaptures } from './recaptures'; import { SITE_DEFS } from './sites'; import { readWorkbook } from './xlsx'; /** Bump when import logic changes so existing caches rebuild on the next start. */ -export const IMPORTER_VERSION = '10'; +export const IMPORTER_VERSION = '11'; const MIGRATIONS = import.meta.glob('/drizzle/*.sql', { query: '?raw', @@ -488,6 +489,7 @@ export function runImport( else uf.union(o, i); } }); + const labTags = new Set(p.labSharks.flatMap((l) => l.tags)); const comps = new Map(); caps.forEach((_, i) => push(comps as unknown as Map, String(uf.find(i)), i)); const individuals: (typeof schema.individuals.$inferInsert)[] = []; @@ -516,6 +518,11 @@ export function runImport( `Shark ${id}: captures disagree on ${[...new Set(conflicts)].join(', ')}`, first.ref ); + checkRecaptures( + list, + log, + tags.some((t) => labTags.has(t)) + ); for (const c of list) c.individualId = id; const siteSeq: string[] = []; for (const c of list) diff --git a/src/lib/server/import/recaptures.test.ts b/src/lib/server/import/recaptures.test.ts new file mode 100644 index 0000000..5cce9e7 --- /dev/null +++ b/src/lib/server/import/recaptures.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest'; +import { IssueLog } from './issues'; +import { checkRecaptures } from './recaptures'; + +// Synthetic captures only. +const row = ( + date: string, + captureNo: number | null, + o: { + tags?: string[]; + season?: string; + kind?: 'field' | 'hook' | 'lab_release'; + row?: number; + } = {} +) => ({ + season: o.season ?? '2020-2021', + date, + captureNo, + kind: o.kind ?? 'field', + tags: o.tags ?? ['900000000000001'], + ref: { file: 'log.xlsx', sheet: 'Shark Log', row: o.row ?? 2 } +}); + +const run = (rows: ReturnType[], inLab = false) => { + const log = new IssueLog(); + checkRecaptures(rows, log, inLab); + return log.items; +}; + +describe('checkRecaptures (#168)', () => { + it('stays quiet when the numbers follow the tag', () => { + expect(run([row('2020-11-02', 1), row('2020-12-01', 2), row('2021-01-05', 3)])).toEqual([]); + }); + + it('accepts the 2013-14 habit of writing the number of catches on every row', () => { + const s = { tags: [], season: '2013-2014' }; + expect(run([row('2014-01-02', 2, s), row('2014-02-01', 2, s)])).toEqual([]); + expect( + run([row('2014-01-02', 3, s), row('2014-02-01', 3, s), row('2014-03-01', 3, s)]) + ).toEqual([]); + expect(run([row('2014-01-02', 3, s), row('2014-02-01', 3, s)])).toHaveLength(1); + }); + + it('flags a recapture with no earlier capture, tagged or not', () => { + const tagged = run([row('2020-12-01', 2, { row: 9 }), row('2021-01-05', 3)], true); + expect(tagged).toHaveLength(1); + expect(tagged[0]).toMatchObject({ severity: 'warning', category: 'recaptures', row: 9 }); + expect(tagged[0].message).toMatch(/capture 2, but tag 900000000000001 .*lab records/); + const untagged = run([row('2014-03-01', 2, { tags: [], season: '2013-2014' })]); + expect(untagged[0].message).toMatch(/no tag and no earlier capture/); + }); + + it('flags a first capture whose tag was caught earlier that season, not in an earlier one', () => { + const same = run([row('2020-11-02', 1), row('2020-12-01', 1)]); + expect(same).toHaveLength(1); + expect(same[0]).toMatchObject({ severity: 'info' }); + expect(same[0].message).toMatch(/caught on 2020-11-02 this season/); + expect(run([row('2020-11-02', 1), row('2021-11-01', 1, { season: '2021-2022' })])).toEqual([]); + }); + + it('flags one tag on two unnumbered or first captures on the same day', () => { + const twins = run([row('2021-02-01', 1), row('2021-02-01', 1, { row: 7 })]); + expect(twins).toHaveLength(1); + expect(twins[0]).toMatchObject({ severity: 'warning', row: 7 }); + expect(twins[0].message).toMatch(/two captures on 2021-02-01/); + expect(run([row('2021-02-01', null), row('2021-02-01', null)])).toHaveLength(1); + expect(run([row('2021-02-01', 1), row('2021-02-01', 2)])).toEqual([]); + }); + + it('ignores hook-and-line and lab-release rows', () => { + expect(run([row('2020-11-02', 1, { kind: 'lab_release' }), row('2020-12-01', 2)])).toHaveLength( + 1 + ); + expect(run([row('2020-11-02', 2, { kind: 'hook' })])).toEqual([]); + }); +}); diff --git a/src/lib/server/import/recaptures.ts b/src/lib/server/import/recaptures.ts new file mode 100644 index 0000000..e991d45 --- /dev/null +++ b/src/lib/server/import/recaptures.ts @@ -0,0 +1,65 @@ +/** + * Capture numbers in the sheets vs the captures linked by tag (#168). The site counts a shark once + * per tag and never reads the capture numbers, so a disagreement changes nothing here; it goes on + * the Data quality page so the lab can find a first capture that was never entered, a mistyped tag + * or a missing number. Capture numbers restart each season, so repeats across seasons are expected. + */ +import type { IssueLog } from './issues'; +import type { RawCapture } from './logs'; + +type Row = Pick; + +const tagText = (tags: string[]) => `tag ${tags.join(' / ')}`; + +/** Check one shark's captures (`inLab`: one of its tags is in the lab records). */ +export function checkRecaptures(list: Row[], log: IssueLog, inLab = false): void { + const field = list + .filter((c) => c.kind === 'field' && c.date) + .sort((a, b) => a.date!.localeCompare(b.date!)); + field.forEach((c, i) => { + const others = field.filter((o) => o !== c); + const n = c.captureNo; + if (n != null && n > 1) { + const earlier = others.some( + (o) => o.date! < c.date! || (o.date === c.date && (o.captureNo ?? 1) < n) + ); + // 2013-14 writes how many times the shark was caught on every row: 2, 2 is fine. + const total = field.filter((o) => o.season === c.season && o.captureNo === n).length >= n; + if (!earlier && !total) + log.add( + 'warning', + 'recaptures', + c.tags.length + ? `Marked as capture ${n}, but ${tagText(c.tags)} isn't on an earlier capture${inLab ? ' (it is in the lab records)' : ''}; counted as a new shark` + : `Marked as capture ${n}, but it has no tag and no earlier capture is linked to it; counted as a new shark`, + c.ref, + 'capture no', + n + ); + return; + } + if (!c.tags.length) return; + const before = others.find((o) => o.season === c.season && o.date! < c.date!); + if (n === 1 && before) + log.add( + 'info', + 'recaptures', + `Marked as capture 1, but ${tagText(c.tags)} was caught on ${before.date} this season`, + c.ref, + 'capture no', + n + ); + const twin = field + .slice(0, i) + .find((o) => o.date === c.date && (o.captureNo == null || o.captureNo <= 1)); + if (twin) + log.add( + 'warning', + 'recaptures', + `Tag ${c.tags.join(' / ')} is on two captures on ${c.date}, neither marked as a recapture: a mistyped tag, or a row entered twice?`, + c.ref, + 'tag', + c.tags[0] + ); + }); +} diff --git a/src/routes/data-quality/+page.svelte b/src/routes/data-quality/+page.svelte index 30fd08e..400e596 100644 --- a/src/routes/data-quality/+page.svelte +++ b/src/routes/data-quality/+page.svelte @@ -18,8 +18,8 @@

The spreadsheets are the source of truth and this site never changes them. While importing, it notes anything odd: unit conversions, impossible measurements, dates outside their season, - tags that disagree, sites it doesn't recognise. This list is for the lab to review and fix in - the source sheets. + tags that disagree, capture numbers the tags don't back up, sites it doesn't recognise. This + list is for the lab to review and fix in the source sheets.