diff --git a/Dockerfile b/Dockerfile
index fce84d8..8c3be0c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -22,6 +22,12 @@ RUN bun install --frozen-lockfile || bun install
FROM base AS runtime
ENV NODE_ENV=production
+# `ntsb-accidents` reads a 558 MB Microsoft Access database out of a zip,
+# because the NTSB publishes no working API. mdbtools turns that into NDJSON.
+# Nothing else in the image needs either, and both are a few hundred kilobytes.
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends mdbtools unzip \
+ && rm -rf /var/lib/apt/lists/*
# Bun's isolated linker keeps each workspace's node_modules beside it, so the
# whole deps stage comes across rather than only /app/node_modules.
COPY --from=deps /app /app
diff --git a/README.md b/README.md
index 7cfc5ad..f87749f 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,7 @@ The first deployment is [nichedb.dev](https://nichedb.dev). Run your own on anyt
| **item** | One row a source produced. Title, URL, when (with `time_known` and `precision`), tags, and the adapter's payload in `data`. |
| **feed** | A saved query over a collection. Has a page, RSS and JSON Feed renderings, an API endpoint, and followers who are told when it changes by push, email or signed webhook. |
-Adapters are one file each in `packages/adapters/src`. Seventy-five ship today across twenty-seven collections:
+Adapters are one file each in `packages/adapters/src`. Seventy-six ship today across twenty-seven collections:
| Collection | Adapters | Key needed |
| --- | --- | --- |
@@ -41,7 +41,7 @@ Adapters are one file each in `packages/adapters/src`. Seventy-five ship today a
| news | `newsfeed`, `gdelt`, `rssamplifier`, `brisk`, `news-channels` | no |
| domains | `ntld-totals`, `ntld-tlds`, `ntld-launches`, `ntld-changes` | no |
| podcasts | `podcasts` | no |
-| aviation | `faa-nas-status`, `aviation-hazards`, `aviation-metar` | no |
+| aviation | `faa-nas-status`, `aviation-hazards`, `aviation-metar`, `ntsb-accidents` | no (needs mdbtools + unzip, in the Dockerfile) |
| water | `nwps-river-gauges`, `coops-water-levels`, `drought-monitor` | no |
| consumer-finance | `cfpb-complaints`, `fdic-institutions`, `fdic-structure-changes` | no |
diff --git a/packages/adapters/src/index.js b/packages/adapters/src/index.js
index 250abcb..0b1cd65 100644
--- a/packages/adapters/src/index.js
+++ b/packages/adapters/src/index.js
@@ -41,6 +41,7 @@ import { nhcCyclones } from './nhc.js';
import { nhtsaComplaints, nhtsaRatings, nhtsaRecalls } from './nhtsa.js';
import { npm } from './npm.js';
import { ntldChanges, ntldLaunches, ntldTlds, ntldTotals } from './ntlddata.js';
+import { ntsbAccidents } from './ntsb.js';
import { nwpsRiverGauges } from './nwps.js';
import { nws } from './nws.js';
import { ocdsTenders } from './ocds.js';
@@ -108,6 +109,7 @@ export const ADAPTERS = [
faaAirportStatus,
aviationHazards,
aviationMetar,
+ ntsbAccidents,
nwpsRiverGauges,
coopsWaterLevels,
droughtMonitor,
diff --git a/packages/adapters/src/ntsb.js b/packages/adapters/src/ntsb.js
new file mode 100644
index 0000000..a49f7b2
--- /dev/null
+++ b/packages/adapters/src/ntsb.js
@@ -0,0 +1,538 @@
+import { mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { defineAdapter, slugify } from '@nichedb/core/adapter';
+
+/**
+ * Every civil aviation accident and incident the NTSB has investigated, from
+ * the bulk database it publishes rather than from an API, because there is no
+ * usable API.
+ *
+ * This is the third leg of the aviation collection and the one that makes the
+ * other two more than a status board. `faa-nas-status` says the airspace is
+ * delayed today; `aviation-metar` says what the weather is doing at the field.
+ * This says what happened the last thirty-one thousand times something went
+ * wrong, and -- the part that matters -- it carries the weather at the moment
+ * it did. The `events` table has a `metar` column holding the raw observation
+ * at the accident, in the identical format `aviation-metar` publishes hourly.
+ * The same string, from the same service, thirty years apart. That is the join.
+ *
+ * WHY A 96 MB DOWNLOAD AND NOT A QUERY
+ *
+ * The NTSB's public query service (CAROL) answers a malformed request with a
+ * 400 that names the problem and a well-formed one with `500 An unknown
+ * exception occured` [sic]. It is not a usable interface. What the NTSB does
+ * publish reliably is `avall.zip`: a 96 MB archive holding a 558 MB Microsoft
+ * Access database, rebuilt monthly, currently carrying 31,436 events with the
+ * newest eight days old. The monthly delta archives beside it stop in December
+ * 2022 and are not a maintained incremental path, so the full file is the file.
+ *
+ * WHY RE-READING IT IS CHEAP ANYWAY
+ *
+ * An NTSB investigation is published long before it is finished: a preliminary
+ * report within days, a factual report months later, and a probable cause that
+ * can take two years. They are all the same event under the same `ev_id`. So
+ * this adapter deliberately re-reads events it has already stored, keyed on
+ * that id, and lets the upsert do the work -- an unchanged accident hashes to
+ * the row already there and costs no write, while one the NTSB has since
+ * ruled on gains its probable cause in place rather than arriving as a second
+ * row that contradicts the first.
+ *
+ * WHAT IT COSTS PER RUN
+ *
+ * The whole database will not go into one statement: `upsertItems` sends every
+ * row in a single insert, and 31,000 accidents carrying their full narratives
+ * is a hundred and forty megabytes of text in one query. So a run emits a
+ * bounded slice, newest first, and remembers where it stopped. The extracted
+ * tables are cached beside the archive and keyed on the file's publication
+ * date, so the slices after the first cost no download at all -- and when the
+ * NTSB publishes a new file, the date changes, the cache misses, and the walk
+ * starts again from the newest accident.
+ *
+ * NEEDS `mdbtools` AND `unzip` ON THE HOST. Both are in the Dockerfile. There
+ * is no pure-JavaScript reader for a 558 MB Access database worth trusting, and
+ * `mdb-json` writes one JSON object per line with the newlines inside a
+ * narrative escaped -- which the CSV export does not, and which is why the
+ * narrative table reads as 522,053 lines and 28,436 rows.
+ */
+
+const LISTING = 'https://data.ntsb.gov/avdata';
+const ARCHIVE =
+ 'https://data.ntsb.gov/avdata/FileDirectory/DownloadFile?fileID=C%3A%5Cavdata%5Cavall.zip';
+
+/** The tables worth reading, and what each is for. */
+const TABLES = ['events', 'aircraft', 'narratives'];
+
+const clean = (v) => {
+ const s = String(v ?? '').trim();
+ return s && s.toLowerCase() !== 'null' ? s : null;
+};
+
+/** A number that may legitimately be zero: a fatality count of 0 is a fact. */
+export function num(v) {
+ if (v === null || v === undefined || v === '') return null;
+ const n = Number(v);
+ return Number.isFinite(n) ? n : null;
+}
+
+/**
+ * When the NTSB last rebuilt the archive, from its own listing page.
+ *
+ * The page prints a `Date created` beside every file. Reading it is one small
+ * request and it is what makes the 96 MB download conditional rather than
+ * monthly-and-hopeful.
+ */
+export function archiveDate(htmlText) {
+ const row =
+ /avall\.zip[^\d]{0,40}(\d{1,2}\/\d{1,2}\/\d{4})[^\d]{0,4}(\d{1,2}:\d{2}:\d{2})?/i.exec(
+ String(htmlText ?? '').replace(/<[^>]+>/g, ' '),
+ );
+ if (!row) return null;
+ const [, date] = row;
+ const [m, d, y] = date.split('/').map(Number);
+ if (!y || !m || !d) return null;
+ return `${y}-${String(m).padStart(2, '0')}-${String(d).padStart(2, '0')}`;
+}
+
+/**
+ * An NTSB date, which arrives as `08/22/26 00:00:00`.
+ *
+ * The two-digit year is the trap: `26` is 2026 and `98` is 1998, and neither
+ * `new Date` nor any amount of hoping gets that right on its own -- handed
+ * `08/22/26` it returns 1926, which would publish every accident in the file a
+ * century early and sort them before everything else in the collection. The
+ * database starts in 1982, so 82 and above is last century.
+ */
+export function ntsbDate(raw, timeHHMM = null) {
+ const s = String(raw ?? '').trim();
+ /* Four digits before two in the alternation. The other order matches the
+ * `20` of `03/04/2015` and reads it as 2020, which is a wrong date that
+ * still parses, still sorts and never raises anything. */
+ const m = /^(\d{1,2})\/(\d{1,2})\/(\d{4}|\d{2})/.exec(s);
+ if (!m) return null;
+ const [, mm, dd, yy] = m;
+ // 82 itself is 1982, the first year in the database, so the boundary is
+ // inclusive: `>` here would file the oldest accidents under 2082.
+ const year =
+ yy.length === 4 ? Number(yy) : Number(yy) >= 82 ? 1900 + Number(yy) : 2000 + Number(yy);
+ const day = `${year}-${mm.padStart(2, '0')}-${dd.padStart(2, '0')}`;
+ const t = num(timeHHMM);
+ if (t === null) return { date: day, timeKnown: false };
+ // `ev_time` is HHMM as an integer, so 1830 is 18:30 and 217 is 02:17.
+ const hh = String(Math.floor(t / 100)).padStart(2, '0');
+ const mi = String(t % 100).padStart(2, '0');
+ if (Number(hh) > 23 || Number(mi) > 59) return { date: day, timeKnown: false };
+ return { date: `${day}T${hh}:${mi}:00Z`, timeKnown: true };
+}
+
+/**
+ * A coordinate from the decimal columns, which are the only trustworthy ones.
+ *
+ * The `latitude`/`longitude` text columns hold packed degrees-minutes-seconds
+ * like `003000N`, and the same row's `dec_latitude` says 30. Reading the packed
+ * pair as a number puts a Texas accident three thousand degrees north.
+ */
+const coord = (v, limit) => {
+ const n = num(v);
+ return n !== null && Math.abs(n) <= limit && n !== 0 ? n : null;
+};
+
+const INJURY = {
+ FATL: 'fatal',
+ SERS: 'serious',
+ MINR: 'minor',
+ NONE: 'no injuries',
+};
+
+const EVENT_TYPE = { ACC: 'Accident', INC: 'Incident' };
+
+/**
+ * What the NTSB did to the aeroplane, from the code it files it under.
+ *
+ * `damage` is `SUBS`, `DEST`, `MINR` or `NONE`, and printed raw it reads
+ * "Aircraft subs." -- which says nothing to anyone who has not read the
+ * codebook. `SUBS` is the threshold that makes an event an accident rather
+ * than an incident, so it is the most important word in the row.
+ */
+const DAMAGE = {
+ DEST: 'destroyed',
+ SUBS: 'substantially damaged',
+ MINR: 'lightly damaged',
+ NONE: 'undamaged',
+ UNK: null,
+};
+
+/**
+ * A name, unless it is one of the register's placeholders.
+ *
+ * The operator columns carry a bare `N` on a great many rows -- a leftover flag
+ * rather than an operator -- and printed straight it produces "operated by N",
+ * which reads like a company. Anything under two characters is not a name.
+ */
+export function operatorName(...candidates) {
+ for (const c of candidates) {
+ const s = clean(c);
+ if (s && s.length > 1 && !/^(n|y|na|n\/a|none|unk|unknown)$/i.test(s)) return s;
+ }
+ return null;
+}
+
+/** VMC and IMC are the whole story in one field: could the pilot see out. */
+const CONDITIONS = { VMC: 'visual conditions', IMC: 'instrument conditions' };
+
+/**
+ * One accident, from its event row plus whatever the other tables add.
+ *
+ * `aircraft` and `narratives` are keyed on the same `ev_id`, so they arrive
+ * here already looked up rather than being searched for per row.
+ */
+export function toItem(ev, { aircraft = null, narrative = null } = {}) {
+ const id = clean(ev?.ev_id);
+ const when = ntsbDate(ev?.ev_date, ev?.ev_time);
+ if (!id || !when) return null;
+
+ const kind = EVENT_TYPE[clean(ev.ev_type)] ?? 'Event';
+ const city = clean(ev.ev_city);
+ const state = clean(ev.ev_state);
+ const country = clean(ev.ev_country);
+ const where = [city, state === 'OF' ? null : state, country === 'USA' ? null : country]
+ .filter(Boolean)
+ .join(', ');
+
+ const fatal = num(ev.inj_tot_f);
+ const serious = num(ev.inj_tot_s);
+ const minor = num(ev.inj_tot_m);
+ const aboard = num(ev.inj_tot_t);
+ const worst = INJURY[clean(ev.ev_highest_injury)] ?? null;
+
+ const make = clean(aircraft?.acft_make);
+ const model = clean(aircraft?.acft_model);
+ const plane = [make, model].filter(Boolean).join(' ') || null;
+ const registration = clean(aircraft?.regis_no);
+ const operator = operatorName(aircraft?.oper_name, aircraft?.oper_individual_name);
+ const damageCode = clean(aircraft?.damage);
+ const damage = damageCode ? (DAMAGE[damageCode.toUpperCase()] ?? null) : null;
+ const phase = clean(aircraft?.phase_flt_spec);
+ const farPart = clean(aircraft?.far_part);
+
+ const cause = clean(narrative?.narr_cause);
+ const account =
+ clean(narrative?.narr_accp) ?? clean(narrative?.narr_accf) ?? clean(narrative?.narr_inc);
+ const metar = clean(ev.metar);
+ const conditions = CONDITIONS[clean(ev.wx_cond_basic)] ?? null;
+
+ const headline = [
+ plane ?? 'Aircraft',
+ registration ? `(${registration})` : null,
+ kind === 'Accident' ? 'accident' : 'incident',
+ where ? `near ${where}` : null,
+ ]
+ .filter(Boolean)
+ .join(' ');
+
+ return {
+ externalId: `ntsb-${id}`,
+ kind: 'accident',
+ title: `${headline}${fatal ? ` — ${fatal} killed` : ''}`,
+ summary: [
+ `${kind} on ${String(when.date).slice(0, 10)}`,
+ where ? ` at ${where}` : '',
+ plane ? `, ${plane}` : '',
+ registration ? ` ${registration}` : '',
+ operator ? `, operated by ${operator}` : '',
+ '. ',
+ fatal !== null || serious !== null
+ ? `${fatal ?? 0} killed, ${serious ?? 0} seriously injured${aboard !== null ? ` of ${aboard} aboard` : ''}. `
+ : '',
+ damage ? `Aircraft ${damage}. ` : '',
+ conditions ? `Flown in ${conditions}. ` : '',
+ cause ? `Probable cause: ${cause}` : account ? account : '',
+ ]
+ .join('')
+ .trim()
+ .slice(0, 3900),
+ url: clean(ev.ntsb_no)
+ ? `https://data.ntsb.gov/carol-main-public/basic-search?ntsbNumber=${encodeURIComponent(ev.ntsb_no)}`
+ : LISTING,
+ publishedAt: when.date,
+ timeKnown: when.timeKnown,
+ precision: when.timeKnown ? 'minute' : 'day',
+ tags: [
+ 'aviation',
+ 'accident',
+ kind.toLowerCase(),
+ country === 'USA' ? 'us' : country ? slugify(country) : null,
+ state && state !== 'OF' ? state.toLowerCase() : null,
+ make ? slugify(make).slice(0, 40) : null,
+ fatal ? 'fatal' : null,
+ worst ? `injury:${slugify(worst)}` : null,
+ damageCode ? `damage:${slugify(DAMAGE[damageCode.toUpperCase()] ?? damageCode)}` : null,
+ phase ? `phase:${slugify(phase).slice(0, 30)}` : null,
+ farPart ? `far:${slugify(farPart)}` : null,
+ cause ? 'probable-cause' : 'under-investigation',
+ metar ? 'has-metar' : null,
+ conditions ? slugify(conditions) : null,
+ ].filter(Boolean),
+ data: {
+ eventId: id,
+ ntsbNumber: clean(ev.ntsb_no),
+ eventType: kind,
+ occurredAt: when.date,
+ aircraft: plane,
+ make,
+ model,
+ registration,
+ serial: clean(aircraft?.acft_serial_no),
+ operator,
+ owner: operatorName(aircraft?.owner_acft),
+ farPart,
+ damage,
+ damageCode,
+ phaseOfFlight: phase,
+ departureAirport: clean(aircraft?.dprt_apt_id),
+ destinationAirport: clean(aircraft?.dest_apt_id),
+ injuries: { fatal, serious, minor, none: num(ev.inj_tot_n), aboard, highest: worst },
+ /*
+ * The weather the NTSB recorded at the accident, including the raw METAR
+ * where one was captured. `aviation-metar` publishes the identical field
+ * for every reporting airport every hour, so an accident and the hourly
+ * record of the sky it happened under are the same shape.
+ */
+ weather: {
+ metar,
+ conditions,
+ conditionsCode: clean(ev.wx_cond_basic),
+ lightConditions: clean(ev.light_cond),
+ temperatureC: num(ev.wx_temp),
+ dewpointC: num(ev.wx_dew_pt),
+ windDirectionDeg: num(ev.wind_dir_deg),
+ windSpeedKt: num(ev.wind_vel_kts),
+ gustKt: num(ev.gust_kts),
+ visibilitySm: num(ev.vis_sm),
+ ceilingFt: num(ev.sky_ceil_ht),
+ observedAtStation: clean(ev.wx_obs_fac_id),
+ },
+ probableCause: cause,
+ narrative: account ? account.slice(0, 20_000) : null,
+ narrativeNote: cause
+ ? 'The NTSB has ruled on this event; `probableCause` is its finding.'
+ : 'No probable cause published yet. An NTSB investigation runs for months and often years, and this row is updated in place when the finding lands.',
+ place: {
+ country: country === 'USA' ? 'US' : country,
+ state: state === 'OF' ? null : state,
+ city,
+ lat: coord(ev.dec_latitude, 90),
+ lon: coord(ev.dec_longitude, 180),
+ nearestAirport: clean(ev.ev_nr_apt_id),
+ airportName: clean(ev.apt_name),
+ },
+ source: 'NTSB aviation accident database (avall.mdb)',
+ dataset: LISTING,
+ },
+ };
+}
+
+/** NDJSON, as `mdb-json` writes it: one object per line, nothing else. */
+export function parseNdjson(text) {
+ const out = [];
+ for (const line of String(text ?? '').split('\n')) {
+ const t = line.trim();
+ if (t?.[0] !== '{') continue;
+ try {
+ out.push(JSON.parse(t));
+ } catch {
+ // A truncated final line is the only way this happens, and it is one row.
+ }
+ }
+ return out;
+}
+
+/** Newest accident first, so a bounded run stores what a reader wants first. */
+export function newestFirst(events) {
+ return [...events].sort((a, b) => {
+ const at = ntsbDate(a?.ev_date)?.date ?? '';
+ const bt = ntsbDate(b?.ev_date)?.date ?? '';
+ return bt.localeCompare(at);
+ });
+}
+
+/** One row per ev_id, preferring the row that actually says something. */
+export function indexByEvent(rows, pick = () => true) {
+ const by = new Map();
+ for (const r of rows) {
+ const id = clean(r?.ev_id);
+ if (!id) continue;
+ const held = by.get(id);
+ if (!held || (!pick(held) && pick(r))) by.set(id, r);
+ }
+ return by;
+}
+
+const hasCause = (n) => Boolean(clean(n?.narr_cause));
+
+async function exists(path) {
+ try {
+ await stat(path);
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+/** Run a binary, or say plainly which one is missing. */
+async function run(cmd, args, { cwd, stdoutFile = null } = {}) {
+ const proc = Bun.spawn([cmd, ...args], {
+ cwd,
+ stdout: stdoutFile ? Bun.file(stdoutFile) : 'pipe',
+ stderr: 'pipe',
+ });
+ const code = await proc.exited;
+ if (code !== 0) {
+ const err = proc.stderr ? await new Response(proc.stderr).text() : '';
+ throw new Error(`${cmd} exited ${code}${err ? `: ${err.slice(0, 200)}` : ''}`);
+ }
+ return stdoutFile ? null : await new Response(proc.stdout).text();
+}
+
+export const ntsbAccidents = defineAdapter({
+ name: 'ntsb-accidents',
+ title: 'NTSB aviation accidents',
+ collection: 'aviation',
+ description:
+ 'Every civil aviation accident and incident the NTSB has investigated — 31,000 of them — with the aircraft, the operator, the injuries, the probable cause once it is ruled, and the weather at the moment it happened including the raw METAR. Read from the bulk Access database the NTSB rebuilds monthly, because its query API does not work. Keyless; needs mdbtools and unzip on the host.',
+ docs: 'https://data.ntsb.gov/avdata',
+ kinds: ['accident'],
+ cadenceMinutes: 60,
+ configFields: [
+ {
+ key: 'maxPerRun',
+ label: 'Accidents per run',
+ type: 'number',
+ help: 'The whole database will not fit in one insert. Default 2,000.',
+ },
+ {
+ key: 'minYear',
+ label: 'Earliest year',
+ type: 'number',
+ help: 'Skip events before this year.',
+ },
+ {
+ key: 'accidentsOnly',
+ label: 'Accidents only',
+ type: 'select',
+ options: ['', 'yes'],
+ help: 'Leave incidents out. An incident is the far more common and far less serious half.',
+ },
+ {
+ key: 'cacheDir',
+ label: 'Working directory',
+ help: 'Where the archive is unpacked. A temp directory unless set.',
+ },
+ ],
+ defaults: {},
+ defaultSources: [
+ { slug: 'ntsb-accidents', name: 'NTSB aviation accidents and incidents' },
+ {
+ slug: 'ntsb-fatal-accidents',
+ name: 'NTSB fatal aviation accidents',
+ config: { accidentsOnly: 'yes', minYear: 2010 },
+ cadenceMinutes: 60 * 6,
+ },
+ ],
+ async pull({ config, cursor, http, log, deadline }) {
+ const maxPerRun = Math.max(50, Math.min(Number(config.maxPerRun) || 2000, 5000));
+ const minYear = Number(config.minYear) || 0;
+ const accidentsOnly = String(config.accidentsOnly ?? '') === 'yes';
+ const dir = clean(config.cacheDir) ?? join(tmpdir(), 'nichedb-ntsb');
+
+ const listing = await http.text(LISTING, { timeoutMs: 60_000 });
+ const published = archiveDate(listing);
+ if (!published) throw new Error('could not read the archive date from the NTSB listing');
+
+ /* A new file resets the walk; the same file continues it. The offset lives
+ * in the cursor rather than in the directory, because the directory is a
+ * cache that a container restart is allowed to lose and the walk is not. */
+ const fresh = cursor.fileDate !== published;
+ const offset = fresh ? 0 : Number(cursor.offset) || 0;
+
+ if (!fresh && cursor.done) {
+ log(`archive unchanged (${published}) and fully read; nothing to do`);
+ return { items: [], cursor, note: 'unchanged' };
+ }
+
+ await mkdir(dir, { recursive: true });
+ const stamp = join(dir, `avall-${published}`);
+ const eventsFile = `${stamp}.events.ndjson`;
+
+ if (!(await exists(eventsFile))) {
+ log(`fetching the ${published} archive (96 MB)`);
+ const zip = join(dir, 'avall.zip');
+ const res = await http.request(ARCHIVE, { timeoutMs: 15 * 60_000 });
+ if (!res.ok) throw new Error(`${res.status} fetching the NTSB archive`);
+ await writeFile(zip, Buffer.from(await res.arrayBuffer()));
+
+ await run('unzip', ['-o', '-q', zip, '-d', dir]);
+ const mdb = join(dir, 'avall.mdb');
+ for (const table of TABLES) {
+ await run('mdb-json', [mdb, table], { stdoutFile: `${stamp}.${table}.ndjson` });
+ }
+ // The 558 MB database and the archive are not needed once exported, and a
+ // container that keeps both has 650 MB of disk it cannot use for anything.
+ await rm(mdb, { force: true });
+ await rm(zip, { force: true });
+ log(`extracted ${TABLES.join(', ')} from the ${published} archive`);
+ }
+
+ const events = newestFirst(
+ parseNdjson(await readFile(eventsFile, 'utf8')).filter((e) => {
+ if (accidentsOnly && clean(e.ev_type) !== 'ACC') return false;
+ if (minYear && Number(e.ev_year) < minYear) return false;
+ return true;
+ }),
+ );
+
+ const slice = events.slice(offset, offset + maxPerRun);
+ if (!slice.length) {
+ log(`all ${events.length} event(s) from the ${published} archive are stored`);
+ return { items: [], cursor: { fileDate: published, offset, done: true }, note: 'complete' };
+ }
+
+ const wanted = new Set(slice.map((e) => clean(e.ev_id)).filter(Boolean));
+ const only = (rows) => rows.filter((r) => wanted.has(clean(r?.ev_id)));
+ const aircraft = indexByEvent(
+ only(parseNdjson(await readFile(`${stamp}.aircraft.ndjson`, 'utf8'))),
+ );
+ const narratives = indexByEvent(
+ only(parseNdjson(await readFile(`${stamp}.narratives.ndjson`, 'utf8'))),
+ hasCause,
+ );
+
+ const items = [];
+ for (const ev of slice) {
+ if (Date.now() > deadline) {
+ log(`out of time after ${items.length} of ${slice.length}`);
+ break;
+ }
+ const item = toItem(ev, {
+ aircraft: aircraft.get(clean(ev.ev_id)) ?? null,
+ narrative: narratives.get(clean(ev.ev_id)) ?? null,
+ });
+ if (item) items.push(item);
+ }
+
+ const next = offset + items.length;
+ const done = next >= events.length;
+ log(
+ `${items.length} accident(s) from the ${published} archive, ${next} of ${events.length} read${
+ done ? ' — complete' : ''
+ }`,
+ );
+ return {
+ items,
+ cursor: { fileDate: published, offset: next, done },
+ // Keep walking straight away while there is more of the file to read.
+ nextInMinutes: done ? undefined : 5,
+ note: `${items.length} accidents (${next}/${events.length})`,
+ };
+ },
+});
diff --git a/packages/core/src/seed.js b/packages/core/src/seed.js
index d226c4e..9478eaf 100644
--- a/packages/core/src/seed.js
+++ b/packages/core/src/seed.js
@@ -157,7 +157,7 @@ export const COLLECTIONS = [
slug: 'aviation',
name: 'Aviation',
description:
- 'Why flights are late, from the three feeds that between them answer it: the traffic management initiatives the FAA has in force right now — ground stops, ground delay programs, airport closures — each written once more when it ends with how long it ran, which the FAA itself never publishes; every SIGMET and AIRMET in the air over the country; and the decoded observation at the airport underneath. One airport, one hour, three sources. All keyless.',
+ 'Why flights are late and what happens when it goes wrong. The traffic management initiatives the FAA has in force right now — ground stops, ground delay programs, airport closures — each written once more when it ends with how long it ran, which the FAA itself never publishes. Every SIGMET and AIRMET in the air over the country, and the decoded observation at the airport underneath. And every one of the 31,000 accidents the NTSB has investigated, each carrying the raw weather observation at the moment it happened — the same field, from the same service, that the hourly feed publishes today. All keyless.',
},
{
slug: 'water',
@@ -895,6 +895,24 @@ export const DEFAULT_FEEDS = [
name: 'Airports below VFR',
query: { kinds: ['observation'], tags: ['below-vfr'] },
},
+ {
+ collection: 'aviation',
+ slug: 'aviation-accidents',
+ name: 'NTSB accidents and incidents',
+ query: { kinds: ['accident'] },
+ },
+ {
+ collection: 'aviation',
+ slug: 'fatal-aviation-accidents',
+ name: 'Fatal aviation accidents',
+ query: { kinds: ['accident'], tags: ['fatal'] },
+ },
+ {
+ collection: 'aviation',
+ slug: 'accidents-in-instrument-conditions',
+ name: 'Accidents in instrument conditions',
+ query: { kinds: ['accident'], tags: ['instrument-conditions'] },
+ },
{
collection: 'water',
slug: 'rivers-in-flood',
diff --git a/test/aviation.test.js b/test/aviation.test.js
index 0a1c2f1..16e8873 100644
--- a/test/aviation.test.js
+++ b/test/aviation.test.js
@@ -15,7 +15,7 @@ import {
parseStatus,
reasonTags,
} from '../packages/adapters/src/faanas.js';
-import { adapterByName } from '../packages/adapters/src/index.js';
+import { ADAPTERS, adapterByName } from '../packages/adapters/src/index.js';
import { normaliseItem, xmlItems } from '../packages/core/src/adapter.js';
process.env.DATABASE_URL ??= 'postgres://test:test@localhost:5432/test';
@@ -43,9 +43,7 @@ describe('the aviation collection', () => {
test('every aviation feed queries kinds the collection actually emits', () => {
const kinds = new Set(
- ['faa-nas-status', 'aviation-hazards', 'aviation-metar'].flatMap(
- (n) => adapterByName(n).kinds,
- ),
+ ADAPTERS.filter((a) => a.collection === 'aviation').flatMap((a) => a.kinds),
);
const feeds = DEFAULT_FEEDS.filter((f) => f.collection === 'aviation');
expect(feeds.length).toBeGreaterThan(0);
diff --git a/test/consumer-finance.test.js b/test/consumer-finance.test.js
index 3123f8a..99d4993 100644
--- a/test/consumer-finance.test.js
+++ b/test/consumer-finance.test.js
@@ -14,7 +14,7 @@ import {
money,
thousands,
} from '../packages/adapters/src/fdic.js';
-import { adapterByName } from '../packages/adapters/src/index.js';
+import { ADAPTERS, adapterByName } from '../packages/adapters/src/index.js';
import { normaliseItem, slugify } from '../packages/core/src/adapter.js';
process.env.DATABASE_URL ??= 'postgres://test:test@localhost:5432/test';
@@ -67,9 +67,7 @@ describe('the consumer finance collection', () => {
test('every consumer-finance feed queries kinds these adapters emit', () => {
const kinds = new Set(
- ['cfpb-complaints', 'fdic-institutions', 'fdic-structure-changes'].flatMap(
- (n) => adapterByName(n).kinds,
- ),
+ ADAPTERS.filter((a) => a.collection === 'consumer-finance').flatMap((a) => a.kinds),
);
const feeds = DEFAULT_FEEDS.filter((f) => f.collection === 'consumer-finance');
expect(feeds.length).toBeGreaterThan(0);
diff --git a/test/ntsb.test.js b/test/ntsb.test.js
new file mode 100644
index 0000000..623595d
--- /dev/null
+++ b/test/ntsb.test.js
@@ -0,0 +1,216 @@
+import { describe, expect, test } from 'bun:test';
+import { adapterByName } from '../packages/adapters/src/index.js';
+import {
+ archiveDate,
+ indexByEvent,
+ newestFirst,
+ ntsbDate,
+ operatorName,
+ parseNdjson,
+ toItem,
+} from '../packages/adapters/src/ntsb.js';
+import { normaliseItem } from '../packages/core/src/adapter.js';
+
+process.env.DATABASE_URL ??= 'postgres://test:test@localhost:5432/test';
+process.env.SITE_URL ??= 'https://nichedb.test';
+const { DEFAULT_FEEDS } = await import('../packages/core/src/seed.js');
+
+/* An event row as `mdb-json` emits it out of avall.mdb. */
+const event = (over = {}) => ({
+ ev_id: '20260831203691',
+ ntsb_no: 'WPR26LA319',
+ ev_type: 'ACC',
+ ev_date: '08/22/26 00:00:00',
+ ev_time: 1830,
+ ev_city: 'Brenham',
+ ev_state: 'TX',
+ ev_country: 'USA',
+ ev_year: 2026,
+ latitude: '003000N',
+ longitude: '0009600W',
+ dec_latitude: 30,
+ dec_longitude: -96,
+ wx_cond_basic: 'VMC',
+ light_cond: 'DAYL',
+ wx_temp: 99,
+ wind_dir_deg: 330,
+ wind_vel_kts: 4,
+ gust_kts: 0,
+ vis_sm: 9,
+ metar: 'METAR KTME 221835Z AUTO 33004KT 9SM CLR 37/21 A3005 RMK AO2',
+ ev_highest_injury: 'NONE',
+ inj_tot_f: 0,
+ inj_tot_s: 0,
+ inj_tot_t: 1,
+ ...over,
+});
+
+const plane = (over = {}) => ({
+ ev_id: '20260831203691',
+ Aircraft_Key: 1,
+ regis_no: 'N2285D',
+ acft_make: 'CESSNA',
+ acft_model: '170B',
+ damage: 'SUBS',
+ far_part: '091',
+ oper_name: 'JET CANE CORP',
+ ...over,
+});
+
+describe('the NTSB bulk reader', () => {
+ test('is registered in the aviation collection and has feeds pointed at it', () => {
+ const a = adapterByName('ntsb-accidents');
+ expect(a).not.toBeNull();
+ expect(a.collection).toBe('aviation');
+ expect(a.kinds).toContain('accident');
+ const feeds = DEFAULT_FEEDS.filter(
+ (f) => f.collection === 'aviation' && (f.query.kinds ?? []).includes('accident'),
+ );
+ expect(feeds.length).toBeGreaterThan(0);
+ });
+
+ test('a two-digit year is read against 1982, not against the century', () => {
+ /* The database starts in 1982 and the dates arrive as `08/22/26`. Handing
+ * that to `new Date` yields 1926. Every accident in the file would be
+ * published a hundred years early and sorted before every other row in the
+ * collection. */
+ expect(ntsbDate('08/22/26 00:00:00').date).toBe('2026-08-22');
+ expect(ntsbDate('01/09/98 00:00:00').date).toBe('1998-01-09');
+ expect(ntsbDate('12/31/82 00:00:00').date).toBe('1982-12-31');
+ expect(ntsbDate('03/04/2015').date).toBe('2015-03-04');
+ expect(ntsbDate('')).toBeNull();
+ expect(ntsbDate(null)).toBeNull();
+ });
+
+ test('the time is HHMM as an integer, and an impossible one is not a time', () => {
+ expect(ntsbDate('08/22/26', 1830)).toEqual({ date: '2026-08-22T18:30:00Z', timeKnown: true });
+ expect(ntsbDate('04/19/26', 217)).toEqual({ date: '2026-04-19T02:17:00Z', timeKnown: true });
+ expect(ntsbDate('08/22/26', 0)).toEqual({ date: '2026-08-22T00:00:00Z', timeKnown: true });
+ // 2575 is not a time; the day is still known.
+ expect(ntsbDate('08/22/26', 2575)).toEqual({ date: '2026-08-22', timeKnown: false });
+ expect(ntsbDate('08/22/26', null)).toEqual({ date: '2026-08-22', timeKnown: false });
+ });
+
+ test('the coordinates come from the decimal columns, never the packed ones', () => {
+ /* The same row carries `latitude: "003000N"` and `dec_latitude: 30`.
+ * Reading the packed pair as a number puts a Texas accident three thousand
+ * degrees north of the equator. */
+ const item = toItem(event());
+ expect(item.data.place.lat).toBe(30);
+ expect(item.data.place.lon).toBe(-96);
+ expect(toItem(event({ dec_latitude: 0, dec_longitude: 0 })).data.place.lat).toBeNull();
+ expect(toItem(event({ dec_latitude: 3000 })).data.place.lat).toBeNull();
+ });
+
+ test('the damage code becomes a word, because SUBS is what makes it an accident', () => {
+ const item = normaliseItem(toItem(event(), { aircraft: plane() }));
+ expect(item.summary).toContain('Aircraft substantially damaged');
+ expect(item.data.damage).toBe('substantially damaged');
+ expect(item.data.damageCode).toBe('SUBS');
+ expect(item.tags).toContain('damage:substantially-damaged');
+ expect(toItem(event(), { aircraft: plane({ damage: 'DEST' }) }).data.damage).toBe('destroyed');
+ expect(toItem(event(), { aircraft: plane({ damage: 'ZZZ' }) }).data.damage).toBeNull();
+ });
+
+ test('a placeholder operator is not an operator', () => {
+ // The register writes a bare `N` in the operator columns on many rows, and
+ // printed straight it reads "operated by N", which looks like a company.
+ expect(operatorName('N')).toBeNull();
+ expect(operatorName('N', 'BADINELLI RALPH D')).toBe('BADINELLI RALPH D');
+ expect(operatorName('none', 'UNK')).toBeNull();
+ expect(operatorName('JET CANE CORP')).toBe('JET CANE CORP');
+ expect(toItem(event(), { aircraft: plane({ oper_name: 'N' }) }).summary).not.toContain(
+ 'operated by',
+ );
+ });
+
+ test('an unruled accident says so rather than looking finished', () => {
+ const open = toItem(event(), { aircraft: plane() });
+ expect(open.data.probableCause).toBeNull();
+ expect(open.tags).toContain('under-investigation');
+ expect(open.data.narrativeNote).toContain('updated in place');
+
+ const ruled = toItem(event(), {
+ aircraft: plane(),
+ narrative: { ev_id: event().ev_id, narr_cause: 'The pilot’s failure to maintain control.' },
+ });
+ expect(ruled.tags).toContain('probable-cause');
+ expect(ruled.summary).toContain('Probable cause: The pilot’s failure');
+ });
+
+ test('the same accident keeps one id however many times the file is re-read', () => {
+ /* This is what makes a monthly re-read cheap and a late ruling an update
+ * rather than a second, contradicting row. */
+ const first = normaliseItem(toItem(event(), { aircraft: plane() }));
+ const later = normaliseItem(
+ toItem(event(), {
+ aircraft: plane(),
+ narrative: { ev_id: event().ev_id, narr_cause: 'Loss of control on landing.' },
+ }),
+ );
+ expect(later.externalId).toBe(first.externalId);
+ expect(later.externalId).toBe('ntsb-20260831203691');
+ expect(later.contentHash).not.toBe(first.contentHash);
+ });
+
+ test('the weather at the accident is kept in the shape the live METAR feed uses', () => {
+ // events.metar holds the raw observation, which is the identical field
+ // `aviation-metar` publishes hourly. That is the join between the two.
+ const item = toItem(event());
+ expect(item.data.weather.metar).toStartWith('METAR KTME');
+ expect(item.data.weather.conditions).toBe('visual conditions');
+ expect(item.data.weather.windSpeedKt).toBe(4);
+ expect(item.data.weather.gustKt).toBe(0);
+ expect(item.tags).toContain('has-metar');
+ expect(toItem(event({ metar: null })).tags).not.toContain('has-metar');
+ });
+
+ test('zero deaths is a fact, not a missing number', () => {
+ const item = toItem(event());
+ expect(item.data.injuries.fatal).toBe(0);
+ expect(item.data.injuries.aboard).toBe(1);
+ expect(item.title).not.toContain('killed');
+ expect(toItem(event({ inj_tot_f: 2 })).title).toContain('2 killed');
+ });
+
+ test('an event with no id or no date is not an accident', () => {
+ expect(toItem(event({ ev_id: null }))).toBeNull();
+ expect(toItem(event({ ev_date: null }))).toBeNull();
+ expect(toItem(null)).toBeNull();
+ });
+
+ test('NDJSON survives the lines that are not rows', () => {
+ const text = '{"ev_id":"a"}\n\n{"ev_id":"b"}\nnot json\n{"ev_id":"c"';
+ expect(parseNdjson(text).map((r) => r.ev_id)).toEqual(['a', 'b']);
+ expect(parseNdjson('')).toEqual([]);
+ });
+
+ test('the walk runs newest first, so a bounded run stores what a reader wants', () => {
+ const rows = [
+ { ev_id: '1', ev_date: '01/04/15 00:00:00' },
+ { ev_id: '2', ev_date: '08/22/26 00:00:00' },
+ { ev_id: '3', ev_date: '03/09/98 00:00:00' },
+ ];
+ expect(newestFirst(rows).map((r) => r.ev_id)).toEqual(['2', '1', '3']);
+ });
+
+ test('the narrative chosen for an event is the one that says something', () => {
+ /* An event has a narrative row per aircraft, and the empty ones are as
+ * numerous as the full ones. Taking the first would drop the ruling. */
+ const rows = [
+ { ev_id: 'x', Aircraft_Key: 1, narr_cause: '' },
+ { ev_id: 'x', Aircraft_Key: 2, narr_cause: 'The pilot lost control.' },
+ ];
+ const picked = indexByEvent(rows, (n) => Boolean(String(n?.narr_cause ?? '').trim()));
+ expect(picked.get('x').narr_cause).toBe('The pilot lost control.');
+ expect(indexByEvent(rows).get('x').Aircraft_Key).toBe(1);
+ });
+
+ test('the archive date is read off the listing, so the download is conditional', () => {
+ const page =
+ '| avall.zip | 9/1/2026 7:03:59 AM | 96148686 | avall.zip | ';
+ expect(archiveDate(page)).toBe('2026-09-01');
+ expect(archiveDate('codman.pdf | 9/15/2021 3:32:50 PM | ')).toBeNull();
+ expect(archiveDate('')).toBeNull();
+ });
+});
diff --git a/test/water.test.js b/test/water.test.js
index a6201fa..5518426 100644
--- a/test/water.test.js
+++ b/test/water.test.js
@@ -7,7 +7,7 @@ import {
STATE_FIPS,
worstClass,
} from '../packages/adapters/src/droughtmonitor.js';
-import { adapterByName } from '../packages/adapters/src/index.js';
+import { ADAPTERS, adapterByName } from '../packages/adapters/src/index.js';
import {
boxFor,
category,
@@ -62,11 +62,7 @@ describe('the water collection', () => {
});
test('every water feed queries kinds these adapters emit', () => {
- const kinds = new Set(
- ['nwps-river-gauges', 'coops-water-levels', 'drought-monitor'].flatMap(
- (n) => adapterByName(n).kinds,
- ),
- );
+ const kinds = new Set(ADAPTERS.filter((a) => a.collection === 'water').flatMap((a) => a.kinds));
const feeds = DEFAULT_FEEDS.filter((f) => f.collection === 'water');
expect(feeds.length).toBeGreaterThan(0);
for (const feed of feeds) {