Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 30 additions & 3 deletions packages/adapters/src/ntsb.js
Original file line number Diff line number Diff line change
Expand Up @@ -464,10 +464,24 @@ export const ntsbAccidents = defineAdapter({
await mkdir(dir, { recursive: true });
const stamp = join(dir, `avall-${published}`);
const eventsFile = `${stamp}.events.ndjson`;

if (!(await exists(eventsFile))) {
/*
* The cache is ready when this marker exists, and NOT when the first
* extract does.
*
* Two sources read this archive -- every accident, and the fatal ones --
* and they share the directory, which is the point: one 96 MB download
* serves both. But `events` is written first and `narratives` last, so a
* second source arriving mid-extraction saw `events.ndjson`, concluded the
* cache was warm, skipped the download and then failed with ENOENT on a
* file still being written. That is exactly what `ntsb-fatal-accidents`
* did on its first real run. The marker is written after every table, so
* "ready" means all of them.
*/
const marker = `${stamp}.complete`;

if (!(await exists(marker))) {
log(`fetching the ${published} archive (96 MB)`);
const zip = join(dir, 'avall.zip');
const zip = join(dir, `avall-${published}.zip`);
const res = await http.request(ARCHIVE, { timeoutMs: 15 * 60_000 });
if (!res.ok) throw new Error(`${res.status} fetching the NTSB archive`);
await writeFile(zip, Buffer.from(await res.arrayBuffer()));
Expand All @@ -481,9 +495,22 @@ export const ntsbAccidents = defineAdapter({
// container that keeps both has 650 MB of disk it cannot use for anything.
await rm(mdb, { force: true });
await rm(zip, { force: true });
await writeFile(marker, `${TABLES.join(',')}\n`);
log(`extracted ${TABLES.join(', ')} from the ${published} archive`);
}

/* Belt and braces: the marker can only be missing above, never wrong, but a
* container that died between two exports leaves a directory that looks
* warm to nothing and cold to everything. If a table is genuinely absent
* here, say which one rather than letting `readFile` raise a bare ENOENT
* against a path nobody can interpret. */
for (const table of TABLES) {
if (!(await exists(`${stamp}.${table}.ndjson`))) {
await rm(marker, { force: true });
throw new Error(`the ${published} extract is missing ${table}; it will be re-fetched`);
}
}

const events = newestFirst(
parseNdjson(await readFile(eventsFile, 'utf8')).filter((e) => {
if (accidentsOnly && clean(e.ev_type) !== 'ACC') return false;
Expand Down
16 changes: 16 additions & 0 deletions test/ntsb.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,22 @@ describe('the NTSB bulk reader', () => {
expect(indexByEvent(rows).get('x').Aircraft_Key).toBe(1);
});

test('the two NTSB sources share one archive, so readiness is a marker not a file', () => {
/* `ntsb-accidents` and `ntsb-fatal-accidents` read the same extract from
* the same directory, which is the point: one 96 MB download serves both.
* But `events` is written first and `narratives` last, so treating the
* first extract as "the cache is warm" let the second source skip the
* download and then fail with ENOENT on a file still being written. That
* is what it did on its first real run. Both sources must therefore be
* pointed at the same cache and distinguished only by their config. */
const sources = adapterByName('ntsb-accidents').defaultSources;
expect(sources).toHaveLength(2);
expect(sources.map((s) => s.slug)).toEqual(['ntsb-accidents', 'ntsb-fatal-accidents']);
// Neither pins its own cacheDir, or they would each download the archive.
for (const s of sources) expect(s.config?.cacheDir).toBeUndefined();
expect(sources[1].config.accidentsOnly).toBe('yes');
});

test('the archive date is read off the listing, so the download is conditional', () => {
const page =
'<td>avall.zip</td><td>9/1/2026 7:03:59 AM</td><td>96148686</td><td><a href="x">avall.zip</a></td>';
Expand Down
Loading