diff --git a/README.md b/README.md index f87749f..c6e134f 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ The first deployment is [nichedb.dev](https://nichedb.dev). Run your own on anyt | **item** | One row a source produced. Title, URL, when (with `time_known` and `precision`), tags, and the adapter's payload in `data`. | | **feed** | A saved query over a collection. Has a page, RSS and JSON Feed renderings, an API endpoint, and followers who are told when it changes by push, email or signed webhook. | -Adapters are one file each in `packages/adapters/src`. Seventy-six ship today across twenty-seven collections: +Adapters are one file each in `packages/adapters/src`. Seventy-nine ship today across twenty-seven collections: | Collection | Adapters | Key needed | | --- | --- | --- | @@ -41,8 +41,8 @@ Adapters are one file each in `packages/adapters/src`. Seventy-six ship today ac | news | `newsfeed`, `gdelt`, `rssamplifier`, `brisk`, `news-channels` | no | | domains | `ntld-totals`, `ntld-tlds`, `ntld-launches`, `ntld-changes` | no | | podcasts | `podcasts` | no | -| aviation | `faa-nas-status`, `aviation-hazards`, `aviation-metar`, `ntsb-accidents` | no (needs mdbtools + unzip, in the Dockerfile) | -| water | `nwps-river-gauges`, `coops-water-levels`, `drought-monitor` | no | +| aviation | `faa-nas-status`, `aviation-hazards`, `aviation-metar`, `ntsb-accidents`, `adsb-flights` | no (NTSB needs mdbtools + unzip, in the Dockerfile) | +| water | `nwps-river-gauges`, `coops-water-levels`, `drought-monitor`, `ndbc-buoys`, `nws-surf-zone` | no | | consumer-finance | `cfpb-complaints`, `fdic-institutions`, `fdic-structure-changes` | no | ## Enrichment diff --git a/packages/adapters/src/adsb.js b/packages/adapters/src/adsb.js new file mode 100644 index 0000000..8ed47e2 --- /dev/null +++ b/packages/adapters/src/adsb.js @@ -0,0 +1,310 @@ +import { defineAdapter, slugify } from '@nichedb/core/adapter'; + +/** + * Live aircraft, but only the ones worth a row. + * + * There is a version of this adapter that stores every aeroplane in the sky + * every few minutes. It would be tens of thousands of rows a day, each one a + * position that was true for four seconds, and nobody would ever read one. A + * position is not news. What is news is an aircraft squawking 7700. + * + * So this reads the event surfaces of the ADS-B feed rather than the firehose: + * the emergency squawks (7700 general emergency, 7600 lost radio, 7500 + * unlawful interference) and the military traffic. When this was written the + * three emergency codes returned zero aircraft between them and the military + * query returned 422, which is exactly the shape you want -- the rare thing is + * rare, so its appearance means something. + * + * AN EMERGENCY IS AN EPISODE, NOT A PING + * + * The same trick `faa-nas-status` uses, for the same reason. An aircraft + * squawking 7700 appears in poll after poll and then stops appearing, and + * nothing records that it stopped. So the open episodes live in the cursor + * keyed on the ICAO hex, an item is keyed on the time it was first seen, and + * when the aircraft leaves the feed it is written once more with `endedAt` and + * a duration. "N123AB squawked 7700 for 22 minutes over Colorado" is the row; + * the forty positions in between are not. + * + * WHOSE DATA THIS IS + * + * adsb.lol is a community receiver network, not a government feed, and it is + * keyless and unmetered. It is the only one of the three I tried that answers: + * airplanes.live rejects an unrecognised client outright, and OpenSky's + * anonymous tier works but is rate limited hard enough to be unreliable on a + * schedule. The aircraft register fields it returns (`r`, `t`, `desc`, `ownOp`) + * come from public FAA and international registries. + */ + +const BASE = 'https://api.adsb.lol/v2'; + +const clean = (v) => { + const s = String(v ?? '').trim(); + return s && s.toLowerCase() !== 'null' ? s : null; +}; + +/** A number that may be legitimately zero: an aircraft on the ground is at 0 ft. */ +export function num(v) { + if (v === null || v === undefined || v === '') return null; + const n = Number(v); + return Number.isFinite(n) ? n : null; +} + +/** + * What the transponder code means. + * + * These three are reserved worldwide and a pilot sets them deliberately. 7500 + * in particular is never set by accident, and an aircraft showing it is the + * single most consequential row this adapter can produce. + */ +export const SQUAWKS = { + 7500: { label: 'unlawful interference', tag: 'hijack', severity: 'critical' }, + 7600: { label: 'radio failure', tag: 'radio-failure', severity: 'urgent' }, + 7700: { label: 'general emergency', tag: 'emergency', severity: 'critical' }, +}; + +/** The watch lists this adapter knows how to read. */ +export const WATCHES = { + emergency: { path: '/sqk/7700', label: 'general emergency' }, + 'radio-failure': { path: '/sqk/7600', label: 'radio failure' }, + hijack: { path: '/sqk/7500', label: 'unlawful interference' }, + military: { path: '/mil', label: 'military' }, + ladd: { path: '/ladd', label: 'limited aircraft data display' }, + pia: { path: '/pia', label: 'privacy ICAO address' }, +}; + +/** A flight-level altitude, or "on the ground", which the feed writes as a word. */ +export function altitude(v) { + if (v === 'ground') return { feet: 0, onGround: true }; + const n = num(v); + return { feet: n, onGround: false }; +} + +/** + * One aircraft's episode on a watch list. + * + * `firstSeen` is the identity, so the row survives every poll the aircraft is + * still there for, and the closing write is the same row with an end on it. + */ +export function toItem(ac, { watch, firstSeen, now, ended = false, positions = 1 }) { + const hex = clean(ac?.hex); + if (!hex) return null; + + const callsign = clean(ac.flight); + const registration = clean(ac.r); + const type = clean(ac.t); + const description = clean(ac.desc); + const operator = clean(ac.ownOp); + const squawk = clean(ac.squawk); + const code = SQUAWKS[Number(squawk)] ?? null; + const alt = altitude(ac.alt_baro); + const lat = num(ac.lat); + const lon = num(ac.lon); + + const who = callsign ?? registration ?? hex.toUpperCase(); + const what = code ? code.label : (WATCHES[watch]?.label ?? watch); + const ran = ended ? minutesBetween(firstSeen, now) : null; + + return { + externalId: `adsb-${watch}-${hex}-${firstSeen}`, + kind: code ? 'aircraft-emergency' : 'aircraft-sighting', + title: `${who}${type ? ` (${type})` : ''}: ${what}${ended && ran ? ` — ended after ${ran}` : ''}`, + summary: [ + `${who}`, + registration && registration !== who ? `, registered ${registration}` : '', + description ? `, a ${description}` : type ? `, a ${type}` : '', + operator ? `, operated by ${operator}` : '', + code ? `, was squawking ${squawk} (${code.label})` : `, seen on the ${what} list`, + alt.onGround + ? ' on the ground' + : alt.feet !== null + ? ` at ${alt.feet.toLocaleString()} ft` + : '', + lat !== null && lon !== null ? ` near ${lat.toFixed(2)}, ${lon.toFixed(2)}` : '', + ended + ? `. Tracked from ${firstSeen} to ${now}${ran ? `, ${ran}` : ''}.` + : `. First seen ${firstSeen}, still showing at ${now}.`, + ].join(''), + url: `https://globe.adsb.lol/?icao=${encodeURIComponent(hex)}`, + publishedAt: firstSeen, + timeKnown: true, + precision: 'minute', + tags: [ + 'aviation', + 'aircraft', + watch, + code ? code.tag : null, + code ? `severity:${code.severity}` : null, + squawk ? `squawk:${squawk}` : null, + type ? slugify(type) : null, + registration ? slugify(registration) : null, + ended ? 'ended' : 'active', + alt.onGround ? 'on-ground' : null, + ].filter(Boolean), + data: { + icaoHex: hex, + callsign, + registration, + aircraftType: type, + aircraftDescription: description, + operator, + squawk, + squawkMeaning: code?.label ?? null, + watch, + altitudeFt: alt.feet, + onGround: alt.onGround, + groundSpeedKt: num(ac.gs), + trackDeg: num(ac.track), + verticalRateFpm: num(ac.baro_rate ?? ac.geom_rate), + firstSeenAt: firstSeen, + lastSeenAt: now, + endedAt: ended ? now : null, + durationHuman: ran, + positionsSeen: positions, + status: ended ? 'ended' : 'active', + statusNote: ended + ? 'The aircraft stopped appearing on this list between the previous poll and this one, so it cleared some time in that window rather than exactly at endedAt.' + : 'Still on the list at lastSeenAt.', + place: { lat, lon }, + source: 'adsb.lol community ADS-B network', + dataset: `${BASE}${WATCHES[watch]?.path ?? ''}`, + }, + }; +} + +/** How long an episode ran, in the words a person would use. */ +export function minutesBetween(fromISO, toISO) { + const ms = new Date(toISO).getTime() - new Date(fromISO).getTime(); + if (!Number.isFinite(ms) || ms < 0) return null; + const mins = Math.round(ms / 60_000); + if (mins < 60) return `${mins}m`; + const h = Math.floor(mins / 60); + const m = mins % 60; + return m ? `${h}h ${m}m` : `${h}h`; +} + +export const adsbFlights = defineAdapter({ + name: 'adsb-flights', + title: 'Aircraft on watch', + collection: 'aviation', + description: + 'Live aircraft, but only the ones that mean something: the emergency transponder codes (7700, 7600, 7500) and military traffic. Each aircraft is one row that lasts as long as it is on the list and is written once more when it clears, with how long it ran — not a position every few minutes. Keyless, from the adsb.lol receiver network.', + docs: 'https://api.adsb.lol/docs', + kinds: ['aircraft-emergency', 'aircraft-sighting'], + cadenceMinutes: 5, + configFields: [ + { + key: 'watch', + label: 'Watch list', + type: 'select', + options: Object.keys(WATCHES), + help: 'Which surface of the feed to read.', + }, + { + key: 'types', + label: 'Only these aircraft types', + type: 'list', + help: 'ICAO type codes, e.g. B738. Empty means every aircraft on the list.', + }, + ], + defaults: { watch: 'emergency' }, + defaultSources: [ + { + slug: 'aircraft-emergency', + name: 'Aircraft squawking 7700 (emergency)', + config: { watch: 'emergency' }, + }, + { + slug: 'aircraft-radio-failure', + name: 'Aircraft squawking 7600 (lost radio)', + config: { watch: 'radio-failure' }, + }, + { + slug: 'aircraft-unlawful-interference', + name: 'Aircraft squawking 7500', + config: { watch: 'hijack' }, + }, + { + slug: 'aircraft-military', + name: 'Military aircraft airborne', + config: { watch: 'military' }, + cadenceMinutes: 60, + }, + ], + async pull({ config, cursor, http, log }) { + const watch = String(config.watch ?? 'emergency'); + const spec = WATCHES[watch]; + if (!spec) throw new Error(`adsb-flights does not know the watch list ${watch}`); + + /* The military list is rate limited far harder than the squawk lists -- it + * returns four hundred aircraft rather than none, and it answers 429 while + * `/sqk/7700` beside it answers 200. A limit is not a failure: returning no + * items and asking to be called back later keeps the source green and, more + * importantly, keeps the cursor intact, because a run that threw here would + * leave every open episode untouched and then report them all as ended on + * the run after. */ + const res = await http.request(`${BASE}${spec.path}`, { timeoutMs: 45_000 }); + if (res.status === 429) { + log(`rate limited on the ${watch} list; backing off`); + return { items: [], cursor, nextInMinutes: 30, note: 'rate limited' }; + } + if (!res.ok) throw new Error(`${res.status} from the ${watch} list`); + const body = await res.json(); + const aircraft = Array.isArray(body?.ac) ? body.ac : []; + const now = new Date().toISOString(); + + const types = (config.types ?? []).map((t) => String(t).trim().toUpperCase()).filter(Boolean); + const wanted = types.length + ? aircraft.filter((a) => types.includes(String(a?.t ?? '').toUpperCase())) + : aircraft; + + const open = { ...(cursor.open ?? {}) }; + const items = []; + + for (const ac of wanted) { + const hex = clean(ac.hex); + if (!hex) continue; + const held = open[hex]; + const firstSeen = held?.firstSeen ?? now; + const positions = (held?.positions ?? 0) + 1; + open[hex] = { firstSeen, positions, last: snapshot(ac) }; + const item = toItem(ac, { watch, firstSeen, now, positions }); + if (item) items.push(item); + } + + /* Whatever was on the list last time and is not now has cleared. It is + * written once with its duration and dropped, because keeping it would + * re-emit the same ended episode on every run forever. */ + const live = new Set(wanted.map((a) => clean(a.hex)).filter(Boolean)); + for (const [hex, held] of Object.entries(cursor.open ?? {})) { + if (live.has(hex)) continue; + delete open[hex]; + const item = toItem( + { hex, ...(held.last ?? {}) }, + { watch, firstSeen: held.firstSeen, now, ended: true, positions: held.positions ?? 1 }, + ); + if (item) items.push(item); + } + + const ended = items.length - wanted.length; + log(`${wanted.length} aircraft on the ${watch} list, ${ended > 0 ? ended : 0} cleared`); + return { items, cursor: { open }, note: `${wanted.length} on ${watch}` }; + }, +}); + +/** + * What to remember about an aircraft so the closing row can still describe it. + * + * Only the identifying fields: the position it held when it left the list is + * where it was last seen, not where it is, and storing the whole record would + * put a stale altitude in a row that says the episode ended. + */ +function snapshot(ac) { + return { + flight: ac.flight ?? null, + r: ac.r ?? null, + t: ac.t ?? null, + desc: ac.desc ?? null, + ownOp: ac.ownOp ?? null, + squawk: ac.squawk ?? null, + }; +} diff --git a/packages/adapters/src/index.js b/packages/adapters/src/index.js index 0b1cd65..eabe9e0 100644 --- a/packages/adapters/src/index.js +++ b/packages/adapters/src/index.js @@ -1,3 +1,4 @@ +import { adsbFlights } from './adsb.js'; import { aiid } from './aiid.js'; import { alpacaCorporateActions, alpacaNews } from './alpaca.js'; import { firefoxAddons } from './amo.js'; @@ -35,6 +36,7 @@ import { mcpRegistry } from './mcpregistry.js'; import { isoMicExchanges } from './mic.js'; import { musicbrainz } from './musicbrainz.js'; import { nasdaqHalts } from './nasdaqhalts.js'; +import { ndbcBuoys } from './ndbc.js'; import { newsChannels } from './newschannels.js'; import { newsfeed } from './newsfeed.js'; import { nhcCyclones } from './nhc.js'; @@ -56,6 +58,7 @@ import { scryfallCards, scryfallSets } from './scryfall.js'; import { socrataCrime } from './socratacrime.js'; import { statuspage } from './statuspage.js'; import { steam, steamNews } from './steam.js'; +import { nwsSurfZone } from './surfzone.js'; import { swpcSpaceWeather } from './swpc.js'; import { tedNotices } from './ted.js'; import { ukPoliceCrime } from './ukpolice.js'; @@ -110,9 +113,12 @@ export const ADAPTERS = [ aviationHazards, aviationMetar, ntsbAccidents, + adsbFlights, nwpsRiverGauges, coopsWaterLevels, droughtMonitor, + ndbcBuoys, + nwsSurfZone, cfpbComplaints, fdicInstitutions, fdicStructureChanges, diff --git a/packages/adapters/src/ndbc.js b/packages/adapters/src/ndbc.js new file mode 100644 index 0000000..1897981 --- /dev/null +++ b/packages/adapters/src/ndbc.js @@ -0,0 +1,355 @@ +import { defineAdapter, slugify } from '@nichedb/core/adapter'; + +/** + * Every buoy NOAA is listening to, and what the sea is doing under it. + * + * The National Data Buoy Center publishes the newest observation from all of + * its stations in a single hundred-kilobyte file: 853 reports, 184 of them + * carrying a wave height. Wind, gust, wave height, dominant and average period, + * mean wave direction, pressure, air and water temperature. One request, the + * whole ocean, keyless. + * + * WHY THIS IS ALSO THE SURF REPORT + * + * A surf report is three numbers and everything else is presentation: how big + * the swell is (WVHT), how far apart the waves are (DPD), and where they are + * coming from (MWD). A two-metre swell at 18 seconds and a two-metre swell at 6 + * seconds are completely different days in the water, and the period is what + * separates them. So a buoy reporting waves is published with those three read + * out in words as well as stored as numbers, and `nws-surf-zone` carries the + * forecaster's own prose beside it. + * + * THE MISSING VALUE IS THE WHOLE PROBLEM + * + * NDBC writes a missing reading as `MM`, in a fixed-width table, in every + * column. `Number('MM')` is NaN, which is at least loud -- but a reader that + * reaches for `|| 0` or `parseFloat` without checking turns "this buoy has no + * anemometer" into a flat calm and "no wave sensor" into a dead-flat sea. Most + * stations are missing most columns: of 853 observations, 669 report no wave + * height at all. Every field here is read through one guard that returns null + * for `MM`, and null means the buoy did not say. + */ + +const LATEST = 'https://www.ndbc.noaa.gov/data/latest_obs/latest_obs.txt'; +const STATIONS = 'https://www.ndbc.noaa.gov/activestations.xml'; + +/** + * The columns of `latest_obs.txt`, in order. + * + * The file is whitespace-aligned rather than delimited, and the header names + * are repeated in two comment rows, so the order is the contract. Splitting on + * runs of whitespace gives exactly 22 fields per row. + */ +const COLUMNS = [ + 'station', + 'lat', + 'lon', + 'year', + 'month', + 'day', + 'hour', + 'minute', + 'windDirection', + 'windSpeed', + 'gust', + 'waveHeight', + 'dominantPeriod', + 'averagePeriod', + 'waveDirection', + 'pressure', + 'pressureTendency', + 'airTemp', + 'waterTemp', + 'dewpoint', + 'visibility', + 'tide', +]; + +/** + * A reading, or null. + * + * `MM` is NDBC's missing value and it appears in every column. Anything that + * coerces it to a number silently reports a calm sea at a station with no wave + * sensor. + */ +export function reading(v) { + const s = String(v ?? '').trim(); + if (!s || s === 'MM' || s === 'N/A') return null; + const n = Number(s); + return Number.isFinite(n) ? n : null; +} + +/** One row of the fixed-width table, or null if it is not a row. */ +export function parseRow(line) { + const t = String(line ?? '').trim(); + if (!t || t.startsWith('#')) return null; + const parts = t.split(/\s+/); + if (parts.length < COLUMNS.length) return null; + const row = {}; + COLUMNS.forEach((name, i) => { + row[name] = name === 'station' ? parts[i] : reading(parts[i]); + }); + return row.station ? row : null; +} + +export function parseLatest(text) { + return String(text ?? '') + .split('\n') + .map(parseRow) + .filter(Boolean); +} + +/** + * The station register, which is where the names live. + * + * `latest_obs.txt` identifies a buoy by a five-character id and nothing else, + * so without this every row would be titled `46221`. The register is one + * request for all 1,353 stations and it changes when NOAA moors or retires a + * buoy, so it is read once and kept in the cursor. + * + * Parsed by attribute rather than with the shared XML reader: every station is + * a self-closing tag with no body, and a reader that looks for `` and + * a matching close tag finds nothing at all. + */ +export function parseStations(xml) { + const out = {}; + for (const m of String(xml ?? '').matchAll(/]*?)\/?>/g)) { + const attrs = {}; + for (const a of m[1].matchAll(/([\w:.-]+)="([^"]*)"/g)) attrs[a[1]] = a[2]; + if (!attrs.id) continue; + /* The register really does carry `name=""` -- 15009 in the Atlantic array + * is one of many. An empty string is not a name, and left as one it wins + * over the fallback and titles the row with nothing at all. */ + const named = (v) => { + const t = String(v ?? '').trim(); + return t || null; + }; + out[attrs.id] = { + name: named(attrs.name), + owner: named(attrs.owner), + type: named(attrs.type), + program: named(attrs.pgm), + }; + } + return out; +} + +const metresToFeet = (m) => (m === null ? null : Number((m * 3.28084).toFixed(1))); +const msToKnots = (m) => (m === null ? null : Number((m * 1.94384).toFixed(1))); + +/** The compass point a swell is coming from, which is how a surf report says it. */ +export function compass(deg) { + if (deg === null || !Number.isFinite(deg)) return null; + const points = [ + 'N', + 'NNE', + 'NE', + 'ENE', + 'E', + 'ESE', + 'SE', + 'SSE', + 'S', + 'SSW', + 'SW', + 'WSW', + 'W', + 'WNW', + 'NW', + 'NNW', + ]; + return points[Math.round((((deg % 360) + 360) % 360) / 22.5) % 16]; +} + +/** + * How a surfer would describe the swell, from the three numbers that decide it. + * + * Period is the discriminator and it is the one people leave out. Under 8 + * seconds is local windswell; over 14 is groundswell that has travelled and + * will break with some force. + */ +export function swellDescription(heightM, periodS) { + if (heightM === null) return null; + const ft = metresToFeet(heightM); + /* A period under two seconds is not a wave. Some stations report 0 or 1 for + * a flat sea or a sensor that is not measuring period, and classifying that + * as "windswell" states something about the water that nobody measured. */ + if (periodS === null || periodS < 2) return `${ft} ft`; + const kind = + periodS >= 14 + ? 'long-period groundswell' + : periodS >= 10 + ? 'groundswell' + : periodS >= 8 + ? 'mixed swell' + : 'windswell'; + return `${ft} ft at ${periodS} s, ${kind}`; +} + +function observedAt(row) { + const { year, month, day, hour, minute } = row; + if ([year, month, day, hour, minute].some((v) => v === null)) return null; + const p = (n) => String(n).padStart(2, '0'); + return `${year}-${p(month)}-${p(day)}T${p(hour)}:${p(minute)}:00Z`; +} + +export function toItem(row, station = null) { + const when = observedAt(row); + if (!when) return null; + + const name = station?.name ?? `Station ${row.station}`; + const flat = row.waveHeight === 0; + const waves = row.waveHeight !== null; + const swell = swellDescription(row.waveHeight, row.dominantPeriod); + const from = compass(row.waveDirection); + const windKt = msToKnots(row.windSpeed); + const gustKt = msToKnots(row.gust); + + return { + externalId: `ndbc-${row.station}-${when}`, + kind: waves ? 'sea-state' : 'marine-observation', + title: `${name}: ${waves ? swell : 'marine observation'}${from && waves ? ` from the ${from}` : ''}`, + summary: [ + `${name} reported at ${when.replace('T', ' ').slice(0, 16)}Z:`, + waves ? ` ${swell}${from ? ` out of the ${from}` : ''}.` : '', + windKt !== null + ? ` Wind ${compass(row.windDirection) ?? 'variable'} at ${windKt} kt${gustKt !== null ? `, gusting ${gustKt}` : ''}.` + : '', + row.waterTemp !== null ? ` Water ${row.waterTemp}°C.` : '', + row.airTemp !== null ? ` Air ${row.airTemp}°C.` : '', + ] + .join('') + .trim(), + url: `https://www.ndbc.noaa.gov/station_page.php?station=${encodeURIComponent(row.station)}`, + publishedAt: when, + timeKnown: true, + precision: 'minute', + tags: [ + 'water', + 'marine', + 'buoy', + `station:${slugify(row.station)}`, + waves ? 'waves' : null, + flat ? 'flat' : null, + waves && row.dominantPeriod !== null && row.dominantPeriod >= 14 ? 'groundswell' : null, + waves && row.waveHeight !== null && row.waveHeight >= 2.5 ? 'big-surf' : null, + from ? `swell:${from.toLowerCase()}` : null, + station?.type ? slugify(station.type) : null, + ].filter(Boolean), + data: { + station: row.station, + stationName: name, + owner: station?.owner ?? null, + stationType: station?.type ?? null, + observedAt: when, + /* + * Waves in metres as NDBC publishes them and in feet beside it, because + * every surf forecast in the United States is in feet and converting at + * read time is how a six-foot day becomes a two-foot day. + */ + waveHeightM: row.waveHeight, + waveHeightFt: metresToFeet(row.waveHeight), + dominantPeriodS: row.dominantPeriod, + averagePeriodS: row.averagePeriod, + waveDirectionDeg: row.waveDirection, + waveDirection: from, + swell, + windSpeedKt: windKt, + windGustKt: gustKt, + windDirectionDeg: row.windDirection, + pressureHpa: row.pressure, + pressureTendencyHpa: row.pressureTendency, + airTempC: row.airTemp, + waterTempC: row.waterTemp, + dewpointC: row.dewpoint, + visibilityNmi: row.visibility, + tideFt: row.tide, + missingNote: + 'NDBC writes an absent reading as MM and most stations carry no wave sensor; a null here means the buoy did not report that field, never that the value was zero.', + place: { country: null, lat: row.lat, lon: row.lon }, + source: 'NOAA National Data Buoy Center', + dataset: LATEST, + }, + }; +} + +export const ndbcBuoys = defineAdapter({ + name: 'ndbc-buoys', + title: 'Ocean buoys', + collection: 'water', + description: + 'The newest observation from every NOAA buoy, in one request: wave height, the period that decides whether it is groundswell or windswell, the direction it is coming from, wind, pressure and water temperature. The raw material of every surf report, keyless.', + docs: 'https://www.ndbc.noaa.gov/docs/ndbc_web_data_guide.pdf', + kinds: ['sea-state', 'marine-observation'], + cadenceMinutes: 60, + configFields: [ + { + key: 'wavesOnly', + label: 'Only buoys reporting waves', + type: 'select', + options: ['', 'yes'], + help: 'Two thirds of stations carry no wave sensor.', + }, + { + key: 'minWaveHeightM', + label: 'Minimum wave height (m)', + type: 'number', + help: 'Keep only seas at or above this.', + }, + { key: 'stations', label: 'Only these stations', type: 'list', help: 'NDBC station ids.' }, + ], + defaults: {}, + defaultSources: [ + { slug: 'buoys-all', name: 'Every NOAA buoy' }, + { + slug: 'buoys-waves', + name: 'Buoys reporting waves', + config: { wavesOnly: 'yes' }, + cadenceMinutes: 30, + }, + { + slug: 'buoys-big-seas', + name: 'Big seas (2.5 m and over)', + config: { wavesOnly: 'yes', minWaveHeightM: 2.5 }, + cadenceMinutes: 30, + }, + ], + async pull({ config, cursor, http, log }) { + /* The register is read once and kept; it changes when NOAA moors or + * retires a buoy, which is not something that happens between two polls. */ + let stations = cursor.stations ?? null; + const stale = + !cursor.stationsAt || Date.now() - new Date(cursor.stationsAt).getTime() > 7 * 24 * 3_600_000; + if (!stations || stale) { + const xml = await http.text(STATIONS, { timeoutMs: 60_000 }); + const parsed = parseStations(xml); + if (Object.keys(parsed).length) stations = parsed; + } + + const text = await http.text(LATEST, { timeoutMs: 60_000 }); + const rows = parseLatest(text); + if (!rows.length) throw new Error('the buoy file held no observations'); + + const only = (config.stations ?? []).map((s) => String(s).trim().toUpperCase()).filter(Boolean); + const wavesOnly = String(config.wavesOnly ?? '') === 'yes'; + const floor = Number(config.minWaveHeightM) || 0; + + const items = rows + .filter((r) => !only.length || only.includes(r.station.toUpperCase())) + .filter((r) => !wavesOnly || r.waveHeight !== null) + .filter((r) => !floor || (r.waveHeight ?? 0) >= floor) + .map((r) => toItem(r, stations?.[r.station] ?? null)) + .filter(Boolean); + + const withWaves = items.filter((i) => i.data.waveHeightM !== null).length; + log(`${items.length} buoy observation(s) of ${rows.length}, ${withWaves} reporting waves`); + return { + items, + cursor: { + stations, + stationsAt: stale || !cursor.stationsAt ? new Date().toISOString() : cursor.stationsAt, + }, + note: `${items.length} buoys, ${withWaves} with waves`, + }; + }, +}); diff --git a/packages/adapters/src/ntsb.js b/packages/adapters/src/ntsb.js index a49f7b2..2cc9cc5 100644 --- a/packages/adapters/src/ntsb.js +++ b/packages/adapters/src/ntsb.js @@ -40,14 +40,15 @@ import { defineAdapter, slugify } from '@nichedb/core/adapter'; * * WHAT IT COSTS PER RUN * - * The whole database will not go into one statement: `upsertItems` sends every - * row in a single insert, and 31,000 accidents carrying their full narratives - * is a hundred and forty megabytes of text in one query. So a run emits a - * bounded slice, newest first, and remembers where it stopped. The extracted - * tables are cached beside the archive and keyed on the file's publication - * date, so the slices after the first cost no download at all -- and when the - * NTSB publishes a new file, the date changes, the cache misses, and the walk - * starts again from the newest accident. + * A run emits a bounded slice, newest first, and remembers where it stopped. + * Not because of the database -- `runSource` already chunks its writes into + * batches of 200 -- but because the whole file is 31,000 accidents carrying a + * hundred and forty megabytes of narrative text, and building all of that into + * items in one pass would hold the lot in memory and run past the four-minute + * ingest deadline. The extracted tables are cached beside the archive and keyed + * on the file's publication date, so the slices after the first cost no + * download at all -- and when the NTSB publishes a new file, the date changes, + * the cache misses, and the walk starts again from the newest accident. * * NEEDS `mdbtools` AND `unzip` ON THE HOST. Both are in the Dockerfile. There * is no pure-JavaScript reader for a 558 MB Access database worth trusting, and @@ -408,7 +409,7 @@ export const ntsbAccidents = defineAdapter({ key: 'maxPerRun', label: 'Accidents per run', type: 'number', - help: 'The whole database will not fit in one insert. Default 2,000.', + help: 'Bounded so a run stays inside the ingest deadline. Default 2,000.', }, { key: 'minYear', diff --git a/packages/adapters/src/surfzone.js b/packages/adapters/src/surfzone.js new file mode 100644 index 0000000..86acb66 --- /dev/null +++ b/packages/adapters/src/surfzone.js @@ -0,0 +1,244 @@ +import { defineAdapter, slugify } from '@nichedb/core/adapter'; + +/** + * The surf forecast, in the forecaster's own words. + * + * `ndbc-buoys` has the three numbers a surf report is made of. This has the + * sentence a human wrote about them: which swell is filling in, which is + * fading, whether the advisory is up, and what the shore looks like tomorrow. + * The National Weather Service issues it as the Surf Zone Forecast (`SRF`) + * from every coastal office, and api.weather.gov serves the full text keyless. + * + * The two belong in one collection for the same reason the aviation feeds do: + * a measurement and the judgement made about it answer different questions. A + * buoy off Oahu reading 2.4 m at 16 seconds is a fact. "Advisory level surf + * along south facing shores through today" is what it means for anyone + * standing on the beach, and no amount of arithmetic over the buoy produces it. + * + * WHAT AN ITEM IS + * + * One issuance, from one office. The NWS reissues the product several times a + * day and each issuance supersedes the last, so the id is the product's own + * uuid rather than the office: a reissue is a new row, and the previous + * forecast stays in the record next to what actually happened. 851 issuances + * were in the recent list when this was written. + */ + +const BASE = 'https://api.weather.gov'; + +const clean = (v) => { + const s = String(v ?? '').trim(); + return s && s.toLowerCase() !== 'null' ? s : null; +}; + +/** + * The offices that issue a surf forecast, with the coast each one covers. + * + * Kept as a lookup so a row can say "Honolulu" rather than "PHFO", and so a + * reader can ask for one coast. The NWS identifier is the authority; this only + * adds the words. + */ +export const OFFICES = { + PHFO: { name: 'Honolulu', coast: 'hawaii' }, + KMTR: { name: 'San Francisco Bay Area', coast: 'pacific' }, + KLOX: { name: 'Los Angeles/Oxnard', coast: 'pacific' }, + KSGX: { name: 'San Diego', coast: 'pacific' }, + KEKA: { name: 'Eureka', coast: 'pacific' }, + KMFR: { name: 'Medford', coast: 'pacific' }, + KPQR: { name: 'Portland', coast: 'pacific' }, + KSEW: { name: 'Seattle', coast: 'pacific' }, + KGYX: { name: 'Gray/Portland ME', coast: 'atlantic' }, + KBOX: { name: 'Boston', coast: 'atlantic' }, + KOKX: { name: 'New York', coast: 'atlantic' }, + KPHI: { name: 'Philadelphia/Mount Holly', coast: 'atlantic' }, + KAKQ: { name: 'Wakefield', coast: 'atlantic' }, + KMHX: { name: 'Newport/Morehead City', coast: 'atlantic' }, + KILM: { name: 'Wilmington', coast: 'atlantic' }, + KCHS: { name: 'Charleston', coast: 'atlantic' }, + KJAX: { name: 'Jacksonville', coast: 'atlantic' }, + KMLB: { name: 'Melbourne', coast: 'atlantic' }, + KMFL: { name: 'Miami', coast: 'atlantic' }, + KKEY: { name: 'Key West', coast: 'atlantic' }, + KTBW: { name: 'Tampa Bay', coast: 'gulf' }, + KTAE: { name: 'Tallahassee', coast: 'gulf' }, + KMOB: { name: 'Mobile', coast: 'gulf' }, + KLIX: { name: 'New Orleans', coast: 'gulf' }, + KLCH: { name: 'Lake Charles', coast: 'gulf' }, + KHGX: { name: 'Houston/Galveston', coast: 'gulf' }, + KCRP: { name: 'Corpus Christi', coast: 'gulf' }, + KBRO: { name: 'Brownsville', coast: 'gulf' }, + TJSJ: { name: 'San Juan', coast: 'caribbean' }, + PAJK: { name: 'Juneau', coast: 'alaska' }, + PAFC: { name: 'Anchorage', coast: 'alaska' }, + PGUM: { name: 'Guam', coast: 'pacific-islands' }, +}; + +/** + * The risk the product is actually warning about, from its own wording. + * + * A surf zone forecast is prose, and the two phrases that change what someone + * does are the rip current risk and whether a high surf advisory or warning is + * up. Both are written in a stable vocabulary, so they can be lifted out + * without pretending to parse the forecast. + */ +export function hazards(text) { + const s = String(text ?? ''); + const found = []; + if (/high surf warning/i.test(s)) found.push('high-surf-warning'); + else if (/high surf advisory/i.test(s)) found.push('high-surf-advisory'); + if (/rip current statement|high risk of rip/i.test(s)) found.push('rip-current-risk:high'); + else if (/moderate risk of rip/i.test(s)) found.push('rip-current-risk:moderate'); + else if (/low risk of rip/i.test(s)) found.push('rip-current-risk:low'); + if (/beach hazards statement/i.test(s)) found.push('beach-hazards'); + if (/sneaker wave/i.test(s)) found.push('sneaker-waves'); + return found; +} + +/** The first paragraph that reads like a forecast rather than like a header. */ +export function firstParagraph(text) { + const body = String(text ?? '').replace(/\r/g, ''); + const start = body.search(/\n\.[A-Z][A-Z ]{2,}\.\.\./); + const from = start === -1 ? body : body.slice(start); + for (const block of from.split(/\n\s*\n/)) { + const t = block + .replace(/^\.[A-Z][A-Z ]*\.\.\./, '') + .replace(/\s+/g, ' ') + .trim(); + if (t.length > 60) return t; + } + return body.replace(/\s+/g, ' ').trim().slice(0, 600); +} + +export function toItem(product, text) { + const id = clean(product?.id); + const office = clean(product?.issuingOffice); + const issued = clean(product?.issuanceTime); + if (!id || !issued) return null; + + const where = OFFICES[office ?? ''] ?? null; + const name = where?.name ?? office ?? 'the coast'; + const risks = hazards(text); + const lead = firstParagraph(text); + + return { + externalId: `srf-${id}`, + kind: 'surf-forecast', + title: `Surf forecast: ${name}${risks.includes('high-surf-warning') ? ' — high surf warning' : risks.includes('high-surf-advisory') ? ' — high surf advisory' : ''}`, + summary: lead.slice(0, 1200), + url: `${BASE}/products/${id}`, + publishedAt: issued, + timeKnown: true, + precision: 'minute', + tags: [ + 'water', + 'surf', + 'forecast', + 'us', + office ? slugify(office) : null, + where?.coast ?? null, + ...risks, + ].filter(Boolean), + data: { + productId: id, + office, + officeName: where?.name ?? null, + coast: where?.coast ?? null, + issuedAt: issued, + hazards: risks, + hazardBasis: + 'Lifted from the forecaster’s own wording. The full text is in `text`, and it is the authority; these tags exist so a coast under an advisory can be found without reading every product.', + text: String(text ?? '').slice(0, 20_000), + source: 'NWS Surf Zone Forecast (SRF)', + dataset: `${BASE}/products/types/SRF`, + }, + }; +} + +export const nwsSurfZone = defineAdapter({ + name: 'nws-surf-zone', + title: 'Surf zone forecasts', + collection: 'water', + description: + 'The National Weather Service surf zone forecast from every coastal office, in full: which swell is filling in and which is fading, high surf advisories and warnings, and the rip current risk for the day. The forecaster’s words beside the buoy’s numbers. Keyless.', + docs: 'https://www.weather.gov/documentation/services-web-api', + kinds: ['surf-forecast'], + cadenceMinutes: 60, + configFields: [ + { + key: 'offices', + label: 'Only these offices', + type: 'list', + help: 'NWS office ids, e.g. PHFO, KLOX.', + }, + { + key: 'coast', + label: 'Only this coast', + type: 'select', + options: [ + '', + 'pacific', + 'atlantic', + 'gulf', + 'hawaii', + 'alaska', + 'caribbean', + 'pacific-islands', + ], + }, + { key: 'maxProducts', label: 'Forecasts per run', type: 'number', help: 'Default 40.' }, + ], + defaults: {}, + defaultSources: [ + { slug: 'surf-forecasts', name: 'Surf zone forecasts (all coasts)' }, + { + slug: 'surf-forecasts-pacific', + name: 'Surf forecasts: Pacific', + config: { coast: 'pacific' }, + }, + { slug: 'surf-forecasts-hawaii', name: 'Surf forecasts: Hawaii', config: { coast: 'hawaii' } }, + ], + async pull({ config, cursor, http, log, deadline }) { + const max = Math.max(5, Math.min(Number(config.maxProducts) || 40, 100)); + const offices = (config.offices ?? []) + .map((o) => String(o).trim().toUpperCase()) + .filter(Boolean); + const coast = clean(config.coast); + + const list = await http.json(`${BASE}/products/types/SRF`, { timeoutMs: 60_000 }); + const all = Array.isArray(list?.['@graph']) ? list['@graph'] : []; + if (!all.length) throw new Error('the weather API returned no surf forecasts'); + + const wanted = all + .filter( + (p) => !offices.length || offices.includes(String(p.issuingOffice ?? '').toUpperCase()), + ) + .filter((p) => !coast || OFFICES[String(p.issuingOffice ?? '')]?.coast === coast) + /* Each issuance is its own product with its own uuid, so the watermark is + * the issuance time: everything newer than the last run is new, and a + * reissue of the same office is a new row rather than an overwrite. */ + .filter((p) => !cursor.since || String(p.issuanceTime ?? '') > cursor.since) + .slice(0, max); + + const items = []; + for (const p of wanted) { + if (Date.now() > deadline) { + log(`out of time after ${items.length} forecast(s)`); + break; + } + const full = await http.jsonOrNull(p['@id'], { timeoutMs: 30_000 }); + const item = toItem(p, full?.productText ?? ''); + if (item) items.push(item); + } + + const newest = + all + .map((p) => String(p.issuanceTime ?? '')) + .filter(Boolean) + .sort() + .at(-1) ?? cursor.since; + log( + `${items.length} surf forecast(s) of ${all.length} listed${newest ? `, newest ${newest}` : ''}`, + ); + return { items, cursor: { since: newest ?? null }, note: `${items.length} forecasts` }; + }, +}); diff --git a/packages/core/src/seed.js b/packages/core/src/seed.js index 9478eaf..8caa8c3 100644 --- a/packages/core/src/seed.js +++ b/packages/core/src/seed.js @@ -157,13 +157,13 @@ export const COLLECTIONS = [ slug: 'aviation', name: 'Aviation', description: - 'Why flights are late and what happens when it goes wrong. The traffic management initiatives the FAA has in force right now — ground stops, ground delay programs, airport closures — each written once more when it ends with how long it ran, which the FAA itself never publishes. Every SIGMET and AIRMET in the air over the country, and the decoded observation at the airport underneath. And every one of the 31,000 accidents the NTSB has investigated, each carrying the raw weather observation at the moment it happened — the same field, from the same service, that the hourly feed publishes today. All keyless.', + 'Why flights are late, what is flying, and what happens when it goes wrong. The traffic management initiatives the FAA has in force right now — ground stops, ground delay programs, airport closures — each written once more when it ends with how long it ran, which the FAA itself never publishes. Every SIGMET and AIRMET in the air over the country, and the decoded observation at the airport underneath. And every one of the 31,000 accidents the NTSB has investigated, each carrying the raw weather observation at the moment it happened — the same field, from the same service, that the hourly feed publishes today. All keyless.', }, { slug: 'water', name: 'Water', description: - 'Too much water and too little, measured rather than forecast: every NOAA river gauge at or above its action stage with the height in feet and the flood category behind the warning, the observed level at tide stations on every US coast against the height at which each one floods, and the US Drought Monitor’s weekly read on how much of each state is dry and how badly. The fast half and the slow half of the same system.', + 'Too much water and too little, measured rather than forecast: every NOAA river gauge at or above its action stage with the height in feet and the flood category behind the warning, the observed level at tide stations on every US coast against the height at which each one floods, and the US Drought Monitor’s weekly read on how much of each state is dry and how badly. Then the sea itself — every NOAA buoy’s wave height, period and direction, which is what a surf report is made of, beside the National Weather Service’s own surf zone forecast saying what it means for anyone standing on the beach.', }, { slug: 'consumer-finance', @@ -949,6 +949,60 @@ export const DEFAULT_FEEDS = [ name: 'Extreme and exceptional drought', query: { tags: ['extreme-drought'] }, }, + { + collection: 'water', + slug: 'surf-report', + name: 'Surf report', + query: { kinds: ['sea-state'], tags: ['waves'] }, + }, + { + collection: 'water', + slug: 'big-surf', + name: 'Big surf', + query: { tags: ['big-surf'] }, + }, + { + collection: 'water', + slug: 'groundswell', + name: 'Long-period groundswell', + query: { tags: ['groundswell'] }, + }, + { + collection: 'water', + slug: 'surf-forecasts', + name: 'Surf zone forecasts', + query: { kinds: ['surf-forecast'] }, + }, + { + collection: 'water', + slug: 'high-surf-advisories', + name: 'High surf advisories and warnings', + query: { tags: ['high-surf-advisory', 'high-surf-warning'] }, + }, + { + collection: 'water', + slug: 'rip-current-risk', + name: 'High rip current risk', + query: { tags: ['rip-current-risk:high'] }, + }, + { + collection: 'water', + slug: 'sea-temperature', + name: 'Buoys and sea temperature', + query: { kinds: ['sea-state', 'marine-observation'] }, + }, + { + collection: 'aviation', + slug: 'aircraft-emergencies', + name: 'Aircraft declaring an emergency', + query: { kinds: ['aircraft-emergency'] }, + }, + { + collection: 'aviation', + slug: 'military-aircraft', + name: 'Military aircraft airborne', + query: { kinds: ['aircraft-sighting'], tags: ['military'] }, + }, { collection: 'consumer-finance', slug: 'consumer-complaints', diff --git a/test/marine-and-flights.test.js b/test/marine-and-flights.test.js new file mode 100644 index 0000000..d6457e4 --- /dev/null +++ b/test/marine-and-flights.test.js @@ -0,0 +1,286 @@ +import { describe, expect, test } from 'bun:test'; +import { + altitude, + toItem as flightItem, + minutesBetween, + SQUAWKS, + WATCHES, +} from '../packages/adapters/src/adsb.js'; +import { ADAPTERS, adapterByName } from '../packages/adapters/src/index.js'; +import { + toItem as buoyItem, + compass, + parseLatest, + parseRow, + parseStations, + reading, + swellDescription, +} from '../packages/adapters/src/ndbc.js'; +import { + firstParagraph, + hazards, + OFFICES, + toItem as surfItem, +} from '../packages/adapters/src/surfzone.js'; +import { normaliseItem } from '../packages/core/src/adapter.js'; + +process.env.DATABASE_URL ??= 'postgres://test:test@localhost:5432/test'; +process.env.SITE_URL ??= 'https://nichedb.test'; +const { DEFAULT_FEEDS } = await import('../packages/core/src/seed.js'); + +/* Rows exactly as latest_obs.txt writes them, MM and all. */ +const HEADER = + '#STN LAT LON YYYY MM DD hh mm WDIR WSPD GST WVHT DPD APD MWD PRES PTDY ATMP WTMP DEWP VIS TIDE'; +const WAVE_ROW = + '46205 54.18 -134.32 2026 09 09 19 00 260 8.0 10.0 2.7 8 6.4 270 1015.0 MM 14.8 13.5 MM MM MM'; +const BARE_ROW = + '15009 0.000 -3.051 2026 09 09 18 00 192 6.0 MM MM MM MM MM 1013.4 MM 24.6 26.0 MM MM MM'; + +describe('ocean buoys', () => { + test('MM is a missing reading, never a zero', () => { + /* This is the whole adapter. `parseFloat('MM')` is NaN and `Number('MM') || 0` + * is 0, and a station with no anemometer would be published as a flat calm. */ + expect(reading('MM')).toBeNull(); + expect(reading('')).toBeNull(); + expect(reading('N/A')).toBeNull(); + expect(reading('0.0')).toBe(0); + expect(reading('2.7')).toBe(2.7); + }); + + test('a row without a wave sensor is not a row reporting a flat sea', () => { + const bare = parseRow(BARE_ROW); + expect(bare.waveHeight).toBeNull(); + expect(bare.windSpeed).toBe(6); + expect(bare.waterTemp).toBe(26); + + const item = normaliseItem(buoyItem(bare)); + expect(item.kind).toBe('marine-observation'); + expect(item.data.waveHeightM).toBeNull(); + expect(item.data.waveHeightFt).toBeNull(); + expect(item.tags).not.toContain('waves'); + }); + + test('a wave row carries height in both units and the period that classifies it', () => { + const item = normaliseItem( + buoyItem(parseRow(WAVE_ROW), { name: 'West Dixon Entrance', type: 'buoy' }), + ); + expect(item.kind).toBe('sea-state'); + expect(item.data.waveHeightM).toBe(2.7); + // Every US surf forecast is in feet; converting at read time is how a + // nine-foot day becomes a three-foot day. + expect(item.data.waveHeightFt).toBe(8.9); + expect(item.data.dominantPeriodS).toBe(8); + expect(item.data.waveDirection).toBe('W'); + expect(item.title).toContain('West Dixon Entrance'); + expect(item.tags).toContain('big-surf'); + }); + + test('the period is what separates two swells of the same size', () => { + // Two metres at 18 seconds and two metres at 5 seconds are completely + // different days in the water. + expect(swellDescription(2, 18)).toContain('long-period groundswell'); + expect(swellDescription(2, 11)).toContain('groundswell'); + expect(swellDescription(2, 8)).toContain('mixed swell'); + expect(swellDescription(2, 5)).toContain('windswell'); + // A period of 0 or 1 is a sensor saying nothing, not a wave. + expect(swellDescription(0, 0)).toBe('0 ft'); + expect(swellDescription(0.5, 1)).toBe('1.6 ft'); + expect(swellDescription(null, 12)).toBeNull(); + }); + + test('an empty name in the register is not a name', () => { + /* Station 15009 really is published with name="". Left as an empty string + * it beats the fallback and titles the row with nothing at all. */ + const stations = parseStations( + '' + + '', + ); + expect(stations['15009'].name).toBeNull(); + expect(stations['46205'].name).toBe('West Dixon Entrance'); + expect(buoyItem(parseRow(BARE_ROW), stations['15009']).title).toStartWith('Station 15009:'); + }); + + test('the header rows are not observations', () => { + const rows = parseLatest([HEADER, '#text deg deg', WAVE_ROW, BARE_ROW, ''].join('\n')); + expect(rows).toHaveLength(2); + expect(rows.map((r) => r.station)).toEqual(['46205', '15009']); + }); + + test('a compass point is where the swell comes from', () => { + expect(compass(0)).toBe('N'); + expect(compass(270)).toBe('W'); + expect(compass(200)).toBe('SSW'); + expect(compass(359)).toBe('N'); + expect(compass(null)).toBeNull(); + }); +}); + +describe('surf zone forecasts', () => { + const product = { + id: 'b1b6d224-cfa4-4329-9f2c-aa29e01e4838', + issuingOffice: 'PHFO', + issuanceTime: '2026-09-09T19:03:00+00:00', + }; + const text = [ + '000', + 'FZHW52 PHFO 091903', + 'SRFHFO', + '', + 'Surf Zone Forecast for Hawaii', + 'National Weather Service Honolulu HI', + '', + '.DISCUSSION...', + 'A moderate long-period south swell peaked last night and will produce', + 'advisory level surf through today. A High Surf Advisory remains in effect', + 'for south facing shores. There is a High Risk of rip currents.', + ].join('\n'); + + test('the hazard is lifted from the forecaster’s own wording', () => { + expect(hazards(text)).toContain('high-surf-advisory'); + expect(hazards(text)).toContain('rip-current-risk:high'); + expect(hazards('High Surf Warning in effect')).toContain('high-surf-warning'); + expect(hazards('nothing much happening')).toEqual([]); + }); + + test('an advisory shows in the title, and the full text is kept as the authority', () => { + const item = normaliseItem(surfItem(product, text)); + expect(item.title).toContain('Honolulu'); + expect(item.title).toContain('high surf advisory'); + expect(item.data.coast).toBe('hawaii'); + expect(item.data.text).toContain('High Surf Advisory'); + expect(item.data.hazardBasis).toContain('the authority'); + expect(item.tags).toContain('hawaii'); + }); + + test('the summary is the forecast, not the teletype header', () => { + const lead = firstParagraph(text); + expect(lead).not.toContain('FZHW52'); + expect(lead).not.toContain('SRFHFO'); + expect(lead).toContain('south swell'); + }); + + test('each issuance is its own row, so a reissue does not overwrite the last', () => { + const a = normaliseItem(surfItem(product, text)); + const b = normaliseItem( + surfItem({ ...product, id: 'other-uuid', issuanceTime: '2026-09-09T22:00:00+00:00' }, text), + ); + expect(a.externalId).not.toBe(b.externalId); + }); + + test('every office in the lookup names a coast a feed can ask for', () => { + const coasts = new Set(Object.values(OFFICES).map((o) => o.coast)); + for (const c of coasts) expect(typeof c).toBe('string'); + expect(OFFICES.PHFO.coast).toBe('hawaii'); + expect(OFFICES.KLOX.coast).toBe('pacific'); + }); +}); + +describe('aircraft on watch', () => { + const ac = (over = {}) => ({ + hex: 'a06115', + flight: 'AAL1234 ', + r: 'N123AB', + t: 'B738', + desc: 'BOEING 737-800', + squawk: '7700', + alt_baro: 31000, + gs: 420, + lat: 39.7, + lon: -104.9, + ...over, + }); + + test('is registered in aviation, with feeds pointed at both kinds', () => { + const a = adapterByName('adsb-flights'); + expect(a.collection).toBe('aviation'); + const kinds = new Set( + ADAPTERS.filter((x) => x.collection === 'aviation').flatMap((x) => x.kinds), + ); + for (const feed of DEFAULT_FEEDS.filter((f) => f.collection === 'aviation')) { + for (const kind of feed.query.kinds ?? []) expect(kinds.has(kind)).toBe(true); + } + }); + + test('an emergency squawk is read as what it means', () => { + const item = normaliseItem( + flightItem(ac(), { + watch: 'emergency', + firstSeen: '2026-09-09T19:00:00.000Z', + now: '2026-09-09T19:00:00.000Z', + }), + ); + expect(item.kind).toBe('aircraft-emergency'); + expect(item.data.squawkMeaning).toBe('general emergency'); + expect(item.tags).toContain('emergency'); + expect(item.tags).toContain('severity:critical'); + expect(item.title).toContain('AAL1234'); + expect(SQUAWKS[7500].tag).toBe('hijack'); + }); + + test('an aircraft is one row while it is on the list, not one row per poll', () => { + const first = normaliseItem( + flightItem(ac(), { + watch: 'emergency', + firstSeen: '2026-09-09T19:00:00.000Z', + now: '2026-09-09T19:00:00.000Z', + }), + ); + const later = normaliseItem( + flightItem(ac({ alt_baro: 12000 }), { + watch: 'emergency', + firstSeen: '2026-09-09T19:00:00.000Z', + now: '2026-09-09T19:20:00.000Z', + }), + ); + expect(later.externalId).toBe(first.externalId); + expect(later.contentHash).not.toBe(first.contentHash); + expect(later.tags).toContain('active'); + }); + + test('when it clears, the row is written once more with how long it ran', () => { + const ended = normaliseItem( + flightItem(ac(), { + watch: 'emergency', + firstSeen: '2026-09-09T19:00:00.000Z', + now: '2026-09-09T19:22:00.000Z', + ended: true, + }), + ); + expect(ended.title).toContain('ended after 22m'); + expect(ended.data.endedAt).toBe('2026-09-09T19:22:00.000Z'); + expect(ended.data.status).toBe('ended'); + expect(ended.tags).toContain('ended'); + expect(minutesBetween('2026-09-09T19:00:00Z', '2026-09-09T21:30:00Z')).toBe('2h 30m'); + expect(minutesBetween('2026-09-09T21:00:00Z', '2026-09-09T19:00:00Z')).toBeNull(); + }); + + test('on the ground is an altitude, not a missing one', () => { + // The feed writes the word `ground` where a number would be, and coercing + // that yields NaN, which reads as "we do not know" for an aircraft whose + // altitude is known exactly. + expect(altitude('ground')).toEqual({ feet: 0, onGround: true }); + expect(altitude(31000)).toEqual({ feet: 31000, onGround: false }); + expect(altitude(null)).toEqual({ feet: null, onGround: false }); + expect( + flightItem(ac({ alt_baro: 'ground' }), { watch: 'emergency', firstSeen: 'x', now: 'x' }).tags, + ).toContain('on-ground'); + }); + + test('a military sighting is not an emergency', () => { + const item = flightItem(ac({ squawk: '5564', flight: 'RCH123' }), { + watch: 'military', + firstSeen: '2026-09-09T19:00:00.000Z', + now: '2026-09-09T19:00:00.000Z', + }); + expect(item.kind).toBe('aircraft-sighting'); + expect(item.data.squawkMeaning).toBeNull(); + expect(item.tags).toContain('military'); + expect(item.tags).not.toContain('emergency'); + }); + + test('every default source names a watch list the adapter knows', () => { + for (const s of adapterByName('adsb-flights').defaultSources) { + expect(Object.keys(WATCHES)).toContain(s.config.watch); + } + }); +});