From 3235e664eb59549fe593663bf3121d060c9d71cc Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 6 Jul 2026 23:44:59 +0000 Subject: [PATCH] feat(prober): scan all 65535 ports (nmap -p-), stretch timeouts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Full TCP port scan instead of top-100 — the point of exposed-services drift is to catch anything open. Safe because targets are owner-verified and scanning runs on the dedicated off-Railway droplet. - prober: nmap -sT -Pn -p- -T4 --host-timeout 1500s; execFile ceiling 30m. - prober-queue: RUNNING_TIMEOUT_MS 15m -> 40m for long scans. - SSE route: MAX_MS -> 45m + 15s heartbeat so idle proxies don't drop the connection while a long scan sits in 'running'. - PRD §12: document full-range scan + ownership/off-Railway justification. Co-Authored-By: Claude Opus 4.8 --- app/(app)/projects/[id]/security/stream/route.ts | 14 ++++++++++++-- docs/uptime-monitoring-prd.md | 13 ++++++++----- lib/prober-queue.ts | 5 +++-- prober/src/index.ts | 12 +++++++----- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/app/(app)/projects/[id]/security/stream/route.ts b/app/(app)/projects/[id]/security/stream/route.ts index f82de0fe..b32ed904 100644 --- a/app/(app)/projects/[id]/security/stream/route.ts +++ b/app/(app)/projects/[id]/security/stream/route.ts @@ -8,8 +8,13 @@ import { createClient } from "@/lib/supabase/server"; export const dynamic = "force-dynamic"; -const POLL_MS = 1500; -const MAX_MS = 5 * 60 * 1000; +const POLL_MS = 2000; +// A full 65535-port scan can run many minutes; keep the stream open long +// enough to see it finish. +const MAX_MS = 45 * 60 * 1000; +// Emit an SSE comment periodically so idle proxies don't drop the connection +// while a long scan sits in `running` with no status change. +const HEARTBEAT_MS = 15_000; export async function GET( req: NextRequest, @@ -48,8 +53,13 @@ export async function GET( const deadline = Date.now() + MAX_MS; let lastStatus = ""; + let lastBeat = Date.now(); try { while (!closed && Date.now() < deadline) { + if (Date.now() - lastBeat >= HEARTBEAT_MS) { + if (!closed) controller.enqueue(encoder.encode(`: keep-alive\n\n`)); + lastBeat = Date.now(); + } const base = supabase .from("port_scans") .select("id, status, open_ports, completed_at") diff --git a/docs/uptime-monitoring-prd.md b/docs/uptime-monitoring-prd.md index 98615665..2f803444 100644 --- a/docs/uptime-monitoring-prd.md +++ b/docs/uptime-monitoring-prd.md @@ -251,9 +251,12 @@ a scanner. can't take down production. GCP and Railway AUPs prohibit network scanning; scanning from the app IP risks the service's IP. The droplet has its own IP reputation and raw-socket access. See §12.3 for the job-transport design. -- **Bounded + TCP-connect only.** Curated **top-~100 common service ports**, never - full 65535; TCP `connect()` only (containers lack `CAP_NET_RAW` for SYN scans - anyway); ICMP discovery skipped (`-Pn`-equivalent). +- **Full port range, TCP-connect only.** Scans **all 65535 TCP ports** (`nmap + -sT -Pn -p- -T4`) so nothing exposed is missed — the point of the feature. Only + safe because targets are owner-verified and the scan runs on a dedicated + off-Railway droplet. TCP `connect()` only (containers lack `CAP_NET_RAW` for SYN + scans anyway); ICMP discovery skipped (`-Pn`); a 25-min nmap `--host-timeout` + bounds heavily-filtered hosts. - **Rate-limited + infrequent.** Daily cadence, not per-minute; per-org caps. ### 12.2 Behavior @@ -276,8 +279,8 @@ Railway (producer) Redis (broker) DO droplet (prober) ────────────────── ────────────── ─────────────────── API / worker enqueues ──▶ "prober" queue ◀── BullMQ Worker dials OUT port-scan jobs (rediss:// TLS) over rediss://, runs -(repeatable = daily) nmap -sT -Pn --top-ports - 100, returns result +(repeatable = daily) nmap -sT -Pn -p- -T4 + (all ports), returns result Railway QueueEvents ◀─── job "completed" ◀── (result = job return value) handler persists to Supabase, diffs diff --git a/lib/prober-queue.ts b/lib/prober-queue.ts index 12b8f90c..11686060 100644 --- a/lib/prober-queue.ts +++ b/lib/prober-queue.ts @@ -17,8 +17,9 @@ const HIGH_RISK_PORTS = new Set([ ]); const LOW_RISK_PORTS = new Set([80, 443]); -// A running scan older than this is treated as timed out. -const RUNNING_TIMEOUT_MS = 15 * 60 * 1000; +// A running scan older than this is treated as timed out. Generous because a +// full 65535-port scan (nmap -p-) can take many minutes on a filtered host. +const RUNNING_TIMEOUT_MS = 40 * 60 * 1000; let queue: Queue | null = null; diff --git a/prober/src/index.ts b/prober/src/index.ts index 0473adde..beaa7e13 100644 --- a/prober/src/index.ts +++ b/prober/src/index.ts @@ -2,7 +2,7 @@ // // A BullMQ Worker running on a self-hosted DigitalOcean droplet. It dials OUT // to Redis (rediss://) — no inbound port — pulls port-scan jobs off the -// "prober" queue, runs a bounded `nmap -sT -Pn --top-ports 100` TCP-connect +// "prober" queue, runs a full `nmap -sT -Pn -p- -T4` (all 65535) TCP-connect // scan, and returns the open-port set as the job's return value. It writes no // database; the Railway side handles results (baseline diff + alerts). import { Worker, type Job, type ConnectionOptions } from "bullmq"; @@ -60,12 +60,14 @@ function assertScannableHost(host: string): void { async function scan(job: PortScanJob): Promise { assertScannableHost(job.host); - // -sT connect scan (no CAP_NET_RAW needed), -Pn skip host discovery, - // --top-ports 100 bounded set, -oG - greppable output on stdout. + // Full TCP port scan (all 65535): -sT connect scan (no CAP_NET_RAW needed), + // -Pn skip host discovery, -p- every port, -T4 faster timing, and a 25-min + // nmap-side host timeout so a heavily-filtered host returns what it found + // instead of hanging. -oG - greppable output on stdout. const { stdout } = await pexecFile( "nmap", - ["-sT", "-Pn", "--top-ports", "100", "-oG", "-", job.host], - { timeout: 180_000, maxBuffer: 8 * 1024 * 1024 }, + ["-sT", "-Pn", "-p-", "-T4", "--host-timeout", "1500s", "-oG", "-", job.host], + { timeout: 30 * 60_000, maxBuffer: 16 * 1024 * 1024 }, ); const openPorts: OpenPort[] = [];