From 714491bbf16859f2b11d4e79d0f2d0c5d3fe629a Mon Sep 17 00:00:00 2001 From: Jerome Ludmann Date: Fri, 3 Apr 2026 19:20:22 -0300 Subject: [PATCH] feat(bench): add benchmarks and stress tests --- bench/README.md | 67 ++++++++++++++ bench/emitter.bench.ts | 99 ++++++++++++++++++++ bench/fixtures.ts | 32 +++++++ bench/parser.bench.ts | 45 +++++++++ bench/pipeline.bench.ts | 77 ++++++++++++++++ bench/stress/backpressure.ts | 89 ++++++++++++++++++ bench/stress/flood.ts | 157 ++++++++++++++++++++++++++++++++ deno.json | 12 ++- integration/integration_test.ts | 2 +- package-lock.json | 4 +- package.json | 6 +- 11 files changed, 581 insertions(+), 9 deletions(-) create mode 100644 bench/README.md create mode 100644 bench/emitter.bench.ts create mode 100644 bench/fixtures.ts create mode 100644 bench/parser.bench.ts create mode 100644 bench/pipeline.bench.ts create mode 100644 bench/stress/backpressure.ts create mode 100644 bench/stress/flood.ts diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 00000000..6c581bb0 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,67 @@ +# Benchmarks & Stress Tests + +## Quick start + +```bash +# All benchmarks (parser, emitter, pipeline) +deno task bench + +# Parser only +deno task bench:parser + +# Stress test — sustained flood for 30s +deno task stress + +# Stress test — slow consumer / backpressure +deno task stress:backpressure +``` + +## What is measured + +### `parser.bench.ts` — Parser throughput + +Measures `parseChunk()` throughput on different message types: + +- **Single message**: one plain or tagged PRIVMSG per iteration +- **Batch 1000**: 1000 messages in one chunk (realistic TCP read) + +### `emitter.bench.ts` — EventEmitter throughput + +Internal characterization of the event system: + +- Emit with 0, 1, 10, 50 listeners +- Multi-event dispatch (e.g. `privmsg` → `privmsg:channel` + `privmsg:private`) +- `on`/`off` registration cycle +- Promise-based `once` + +### `pipeline.bench.ts` — Full message pipeline + +End-to-end: `MockConn → read → decode → parse → emit(raw:*) → listener` + +- **CoreClient** (no plugins): raw protocol overhead +- **Full Client** (50+ plugins): measures plugin dispatch overhead +- Single message and 80-message batch variants + +### `stress/flood.ts` — Sustained flood + +Sends messages continuously for 30 seconds via chained microtask sends. +Reports per-second throughput and heap usage. Detects memory leaks by checking +for monotonically growing heap over 10+ consecutive seconds. + +Runs two passes: CoreClient (no plugins) then full Client (50+ plugins). + +### `stress/backpressure.ts` — Slow consumer + +Injects 1ms artificial delay in the message listener. Verifies that the read +loop stalls naturally (pull-based `once("read")`) without accumulating an +unbounded event queue. Reports send-to-receive latency percentiles. + +## Interpreting results + +`deno bench` outputs: `avg`, `min`, `max`, `p75`, `p99`, `p995`, `p999`, and +iterations/second. + +For batch benchmarks, multiply iterations/sec by the batch size (80 or 1000) to +get messages/sec. + +Use `--json` and pipe to `bench/report.ts` for a formatted comparison table. diff --git a/bench/emitter.bench.ts b/bench/emitter.bench.ts new file mode 100644 index 00000000..bfb45751 --- /dev/null +++ b/bench/emitter.bench.ts @@ -0,0 +1,99 @@ +import { EventEmitter } from "../core/events.ts"; + +interface Payload { + command: string; + params: string[]; +} + +interface BenchEvents { + "ev": Payload; + "privmsg": Payload; + "privmsg:channel": Payload; + "privmsg:private": Payload; +} + +const PAYLOAD: Payload = { command: "privmsg", params: ["#ch", "hello"] }; + +// Isolated emitters per listener count +const emitter0 = new EventEmitter(); + +const emitter1 = new EventEmitter(); +emitter1.on("ev", () => {}); + +const emitter10 = new EventEmitter(); +for (let i = 0; i < 10; i++) emitter10.on("ev", () => {}); + +const emitter50 = new EventEmitter(); +for (let i = 0; i < 50; i++) emitter50.on("ev", () => {}); + +// Multi-event: simulates privmsg → [privmsg:channel, privmsg:private] +const emitterMulti = new EventEmitter(); +emitterMulti.createMultiEvent("privmsg", [ + "privmsg:channel", + "privmsg:private", +]); +for (let i = 0; i < 5; i++) emitterMulti.on("privmsg:channel", () => {}); +for (let i = 0; i < 5; i++) emitterMulti.on("privmsg:private", () => {}); + +const emitterReg = new EventEmitter(); + +Deno.bench({ + name: "emit: no listener (fast path)", + group: "emit", + baseline: true, + fn() { + emitter0.emit("ev", PAYLOAD); + }, +}); + +Deno.bench({ + name: "emit: 1 listener", + group: "emit", + fn() { + emitter1.emit("ev", PAYLOAD); + }, +}); + +Deno.bench({ + name: "emit: 10 listeners", + group: "emit", + fn() { + emitter10.emit("ev", PAYLOAD); + }, +}); + +Deno.bench({ + name: "emit: 50 listeners", + group: "emit", + fn() { + emitter50.emit("ev", PAYLOAD); + }, +}); + +Deno.bench({ + name: "emit: multi-event (2 targets × 5 listeners)", + group: "emit", + fn() { + emitterMulti.emit("privmsg", PAYLOAD); + }, +}); + +Deno.bench({ + name: "on + off cycle", + group: "registration", + fn() { + const fn = () => {}; + const off = emitterReg.on("ev", fn); + off(); + }, +}); + +Deno.bench({ + name: "once (promise-based)", + group: "registration", + async fn() { + const p = emitterReg.once("ev"); + emitterReg.emit("ev", PAYLOAD); + await p; + }, +}); diff --git a/bench/fixtures.ts b/bench/fixtures.ts new file mode 100644 index 00000000..63864e24 --- /dev/null +++ b/bench/fixtures.ts @@ -0,0 +1,32 @@ +// Pre-built IRC message corpus for all benchmarks. +// Defined once at module load to avoid GC pressure inside measured loops. + +export const MSG_PLAIN = ":nick!user@host PRIVMSG #channel :Hello world"; + +export const MSG_TAGGED = + "@time=2024-01-01T00:00:00Z;msgid=abc123 :nick!user@host PRIVMSG #channel :Hello"; + +export const MSG_NUMERIC = + ":server.example.com 001 me :Welcome to the IRC Network"; + +export const MSG_NOTICE = + ":server.example.com NOTICE * :*** Looking up your hostname..."; + +function batch(msgs: string[]): string { + return msgs.join("\r\n") + "\r\n"; +} + +// Batches terminated with \r\n for Parser.parseMessages() +export const BATCH_1000_PLAIN = batch( + Array.from({ length: 1000 }, () => MSG_PLAIN), +); + +export const BATCH_1000_MIXED = batch([ + ...Array.from({ length: 250 }, () => MSG_PLAIN), + ...Array.from({ length: 250 }, () => MSG_TAGGED), + ...Array.from({ length: 250 }, () => MSG_NUMERIC), + ...Array.from({ length: 250 }, () => MSG_NOTICE), +]); + +// Arrays of raw lines for MockServer.send() — fits in default 4096-byte buffer +export const SEND_80_PLAIN = Array.from({ length: 80 }, () => MSG_PLAIN); diff --git a/bench/parser.bench.ts b/bench/parser.bench.ts new file mode 100644 index 00000000..1e67490a --- /dev/null +++ b/bench/parser.bench.ts @@ -0,0 +1,45 @@ +import { parseChunk } from "../core/parsers.ts"; +import { + BATCH_1000_MIXED, + BATCH_1000_PLAIN, + MSG_PLAIN, + MSG_TAGGED, +} from "./fixtures.ts"; + +// Single message --- + +Deno.bench({ + name: "single plain PRIVMSG", + group: "single", + baseline: true, + fn() { + parseChunk(MSG_PLAIN + "\r\n"); + }, +}); + +Deno.bench({ + name: "single tagged PRIVMSG", + group: "single", + fn() { + parseChunk(MSG_TAGGED + "\r\n"); + }, +}); + +// Batch 1000 messages --- + +Deno.bench({ + name: "1000 plain PRIVMSG", + group: "batch", + baseline: true, + fn() { + parseChunk(BATCH_1000_PLAIN); + }, +}); + +Deno.bench({ + name: "1000 mixed messages", + group: "batch", + fn() { + parseChunk(BATCH_1000_MIXED); + }, +}); diff --git a/bench/pipeline.bench.ts b/bench/pipeline.bench.ts new file mode 100644 index 00000000..14ca17f7 --- /dev/null +++ b/bench/pipeline.bench.ts @@ -0,0 +1,77 @@ +import { MockClient, MockCoreClient } from "../testing/client.ts"; +import { MockServer } from "../testing/server.ts"; +import { MSG_PLAIN, SEND_80_PLAIN } from "./fixtures.ts"; + +// CoreClient (no plugins) --- + +const coreClient = new MockCoreClient([], {}); +const coreServer = new MockServer(coreClient); +await coreClient.connect(""); + +Deno.bench({ + name: "pipeline: single PRIVMSG (CoreClient)", + group: "pipeline-single", + baseline: true, + async fn() { + const p = coreClient.once("raw:privmsg"); + coreServer.send(MSG_PLAIN); + await p; + }, +}); + +Deno.bench({ + name: "pipeline: 80 PRIVMSG batch (CoreClient)", + group: "pipeline-batch", + baseline: true, + async fn() { + let count = 0; + const done = new Promise((resolve) => { + const off = coreClient.on("raw:privmsg", () => { + if (++count === 80) { + off(); + resolve(); + } + }); + }); + coreServer.send(SEND_80_PLAIN); + await done; + }, +}); + +// Full Client (50+ plugins) --- + +const fullClient = new MockClient({ + nick: "me", + pingTimeout: false, +}); +const fullServer = new MockServer(fullClient); +await fullClient.connect(""); +fullServer.receive(); // drain registration messages + +Deno.bench({ + name: "pipeline: single PRIVMSG (full Client)", + group: "pipeline-single", + async fn() { + const p = fullClient.once("raw:privmsg"); + fullServer.send(MSG_PLAIN); + await p; + }, +}); + +Deno.bench({ + name: "pipeline: 80 PRIVMSG batch (full Client)", + group: "pipeline-batch", + async fn() { + let count = 0; + const done = new Promise((resolve) => { + const off = fullClient.on("raw:privmsg", () => { + if (++count === 80) { + off(); + resolve(); + } + }); + }); + fullServer.send(SEND_80_PLAIN); + await done; + }, +}); diff --git a/bench/stress/backpressure.ts b/bench/stress/backpressure.ts new file mode 100644 index 00000000..dc5a8ec6 --- /dev/null +++ b/bench/stress/backpressure.ts @@ -0,0 +1,89 @@ +import { MockCoreClient } from "../../testing/client.ts"; +import { MockServer } from "../../testing/server.ts"; +import { MSG_PLAIN } from "../fixtures.ts"; + +const N = 5000; +const CONSUMER_DELAY_MS = 1; + +console.log( + `Backpressure test: ${N} messages, ${CONSUMER_DELAY_MS}ms consumer delay`, +); +console.log("=".repeat(60)); + +const client = new MockCoreClient([], {}); +const server = new MockServer(client); +await client.connect(""); + +const sendTimestamps: number[] = []; +const receiveTimestamps: number[] = []; +let received = 0; + +const done = new Promise((resolve) => { + client.on("raw:privmsg", async () => { + receiveTimestamps.push(performance.now()); + await new Promise((r) => setTimeout(r, CONSUMER_DELAY_MS)); + received++; + if (received === N) resolve(); + else { + sendTimestamps.push(performance.now()); + server.send(MSG_PLAIN); + } + }); +}); + +// Start the chain +const start = performance.now(); +sendTimestamps.push(start); +server.send(MSG_PLAIN); + +await done; + +const totalTime = performance.now() - start; + +// Compute lag: time between send and receive for each message +const lags: number[] = []; +for ( + let i = 0; + i < Math.min(sendTimestamps.length, receiveTimestamps.length); + i++ +) { + lags.push(receiveTimestamps[i] - sendTimestamps[i]); +} + +const avgLag = lags.reduce((a, b) => a + b, 0) / lags.length; +const maxLag = Math.max(...lags); +const minLag = Math.min(...lags); +const p99Idx = Math.floor(lags.length * 0.99); +const sortedLags = [...lags].sort((a, b) => a - b); +const p99Lag = sortedLags[p99Idx] ?? maxLag; + +const heapMB = Deno.memoryUsage().heapUsed / 1024 / 1024; + +console.log(""); +console.log("RESULTS"); +console.log(` Messages processed: ${received.toLocaleString()}`); +console.log(` Total time: ${(totalTime / 1000).toFixed(2)}s`); +console.log( + ` Effective rate: ${ + Math.round(received / (totalTime / 1000)).toLocaleString() + } msgs/sec`, +); +console.log(` Latency (send→recv):`); +console.log(` min: ${minLag.toFixed(3)}ms`); +console.log(` avg: ${avgLag.toFixed(3)}ms`); +console.log(` p99: ${p99Lag.toFixed(3)}ms`); +console.log(` max: ${maxLag.toFixed(3)}ms`); +console.log(` Heap: ${heapMB.toFixed(1)} MB`); +console.log(""); + +if (maxLag > CONSUMER_DELAY_MS * 10) { + console.log( + ` ⚠ High max latency (${maxLag.toFixed(1)}ms) — possible queue buildup`, + ); +} else { + console.log( + " ✓ Latency stable — read loop stalls naturally (no unbounded queue)", + ); +} + +client.conn?.close(); diff --git a/bench/stress/flood.ts b/bench/stress/flood.ts new file mode 100644 index 00000000..f989e0c3 --- /dev/null +++ b/bench/stress/flood.ts @@ -0,0 +1,157 @@ +import { MockClient, MockCoreClient } from "../../testing/client.ts"; +import { MockServer } from "../../testing/server.ts"; +import type { CoreFeatures } from "../../core/client.ts"; +import type { EventEmitter } from "../../core/events.ts"; +import { MSG_PLAIN } from "../fixtures.ts"; + +const DURATION_SEC = 30; +const REPORT_INTERVAL_MS = 1000; + +interface Sample { + elapsed: number; + msgsPerSec: number; + heapMB: number; +} + +async function runFlood( + label: string, + client: EventEmitter, + server: MockServer, +) { + const samples: Sample[] = []; + let totalCount = 0; + let intervalCount = 0; + let running = true; + + const startHeap = Deno.memoryUsage().heapUsed; + const start = performance.now(); + let lastReport = start; + + const YIELD_EVERY = 1000; + let batchCount = 0; + + client.on("raw:privmsg", () => { + totalCount++; + intervalCount++; + if (!running) return; + batchCount++; + if (batchCount >= YIELD_EVERY) { + // Yield to macrotask queue so setTimeout/setInterval can fire + batchCount = 0; + setTimeout(() => server.send(MSG_PLAIN), 0); + } else { + queueMicrotask(() => server.send(MSG_PLAIN)); + } + }); + + // Start the chain + server.send(MSG_PLAIN); + + // Report loop + const interval = setInterval(() => { + const now = performance.now(); + const dt = (now - lastReport) / 1000; + const msgsPerSec = Math.round(intervalCount / dt); + const heapMB = Deno.memoryUsage().heapUsed / 1024 / 1024; + + samples.push({ + elapsed: Math.round((now - start) / 1000), + msgsPerSec, + heapMB: Math.round(heapMB * 10) / 10, + }); + + intervalCount = 0; + lastReport = now; + }, REPORT_INTERVAL_MS); + + // Run for DURATION_SEC + await new Promise((resolve) => { + setTimeout(() => { + running = false; + clearInterval(interval); + resolve(); + }, DURATION_SEC * 1000); + }); + + // Wait for remaining in-flight messages + await new Promise((r) => setTimeout(r, 100)); + + const endHeap = Deno.memoryUsage().heapUsed; + const totalTime = (performance.now() - start) / 1000; + + // Print report + console.log(`\n${"=".repeat(60)}`); + console.log(`Flood stress test: ${label} (${DURATION_SEC}s)`); + console.log("=".repeat(60)); + console.log(""); + console.log("Time(s) Msgs/sec Heap(MB)"); + console.log("-".repeat(40)); + + for (const s of samples) { + console.log( + `${String(s.elapsed).padStart(4)} ${ + String(s.msgsPerSec).padStart(10) + } ${s.heapMB.toFixed(1).padStart(7)}`, + ); + } + + const peak = Math.max(...samples.map((s) => s.msgsPerSec)); + const avg = Math.round( + samples.reduce((a, s) => a + s.msgsPerSec, 0) / samples.length, + ); + const heapStart = Math.round((startHeap / 1024 / 1024) * 10) / 10; + const heapEnd = Math.round((endHeap / 1024 / 1024) * 10) / 10; + + // Leak detection: check if heap grew monotonically for >10 consecutive samples + let monotonic = 0; + let maxMonotonic = 0; + for (let i = 1; i < samples.length; i++) { + if (samples[i].heapMB > samples[i - 1].heapMB) { + monotonic++; + maxMonotonic = Math.max(maxMonotonic, monotonic); + } else { + monotonic = 0; + } + } + + console.log(""); + console.log("SUMMARY"); + console.log(` Peak throughput: ${peak.toLocaleString()} msgs/sec`); + console.log(` Avg throughput: ${avg.toLocaleString()} msgs/sec`); + console.log(` Heap at start: ${heapStart} MB`); + console.log(` Heap at end: ${heapEnd} MB`); + console.log(` Heap growth: ${(heapEnd - heapStart).toFixed(1)} MB`); + console.log(` Total processed: ${totalCount.toLocaleString()} messages`); + console.log(` Duration: ${totalTime.toFixed(1)}s`); + + if (maxMonotonic >= 10) { + console.log( + ` ⚠ POSSIBLE LEAK: heap grew monotonically for ${maxMonotonic} consecutive seconds`, + ); + } else { + console.log(" Heap: stable (no leak detected)"); + } +} + +// CoreClient (no plugins) --- +console.log("Setting up CoreClient..."); +const coreClient = new MockCoreClient([], {}); +const coreServer = new MockServer(coreClient); +await coreClient.connect(""); + +await runFlood("CoreClient (no plugins)", coreClient, coreServer); + +// Shutdown and set up full client +coreClient.conn?.close(); +await new Promise((r) => setTimeout(r, 200)); + +// Full Client (50+ plugins) --- +console.log("\nSetting up full Client..."); +const fullClient = new MockClient({ nick: "me", pingTimeout: false }); +const fullServer = new MockServer(fullClient); +await fullClient.connect(""); +fullServer.receive(); + +await runFlood("Full Client (50+ plugins)", fullClient, fullServer); + +fullClient.conn?.close(); diff --git a/deno.json b/deno.json index 5d4f3b8f..1550ff76 100644 --- a/deno.json +++ b/deno.json @@ -16,7 +16,8 @@ "plugins/", "runtime/", "testing/", - "integration/" + "integration/", + "bench/" ], "exclude": [ "integration/ergo/", @@ -37,7 +38,8 @@ "plugins", "runtime", "testing", - "integration" + "integration", + "bench" ], "exclude": [ "runtime/node.ts", @@ -62,7 +64,11 @@ "test:integration": "deno task test:integration:ergo && deno task test:integration:inspircd && deno task test:integration:unrealircd", "test:integration:ergo": "docker compose -f integration/ergo/docker-compose.yml down -v 2>/dev/null; docker compose -f integration/ergo/docker-compose.yml up -d --force-recreate && sleep 2 && IRCD=ergo deno test --allow-net --allow-read --allow-env integration/integration_test.ts --failfast ; docker compose -f integration/ergo/docker-compose.yml down -v", "test:integration:inspircd": "docker compose -f integration/inspircd/docker-compose.yml down -v 2>/dev/null; docker compose -f integration/inspircd/docker-compose.yml up -d --force-recreate && sleep 3 && IRCD=inspircd deno test --allow-net --allow-read --allow-env integration/integration_test.ts --failfast ; docker compose -f integration/inspircd/docker-compose.yml down -v", - "test:integration:unrealircd": "docker compose -f integration/unrealircd/docker-compose.yml down -v 2>/dev/null; docker compose -f integration/unrealircd/docker-compose.yml up -d --force-recreate && sleep 3 && IRCD=unrealircd deno test --allow-net --allow-read --allow-env integration/integration_test.ts --failfast ; docker compose -f integration/unrealircd/docker-compose.yml down -v" + "test:integration:unrealircd": "docker compose -f integration/unrealircd/docker-compose.yml down -v 2>/dev/null; docker compose -f integration/unrealircd/docker-compose.yml up -d --force-recreate && sleep 3 && IRCD=unrealircd deno test --allow-net --allow-read --allow-env integration/integration_test.ts --failfast ; docker compose -f integration/unrealircd/docker-compose.yml down -v", + "bench": "deno bench bench/parser.bench.ts bench/emitter.bench.ts bench/pipeline.bench.ts", + "bench:parser": "deno bench bench/parser.bench.ts", + "stress": "deno run --allow-read bench/stress/flood.ts", + "stress:backpressure": "deno run --allow-read bench/stress/backpressure.ts" }, "publish": { "include": [ diff --git a/integration/integration_test.ts b/integration/integration_test.ts index 9524e3be..3d6eb0d6 100644 --- a/integration/integration_test.ts +++ b/integration/integration_test.ts @@ -9,7 +9,7 @@ const IRCD = typeof Deno !== "undefined" // deno-lint-ignore no-process-global : process.env.IRCD ?? "ergo"; -describe("integration", (test) => { +describe(`integration/${IRCD}`, (test) => { let counter = 0; const connect = async (base: string) => { diff --git a/package-lock.json b/package-lock.json index 69ec15c9..32a786df 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "irc-client-ts", - "version": "0.22.1", + "version": "0.23.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "irc-client-ts", - "version": "0.22.1", + "version": "0.23.0", "license": "MIT", "devDependencies": { "@std/assert": "npm:@jsr/std__assert@^1.0.19", diff --git a/package.json b/package.json index 60f64004..ebd4097f 100644 --- a/package.json +++ b/package.json @@ -30,12 +30,12 @@ "url": "https://github.com/jeromeludmann/irc.git" }, "devDependencies": { - "@types/node": "^24.7.2", "@std/assert": "npm:@jsr/std__assert@^1.0.19", "@std/fmt": "npm:@jsr/std__fmt@^1.0.9", + "@types/node": "^24.7.2", "bun-types": "latest", - "typescript": "^5.9.3", - "tsx": "^4" + "tsx": "^4", + "typescript": "^5.9.3" }, "scripts": { "build": "tsc -p tsconfig.build.json",