diff --git a/.gitignore b/.gitignore index 48e0444d..48e0b513 100644 --- a/.gitignore +++ b/.gitignore @@ -109,4 +109,6 @@ dist .tern-port playground/bundle.js -test/debug.ts \ No newline at end of file +test/debug.ts + +benchmark/baseline.json \ No newline at end of file diff --git a/.npmignore b/.npmignore index 0e9d3a76..9fcff87f 100644 --- a/.npmignore +++ b/.npmignore @@ -14,4 +14,5 @@ playground/ .idea/ www/ *.yml -z/ \ No newline at end of file +z/ +benchmark/ \ No newline at end of file diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 00000000..eec3dff3 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,168 @@ +# ElectroDB benchmarks + +Manual/local micro-benchmarks for ElectroDB's hot paths, built on +[tinybench](https://github.com/tinylibs/tinybench). They are **not** part of +the test suite or CI — nothing here gates a build. + +## Running + +```sh +npm run benchmark # run everything, print a table +npm run benchmark:json # machine-readable results (operations per second, mean, 99th percentile, margin of error) +npm run benchmark:update # rewrite baseline.json from a fresh run +npm run benchmark:compare # run + diff against baseline.json (advisory) +``` + +`benchmark:compare` always exits 0 (advisory); add `--strict` to exit 1 on +regressions — that one flag is the hook for promoting the comparison to CI +later: + +```sh +npx ts-node ./benchmark/run.ts --compare --strict +``` + +Other flags: `--filter ` runs a subset while iterating, +`--threshold ` sets the practical-significance bar, +`--time ` sets within-run sampling time per task, and +`--runs ` repeats the whole suite that many times to derive error bars from the +between-run spread. + +All defaults and tuning knobs — sampling (`runs`/`time`/warmup), the verdict +gates, the reference task, color bands, and lane widths — live in +[`config.ts`](./config.ts), each documented with how it is used and how the +knobs interact (the file header walks the whole pipeline). + +Human-readable output includes terminal visualizations (suppressed by +`--json`; colors respect `NO_COLOR`/non-TTY): plain runs print log-scale +throughput bars; multi-run invocations plot each run's normalized score as a +dot, with every task's lane sharing one percent scale centered on that +task's own mean (deviation-from-own-mean is dimensionless, so rows are +comparable across otherwise apples-and-oranges tasks — sorted noisiest +first for triage, with ├ ┤ marking the CI of the mean over the scatter); +and compare draws each task's baseline and current confidence intervals as +two lanes on a shared scale — the overlap test from the section below, made +literal. Color carries +meaning everywhere: green = good (improved, steady sampling), red = bad +(regression, jittery sampling), yellow = caution (real-but-small change, +moderate variance), cyan = neutral measurement, and magenta = abnormal — +the instrument itself is suspect (a task's noise floor exceeds the +threshold, or machine conditions drifted since the baseline). + +``` +~noise params-chain/get Δ -3.9% vs noise ±11.8% + baseline 160.1759 ±5.4% ··········├──────────────●───────────────┤·· + current 153.8809 ±6.1% ··├─────────────●────────────┤·············· +``` + +Each lane also prints its own CI width (`±`), which is the consistency +signal: a current lane much narrower than its baseline lane means the change +made the benchmark steadier run-to-run, not just faster. When the underlying +spreads differ by ≥2.5x the header calls it out (`· consistency ×N steadier` +/ `noisier`) — a deliberate hint rather than a gated verdict, since spread +estimates from a handful of runs are themselves noisy. Note that a raw CI +width shrinks with more `--runs` regardless of the code (`width = +t(n−1)·sd/√n` — sampling effort, not steadiness), so the baseline records +its run count and the hint converts both sides back to underlying spread +before comparing; compare also warns when the two sides used different +`--runs`. + +## Baseline + normalization + +Raw ops/sec is machine-dependent, so the committed `baseline.json` stores each +task's throughput **normalized** to a fixed reference task +(`reference/parse-500-items`, a representative formatting workload) that runs +with every invocation: `normalized = task operations per second ÷ reference operations per second`. Comparing +normalized values makes the baseline meaningful across reasonably similar +machines; for precise A/B work, regenerate the baseline on your own machine +first (`npm run benchmark:update`), apply your change, then +`npm run benchmark:compare`. + +## Interpreting a compare: is the difference significant? + +Micro-benchmark deltas are meaningless without an error model, so every +compare applies **two gates** per task and prints a verdict: + +1. **Statistical — is the change real?** Both the baseline and the current + measurement carry a 95% confidence interval, and a change counts as _real_ + only when the two intervals do **not** overlap. The table prints this as + the task's **noise ±%** — if `|Δ%|` is below it, the verdict is `~noise` + no matter how the raw numbers look. Where the interval comes from matters: + - With `--runs > 1` (the default for compare/update) the suite executes + N times and the interval is a t-based CI of the normalized value's + **between-run spread**. This is the trustworthy estimate: it captures + GC phasing, JIT state, and thermal drift, which dominate for long + async tasks (the pagination scenarios can swing ±20% between runs + while their within-run margin reads ±1%). + - With `--runs 1` it falls back to tinybench's within-run relative margin of error, combined + with the reference task's (combining both sides as `√(taskMargin² + referenceMargin²)`, since a + normalized value is a ratio of two measurements). Fine for quick + iteration; too optimistic for decisions. +2. **Practical — is it big enough to care?** A real change must also exceed + `--threshold` (default 5%) to be labeled `REGRESSION` or `improved`; + otherwise it reports as `within threshold`. + +| Verdict | Meaning | +| ------------------ | -------------------------------------------------------------- | +| `REGRESSION` | Real (beats the noise floor) **and** ≥ threshold slower | +| `improved` | Real **and** ≥ threshold faster | +| `within threshold` | Real, but smaller than the threshold — judgment call | +| `~noise` | Indistinguishable from sampling error — not evidence of change | + +### Evaluating one of your own changes + +```sh +git stash # or check out the pre-change tree +npm run benchmark:update # capture the "before" baseline +git stash pop +npm run benchmark:compare # verdicts tell you if the change is real +``` + +Practical notes: + +- If a task you care about shows a high noise floor (the runner warns when it + exceeds the threshold), increase repetitions on **both** sides, e.g. + `--runs 8`. More runs shrink the between-run confidence interval, which is + the only honest way to detect small differences; `--time` only tightens + the (already optimistic) within-run estimate. +- The compare prints the reference task's raw throughput drift since the + baseline. Normalization absorbs uniform machine slowdown, but drift is + rarely uniform — when the drift is large the runner warns and borderline + verdicts deserve suspicion. +- The statistical gate assumes both sides came from the same machine under + broadly similar load. Close heavy apps, and re-run a suspicious result once + before believing it — two consistent verdicts beat one. +- The noise-floor test is deliberately conservative (CI overlap). A `~noise` + verdict doesn't prove there is no difference — only that this run can't + detect one at its current sensitivity. + +## Adding a scenario + +Drop a file matching `benchmark/scenarios/*.benchmark.ts` that default-exports an +array of entries — there is no registration list to edit: + +```ts +import type { ScenarioEntry } from "../run"; +import { makeFixtureEntity } from "../../test/fixtures/entities"; + +const entity = makeFixtureEntity(); + +const scenarios: ScenarioEntry[] = [ + { + name: "my-scenario/some-task", // group/task; sizes get their own task names + fn: () => entity.query.records({ org: "org1" }).params(), + // opts: optional tinybench task options (beforeAll/beforeEach/...) + }, +]; + +export default scenarios; +``` + +Conventions: + +- Do setup (entities, stored items, mock clients) at module scope or in + `opts.beforeAll` so it stays outside the timed region; `fn` should measure + only the operation under test. `fn` may be async. +- Express sizes as separate named tasks (`parse-format/1000-items`) rather + than parameters, so the baseline tracks each size independently. +- Reuse `test/fixtures/` (mock clients, fixture entities) rather than talking + to a real DynamoDB — benchmarks must run offline. diff --git a/benchmark/config.ts b/benchmark/config.ts new file mode 100644 index 00000000..f6dffea8 --- /dev/null +++ b/benchmark/config.ts @@ -0,0 +1,153 @@ +/** + * Tuning knobs for the benchmark runner, grouped by the pipeline stage they + * affect. How they play together, end to end: + * + * 1. sampling Each invocation repeats the whole suite `--runs` times + * (DEFAULT_DECISION_RUNS for compare/update, 1 otherwise); + * within each run tinybench warms a task for + * WARMUP_TIME_MILLISECONDS and then samples it for `--time` + * milliseconds (DEFAULT_SAMPLING_TIME_MILLISECONDS). + * RUNS buys narrower between-run confidence intervals — the + * error bars every decision is made with. TIME buys steadier + * within-run numbers — mostly cosmetic at runs > 1 (it feeds + * the stability colors and the runs=1 fallback interval). + * 2. normalize Every task's operations/second is divided by + * REFERENCE_TASK's, so stored scores are machine-portable + * ratios. + * 3. verdicts compare labels a task REGRESSION/improved only when |Δ| + * beats BOTH the measured noise floor (from the two + * confidence intervals — shrinks as runs grow) AND + * DEFAULT_THRESHOLD_PERCENT. CONSISTENCY_NOTE_RATIO and + * CONSISTENCY_MINIMUM_RUNS separately gate the + * spread-change hint. DRIFT_* only annotate machine + * conditions; they never change a verdict. + * 4. rendering STABILITY_* bands pick green/yellow/red for ± values, and + * the *_WIDTH knobs size the ASCII lanes and bars. + * + * Rules of thumb: to detect smaller effects raise --runs (not --time); to + * make verdicts stricter or looser change DEFAULT_THRESHOLD_PERCENT; + * everything else is presentation. + */ + +// --------------------------------------------------------------------------- +// sampling — how much data an invocation collects +// --------------------------------------------------------------------------- + +/** + * Whole-suite repetitions when `--runs` is not given to compare/update + * (plain/exploratory runs default to 1). This is the dominant cost AND the + * dominant sensitivity lever: the between-run 95% confidence interval + * shrinks roughly as t(n−1)/√n, so 5→10 runs is ~1.7x sharper, each + * doubling ~1.5x more. The consistency hint needs CONSISTENCY_MINIMUM_RUNS + * on both sides to fire at all. + */ +export const DEFAULT_DECISION_RUNS = 10; + +/** + * tinybench sampling window per task per run, milliseconds, when `--time` is + * not given. Total invocation cost ≈ runs × tasks × (sampling time + + * WARMUP_TIME_MILLISECONDS). Raising it tightens the within-run margin of + * error — which colors the throughput-bar stability and is the confidence + * interval fallback at runs=1 — but does NOT tighten the between-run + * interval that verdicts use; raise runs for that. + */ +export const DEFAULT_SAMPLING_TIME_MILLISECONDS = 500; + +/** + * tinybench per-task warmup before sampling starts, milliseconds, within + * every run. This warms a task immediately before ITS measurement; it does + * not prevent the first whole-suite run from being JIT-cold relative to + * later runs — that effect is what the ○ marker in the spread view shows. + */ +export const WARMUP_TIME_MILLISECONDS = 100; + +// --------------------------------------------------------------------------- +// normalization — what raw operations/second is divided by +// --------------------------------------------------------------------------- + +/** + * The yardstick task every score is normalized against + * (normalized = task speed ÷ reference speed). It runs in every invocation, + * is excluded from verdicts/spread rows, and its raw drift between baseline + * and compare is reported as the machine-condition indicator. Pick a + * workload representative of the library's hot path; changing it invalidates + * committed baselines (regenerate with benchmark:update). + */ +export const REFERENCE_TASK = "reference/parse-500-items"; + +// --------------------------------------------------------------------------- +// verdict gates — when compare is allowed to claim something +// --------------------------------------------------------------------------- + +/** + * Practical-significance gate, percent, when `--threshold` is not given. + * A delta must beat BOTH the statistical noise floor and this value to earn + * REGRESSION/improved; real-but-smaller deltas report "within threshold". + * It also drives the undersensitivity warnings: tasks whose noise floor + * exceeds it are flagged magenta (the run cannot detect changes this small). + */ +export const DEFAULT_THRESHOLD_PERCENT = 5; + +/** + * Underlying-spread ratio before compare calls out a consistency change + * (`consistency ×N steadier/noisier`). Spread estimates from a handful of + * runs are themselves noisy — ~2.5x is what an F-test needs to clear 95% + * confidence at 5 runs — so this stays conservative; lower it only if you + * also raise runs. The hint converts confidence-interval widths back to + * spread (spread ∝ width·√n/t(n−1)) so unequal --runs between baseline and + * current do not fake a change. + */ +export const CONSISTENCY_NOTE_RATIO = 2.5; + +/** + * Minimum --runs on BOTH sides before the consistency hint may fire. A + * spread estimated from 2 runs has one degree of freedom and swings wildly; + * 3+ keeps the hint from reacting to pure sampling luck. + */ +export const CONSISTENCY_MINIMUM_RUNS = 3; + +/** + * Reference-task raw-drift bands, percent: ≤ ACCEPTABLE green, ≤ CAUTION + * yellow, beyond magenta plus a warning. Drift means machine conditions + * changed between baseline and compare (thermal, load); normalization + * absorbs uniform drift but not GC/async-heavy skew, so large drift makes + * borderline verdicts suspect. Advisory only — never changes a verdict. + */ +export const DRIFT_ACCEPTABLE_PERCENT = 3; +export const DRIFT_CAUTION_PERCENT = 10; + +// --------------------------------------------------------------------------- +// rendering — colors and layout only; no effect on measurements or verdicts +// --------------------------------------------------------------------------- + +/** + * Variance bands, percent, for coloring ± values green/yellow/red across all + * views (throughput bars, spread rows, compare lanes) and per-dot deviation + * colors in the spread view. Purely presentational. + */ +export const STABILITY_STEADY_PERCENT = 3; +export const STABILITY_MODERATE_PERCENT = 10; + +/** column width for verdict labels; lane rows indent to clear it */ +export const VERDICT_LABEL_WIDTH = 17; + +/** character width of compare confidence-interval lanes */ +export const LANE_WIDTH = 44; + +/** character width of spread-view lanes; odd so the mean has an exact center */ +export const SPREAD_LANE_WIDTH = 45; + +/** character width of throughput bars */ +export const BAR_WIDTH = 28; + +// --------------------------------------------------------------------------- +// wiring — not tuning knobs +// --------------------------------------------------------------------------- + +/** + * Bump when baseline.json's shape changes incompatibly; compare refuses + * baselines with a different version and asks for benchmark:update. + * (Version 3: field names spelled out — operationsPerSecond, + * normalizedRelativeMarginOfErrorPercent, referenceOperationsPerSecond.) + */ +export const BASELINE_SCHEMA_VERSION = 3; diff --git a/benchmark/run.ts b/benchmark/run.ts new file mode 100644 index 00000000..87f95f4b --- /dev/null +++ b/benchmark/run.ts @@ -0,0 +1,1013 @@ +#!/usr/bin/env ts-node +/** + * ElectroDB benchmark runner (manual/local — not wired into the test gate). + * + * npm run benchmark run all scenarios, print a table + * npm run benchmark:json print machine-readable results + * npm run benchmark:update write benchmark/baseline.json + * npm run benchmark:compare compare against baseline.json (advisory) + * --strict exit 1 when the compare finds regressions + * --filter only run tasks whose name includes substring + * --threshold practical-significance threshold + * --time within-run sampling time per task + * --runs repeat the whole suite this many times and + * derive error bars from the between-run spread + * (defaults for all of these live in config.ts, with documentation on + * how the knobs interact) + * + * Scenarios live in benchmark/scenarios/*.benchmark.ts; each default-exports an + * array of `{ name, fn, options? }` entries (see README.md). A fixed reference + * task runs with every invocation and task throughput is recorded relative to + * it (`normalized = task operations/second ÷ reference operations/second`), + * which makes the committed baseline + * roughly machine-independent. + * + * Compare verdicts use two gates (see README "Interpreting a compare"): + * statistical — the change must exceed the noise floor derived from both + * sides' 95% confidence intervals. With --runs > 1 the interval comes + * from the between-run spread of the normalized value (which captures + * GC/JIT/thermal variance that within-run sampling underestimates, + * especially for long async tasks); with --runs 1 it falls back to the + * within-run tinybench relative margin of error combined with the + * reference task's. + * practical — the change must also exceed --threshold percent to be worth + * acting on. Real-but-small changes report as "within threshold", and + * changes inside the noise floor report as "~noise". + */ +import * as fs from "fs"; +import * as path from "path"; +import { Bench } from "tinybench"; +import { makeFixtureEntity, makeStoredItem } from "../test/fixtures/entities"; +import { + BAR_WIDTH, + BASELINE_SCHEMA_VERSION, + CONSISTENCY_MINIMUM_RUNS, + CONSISTENCY_NOTE_RATIO, + DEFAULT_DECISION_RUNS, + DEFAULT_THRESHOLD_PERCENT, + DEFAULT_SAMPLING_TIME_MILLISECONDS, + DRIFT_CAUTION_PERCENT, + DRIFT_ACCEPTABLE_PERCENT, + LANE_WIDTH, + REFERENCE_TASK, + SPREAD_LANE_WIDTH, + STABILITY_MODERATE_PERCENT, + STABILITY_STEADY_PERCENT, + VERDICT_LABEL_WIDTH, + WARMUP_TIME_MILLISECONDS, +} from "./config"; + +export interface ScenarioEntry { + /** task name, conventionally `group/task` */ + name: string; + /** the operation under test; may be async */ + fn: () => unknown | Promise; + /** optional tinybench task options (beforeAll/beforeEach/...) */ + options?: Record; +} + +interface TaskStatistics { + operationsPerSecond: number; + mean: number; + percentile99: number; + /** tinybench's relative margin of error for the task itself (95%, percent) */ + relativeMarginOfErrorPercent: number; + samples: number; + normalized?: number; + /** + * relative 95% margin of error of `normalized` (percent). `normalized` is a + * ratio of two independent estimates (task speed ÷ reference speed), so + * its relative error combines both sides' margins: + * sqrt(taskMargin² + referenceMargin²). + */ + normalizedRelativeMarginOfErrorPercent?: number; +} + +interface RunResults { + referenceOperationsPerSecond: number | null; + tasks: Record; +} + +interface BaselineTask { + operationsPerSecond: number; + mean: number; + normalized: number; + /** relative 95% margin of error of `normalized` (percent) */ + normalizedRelativeMarginOfErrorPercent: number; + samples: number; +} + +interface BaselineFile { + schemaVersion: number; + generatedAt: string; + node: string; + /** --runs the baseline was captured at; absent on older baselines */ + runs?: number; + referenceOperationsPerSecond: number | null; + tasks: Record; +} + +interface CommandLineArguments { + json: boolean; + compare: boolean; + updateBaseline: boolean; + strict: boolean; + filter: string | null; + /** practical-significance threshold, percent */ + threshold: number; + /** tinybench sampling time per task, milliseconds */ + time: number; + /** full-suite repetitions; >1 derives error bars from between-run spread */ + runs: number; +} + +const SCENARIO_DIRECTORY = path.join(__dirname, "scenarios"); +const BASELINE_PATH = path.join(__dirname, "baseline.json"); + +function parseCommandLineArguments( + processArguments: string[], +): CommandLineArguments { + const commandLineArguments: CommandLineArguments = { + json: false, + compare: false, + updateBaseline: false, + strict: false, + filter: null, + threshold: DEFAULT_THRESHOLD_PERCENT, + time: DEFAULT_SAMPLING_TIME_MILLISECONDS, + runs: 0, // resolved after flags are read; see below + }; + for (let i = 2; i < processArguments.length; i++) { + const argument = processArguments[i]; + if (argument === "--json") commandLineArguments.json = true; + else if (argument === "--compare") commandLineArguments.compare = true; + else if (argument === "--update-baseline") + commandLineArguments.updateBaseline = true; + else if (argument === "--strict") commandLineArguments.strict = true; + else if (argument === "--filter") + commandLineArguments.filter = processArguments[++i] ?? null; + else if ( + argument === "--threshold" || + argument === "--time" || + argument === "--runs" + ) { + const value = Number(processArguments[++i]); + if (!Number.isFinite(value) || value <= 0) { + console.error(`${argument} requires a positive number`); + process.exit(1); + } + if (argument === "--threshold") commandLineArguments.threshold = value; + else if (argument === "--time") commandLineArguments.time = value; + else commandLineArguments.runs = Math.floor(value); + } else { + console.error(`Unknown argument: ${argument}`); + process.exit(1); + } + } + if (commandLineArguments.runs === 0) { + // compare/update need trustworthy error bars, which only between-run + // spread provides; plain exploratory runs stay fast by default + commandLineArguments.runs = + commandLineArguments.compare || commandLineArguments.updateBaseline + ? DEFAULT_DECISION_RUNS + : 1; + } + return commandLineArguments; +} + +function makeReferenceTask(): ScenarioEntry { + const entity = makeFixtureEntity(); + const Items: Record[] = []; + for (let i = 0; i < 500; i++) { + Items.push(makeStoredItem(entity, i)); + } + return { + name: REFERENCE_TASK, + fn: () => entity.parse({ Items }), + }; +} + +function loadScenarios(): ScenarioEntry[] { + const entries: ScenarioEntry[] = [makeReferenceTask()]; + const files = fs + .readdirSync(SCENARIO_DIRECTORY) + .filter( + (file) => + file.endsWith(".benchmark.ts") || file.endsWith(".benchmark.js"), + ) + .sort(); + for (const file of files) { + const loadedModule = require(path.join(SCENARIO_DIRECTORY, file)); + const scenario: unknown = loadedModule.default ?? loadedModule; + if (!Array.isArray(scenario)) { + throw new Error( + `${file} must export an array of {name, fn, options?}`, + ); + } + for (const entry of scenario) { + if ( + !entry || + typeof entry.name !== "string" || + typeof entry.fn !== "function" + ) { + throw new Error( + `${file} exported an entry without {name, fn}`, + ); + } + entries.push(entry); + } + } + return entries; +} + +function collectResults(benchmarkSuite: Bench): RunResults { + const tasks: Record = {}; + let failed = false; + for (const task of benchmarkSuite.tasks) { + if (!task.result || task.result.error) { + console.error(`Task "${task.name}" failed:`); + console.error(task.result && task.result.error); + failed = true; + continue; + } + tasks[task.name] = { + operationsPerSecond: task.result.hz, + mean: task.result.mean, + percentile99: task.result.p99, + relativeMarginOfErrorPercent: task.result.rme, + samples: task.result.samples.length, + }; + } + if (failed) { + process.exit(1); + } + const reference = tasks[REFERENCE_TASK]; + if (reference) { + for (const name of Object.keys(tasks)) { + const task = tasks[name]; + task.normalized = + task.operationsPerSecond / reference.operationsPerSecond; + task.normalizedRelativeMarginOfErrorPercent = Math.sqrt( + task.relativeMarginOfErrorPercent * task.relativeMarginOfErrorPercent + + reference.relativeMarginOfErrorPercent * + reference.relativeMarginOfErrorPercent, + ); + } + } + return { + referenceOperationsPerSecond: reference + ? reference.operationsPerSecond + : null, + tasks, + }; +} + +const Verdicts = { + regression: "regression", + improved: "improved", + within_threshold: "within_threshold", + noise: "noise", + added: "added", + removed: "removed", +} as const; + +type Verdict = keyof typeof Verdicts; + +const VerdictDisplayText: Record = { + regression: "REGRESSION", + improved: "improved", + within_threshold: "within threshold", + noise: "~noise", + added: "added (not in baseline)", + removed: "removed", +}; + +// --------------------------------------------------------------------------- +// terminal rendering — ANSI colors, CI lanes, and throughput bars +// --------------------------------------------------------------------------- + +const useColor = + process.stdout.isTTY && process.env.NO_COLOR === undefined; + +function paint(code: number): (text: string) => string { + return (text) => (useColor ? `\u001b[${code}m${text}\u001b[0m` : text); +} + +const bold = paint(1); +const dim = paint(2); +const red = paint(31); +const green = paint(32); +const yellow = paint(33); +const magenta = paint(35); +const cyan = paint(36); + +// color semantics used across all output: +// green = good (improved / steady) red = bad (regression / jittery) +// yellow = caution (real-but-small / moderate variance) +// cyan = neutral measurement magenta = abnormal (instrument too +// blunt: noise floor above the threshold, or large machine drift) +const VerdictPaint: Record string> = { + regression: (text) => bold(red(text)), + improved: green, + within_threshold: yellow, + noise: dim, + added: cyan, + removed: dim, +}; + +// the lane/delta color for a verdict; ~noise stays a visible neutral so the +// current lane never disappears into the dim baseline lane +const VerdictEmphasisPaint: Record string> = { + regression: red, + improved: green, + within_threshold: yellow, + noise: cyan, + added: cyan, + removed: dim, +}; + +function stabilityPaint( + relativeMarginOfErrorPercent: number, +): (text: string) => string { + if (relativeMarginOfErrorPercent <= STABILITY_STEADY_PERCENT) return green; + if (relativeMarginOfErrorPercent <= STABILITY_MODERATE_PERCENT) return yellow; + return red; +} + +const LANE_INDENT = " ".repeat(VERDICT_LABEL_WIDTH + 1); + +function verdictLabel(verdict: Verdict): string { + return VerdictPaint[verdict]( + VerdictDisplayText[verdict].padEnd(VERDICT_LABEL_WIDTH), + ); +} + +// dim, single-line "how to read this" bullets under a section header +function explain(bullets: string[]): void { + for (const bullet of bullets) { + console.log(dim(` • ${bullet}`)); + } +} + +interface Interval { + lowerBound: number; + mean: number; + upperBound: number; +} + +function intervalOf( + mean: number, + relativeMarginOfErrorPercent: number, +): Interval { + const half = mean * (relativeMarginOfErrorPercent / 100); + return { + lowerBound: Math.max(0, mean - half), + mean, + upperBound: mean + half, + }; +} + +// one fixed-width lane — ····├────●────┤···· — the 95% CI drawn over a dotted +// track; two lanes rendered on the same scale make interval overlap (or the +// gap between them) directly visible in the terminal +function renderLane( + interval: Interval, + scaleLowerBound: number, + scaleUpperBound: number, + color: (text: string) => string, +): string { + const cells: string[] = new Array(LANE_WIDTH).fill("·"); + const span = scaleUpperBound - scaleLowerBound; + const at = (value: number) => + span <= 0 + ? 0 + : Math.max( + 0, + Math.min( + LANE_WIDTH - 1, + Math.round(((value - scaleLowerBound) / span) * (LANE_WIDTH - 1)), + ), + ); + const lowerBoundPosition = at(interval.lowerBound); + const upperBoundPosition = at(interval.upperBound); + if (upperBoundPosition - lowerBoundPosition < 2) { + // interval narrower than the lane resolution: render a point, not caps + cells[at(interval.mean)] = "●"; + } else { + for (let i = lowerBoundPosition; i <= upperBoundPosition; i++) { + cells[i] = "─"; + } + cells[lowerBoundPosition] = "├"; + cells[upperBoundPosition] = "┤"; + cells[at(interval.mean)] = "●"; + } + const track = cells.join(""); + return ( + dim(track.slice(0, lowerBoundPosition)) + + color(track.slice(lowerBoundPosition, upperBoundPosition + 1)) + + dim(track.slice(upperBoundPosition + 1)) + ); +} + +// log scale: tasks span ~140 to ~450k ops/sec, so linear bars would flatten +// everything but the fastest group +function renderThroughputBars(tasks: Record): void { + const names = Object.keys(tasks); + if (names.length === 0) { + return; + } + const nameWidth = Math.max(...names.map((name) => name.length)); + const logarithms = names.map((name) => + Math.log10(tasks[name].operationsPerSecond), + ); + const logarithmLowerBound = Math.min(...logarithms); + const logarithmUpperBound = Math.max(...logarithms); + console.log(""); + console.log("throughput — operations per second"); + explain([ + "longer bar = faster (log scale: small bar gaps are big speed gaps)", + "±% = sampling wobble — smaller is more trustworthy", + ]); + console.log( + ` • bar color = stability: ${green( + `steady ≤${STABILITY_STEADY_PERCENT}%`, + )} · ${yellow(`moderate ≤${STABILITY_MODERATE_PERCENT}%`)} · ${red( + "jittery above", + )}`, + ); + for (const name of names) { + const task = tasks[name]; + const fraction = + logarithmUpperBound > logarithmLowerBound + ? (Math.log10(task.operationsPerSecond) - logarithmLowerBound) / + (logarithmUpperBound - logarithmLowerBound) + : 1; + const filled = Math.max(1, Math.round(fraction * BAR_WIDTH)); + const stability = stabilityPaint(task.relativeMarginOfErrorPercent); + const bar = + stability("█".repeat(filled)) + dim("░".repeat(BAR_WIDTH - filled)); + const operationsPerSecondLabel = Math.round(task.operationsPerSecond) + .toLocaleString("en-US") + .padStart(9); + const relativeMarginOfErrorLabel = stability( + `±${task.relativeMarginOfErrorPercent.toFixed(1)}%`.padStart(7), + ); + console.log( + ` ${name.padEnd( + nameWidth, + )} ${bar} ${operationsPerSecondLabel} ${relativeMarginOfErrorLabel}`, + ); + } +} + +// terminal version of the "same scatter, shrinking bracket" picture: every +// lane shares ONE percent scale centered on each task's own mean, so rows +// are directly comparable even across apples-and-oranges tasks (deviation +// from your own mean is dimensionless). Dots = individual runs (colored by +// deviation band; ○ marks the first, usually-cold run), │ = the task's +// mean, ├ ┤ = the 95% CI of the mean — caps tighten with --runs while the +// scatter does not. Rows sort noisiest-first for triage. +function renderRunSpread(allRuns: RunResults[], current: RunResults): void { + const names = Object.keys(current.tasks).filter( + (name) => name !== REFERENCE_TASK, + ); + if (names.length === 0) { + return; + } + const rows = names.map((name) => { + const stats = current.tasks[name]; + const mean = stats.normalized ?? 0; + const values = allRuns.map((run) => run.tasks[name].normalized ?? 0); + const deviations = values.map((value) => + mean > 0 ? (value / mean - 1) * 100 : 0, + ); + return { + name, + mean, + relativeMarginOfErrorPercent: + stats.normalizedRelativeMarginOfErrorPercent ?? 0, + deviations, + }; + }); + rows.sort( + (a, b) => b.relativeMarginOfErrorPercent - a.relativeMarginOfErrorPercent, + ); + const scaleMaximum = Math.max( + 1, + ...rows.map((row) => + Math.max( + row.relativeMarginOfErrorPercent, + ...row.deviations.map(Math.abs), + ), + ), + ); + const nameWidth = Math.max(...names.map((name) => name.length)); + const center = Math.floor(SPREAD_LANE_WIDTH / 2); + const at = (percent: number) => + center + + Math.max( + -center, + Math.min(center, Math.round((percent / scaleMaximum) * center)), + ); + console.log(""); + console.log( + `normalized score per run — each dot is one of ${ + allRuns.length + } runs, every lane spans ±${scaleMaximum.toFixed(1)}% of its task's mean`, + ); + explain([ + "one shared scale: wider dot scatter = jitterier task — rows sorted noisiest first for triage", + ]); + console.log( + ` • dots = individual runs, colored by deviation from their mean: ${green( + `≤${STABILITY_STEADY_PERCENT}%`, + )} · ${yellow(`≤${STABILITY_MODERATE_PERCENT}%`)} · ${red( + "beyond", + )}; ${cyan("○")} = first run (often cold)`, + ); + explain([ + "│ = the mean · ├ ┤ = how precisely that mean is known (the ±% column) — more --runs pulls the caps inward", + "the dot scatter is the code's actual jitter — more runs measure it better but don't shrink it", + ]); + for (const row of rows) { + const cells: string[] = new Array(SPREAD_LANE_WIDTH).fill("·"); + const paints: ((text: string) => string)[] = new Array( + SPREAD_LANE_WIDTH, + ).fill(dim); + const capPaint = stabilityPaint(row.relativeMarginOfErrorPercent); + // the mean line stays un-dimmed so the lane has a visible anchor + cells[center] = "│"; + paints[center] = (text) => text; + if (at(row.relativeMarginOfErrorPercent) !== center) { + cells[at(-row.relativeMarginOfErrorPercent)] = "├"; + paints[at(-row.relativeMarginOfErrorPercent)] = capPaint; + cells[at(row.relativeMarginOfErrorPercent)] = "┤"; + paints[at(row.relativeMarginOfErrorPercent)] = capPaint; + } + row.deviations.forEach((deviation, runIndex) => { + const position = at(deviation); + // first run is marked distinctly: it is the usual cold-JIT outlier + cells[position] = runIndex === 0 ? "○" : "●"; + paints[position] = stabilityPaint(Math.abs(deviation)); + }); + const lane = cells.map((cell, index) => paints[index](cell)).join(""); + const normalized = row.mean.toFixed(4).padStart(10); + const confidenceIntervalLabel = capPaint( + `±${row.relativeMarginOfErrorPercent.toFixed(1)}%`.padStart(7), + ); + console.log( + ` ${row.name.padEnd( + nameWidth, + )} ${normalized} ${confidenceIntervalLabel} ${lane}`, + ); + } +} + +/** + * Two-gate comparison per task: + * + * 1. statistical -- is the change distinguishable from sampling error? Each + * run's normalized value carries a 95% confidence interval (normalizedRelativeMarginOfErrorPercent). + * The change is significant only when the two intervals do not overlap, + * i.e. |delta| exceeds the "noise floor" (the sum of both CI half-widths, + * expressed in percent of the baseline value). This is conservative: when + * in doubt it says "~noise". + * 2. practical -- a real change must also exceed `threshold` percent before + * it is labeled REGRESSION/improved; real-but-small changes report as + * "within threshold". + */ +function compare( + baseline: BaselineFile, + current: RunResults, + { + strict, + threshold, + filter, + runs, + }: { + strict: boolean; + threshold: number; + filter: string | null; + runs: number; + }, +): void { + if (baseline.schemaVersion !== BASELINE_SCHEMA_VERSION) { + console.error( + `baseline.json has schemaVersion ${baseline.schemaVersion}; expected ${BASELINE_SCHEMA_VERSION}. Regenerate it with \`npm run benchmark:update\`.`, + ); + process.exit(1); + } + const regressions: string[] = []; + let insensitive = 0; + console.log(""); + console.log( + `compare vs baseline from ${baseline.generatedAt} (node ${baseline.node})`, + ); + explain([ + "each lane = 95% CI of normalized speed (\u25cf mean) \u2014 further right is faster/better", + "overlapping lanes \u2192 no provable change (~noise)", + `\u0394 = % vs baseline (positive is better); must beat noise \u00b1 and the ${threshold}% threshold for a verdict`, + "noise \u00b1 = smallest provable change \u2014 smaller is sharper (raise --runs to sharpen)", + "\u00b1 per lane = that side's CI width: current much narrower than baseline = consistency improved (hint, not a verdict)", + ]); + console.log( + ` \u2022 current lane color: ${red("regression")} \u00b7 ${green( + "improved", + )} \u00b7 ${yellow("within threshold")} \u00b7 ${cyan("~noise")}; ${magenta( + "magenta \u00b1", + )} = noise floor above threshold`, + ); + if (baseline.runs !== undefined && baseline.runs !== runs) { + console.log( + yellow( + ` \u2022 run counts differ (baseline ${baseline.runs}, current ${runs}) \u2014 \u00b1 widths partly reflect sampling effort; the consistency hint corrects for this, the raw \u00b1 columns do not`, + ), + ); + } + console.log(""); + for (const [name, base] of Object.entries(baseline.tasks)) { + if (name === REFERENCE_TASK) continue; + // a filtered run intentionally skips tasks; don't report those as removed + if (filter !== null && !name.includes(filter)) continue; + const task = current.tasks[name]; + if ( + !task || + task.normalized === undefined || + task.normalizedRelativeMarginOfErrorPercent === undefined + ) { + console.log(`${verdictLabel(Verdicts.removed)} ${name}`); + continue; + } + const deltaPercent = (task.normalized / base.normalized - 1) * 100; + const baseHalfWidth = + base.normalized * (base.normalizedRelativeMarginOfErrorPercent / 100); + const currentHalfWidth = + task.normalized * (task.normalizedRelativeMarginOfErrorPercent / 100); + const noiseFloorPercent = + ((baseHalfWidth + currentHalfWidth) / base.normalized) * 100; + const significant = Math.abs(deltaPercent) > noiseFloorPercent; + const material = Math.abs(deltaPercent) >= threshold; + const verdict: Verdict = !significant + ? Verdicts.noise + : !material + ? Verdicts.within_threshold + : deltaPercent < 0 + ? Verdicts.regression + : Verdicts.improved; + if (noiseFloorPercent > threshold) { + insensitive++; + } + if (verdict === Verdicts.regression) { + regressions.push(name); + } + const baseInterval = intervalOf( + base.normalized, + base.normalizedRelativeMarginOfErrorPercent, + ); + const currentInterval = intervalOf( + task.normalized, + task.normalizedRelativeMarginOfErrorPercent, + ); + const scaleLowerBound = Math.min( + baseInterval.lowerBound, + currentInterval.lowerBound, + ); + const scaleUpperBound = Math.max( + baseInterval.upperBound, + currentInterval.upperBound, + ); + const margin = + (scaleUpperBound - scaleLowerBound) * 0.05 || scaleUpperBound * 0.01 || 1; + const emphasis = VerdictEmphasisPaint[verdict]; + const deltaText = emphasis( + `\u0394 ${deltaPercent >= 0 ? "+" : ""}${deltaPercent.toFixed(1)}%`, + ); + // magenta marks the abnormal case: the comparison itself is too blunt to + // detect threshold-sized changes for this task + const noisePaint = noiseFloorPercent > threshold ? magenta : dim; + const noiseText = noisePaint( + `vs noise \u00b1${noiseFloorPercent.toFixed(1)}%`, + ); + // consistency hint: a much narrower/wider current CI than the baseline's + // means the change altered run-to-run variability, not just the mean. + // CI width = t(n-1)\u00b7sd/\u221an, so each side is converted back to its + // underlying spread (sd \u221d width\u00b7\u221an/t) before comparing \u2014 otherwise a + // higher --runs on one side fakes a consistency change. Spread estimates + // from a handful of runs are themselves noisy, so only call it out + // beyond a conservative ratio (\u2248 what an F-test needs at 5 runs). + // Requires the per-side run count, and CONSISTENCY_MINIMUM_RUNS runs (at 1 + // run the \u00b1 is a within-run margin, not a between-run spread; at 2 the + // spread estimate has a single degree of freedom and swings wildly). + let consistencyText = ""; + if ( + baseline.runs !== undefined && + baseline.runs >= CONSISTENCY_MINIMUM_RUNS && + runs >= CONSISTENCY_MINIMUM_RUNS && + base.normalizedRelativeMarginOfErrorPercent > 0 && + task.normalizedRelativeMarginOfErrorPercent > 0 + ) { + const confidenceIntervalWidthToSpread = ( + relativeMarginOfErrorPercent: number, + runCount: number, + ) => + (relativeMarginOfErrorPercent * Math.sqrt(runCount)) / + studentTCriticalValue95(runCount - 1); + const ratio = + confidenceIntervalWidthToSpread( + base.normalizedRelativeMarginOfErrorPercent, + baseline.runs, + ) / + confidenceIntervalWidthToSpread( + task.normalizedRelativeMarginOfErrorPercent, + runs, + ); + if (ratio >= CONSISTENCY_NOTE_RATIO) { + consistencyText = ` ${green( + `\u00b7 consistency \u00d7${ratio.toFixed(1)} steadier`, + )}`; + } else if (ratio <= 1 / CONSISTENCY_NOTE_RATIO) { + consistencyText = ` ${red( + `\u00b7 consistency \u00d7${(1 / ratio).toFixed(1)} noisier`, + )}`; + } + } + console.log( + `${verdictLabel( + verdict, + )} ${name} ${deltaText} ${noiseText}${consistencyText}`, + ); + console.log( + `${LANE_INDENT}baseline ${base.normalized + .toFixed(4) + .padStart(10)} ${stabilityPaint( + base.normalizedRelativeMarginOfErrorPercent, + )( + `\u00b1${base.normalizedRelativeMarginOfErrorPercent.toFixed( + 1, + )}%`.padStart(7), + )} ${renderLane( + baseInterval, + scaleLowerBound - margin, + scaleUpperBound + margin, + dim, + )}`, + ); + console.log( + `${LANE_INDENT}current ${task.normalized + .toFixed(4) + .padStart(10)} ${stabilityPaint( + task.normalizedRelativeMarginOfErrorPercent, + )( + `\u00b1${task.normalizedRelativeMarginOfErrorPercent.toFixed( + 1, + )}%`.padStart(7), + )} ${renderLane( + currentInterval, + scaleLowerBound - margin, + scaleUpperBound + margin, + emphasis, + )}`, + ); + console.log(""); + } + for (const name of Object.keys(current.tasks)) { + if (name !== REFERENCE_TASK && !baseline.tasks[name]) { + console.log(`${verdictLabel(Verdicts.added)} ${name}`); + } + } + // Normalization absorbs *uniform* machine slowdown, but drift is rarely + // uniform (GC/async-heavy tasks throttle harder), so large reference drift + // means conditions changed between runs and borderline verdicts are suspect. + if ( + baseline.referenceOperationsPerSecond && + current.referenceOperationsPerSecond + ) { + const driftPercent = + (current.referenceOperationsPerSecond / + baseline.referenceOperationsPerSecond - + 1) * + 100; + const driftPaint = + Math.abs(driftPercent) <= DRIFT_ACCEPTABLE_PERCENT + ? green + : Math.abs(driftPercent) <= DRIFT_CAUTION_PERCENT + ? yellow + : magenta; + console.log( + `Reference-task raw throughput drift since baseline: ${driftPaint( + `${driftPercent >= 0 ? "+" : ""}${driftPercent.toFixed(1)}%`, + )}.`, + ); + if (Math.abs(driftPercent) > DRIFT_CAUTION_PERCENT) { + console.warn( + magenta( + "Machine conditions differ noticeably from the baseline run (thermal throttling, background load, or a different machine). Treat borderline verdicts with suspicion; re-run when idle or regenerate the baseline.", + ), + ); + } + } + if (insensitive > 0) { + console.warn( + magenta( + `Note: ${insensitive} task(s) have a noise floor above ${threshold}% -- a change of that size could hide in noise there. Increase --runs on both sides (and regenerate the baseline) for more sensitivity.`, + ), + ); + } + if (regressions.length) { + console.error( + bold( + red(`${regressions.length} regression(s): ${regressions.join(", ")}.`), + ), + ); + if (strict) process.exit(1); + } else { + console.log(green("No regressions.")); + } +} + +// two-sided 95% critical values of Student's t, indexed by degrees of freedom +const STUDENT_T_CRITICAL_VALUES_95_TWO_SIDED = [ + 12.71, 4.3, 3.18, 2.78, 2.57, 2.45, 2.36, 2.31, 2.26, 2.23, 2.2, 2.18, 2.16, + 2.14, 2.13, +]; +function studentTCriticalValue95(degreesOfFreedom: number): number { + return degreesOfFreedom <= 0 + ? Infinity + : degreesOfFreedom <= STUDENT_T_CRITICAL_VALUES_95_TWO_SIDED.length + ? STUDENT_T_CRITICAL_VALUES_95_TWO_SIDED[degreesOfFreedom - 1] + : 1.96; +} + +function average(values: number[]): number { + return values.reduce((sum, value) => sum + value, 0) / values.length; +} + +/** + * Collapse N independent runs into one result. For N > 1, each task's + * normalized value is averaged across runs and its margin of error is + * computed from the BETWEEN-run spread (t-based 95% CI of the mean). This + * captures variance that within-run sampling cannot see — GC phasing, JIT + * state, thermal drift — which dominates for long async tasks. For N = 1 the + * within-run estimate from collectResults is kept as-is. + */ +function aggregateRuns(runs: RunResults[]): RunResults { + if (runs.length === 1) { + return runs[0]; + } + // collectResults aborts the process on any task failure, so every run + // contains every task + const runCount = runs.length; + const tasks: Record = {}; + for (const name of Object.keys(runs[0].tasks)) { + const perRun = runs.map((run) => run.tasks[name]); + const normalizedValues = perRun.map((task) => task.normalized ?? 0); + const normalizedMean = average(normalizedValues); + const variance = + normalizedValues.reduce( + (sum, value) => sum + (value - normalizedMean) ** 2, + 0, + ) / + (runCount - 1); + const confidenceIntervalHalfWidth = + studentTCriticalValue95(runCount - 1) * Math.sqrt(variance / runCount); + tasks[name] = { + operationsPerSecond: average( + perRun.map((task) => task.operationsPerSecond), + ), + mean: average(perRun.map((task) => task.mean)), + percentile99: average(perRun.map((task) => task.percentile99)), + relativeMarginOfErrorPercent: average( + perRun.map((task) => task.relativeMarginOfErrorPercent), + ), + samples: perRun.reduce((sum, task) => sum + task.samples, 0), + normalized: normalizedMean, + normalizedRelativeMarginOfErrorPercent: + normalizedMean > 0 + ? (confidenceIntervalHalfWidth / normalizedMean) * 100 + : 0, + }; + } + return { + referenceOperationsPerSecond: average( + runs.map((run) => run.referenceOperationsPerSecond ?? 0), + ), + tasks, + }; +} + +async function main(): Promise { + const commandLineArguments = parseCommandLineArguments(process.argv); + let entries = loadScenarios(); + if (commandLineArguments.filter !== null) { + const filter = commandLineArguments.filter; + entries = entries.filter( + (entry) => entry.name === REFERENCE_TASK || entry.name.includes(filter), + ); + } + + const allRuns: RunResults[] = []; + for (let run = 0; run < commandLineArguments.runs; run++) { + const benchmarkSuite = new Bench({ + time: commandLineArguments.time, + warmupTime: WARMUP_TIME_MILLISECONDS, + }); + for (const { name, fn, options } of entries) { + benchmarkSuite.add(name, fn, options); + } + await benchmarkSuite.warmup(); + await benchmarkSuite.run(); + allRuns.push(collectResults(benchmarkSuite)); + if (commandLineArguments.runs > 1) { + console.error(`run ${run + 1}/${commandLineArguments.runs} complete`); + } + if (!commandLineArguments.json && run === commandLineArguments.runs - 1) { + console.table(benchmarkSuite.table()); + } + } + + const current = aggregateRuns(allRuns); + + if (!commandLineArguments.json) { + renderThroughputBars(current.tasks); + if (commandLineArguments.runs > 1) { + renderRunSpread(allRuns, current); + } + } + + if (commandLineArguments.json) { + console.log( + JSON.stringify( + { + generatedAt: new Date().toISOString(), + node: process.version, + runs: commandLineArguments.runs, + referenceOperationsPerSecond: current.referenceOperationsPerSecond, + tasks: current.tasks, + }, + null, + 2, + ), + ); + } + + if (commandLineArguments.updateBaseline) { + const baseline: BaselineFile = { + schemaVersion: BASELINE_SCHEMA_VERSION, + generatedAt: new Date().toISOString(), + node: process.version, + runs: commandLineArguments.runs, + referenceOperationsPerSecond: current.referenceOperationsPerSecond, + tasks: {}, + }; + let noisy = 0; + for (const [name, task] of Object.entries(current.tasks)) { + baseline.tasks[name] = { + operationsPerSecond: task.operationsPerSecond, + mean: task.mean, + normalized: task.normalized ?? 0, + normalizedRelativeMarginOfErrorPercent: + task.normalizedRelativeMarginOfErrorPercent ?? 0, + samples: task.samples, + }; + if ( + name !== REFERENCE_TASK && + (task.normalizedRelativeMarginOfErrorPercent ?? 0) > + commandLineArguments.threshold + ) { + noisy++; + } + } + fs.writeFileSync(BASELINE_PATH, `${JSON.stringify(baseline, null, 2)}\n`); + console.log( + `Baseline written to ${path.relative(process.cwd(), BASELINE_PATH)}`, + ); + if (noisy > 0) { + console.warn( + `Note: ${noisy} task(s) were captured with a margin of error above ${commandLineArguments.threshold}% — comparisons against them will have a high noise floor. Consider regenerating with more --runs.`, + ); + } + } + + if (commandLineArguments.compare) { + if (!fs.existsSync(BASELINE_PATH)) { + console.error( + "No baseline.json found — run `npm run benchmark:update` first.", + ); + process.exit(1); + } + const baseline: BaselineFile = JSON.parse( + fs.readFileSync(BASELINE_PATH, "utf8"), + ); + compare(baseline, current, { + strict: commandLineArguments.strict, + threshold: commandLineArguments.threshold, + filter: commandLineArguments.filter, + runs: commandLineArguments.runs, + }); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/benchmark/scenarios/batch-get-format.benchmark.ts b/benchmark/scenarios/batch-get-format.benchmark.ts new file mode 100644 index 00000000..2e5cbb5a --- /dev/null +++ b/benchmark/scenarios/batch-get-format.benchmark.ts @@ -0,0 +1,38 @@ +import type { ScenarioEntry } from "../run"; +import { makeMockV2Client, StoredItem } from "../../test/fixtures/mock-client"; +import { + table, + makeFixtureEntity, + makeStoredItem, + makeItemData, +} from "../../test/fixtures/entities"; + +const entity = makeFixtureEntity(); + +function makeBatch(count: number) { + const Responses: Record = { [table]: [] }; + const keys: { org: string; id: string }[] = []; + for (let i = 0; i < count; i++) { + Responses[table].push(makeStoredItem(entity, i)); + const { org, id } = makeItemData(i); + keys.push({ org, id }); + } + const { client, calls } = makeMockV2Client({ + batchGet: { Responses, UnprocessedKeys: {} }, + }); + return { client, calls, keys }; +} + +const batch100 = makeBatch(100); + +const scenarios: ScenarioEntry[] = [ + { + name: "batch-get-format/100-items", + fn: async () => { + batch100.calls.length = 0; // keep the mock's call log from growing + return entity.get(batch100.keys).go({ client: batch100.client }); + }, + }, +]; + +export default scenarios; diff --git a/benchmark/scenarios/entity-construction.benchmark.ts b/benchmark/scenarios/entity-construction.benchmark.ts new file mode 100644 index 00000000..2705fb33 --- /dev/null +++ b/benchmark/scenarios/entity-construction.benchmark.ts @@ -0,0 +1,20 @@ +import type { ScenarioEntry } from "../run"; +import { table, makeFixtureModel } from "../../test/fixtures/entities"; + +const { Entity } = require("../../src/entity"); + +const model = makeFixtureModel(); +const watcherModel = makeFixtureModel({ withWatchers: true }); + +const scenarios: ScenarioEntry[] = [ + { + name: "entity-construction/new-entity", + fn: () => new Entity(model, { table }), + }, + { + name: "entity-construction/new-entity-watchers", + fn: () => new Entity(watcherModel, { table }), + }, +]; + +export default scenarios; diff --git a/benchmark/scenarios/params-chain.benchmark.ts b/benchmark/scenarios/params-chain.benchmark.ts new file mode 100644 index 00000000..ed32311c --- /dev/null +++ b/benchmark/scenarios/params-chain.benchmark.ts @@ -0,0 +1,36 @@ +import type { ScenarioEntry } from "../run"; +import { makeFixtureEntity } from "../../test/fixtures/entities"; + +const entity = makeFixtureEntity(); +const key = { org: "org1", id: "id1" }; + +const scenarios: ScenarioEntry[] = [ + { + name: "params-chain/get", + fn: () => entity.get(key).params(), + }, + { + name: "params-chain/query", + fn: () => entity.query.records({ org: "org1" }).params(), + }, + { + name: "params-chain/query-where", + fn: () => + entity.query + .records({ org: "org1" }) + .where((attributes: any, operations: any) => + operations.gt(attributes.count, 10), + ) + .params(), + }, + { + name: "params-chain/update-set", + fn: () => entity.update(key).set({ name: "x" }).params(), + }, + { + name: "params-chain/upsert", + fn: () => entity.upsert({ ...key, name: "x", count: 1 }).params(), + }, +]; + +export default scenarios; diff --git a/benchmark/scenarios/parse-format.benchmark.ts b/benchmark/scenarios/parse-format.benchmark.ts new file mode 100644 index 00000000..d298206b --- /dev/null +++ b/benchmark/scenarios/parse-format.benchmark.ts @@ -0,0 +1,38 @@ +import type { ScenarioEntry } from "../run"; +import type { StoredItem } from "../../test/fixtures/mock-client"; +import { + makeFixtureEntity, + makeStoredItem, +} from "../../test/fixtures/entities"; + +const entity = makeFixtureEntity(); +const watcherEntity = makeFixtureEntity({ withWatchers: true }); + +function makeItems(owner: any, count: number): StoredItem[] { + const Items: StoredItem[] = []; + for (let i = 0; i < count; i++) { + Items.push(makeStoredItem(owner, i)); + } + return Items; +} + +const items100 = makeItems(entity, 100); +const items1000 = makeItems(entity, 1000); +const watcherItems1000 = makeItems(watcherEntity, 1000); + +const scenarios: ScenarioEntry[] = [ + { + name: "parse-format/100-items", + fn: () => entity.parse({ Items: items100 }), + }, + { + name: "parse-format/1000-items", + fn: () => entity.parse({ Items: items1000 }), + }, + { + name: "parse-format/1000-items-watchers", + fn: () => watcherEntity.parse({ Items: watcherItems1000 }), + }, +]; + +export default scenarios; diff --git a/benchmark/scenarios/query-pagination.benchmark.ts b/benchmark/scenarios/query-pagination.benchmark.ts new file mode 100644 index 00000000..60dd913a --- /dev/null +++ b/benchmark/scenarios/query-pagination.benchmark.ts @@ -0,0 +1,50 @@ +import type { ScenarioEntry } from "../run"; +import { + makeMockV2Client, + makePagingQueryHandler, + MockV2Client, +} from "../../test/fixtures/mock-client"; +import { + makeFixtureEntity, + makeStoredItem, +} from "../../test/fixtures/entities"; + +const entity = makeFixtureEntity(); + +function makePagingClient({ + pages, + perPage, +}: { + pages: number; + perPage: number; +}): MockV2Client { + const query = makePagingQueryHandler({ + pages, + perPage, + makeItem: (i) => makeStoredItem(entity, i), + }); + return makeMockV2Client({ query }); +} + +const mock10 = makePagingClient({ pages: 10, perPage: 100 }); +const mock100 = makePagingClient({ pages: 100, perPage: 100 }); + +function runAllPages(mock: MockV2Client) { + mock.calls.length = 0; // keep the mock's call log from growing + return entity.query + .records({ org: "org1" }) + .go({ client: mock.client, pages: "all" }); +} + +const scenarios: ScenarioEntry[] = [ + { + name: "query-pagination/10-pages-x100", + fn: async () => runAllPages(mock10), + }, + { + name: "query-pagination/100-pages-x100", + fn: async () => runAllPages(mock100), + }, +]; + +export default scenarios; diff --git a/package-lock.json b/package-lock.json index e4fa0761..76ffa4d1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,7 @@ "prettier-plugin-astro": "^0.12.0", "source-map-support": "^0.5.19", "sst": "^2.28.0", + "tinybench": "^2.9.0", "ts-node": "^10.9.1", "tsd": "^0.28.1", "typescript": "^5.2.2", @@ -29437,6 +29438,13 @@ "node": ">=0.6.0" } }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, "node_modules/tmpl": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.4.tgz", @@ -54336,6 +54344,12 @@ "process": "~0.11.0" } }, + "tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true + }, "tmpl": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.4.tgz", diff --git a/package.json b/package.json index cae239b6..569c21e4 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,10 @@ "local:stop": "npm run ddb:stop", "local:exec": "LOCAL_DYNAMO_ENDPOINT='http://localhost:8000' ts-node ./test/debug.ts", "local:debug": "npm run local:start && npm run local:exec", + "benchmark": "ts-node ./benchmark/run.ts", + "benchmark:json": "ts-node ./benchmark/run.ts --json", + "benchmark:compare": "ts-node ./benchmark/run.ts --compare", + "benchmark:update": "ts-node ./benchmark/run.ts --update-baseline", "format": "prettier -w src/**/*.js examples/**/* --log-level=error" }, "repository": { @@ -70,6 +74,7 @@ "prettier-plugin-astro": "^0.12.0", "source-map-support": "^0.5.19", "sst": "^2.28.0", + "tinybench": "^2.9.0", "ts-node": "^10.9.1", "tsd": "^0.28.1", "typescript": "^5.2.2", diff --git a/src/clauses.js b/src/clauses.js index 1c9c1507..23b46698 100644 --- a/src/clauses.js +++ b/src/clauses.js @@ -1392,6 +1392,11 @@ class ChainState { parentState = null, } = {}) { const update = new UpdateExpression({ prefix: "_u" }); + // AttributeOperationProxy defines a property per attribute and per + // operation, which makes it the dominant chain-construction cost; read + // chains (get/query/scan) never touch it, so it is built lazily and + // cached so write clauses keep accumulating into one instance + let updateProxy; this.parentState = parentState; this.error = null; this.attributes = attributes; @@ -1402,11 +1407,16 @@ class ChainState { method: "", facets: { ...compositeAttributes }, update, - updateProxy: new AttributeOperationProxy({ - builder: update, - attributes: attributes, - operations: UpdateOperations, - }), + get updateProxy() { + updateProxy = + updateProxy || + new AttributeOperationProxy({ + builder: update, + attributes: attributes, + operations: UpdateOperations, + }); + return updateProxy; + }, put: { data: {}, }, diff --git a/src/entity.js b/src/entity.js index 0bdd109c..9b6072c2 100644 --- a/src/entity.js +++ b/src/entity.js @@ -753,12 +753,14 @@ class Entity { config, ); items = hydrated.data; - hydratedUnprocessed = hydratedUnprocessed.concat( - hydrated.unprocessed, - ); + for (const unprocessed of hydrated.unprocessed) { + hydratedUnprocessed.push(unprocessed); + } + } + const entityResults = (results[entity] = results[entity] || []); + for (const item of items) { + entityResults.push(item); } - results[entity] = results[entity] || []; - results[entity] = [...results[entity], ...items]; } } else if (Array.isArray(response.data)) { let prevCount = count; @@ -777,11 +779,13 @@ class Entity { config, ); items = hydrated.data; - hydratedUnprocessed = hydratedUnprocessed.concat( - hydrated.unprocessed, - ); + for (const unprocessed of hydrated.unprocessed) { + hydratedUnprocessed.push(unprocessed); + } + } + for (const item of items) { + results.push(item); } - results = [...results, ...items]; if (moreItemsThanRequired || count === config.count) { const lastItem = results[results.length - 1]; ExclusiveStartKey = this._fromCompositeToKeysByIndex({ @@ -989,10 +993,6 @@ class Entity { } formatResponse(response, index, config = {}) { - let stackTrace; - if (!config.originalErr) { - stackTrace = new e.ElectroError(e.ErrorCodes.AWSError); - } try { let results = {}; if (validations.isFunction(config.parse)) { @@ -1058,13 +1058,12 @@ class Entity { return { data: results }; } catch (err) { - if ( - config.originalErr || - stackTrace === undefined || - err.isElectroError - ) { + if (config.originalErr || err.isElectroError) { throw err; } else { + // built only on the throw path; formatResponse is synchronous so the + // stack captured here matches what eager construction produced + const stackTrace = new e.ElectroError(e.ErrorCodes.AWSError); stackTrace.message = `Error thrown by DynamoDB client: "${err.message}" - For more detail on this error reference: https://electrodb.dev/en/reference/errors/#aws-error`; stackTrace.cause = err; throw stackTrace; @@ -2119,7 +2118,6 @@ class Entity { update = {}, filter = {}, upsert, - updateProxy, } = state.query; let consolidatedQueryFacets = this._consolidateQueryFacets(keys.sk); let params = {}; @@ -2134,8 +2132,10 @@ class Entity { ); break; case MethodTypes.upsert: + // updateProxy is read inside the case (not destructured above) so + // read methods never trigger its lazy construction params = this._makeUpsertParams( - { update, upsert, updateProxy }, + { update, upsert, updateProxy: state.query.updateProxy }, keys.pk, ...keys.sk, ); diff --git a/src/schema.js b/src/schema.js index 67f99731..9d5e1ec0 100644 --- a/src/schema.js +++ b/src/schema.js @@ -1656,9 +1656,13 @@ class Schema { _applyAttributeMutation(method, include, avoid, payload) { let data = { ...payload }; - const getSiblings = () => ({ ...payload }); + // one read-only snapshot shared across the pass; `payload` is never + // mutated inside the loop (mutations land on `data`), so building it + // lazily on first use returns the same values a per-call copy would + let siblings; + const getSiblings = () => (siblings = siblings || { ...payload }); for (let path of Object.keys(include)) { - // this.attributes[attribute] !== undefined | Attribute exists as actual attribute. If `includeKeys` is turned on for example this will include values that do not have a presence in the model and therefore will not have a `.get()` method + // this.attributes[attribute] !== undefined | Attribute exists as an actual attribute. If `includeKeys` is turned on for example this will include values that do not have a presence in the model and therefore will not have a `.get()` method // avoid[attribute] === undefined | Attribute shouldn't be in the avoided const attribute = this.getAttribute(path); if (attribute !== undefined && avoid[path] === undefined) { diff --git a/src/util.js b/src/util.js index 1e53218d..c1a8c03e 100644 --- a/src/util.js +++ b/src/util.js @@ -11,24 +11,31 @@ function parseJSONPath(path = "") { } function genericizeJSONPath(path = "") { + // the regex only rewrites `[digits]` segments; skip it for the common + // bracket-free attribute paths resolved on every getter/setter pass + if (path.indexOf("[") === -1) { + return path; + } return path.replace(/\[\d+\]/g, "[*]"); } function getInstanceType(instance = {}) { - let [isModel, errors] = v.testModel(instance); + // check the cheap instance symbols before falling back to testModel, which + // runs full jsonschema validation just to be discarded for non-models if (!instance || Object.keys(instance).length === 0) { return ""; - } else if (isModel) { - return t.ElectroInstanceTypes.model; } else if (instance._instance === t.ElectroInstance.entity) { return t.ElectroInstanceTypes.entity; } else if (instance._instance === t.ElectroInstance.service) { return t.ElectroInstanceTypes.service; } else if (instance._instance === t.ElectroInstance.electro) { return t.ElectroInstanceTypes.electro; - } else { - return ""; } + let [isModel] = v.testModel(instance); + if (isModel) { + return t.ElectroInstanceTypes.model; + } + return ""; } function getModelVersion(model = {}) { diff --git a/src/validations.js b/src/validations.js index 54a5e30d..74b9c528 100644 --- a/src/validations.js +++ b/src/validations.js @@ -284,8 +284,13 @@ v.addSchema(Modelv1, "/Modelv1"); function validateModel(model = {}) { /** start beta/v1 condition **/ - let betaErrors = v.validate(model, ModelBeta).errors; - if (betaErrors.length) { + // ModelBeta requires a root `entity` string; without one that schema can + // only fail, so modern (v1-shaped) models skip straight to Modelv1 + // validation instead of enumerating-and-discarding beta errors on every + // construction. Any model that fails (or skips) the beta pass throws the + // Modelv1 errors, exactly as before. + const couldBeBeta = !!model && isStringHasLength(model.entity); + if (!couldBeBeta || v.validate(model, ModelBeta).errors.length) { /** end/v1 condition **/ let errors = v.validate(model, Modelv1).errors; if (errors.length) { diff --git a/test/fixtures/entities.ts b/test/fixtures/entities.ts new file mode 100644 index 00000000..dc335d01 --- /dev/null +++ b/test/fixtures/entities.ts @@ -0,0 +1,108 @@ +/** + * Shared entity fixtures for offline tests and benchmarks. A representative + * single-index entity exercising the common attribute types, plus helpers to + * produce from-DynamoDB-shaped stored items for feeding `parse`/ + * `formatResponse` without a database. + */ +import type { StoredItem } from "./mock-client"; + +// internals are untyped; required (not imported) per repo test convention +const { Entity } = require("../../src/entity"); + +export const table = "electro"; + +export interface FixtureModelOptions { + entity?: string; + service?: string; + withWatchers?: boolean; +} + +export interface FixtureItemData { + org: string; + id: string; + name: string; + count: number; + active: boolean; + tags: string[]; + profile: { nickname: string; age: number }; + notes: { body: string }[]; +} + +export function makeFixtureModel({ + entity = "perfFixture", + service = "perfService", + withWatchers = false, +}: FixtureModelOptions = {}): Record { + const attributes: Record = { + org: { type: "string" }, + id: { type: "string" }, + name: { type: "string" }, + count: { type: "number" }, + active: { type: "boolean" }, + tags: { type: "set", items: "string" }, + profile: { + type: "map", + properties: { + nickname: { type: "string" }, + age: { type: "number" }, + }, + }, + notes: { + type: "list", + items: { + type: "map", + properties: { + body: { type: "string" }, + }, + }, + }, + }; + if (withWatchers) { + attributes.displayName = { + type: "string", + watch: ["name"], + set: (_: unknown, item: Record) => + typeof item.name === "string" ? item.name.toUpperCase() : undefined, + }; + } + return { + model: { entity, service, version: "1" }, + table, + attributes, + indexes: { + records: { + pk: { field: "pk", composite: ["org"] }, + sk: { field: "sk", composite: ["id"] }, + }, + }, + }; +} + +export function makeFixtureEntity(options: FixtureModelOptions = {}): any { + return new Entity(makeFixtureModel(options), { table }); +} + +export function makeItemData(i: number): FixtureItemData { + return { + org: "org1", + id: `id${String(i).padStart(8, "0")}`, + name: `name${i}`, + count: i, + active: i % 2 === 0, + tags: [`tag${i % 5}`, "common"], + profile: { nickname: `nick${i}`, age: 20 + (i % 50) }, + notes: [{ body: `note${i}` }], + }; +} + +// Builds the i-th item exactly as it would be stored in DynamoDB (keys, +// `__edb_e__`/`__edb_v__` identifiers, field names) by reusing the entity's +// own put-params formatting, then normalizing set attributes to plain arrays +// (the shape a v3 DocumentClient unmarshalls to). +export function makeStoredItem(entity: any, i: number): StoredItem { + const { Item } = entity.put(makeItemData(i)).params(); + if (Item.tags && Array.isArray(Item.tags.values)) { + Item.tags = [...Item.tags.values]; + } + return Item; +} diff --git a/test/offline.audit-fixes.spec.js b/test/offline.audit-fixes.spec.js new file mode 100644 index 00000000..72f6e22e --- /dev/null +++ b/test/offline.audit-fixes.spec.js @@ -0,0 +1,294 @@ +/** + * Regression tests for the "stage 1" batch of audit fixes. + * + * Each describe block documents the bug it pins, and is written to FAIL against + * the pre-fix source and PASS once the corresponding fix is applied. + */ +const { Entity } = require("../src/entity"); +const { + cleanseTransactionData, + cleanseCanceledData, + createWriteTransaction, +} = require("../src/transaction"); +const { DynamoDBSet } = require("../src/set"); +const { TableIndex } = require("../src/types"); +const { expect } = require("chai"); +const { makeMockV2Client } = require("./fixtures/mock-client"); + +const table = "electro"; + +// --------------------------------------------------------------------------- +// B1: `unprocessed: "raw"` execution option is dropped due to a property typo +// (`config.unproessed`) in `_normalizeExecutionOptions`. +// --------------------------------------------------------------------------- +describe("B1: unprocessed:'raw' execution option", () => { + const MallStores = new Entity( + { + model: { service: "BugBeater", entity: "TEST_ENTITY", version: "1" }, + table, + attributes: { + id: { type: "string", field: "storeLocationId" }, + sector: { type: "string" }, + }, + indexes: { + store: { + pk: { field: "pk", facets: ["sector"] }, + sk: { field: "sk", facets: ["id"] }, + }, + }, + }, + {}, + ); + + const rawKey = { + pk: "$bugbeater#sector_a1", + sk: "$test_entity_1#id_abc", + }; + const cannedBatchGet = { + Responses: { electro: [] }, + UnprocessedKeys: { electro: { Keys: [rawKey] } }, + }; + + it("normalizes onto config.unprocessed (not a typo'd property)", () => { + const config = MallStores._normalizeExecutionOptions({ + provided: [{ unprocessed: "raw" }], + }); + expect(config.unprocessed).to.equal("raw"); + expect(config).to.not.have.property("unproessed"); + }); + + it("returns raw DynamoDB keys when unprocessed:'raw'", async () => { + const { client } = makeMockV2Client({ batchGet: cannedBatchGet }); + const res = await MallStores.get([{ sector: "a1", id: "abc" }]).go({ + client, + unprocessed: "raw", + }); + expect(res.unprocessed).to.deep.equal([rawKey]); + }); + + it("still returns deconstructed composite attributes by default", async () => { + const { client } = makeMockV2Client({ batchGet: cannedBatchGet }); + const res = await MallStores.get([{ sector: "a1", id: "abc" }]).go({ + client, + }); + expect(res.unprocessed).to.deep.equal([{ sector: "a1", id: "abc" }]); + }); +}); + +// --------------------------------------------------------------------------- +// S3: Hidden Set attributes nested inside a map leak on read because +// SetAttribute._makeGet lacks the `if (this.hidden) return;` guard that the +// string/map/list getters have. +// --------------------------------------------------------------------------- +describe("S3: hidden nested Set attributes on read", () => { + const HiddenSet = new Entity({ + model: { service: "audit", entity: "hiddenset", version: "1" }, + table, + attributes: { + id: { type: "string" }, + secretRootSet: { type: "set", items: "string", hidden: true }, + data: { + type: "map", + properties: { + secretSet: { type: "set", items: "string", hidden: true }, + secretStr: { type: "string", hidden: true }, + visible: { type: "string" }, + }, + }, + }, + indexes: { + primary: { + pk: { field: "pk", composite: ["id"] }, + sk: { field: "sk", composite: [] }, + }, + }, + }); + + it("strips a hidden Set nested inside a map", () => { + const result = HiddenSet.parse({ + Item: { + pk: "$audit#id_1", + sk: "$hiddenset_1", + id: "1", + secretRootSet: ["x", "y"], + data: { secretSet: ["a", "b"], secretStr: "shh", visible: "ok" }, + __edb_e__: "hiddenset", + __edb_v__: "1", + }, + }).data; + expect(result.data).to.deep.equal({ visible: "ok" }); + expect(result.data).to.not.have.property("secretSet"); + // root-level hidden set should also be absent (already true pre-fix) + expect(result).to.not.have.property("secretRootSet"); + }); +}); + +// --------------------------------------------------------------------------- +// set.js: (1) the `Invalid Set type` Error is constructed but never thrown, and +// (2) `enum`-typed set members are not mapped, producing a DynamoDBSet +// with `type: undefined` in client-less `.params()`. +// --------------------------------------------------------------------------- +describe("set.js: DynamoDBSet type handling", () => { + it("throws on an unknown set member type", () => { + expect(() => new DynamoDBSet(["a"], "bogus")).to.throw(/Invalid Set type/); + }); + + it("maps known and enum member types to a DynamoDB set type", () => { + expect(new DynamoDBSet(["a"], "string").type).to.equal("String"); + expect(new DynamoDBSet([1], "number").type).to.equal("Number"); + expect(new DynamoDBSet(["a"], "enum").type).to.equal("String"); + }); + + it("produces a typed set for enum-item sets in client-less params()", () => { + const EnumSet = new Entity({ + model: { service: "audit", entity: "enumset", version: "1" }, + table, + attributes: { + id: { type: "string" }, + tags: { type: "set", items: ["red", "green", "blue"] }, + }, + indexes: { + primary: { + pk: { field: "pk", composite: ["id"] }, + sk: { field: "sk", composite: [] }, + }, + }, + }); + const params = EnumSet.put({ id: "1", tags: ["red", "green"] }).params(); + expect(params.Item.tags.wrapperName).to.equal("Set"); + expect(params.Item.tags.type).to.equal("String"); + }); +}); + +// --------------------------------------------------------------------------- +// WatchAll: an invalid `watch` value should raise an ElectroError, but the +// error message template references an undefined identifier +// (`WatchAll` instead of `AttributeWildCard`), so model construction +// dies with a raw ReferenceError instead. +// --------------------------------------------------------------------------- +describe("WatchAll: invalid watch definition error", () => { + const build = () => + new Entity({ + model: { service: "audit", entity: "watcher", version: "1" }, + table, + attributes: { + id: { type: "string" }, + a: { type: "string", watch: "notvalid" }, + }, + indexes: { + primary: { + pk: { field: "pk", composite: ["id"] }, + sk: { field: "sk", composite: [] }, + }, + }, + }); + + it("throws a descriptive ElectroError, not a ReferenceError", () => { + let err; + try { + build(); + } catch (e) { + err = e; + } + expect(err, "expected model construction to throw").to.exist; + expect(err).to.not.be.an.instanceof(ReferenceError); + expect(err.message).to.match(/array of attribute names/); + }); +}); + +// --------------------------------------------------------------------------- +// B4: Transaction result-handling defects. +// --------------------------------------------------------------------------- +describe("B4: transaction result handling", () => { + const makeEntity = (entity, extraAttr) => + new Entity({ + model: { service: "txtest", entity, version: "1" }, + table, + attributes: { + id: { type: "string" }, + [extraAttr]: { type: "string" }, + }, + indexes: { + primary: { + pk: { field: "pk", composite: ["id"] }, + sk: { field: "sk", composite: [] }, + }, + }, + }); + + const A = makeEntity("alpha", "name"); + const B = makeEntity("bravo", "label"); + const X = makeEntity("xray", "value"); // never joined to the entities map + + const recA = A.put({ id: "a1", name: "alice" }).params().Item; + const recB = B.put({ id: "b1", label: "bob" }).params().Item; + const recX = X.put({ id: "x1", value: "ghost" }).params().Item; + + it("(a) preserves position/size of transactGet results when an item is unmatched", () => { + const results = cleanseTransactionData( + TableIndex, + { a: A, b: B }, + { Items: [recA, recX, recB] }, + {}, + ); + expect(results).to.have.length(3); + expect(results[0].item).to.deep.equal({ id: "a1", name: "alice" }); + expect(results[1].item).to.equal(null); // unmatched -> null placeholder + expect(results[2].item).to.deep.equal({ id: "b1", label: "bob" }); + }); + + it("(b) does not crash on a canceled reason whose Item is unmatched", () => { + let results; + expect(() => { + results = cleanseCanceledData( + TableIndex, + { a: A, b: B }, + { + canceled: [ + { Code: "None", Item: recA }, + { Code: "ConditionalCheckFailed", Item: recX }, + { Code: "None", Item: recB }, + ], + }, + {}, + ); + }).to.not.throw(); + expect(results).to.have.length(3); + expect(results[0].item).to.deep.equal({ id: "a1", name: "alice" }); + expect(results[1]).to.deep.include({ + rejected: true, + code: "ConditionalCheckFailed", + item: null, + }); + expect(results[2].item).to.deep.equal({ id: "b1", label: "bob" }); + }); + + it("(c) does not share a single result object across all slots", async () => { + const { client } = makeMockV2Client({ transactWrite: {} }); + const tx = createWriteTransaction({ alpha: A }, (e) => [ + e.alpha.put({ id: "a1", name: "alice" }).commit(), + e.alpha.put({ id: "a2", name: "amy" }).commit(), + ]); + const result = await tx.go({ client }); + expect(result.data).to.have.length(2); + result.data[0].code = "MUTATED"; + expect(result.data[1].code).to.equal("None"); + }); + + it("(d) does not mutate the caller's options object", async () => { + const { client } = makeMockV2Client({ transactWrite: {} }); + const events = []; + const options = { client, logger: (event) => events.push(event.type) }; + const makeTx = () => + createWriteTransaction({ alpha: A }, (e) => [ + e.alpha.put({ id: "a1", name: "alice" }).commit(), + ]); + await makeTx().go(options); + const afterFirst = events.length; + await makeTx().go(options); + const afterSecond = events.length - afterFirst; + expect(options).to.not.have.property("listeners"); + // logger should fire the same number of times on each identical call + expect(afterSecond).to.equal(afterFirst); + }); +}); diff --git a/test/offline.perf-fixes.spec.ts b/test/offline.perf-fixes.spec.ts new file mode 100644 index 00000000..f4dc45e6 --- /dev/null +++ b/test/offline.perf-fixes.spec.ts @@ -0,0 +1,665 @@ +/** + * Behavioral guard tests for the performance-focused batch of audit fixes + * ("stage 2"). Each describe block pins user-visible behavior that the + * corresponding optimization must preserve; none of these rely on timing. + */ +import { expect } from "chai"; +import { + makeMockV2Client, + makePagingQueryHandler, +} from "./fixtures/mock-client"; +import { + table, + makeFixtureEntity, + makeStoredItem, + makeItemData, +} from "./fixtures/entities"; + +// internals are untyped; required (not imported) per repo test convention +const { Entity } = require("../src/entity"); +const { Service } = require("../src/service"); + +// --------------------------------------------------------------------------- +// P5: validateModel ran the (always-failing) ModelBeta schema pass for every +// modern v1 model before validating Modelv1, and getInstanceType ran full +// jsonschema validation before the cheap `_instance` symbol checks. These +// tests pin the thrown messages byte-for-byte (captured from the pre-fix +// implementation) and the instance-type resolution order. +// --------------------------------------------------------------------------- +describe("P5: model validation short-circuits", () => { + const validations = require("../src/validations"); + const util = require("../src/util"); + const { ElectroInstance, ElectroInstanceTypes } = require("../src/types"); + + // captured verbatim from the pre-fix validateModel implementation + const invalidModelFixtures = [ + { + name: "empty object", + model: {}, + message: + 'instance.model is required, instance requires property "model", instance requires property "attributes", instance requires property "indexes" - For more detail on this error reference: https://electrodb.dev/en/reference/errors/#invalid-model', + }, + { + name: "v1 model missing indexes", + model: { + model: { entity: "e", service: "s", version: "1" }, + attributes: { id: { type: "string" } }, + }, + message: + 'instance requires property "indexes" - For more detail on this error reference: https://electrodb.dev/en/reference/errors/#invalid-model', + }, + { + name: "v1 model with non-function getter", + model: { + model: { entity: "e", service: "s", version: "1" }, + attributes: { id: { type: "string", get: "not-a-function" } }, + indexes: { main: { pk: { field: "pk", composite: ["id"] } } }, + }, + message: + "instance.attributes.id.get must be a function - For more detail on this error reference: https://electrodb.dev/en/reference/errors/#invalid-model", + }, + { + name: "beta-shaped model missing indexes (still throws v1 messages)", + model: { + entity: "e", + service: "s", + attributes: { id: { type: "string" } }, + }, + message: + 'instance.model is required, instance requires property "model", instance requires property "indexes" - For more detail on this error reference: https://electrodb.dev/en/reference/errors/#invalid-model', + }, + { + name: "non-object model namespace", + model: { + model: "nope", + attributes: { id: { type: "string" } }, + indexes: { main: { pk: { field: "pk", composite: ["id"] } } }, + }, + message: + "instance.model is not of a type(s) object - For more detail on this error reference: https://electrodb.dev/en/reference/errors/#invalid-model", + }, + ]; + + for (const { name, model, message } of invalidModelFixtures) { + it(`throws the exact pre-fix message for: ${name}`, () => { + let thrown: any; + try { + validations.model(model); + } catch (err) { + thrown = err; + } + expect(thrown, "expected validateModel to throw").to.not.equal(undefined); + expect(thrown.isElectroError).to.equal(true); + expect(thrown.message).to.equal(message); + }); + } + + it("accepts a valid v1 model and a valid beta model", () => { + const v1Model = { + model: { entity: "e", service: "s", version: "1" }, + attributes: { id: { type: "string" } }, + indexes: { main: { pk: { field: "pk", composite: ["id"] } } }, + }; + const betaModel = { + entity: "e", + service: "s", + version: "1", + attributes: { id: { type: "string" } }, + indexes: { main: { pk: { field: "pk", facets: ["id"] } } }, + }; + expect(() => validations.model(v1Model)).to.not.throw(); + expect(() => validations.model(betaModel)).to.not.throw(); + expect(() => new Entity(v1Model, { table })).to.not.throw(); + }); + + it("resolves instance types via symbols and bare models via validation", () => { + const entity = makeFixtureEntity(); + expect(util.getInstanceType(entity)).to.equal(ElectroInstanceTypes.entity); + expect( + util.getInstanceType({ _instance: ElectroInstance.service }), + ).to.equal(ElectroInstanceTypes.service); + expect( + util.getInstanceType({ _instance: ElectroInstance.electro }), + ).to.equal(ElectroInstanceTypes.electro); + expect( + util.getInstanceType({ + model: { entity: "e", service: "s", version: "1" }, + attributes: { id: { type: "string" } }, + indexes: { main: { pk: { field: "pk", composite: ["id"] } } }, + }), + ).to.equal(ElectroInstanceTypes.model); + expect(util.getInstanceType({})).to.equal(""); + expect(util.getInstanceType(undefined)).to.equal(""); + expect(util.getInstanceType({ anything: "else" })).to.equal(""); + }); +}); + +// --------------------------------------------------------------------------- +// P2: formatResponse eagerly allocated an ElectroError (with stack capture) +// on every call — once per item on batchGet/collection paths — only to +// discard it on success. The error is now built lazily on the throw path; +// these tests pin the wrapping contract, which had no prior coverage. +// --------------------------------------------------------------------------- +describe("P2: formatResponse error wrapping", () => { + const { ElectroError, ErrorCodes } = require("../src/errors"); + + function makeThrowingEntity(makeError: () => Error) { + return new Entity( + { + model: { entity: "thrower", service: "perfService", version: "1" }, + table, + attributes: { + org: { type: "string" }, + id: { type: "string" }, + boom: { + type: "string", + get: () => { + throw makeError(); + }, + }, + }, + indexes: { + records: { + pk: { field: "pk", composite: ["org"] }, + sk: { field: "sk", composite: ["id"] }, + }, + }, + }, + { table }, + ); + } + + function makeGetClient(entity: any) { + const { Item } = entity + .put({ org: "org1", id: "id1", boom: "value" }) + .params(); + return makeMockV2Client({ get: { Item } }); + } + + it("wraps a plain error thrown during formatting in an ElectroError", async () => { + const original = new Error("boom"); + const entity = makeThrowingEntity(() => original); + const { client } = makeGetClient(entity); + let thrown: any; + try { + await entity.get({ org: "org1", id: "id1" }).go({ client }); + } catch (err) { + thrown = err; + } + expect(thrown, "expected go() to reject").to.not.equal(undefined); + expect(thrown.isElectroError).to.equal(true); + expect(thrown.code).to.equal(ErrorCodes.AWSError.code); + expect(thrown.cause).to.equal(original); + expect(thrown.message).to.equal( + 'Error thrown by DynamoDB client: "boom" - For more detail on this error reference: https://electrodb.dev/en/reference/errors/#aws-error', + ); + expect(thrown.stack).to.be.a("string").and.to.have.length.greaterThan(0); + }); + + it("rethrows an ElectroError unwrapped (same instance)", async () => { + const original = new ElectroError( + ErrorCodes.InvalidAttribute, + "already electro", + ); + const entity = makeThrowingEntity(() => original); + const { client } = makeGetClient(entity); + let thrown: any; + try { + await entity.get({ org: "org1", id: "id1" }).go({ client }); + } catch (err) { + thrown = err; + } + expect(thrown).to.equal(original); + }); + + it("rethrows the raw error when originalErr is set", async () => { + const original = new Error("boom"); + const entity = makeThrowingEntity(() => original); + const { client } = makeGetClient(entity); + let thrown: any; + try { + await entity + .get({ org: "org1", id: "id1" }) + .go({ client, originalErr: true }); + } catch (err) { + thrown = err; + } + expect(thrown).to.equal(original); + expect(thrown.isElectroError).to.equal(undefined); + }); + + it("wraps errors surfaced through the public parse() path", () => { + const original = new Error("parse boom"); + const entity = makeThrowingEntity(() => original); + const { Item } = entity + .put({ org: "org1", id: "id1", boom: "value" }) + .params(); + let thrown: any; + try { + entity.parse({ Item }); + } catch (err) { + thrown = err; + } + expect(thrown.isElectroError).to.equal(true); + expect(thrown.cause).to.equal(original); + }); +}); + +// --------------------------------------------------------------------------- +// P4: each getter/setter pass copied the entire payload once per attribute +// (`getSiblings`) and re-ran a regex per path lookup. The snapshot is now +// shared per pass and the regex skipped for bracket-free paths. These +// tests pin the sibling-visibility semantics and bracketed-path +// resolution the optimizations must preserve. +// --------------------------------------------------------------------------- +describe("P4: attribute mutation passes", () => { + it("setters see original sibling values, not other setters' output", () => { + const entity = new Entity( + { + model: { entity: "siblings", service: "perfService", version: "1" }, + table, + attributes: { + org: { type: "string" }, + id: { type: "string" }, + a: { + type: "string", + set: (value: any, item: any) => `${value}:${item.b}`, + }, + b: { + type: "string", + set: (value: any, item: any) => `${value}:${item.a}`, + }, + }, + indexes: { + records: { + pk: { field: "pk", composite: ["org"] }, + sk: { field: "sk", composite: ["id"] }, + }, + }, + }, + { table }, + ); + const { Item } = entity + .put({ org: "org1", id: "id1", a: "1", b: "2" }) + .params(); + // each setter saw the *original* value of its sibling + expect(Item.a).to.equal("1:2"); + expect(Item.b).to.equal("2:1"); + }); + + it("watchers fire exactly once and see the watched attribute's set output", () => { + const counts = { name: 0, display: 0 }; + const entity = new Entity( + { + model: { entity: "watchers", service: "perfService", version: "1" }, + table, + attributes: { + org: { type: "string" }, + id: { type: "string" }, + name: { + type: "string", + set: (value: any) => { + counts.name++; + return `${value}!`; + }, + }, + display: { + type: "string", + watch: ["name"], + set: (_: any, item: any) => { + counts.display++; + return `D:${item.name}`; + }, + }, + }, + indexes: { + records: { + pk: { field: "pk", composite: ["org"] }, + sk: { field: "sk", composite: ["id"] }, + }, + }, + }, + { table }, + ); + const { Item } = entity + .put({ org: "org1", id: "id1", name: "joe" }) + .params(); + expect(counts).to.deep.equal({ name: 1, display: 1 }); + expect(Item.name).to.equal("joe!"); + // the watcher pass runs against the first pass's output + expect(Item.display).to.equal("D:joe!"); + }); + + it("getters see sibling values during retrieval formatting", () => { + const entity = new Entity( + { + model: { entity: "getters", service: "perfService", version: "1" }, + table, + attributes: { + org: { type: "string" }, + id: { type: "string" }, + label: { + type: "string", + get: (value: any, item: any) => `${value}@${item.org}`, + }, + }, + indexes: { + records: { + pk: { field: "pk", composite: ["org"] }, + sk: { field: "sk", composite: ["id"] }, + }, + }, + }, + { table }, + ); + const { Item } = entity + .put({ org: "org1", id: "id1", label: "x" }) + .params(); + const { data } = entity.parse({ Item }); + expect(data.label).to.equal("x@org1"); + }); + + it("resolves bracketed list paths through update validation", () => { + const entity = makeFixtureEntity(); + const params = entity + .update({ org: "org1", id: "id1" }) + .data((attributes: any, operations: any) => + operations.set(attributes.notes[0].body, "edited"), + ) + .params(); + expect(params.UpdateExpression).to.equal( + "SET #notes[0].#body = :body_u0, #org = :org_u0, #id = :id_u0, #__edb_e__ = :__edb_e___u0, #__edb_v__ = :__edb_v___u0", + ); + expect(params.ExpressionAttributeNames).to.deep.equal({ + "#notes": "notes", + "#body": "body", + "#org": "org", + "#id": "id", + "#__edb_e__": "__edb_e__", + "#__edb_v__": "__edb_v__", + }); + expect(params.ExpressionAttributeValues[":body_u0"]).to.equal("edited"); + }); +}); + +// --------------------------------------------------------------------------- +// P3: every chain construction (including get/query/scan) eagerly built an +// AttributeOperationProxy — an Object.defineProperty per attribute and +// per operation — that read chains never use. It is now a cached lazy +// getter on ChainState's query object. These tests pin laziness, instance +// identity (write clauses accumulate into one builder), and exact params +// parity with the eager implementation (fixtures captured pre-change). +// --------------------------------------------------------------------------- +describe("P3: lazy update machinery on chain construction", () => { + const { ChainState } = require("../src/clauses"); + const { AttributeOperationProxy } = require("../src/operations"); + const entity = makeFixtureEntity(); + + it("exposes updateProxy as a cached lazy getter with stable identity", () => { + const state = new ChainState({ + index: "", + attributes: entity.model.schema.attributes, + hasSortKey: true, + options: {}, + }); + const descriptor = Object.getOwnPropertyDescriptor( + state.query, + "updateProxy", + ); + expect(descriptor?.get, "updateProxy should be an accessor").to.be.a( + "function", + ); + const first = state.query.updateProxy; + expect(first).to.be.instanceOf(AttributeOperationProxy); + expect(state.query.updateProxy).to.equal(first); + }); + + it("never builds the proxy for read chains, builds once for writes", () => { + // AttributeOperationProxy's constructor always calls the static + // buildAttributes, so spying on it counts constructions + const original = AttributeOperationProxy.buildAttributes; + let constructions = 0; + AttributeOperationProxy.buildAttributes = function ( + this: any, + ...args: any[] + ) { + constructions++; + return original.apply(this, args); + }; + try { + entity.query.records({ org: "org1" }).params(); + entity.query.records({ org: "org1" }).gt({ id: "id1" }).params(); + entity.get({ org: "org1", id: "id1" }).params(); + entity.scan.params(); + expect(constructions, "read chains should not build the proxy").to.equal( + 0, + ); + entity.update({ org: "org1", id: "id1" }).set({ name: "x" }).params(); + expect(constructions).to.equal(1); + } finally { + AttributeOperationProxy.buildAttributes = original; + } + }); + + it("produces params identical to the eager implementation", () => { + // fixtures captured from the pre-change (eager) implementation + const update = entity + .update({ org: "org1", id: "id1" }) + .set({ name: "newname" }) + .add({ count: 5 }) + .append({ notes: [{ body: "extra" }] }) + .remove(["active"]) + .params(); + expect(update).to.deep.equal({ + UpdateExpression: + "SET #name = :name_u0, #notes = list_append(if_not_exists(#notes, :notes_default_value_u0), :notes_u0), #org = :org_u0, #id = :id_u0, #__edb_e__ = :__edb_e___u0, #__edb_v__ = :__edb_v___u0 REMOVE #active ADD #count :count_u0", + ExpressionAttributeNames: { + "#name": "name", + "#count": "count", + "#notes": "notes", + "#active": "active", + "#org": "org", + "#id": "id", + "#__edb_e__": "__edb_e__", + "#__edb_v__": "__edb_v__", + }, + ExpressionAttributeValues: { + ":name_u0": "newname", + ":count_u0": 5, + ":notes_u0": [{ body: "extra" }], + ":notes_default_value_u0": [], + ":org_u0": "org1", + ":id_u0": "id1", + ":__edb_e___u0": "perfFixture", + ":__edb_v___u0": "1", + }, + TableName: "electro", + Key: { + pk: "$perfservice#org_org1", + sk: "$perffixture_1#id_id1", + }, + }); + + const upsert = entity + .upsert({ org: "org1", id: "id1", name: "n", count: 1 }) + .params(); + expect(upsert).to.deep.equal({ + TableName: "electro", + UpdateExpression: + "SET #__edb_e__ = :__edb_e___u0, #__edb_v__ = :__edb_v___u0, #org = :org_u0, #id = :id_u0, #name = :name_u0, #count = :count_u0", + ExpressionAttributeNames: { + "#__edb_e__": "__edb_e__", + "#__edb_v__": "__edb_v__", + "#org": "org", + "#id": "id", + "#name": "name", + "#count": "count", + }, + ExpressionAttributeValues: { + ":__edb_e___u0": "perfFixture", + ":__edb_v___u0": "1", + ":org_u0": "org1", + ":id_u0": "id1", + ":name_u0": "n", + ":count_u0": 1, + }, + Key: { + pk: "$perfservice#org_org1", + sk: "$perffixture_1#id_id1", + }, + }); + + const query = entity.query + .records({ org: "org1" }) + .where((attributes: any, operations: any) => + operations.gt(attributes.count, 10), + ) + .params(); + expect(query).to.deep.equal({ + KeyConditionExpression: "#pk = :pk and begins_with(#sk1, :sk1)", + TableName: "electro", + ExpressionAttributeNames: { + "#count": "count", + "#pk": "pk", + "#sk1": "sk", + }, + ExpressionAttributeValues: { + ":count0": 10, + ":pk": "$perfservice#org_org1", + ":sk1": "$perffixture_1#id_", + }, + FilterExpression: "#count > :count0", + }); + }); +}); + +// --------------------------------------------------------------------------- +// P1: executeQuery accumulated results by rebuilding the whole accumulator on +// every page (`results = [...results, ...items]`), making auto-paging +// O(pages²). These tests pin paging semantics: order, count truncation, +// cursor resumability, and collection demixing. +// --------------------------------------------------------------------------- +describe("P1: executeQuery result accumulation", () => { + const entity = makeFixtureEntity(); + + function makePagingClient({ + pages, + perPage, + }: { + pages: number; + perPage: number; + }) { + const query = makePagingQueryHandler({ + pages, + perPage, + makeItem: (i) => makeStoredItem(entity, i), + }); + return makeMockV2Client({ query }); + } + + it("returns the in-order union of all pages with pages:'all'", async () => { + const pages = 5; + const perPage = 4; + const { client, calls } = makePagingClient({ pages, perPage }); + const { data, cursor } = await entity.query + .records({ org: "org1" }) + .go({ client, pages: "all" }); + expect(calls.length).to.equal(pages); + expect(cursor).to.equal(null); + expect(data.map((item: any) => item.id)).to.deep.equal( + Array.from({ length: pages * perPage }, (_, i) => makeItemData(i).id), + ); + // items keep full attribute formatting, not just identity + expect(data[0]).to.deep.equal(makeItemData(0)); + expect(data[data.length - 1]).to.deep.equal(makeItemData(19)); + }); + + it("truncates to exactly `count` mid-page and returns a resumable cursor", async () => { + const pages = 3; + const perPage = 4; + const count = 5; // straddles the first page boundary + const { client } = makePagingClient({ pages, perPage }); + const first = await entity.query + .records({ org: "org1" }) + .go({ client, count }); + expect(first.data.length).to.equal(count); + expect(first.data.map((item: any) => item.count)).to.deep.equal([ + 0, 1, 2, 3, 4, + ]); + expect(first.cursor).to.be.a("string"); + + // resuming from the returned cursor picks up at the very next item + const rest = await entity.query + .records({ org: "org1" }) + .go({ client, cursor: first.cursor, pages: "all" }); + expect(rest.data.map((item: any) => item.count)).to.deep.equal([ + 5, 6, 7, 8, 9, 10, 11, + ]); + expect(rest.cursor).to.equal(null); + }); + + describe("collection queries", () => { + function makeCollectionModel(name: string) { + return { + model: { entity: name, service: "perfService", version: "1" }, + table, + attributes: { + org: { type: "string" }, + id: { type: "string" }, + label: { type: "string" }, + }, + indexes: { + records: { + collection: "shared", + pk: { field: "pk", composite: ["org"] }, + sk: { field: "sk", composite: ["id"] }, + }, + }, + }; + } + const alpha = new Entity(makeCollectionModel("alpha"), { table }); + const beta = new Entity(makeCollectionModel("beta"), { table }); + const service = new Service({ alpha, beta }); + + it("accumulates per-entity arrays in order across pages", async () => { + const pages = 4; + const perPage = 3; + const query = makePagingQueryHandler({ + pages, + perPage, + makeItem: (i) => { + const owner = i % 2 === 0 ? alpha : beta; + const { Item } = owner + .put({ + org: "org1", + id: `id${String(i).padStart(4, "0")}`, + label: `label${i}`, + }) + .params(); + return Item; + }, + }); + const { client, calls } = makeMockV2Client({ query }); + const { data, cursor } = await service.collections + .shared({ org: "org1" }) + .go({ client, pages: "all" }); + expect(calls.length).to.equal(pages); + expect(cursor).to.equal(null); + const expectedIds = Array.from( + { length: pages * perPage }, + (_, i) => `id${String(i).padStart(4, "0")}`, + ); + expect(data.alpha.map((item: any) => item.id)).to.deep.equal( + expectedIds.filter((_, i) => i % 2 === 0), + ); + expect(data.beta.map((item: any) => item.id)).to.deep.equal( + expectedIds.filter((_, i) => i % 2 === 1), + ); + expect(data.alpha[0]).to.deep.equal({ + org: "org1", + id: "id0000", + label: "label0", + }); + }); + }); +});