From c7f7612cbc397f3b77a8a7e6a44b8b99cb44b813 Mon Sep 17 00:00:00 2001 From: ty Date: Tue, 9 Jun 2026 22:36:45 -0400 Subject: [PATCH 01/17] Addresses divergencies between documentation and implementation discovered by Claude Fable. --- src/entity.js | 2 +- src/schema.js | 5 +- src/set.js | 3 +- src/transaction.js | 40 +- test/offline.audit-fixes.spec.js | 357 ++++++++++++++++++ .../en/core-concepts/executing-queries.mdx | 4 +- www/src/pages/en/mutations/conditions.mdx | 4 +- www/src/pages/en/queries/filters.mdx | 4 +- www/src/pages/en/queries/match.mdx | 2 +- www/src/partials/query-options.mdx | 4 +- 10 files changed, 398 insertions(+), 27 deletions(-) create mode 100644 test/offline.audit-fixes.spec.js diff --git a/src/entity.js b/src/entity.js index 42345f77..0bdd109c 100644 --- a/src/entity.js +++ b/src/entity.js @@ -1960,7 +1960,7 @@ class Entity { if (typeof option.unprocessed === "string") { if (typeof UnprocessedTypes[option.unprocessed] === "string") { - config.unproessed = UnprocessedTypes[option.unprocessed]; + config.unprocessed = UnprocessedTypes[option.unprocessed]; } else { throw new e.ElectroError( e.ErrorCodes.InvalidOptions, diff --git a/src/schema.js b/src/schema.js index 14f24754..67f99731 100644 --- a/src/schema.js +++ b/src/schema.js @@ -1125,6 +1125,9 @@ class SetAttribute extends Attribute { ? (value, getSiblings) => get(value, getSiblings()) : (attr) => attr; return (values, getSiblings) => { + if (this.hidden) { + return; + } if (values !== undefined) { const data = this.fromDDBSet(values); return getter(data, getSiblings); @@ -1423,7 +1426,7 @@ class Schema { } else { throw new e.ElectroError( e.ErrorCodes.InvalidAttributeWatchDefinition, - `Attribute Validation Error. The attribute '${name}' is defined to "watch" an invalid value of: '${attribute.watch}'. The watch property must either be a an array of attribute names, or the single string value of "${WatchAll}".`, + `Attribute Validation Error. The attribute '${name}' is defined to "watch" an invalid value of: '${attribute.watch}'. The watch property must either be a an array of attribute names, or the single string value of "${AttributeWildCard}".`, ); } } else { diff --git a/src/set.js b/src/set.js index 0e1f9891..700b7e22 100644 --- a/src/set.js +++ b/src/set.js @@ -5,6 +5,7 @@ const memberTypeToSetType = { Binary: "Binary", string: "String", number: "Number", + enum: "String", }; class DynamoDBSet { @@ -12,7 +13,7 @@ class DynamoDBSet { this.wrapperName = "Set"; this.type = memberTypeToSetType[type]; if (this.type === undefined) { - new Error(`Invalid Set type: ${type}`); + throw new Error(`Invalid Set type: ${type}`); } this.values = Array.from(new Set([].concat(list))); } diff --git a/src/transaction.js b/src/transaction.js index 7aef39e9..68c115c4 100644 --- a/src/transaction.js +++ b/src/transaction.js @@ -31,11 +31,15 @@ function cleanseCanceledData( paramItem, identifiers, }); - result.item = entities[entityAlias].formatResponse({ Item }, index, { - ...config, - pager: false, - parse: undefined, - }).data; + if (entityAlias) { + result.item = entities[entityAlias].formatResponse({ Item }, index, { + ...config, + pager: false, + parse: undefined, + }).data; + } else { + result.item = null; + } } else { result.item = null; } @@ -69,6 +73,7 @@ function cleanseTransactionData( const paramItem = paramItems[i]; const entityAlias = matchToEntityAlias({ paramItem, identifiers, record }); if (!entityAlias) { + results.push(null); continue; } @@ -130,14 +135,14 @@ function createTransaction(options) { data: [], }; } + let listeners = + options && options.listeners ? [...options.listeners] : []; if (options && options.logger) { - if (!options.listeners) { - options.listeners = []; - } - options.listeners.push(options.logger); + listeners = [...listeners, options.logger]; } const response = await driver.go(method, params, { ...options, + listeners, parse: (options, data) => { if (options.data === DataOptions.raw) { return data; @@ -162,12 +167,15 @@ function createTransaction(options) { }, ); } else { - return new Array(paramItems ? paramItems.length : 0).fill({ - item: null, - code: "None", - rejected: false, - message: undefined, - }); + return Array.from( + { length: paramItems ? paramItems.length : 0 }, + () => ({ + item: null, + code: "None", + rejected: false, + message: undefined, + }), + ); } }, }); @@ -202,4 +210,6 @@ module.exports = { createTransaction, createWriteTransaction, createGetTransaction, + cleanseTransactionData, + cleanseCanceledData, }; diff --git a/test/offline.audit-fixes.spec.js b/test/offline.audit-fixes.spec.js new file mode 100644 index 00000000..24823ad6 --- /dev/null +++ b/test/offline.audit-fixes.spec.js @@ -0,0 +1,357 @@ +/** + * Regression tests for the "stage 1" batch of audit fixes. + * + * Each describe block documents the bug it pins, and is written to FAIL against + * the pre-fix source and PASS once the corresponding fix is applied. + */ +const { Entity } = require("../src/entity"); +const { + cleanseTransactionData, + cleanseCanceledData, + createWriteTransaction, +} = require("../src/transaction"); +const { DynamoDBSet } = require("../src/set"); +const { TableIndex } = require("../src/types"); +const { expect } = require("chai"); + +const table = "electro"; + +// A minimal v2-style DocumentClient mock. Non-transaction methods return an +// object with `.promise()`; transaction methods return a request-like object +// (`on`/`abort`/`promise`) so it works both directly (via `_exec`) and through +// the v2 wrapper. `handlers` maps a method name to a canned response (or a +// function of the params). Transaction handlers may return +// `{ cancellationReasons: [...] }` to simulate a canceled transaction. +function makeMockV2Client(handlers = {}) { + const calls = []; + const transactMethods = new Set(["transactWrite", "transactGet"]); + const methods = [ + "get", + "put", + "update", + "delete", + "batchWrite", + "batchGet", + "scan", + "query", + "transactWrite", + "transactGet", + ]; + const client = { + createSet: (value) => new Set([].concat(value)), + }; + for (const method of methods) { + client[method] = (params) => { + calls.push({ method, params }); + const handler = handlers[method]; + const value = typeof handler === "function" ? handler(params) : handler; + if (transactMethods.has(method)) { + const stored = {}; + return { + on: (event, cb) => { + stored[event] = cb; + }, + abort: () => {}, + promise: () => { + if (value && value.cancellationReasons) { + if (stored.extractError) { + stored.extractError({ + httpResponse: { + body: { + toString: () => + JSON.stringify({ + CancellationReasons: value.cancellationReasons, + }), + }, + }, + }); + } + return Promise.reject(new Error("TransactionCanceledException")); + } + return Promise.resolve(value === undefined ? {} : value); + }, + }; + } + return { + promise: () => Promise.resolve(value === undefined ? {} : value), + }; + }; + } + return { client, calls }; +} + +// --------------------------------------------------------------------------- +// B1: `unprocessed: "raw"` execution option is dropped due to a property typo +// (`config.unproessed`) in `_normalizeExecutionOptions`. +// --------------------------------------------------------------------------- +describe("B1: unprocessed:'raw' execution option", () => { + const MallStores = new Entity( + { + model: { service: "BugBeater", entity: "TEST_ENTITY", version: "1" }, + table, + attributes: { + id: { type: "string", field: "storeLocationId" }, + sector: { type: "string" }, + }, + indexes: { + store: { + pk: { field: "pk", facets: ["sector"] }, + sk: { field: "sk", facets: ["id"] }, + }, + }, + }, + {}, + ); + + const rawKey = { + pk: "$bugbeater#sector_a1", + sk: "$test_entity_1#id_abc", + }; + const cannedBatchGet = { + Responses: { electro: [] }, + UnprocessedKeys: { electro: { Keys: [rawKey] } }, + }; + + it("normalizes onto config.unprocessed (not a typo'd property)", () => { + const config = MallStores._normalizeExecutionOptions({ + provided: [{ unprocessed: "raw" }], + }); + expect(config.unprocessed).to.equal("raw"); + expect(config).to.not.have.property("unproessed"); + }); + + it("returns raw DynamoDB keys when unprocessed:'raw'", async () => { + const { client } = makeMockV2Client({ batchGet: cannedBatchGet }); + const res = await MallStores.get([{ sector: "a1", id: "abc" }]).go({ + client, + unprocessed: "raw", + }); + expect(res.unprocessed).to.deep.equal([rawKey]); + }); + + it("still returns deconstructed composite attributes by default", async () => { + const { client } = makeMockV2Client({ batchGet: cannedBatchGet }); + const res = await MallStores.get([{ sector: "a1", id: "abc" }]).go({ + client, + }); + expect(res.unprocessed).to.deep.equal([{ sector: "a1", id: "abc" }]); + }); +}); + +// --------------------------------------------------------------------------- +// S3: Hidden Set attributes nested inside a map leak on read because +// SetAttribute._makeGet lacks the `if (this.hidden) return;` guard that the +// string/map/list getters have. +// --------------------------------------------------------------------------- +describe("S3: hidden nested Set attributes on read", () => { + const HiddenSet = new Entity({ + model: { service: "audit", entity: "hiddenset", version: "1" }, + table, + attributes: { + id: { type: "string" }, + secretRootSet: { type: "set", items: "string", hidden: true }, + data: { + type: "map", + properties: { + secretSet: { type: "set", items: "string", hidden: true }, + secretStr: { type: "string", hidden: true }, + visible: { type: "string" }, + }, + }, + }, + indexes: { + primary: { + pk: { field: "pk", composite: ["id"] }, + sk: { field: "sk", composite: [] }, + }, + }, + }); + + it("strips a hidden Set nested inside a map", () => { + const result = HiddenSet.parse({ + Item: { + pk: "$audit#id_1", + sk: "$hiddenset_1", + id: "1", + secretRootSet: ["x", "y"], + data: { secretSet: ["a", "b"], secretStr: "shh", visible: "ok" }, + __edb_e__: "hiddenset", + __edb_v__: "1", + }, + }).data; + expect(result.data).to.deep.equal({ visible: "ok" }); + expect(result.data).to.not.have.property("secretSet"); + // root-level hidden set should also be absent (already true pre-fix) + expect(result).to.not.have.property("secretRootSet"); + }); +}); + +// --------------------------------------------------------------------------- +// set.js: (1) the `Invalid Set type` Error is constructed but never thrown, and +// (2) `enum`-typed set members are not mapped, producing a DynamoDBSet +// with `type: undefined` in client-less `.params()`. +// --------------------------------------------------------------------------- +describe("set.js: DynamoDBSet type handling", () => { + it("throws on an unknown set member type", () => { + expect(() => new DynamoDBSet(["a"], "bogus")).to.throw(/Invalid Set type/); + }); + + it("maps known and enum member types to a DynamoDB set type", () => { + expect(new DynamoDBSet(["a"], "string").type).to.equal("String"); + expect(new DynamoDBSet([1], "number").type).to.equal("Number"); + expect(new DynamoDBSet(["a"], "enum").type).to.equal("String"); + }); + + it("produces a typed set for enum-item sets in client-less params()", () => { + const EnumSet = new Entity({ + model: { service: "audit", entity: "enumset", version: "1" }, + table, + attributes: { + id: { type: "string" }, + tags: { type: "set", items: ["red", "green", "blue"] }, + }, + indexes: { + primary: { + pk: { field: "pk", composite: ["id"] }, + sk: { field: "sk", composite: [] }, + }, + }, + }); + const params = EnumSet.put({ id: "1", tags: ["red", "green"] }).params(); + expect(params.Item.tags.wrapperName).to.equal("Set"); + expect(params.Item.tags.type).to.equal("String"); + }); +}); + +// --------------------------------------------------------------------------- +// WatchAll: an invalid `watch` value should raise an ElectroError, but the +// error message template references an undefined identifier +// (`WatchAll` instead of `AttributeWildCard`), so model construction +// dies with a raw ReferenceError instead. +// --------------------------------------------------------------------------- +describe("WatchAll: invalid watch definition error", () => { + const build = () => + new Entity({ + model: { service: "audit", entity: "watcher", version: "1" }, + table, + attributes: { + id: { type: "string" }, + a: { type: "string", watch: "notvalid" }, + }, + indexes: { + primary: { + pk: { field: "pk", composite: ["id"] }, + sk: { field: "sk", composite: [] }, + }, + }, + }); + + it("throws a descriptive ElectroError, not a ReferenceError", () => { + let err; + try { + build(); + } catch (e) { + err = e; + } + expect(err, "expected model construction to throw").to.exist; + expect(err).to.not.be.an.instanceof(ReferenceError); + expect(err.message).to.match(/array of attribute names/); + }); +}); + +// --------------------------------------------------------------------------- +// B4: Transaction result-handling defects. +// --------------------------------------------------------------------------- +describe("B4: transaction result handling", () => { + const makeEntity = (entity, extraAttr) => + new Entity({ + model: { service: "txtest", entity, version: "1" }, + table, + attributes: { + id: { type: "string" }, + [extraAttr]: { type: "string" }, + }, + indexes: { + primary: { + pk: { field: "pk", composite: ["id"] }, + sk: { field: "sk", composite: [] }, + }, + }, + }); + + const A = makeEntity("alpha", "name"); + const B = makeEntity("bravo", "label"); + const X = makeEntity("xray", "value"); // never joined to the entities map + + const recA = A.put({ id: "a1", name: "alice" }).params().Item; + const recB = B.put({ id: "b1", label: "bob" }).params().Item; + const recX = X.put({ id: "x1", value: "ghost" }).params().Item; + + it("(a) preserves position/size of transactGet results when an item is unmatched", () => { + const results = cleanseTransactionData( + TableIndex, + { a: A, b: B }, + { Items: [recA, recX, recB] }, + {}, + ); + expect(results).to.have.length(3); + expect(results[0].item).to.deep.equal({ id: "a1", name: "alice" }); + expect(results[1].item).to.equal(null); // unmatched -> null placeholder + expect(results[2].item).to.deep.equal({ id: "b1", label: "bob" }); + }); + + it("(b) does not crash on a canceled reason whose Item is unmatched", () => { + let results; + expect(() => { + results = cleanseCanceledData( + TableIndex, + { a: A, b: B }, + { + canceled: [ + { Code: "None", Item: recA }, + { Code: "ConditionalCheckFailed", Item: recX }, + { Code: "None", Item: recB }, + ], + }, + {}, + ); + }).to.not.throw(); + expect(results).to.have.length(3); + expect(results[0].item).to.deep.equal({ id: "a1", name: "alice" }); + expect(results[1]).to.deep.include({ + rejected: true, + code: "ConditionalCheckFailed", + item: null, + }); + expect(results[2].item).to.deep.equal({ id: "b1", label: "bob" }); + }); + + it("(c) does not share a single result object across all slots", async () => { + const { client } = makeMockV2Client({ transactWrite: {} }); + const tx = createWriteTransaction({ alpha: A }, (e) => [ + e.alpha.put({ id: "a1", name: "alice" }).commit(), + e.alpha.put({ id: "a2", name: "amy" }).commit(), + ]); + const result = await tx.go({ client }); + expect(result.data).to.have.length(2); + result.data[0].code = "MUTATED"; + expect(result.data[1].code).to.equal("None"); + }); + + it("(d) does not mutate the caller's options object", async () => { + const { client } = makeMockV2Client({ transactWrite: {} }); + const events = []; + const options = { client, logger: (event) => events.push(event.type) }; + const makeTx = () => + createWriteTransaction({ alpha: A }, (e) => [ + e.alpha.put({ id: "a1", name: "alice" }).commit(), + ]); + await makeTx().go(options); + const afterFirst = events.length; + await makeTx().go(options); + const afterSecond = events.length - afterFirst; + expect(options).to.not.have.property("listeners"); + // logger should fire the same number of times on each identical call + expect(afterSecond).to.equal(afterFirst); + }); +}); diff --git a/www/src/pages/en/core-concepts/executing-queries.mdx b/www/src/pages/en/core-concepts/executing-queries.mdx index 4743b591..26e37d5a 100644 --- a/www/src/pages/en/core-concepts/executing-queries.mdx +++ b/www/src/pages/en/core-concepts/executing-queries.mdx @@ -40,8 +40,8 @@ Additionally, ElectroDB offers a few additional methods built on top of DynamoDB | [batchGet](/en/queries/batch-get) | `batchGet` | Returns a set of attributes for multiple items from one or more tables. | | [query](/en/queries/query) | `query` | Queries a table or index for items that match the specified primary key value. | | [scan](/en/queries/scan) | `scan` | Scans the table and returns all matching items. | -| [match](/en/queries/match) | `query` or `scan` | The `match` method will attempt to build the most performant index keys with the attributes provided. ElectroDB identifies which index it can be "most" fulfilled with the index provided, and will fallback to a `scan` if no index can be built. | -| [find](/en/queries/find) | `query` or `scan` | the `find` method performs the same index seeking behavior but will also apply all provided values as filters instead of only using provided attributes to build index keys. | +| [match](/en/queries/match) | `query` or `scan` | The `match` method performs the same index-seeking behavior as `find` and additionally applies every provided value as a query filter. ElectroDB identifies which index can be "most" fulfilled with the attributes provided, and will fallback to a `scan` if no index can be built. | +| [find](/en/queries/find) | `query` or `scan` | the `find` method builds the most performant index keys it can from the attributes provided, falling back to a `scan` if no index can be built. Unlike `match`, values that do not contribute to a composite key are not applied as query filters. | ## Mutation Methods diff --git a/www/src/pages/en/mutations/conditions.mdx b/www/src/pages/en/mutations/conditions.mdx index 168a38f3..30e00ab8 100644 --- a/www/src/pages/en/mutations/conditions.mdx +++ b/www/src/pages/en/mutations/conditions.mdx @@ -194,8 +194,8 @@ The `attributes` object contains every Attribute defined in the Entity's Model. | `between` | `between(rent, minRent, maxRent)` | `(#rent between :rent1 and :rent2)` | | `name` | `name(rent)` | `#rent` | | `value` | `value(rent, maxRent)` | `:rent1` | -| `escape` | `escape(123)`, `escape('abc')` | `#123`, `#abc` | -| `field` | `field('field_name')` | `:field_name` | +| `escape` | `escape(123)`, `escape('abc')` | `:123`, `:abc` | +| `field` | `field('field_name')` | `#field_name` | ### ElectroDB Functions diff --git a/www/src/pages/en/queries/filters.mdx b/www/src/pages/en/queries/filters.mdx index 79df3924..7291632e 100644 --- a/www/src/pages/en/queries/filters.mdx +++ b/www/src/pages/en/queries/filters.mdx @@ -155,8 +155,8 @@ The `attributes` object contains every Attribute defined in the Entity's Model. | `type` | `type(rent, 'N')` | `attribute_type(#rent, 'N')` | | `name` | `name(rent)` | `#rent` | | `value` | `value(rent, maxRent)` | `:rent1` | -| `escape` | `escape(123)`, `escape('abc')` | `#123`, `#abc` | -| `field` | `field('field_name')` | `:field_name` | +| `escape` | `escape(123)`, `escape('abc')` | `:123`, `:abc` | +| `field` | `field('field_name')` | `#field_name` | ### ElectroDB Functions diff --git a/www/src/pages/en/queries/match.mdx b/www/src/pages/en/queries/match.mdx index d49ce072..1bc6b6b9 100644 --- a/www/src/pages/en/queries/match.mdx +++ b/www/src/pages/en/queries/match.mdx @@ -23,7 +23,7 @@ import ExampleSetup from "../../../partials/entity-query-example-setup.mdx"; ```typescript -await StoreLocations.find({ +await StoreLocations.match({ mallId: "EastPointe", buildingId: "BuildingA1", leaseEndDate: "2020-03-22", diff --git a/www/src/partials/query-options.mdx b/www/src/partials/query-options.mdx index 60264bda..6eb69705 100644 --- a/www/src/partials/query-options.mdx +++ b/www/src/partials/query-options.mdx @@ -92,9 +92,9 @@ A convenience option for a single event listener that semantically can be used f **Default Value:** 'keys' When performing comparison queries (e.g. `gt`, `gte`, `lte`, `lt`, `between`) it is often necessary to add additional filters. Understanding how a comparison query will impact the items returned can be challenging due to the unintuitive nature of sorting and multi-composite attribute Sort Keys. -By default (`comparison: "keys"`), ElectroDB builds comparison queries without applying attribute filters. In this way, the query's comparison applies to the item's "keys". This approach gives you complete control of your indexes, though your query may return unexpected data if you are not careful with item sorting. +By default (`compare: "keys"`), ElectroDB builds comparison queries without applying attribute filters. In this way, the query's comparison applies to the item's "keys". This approach gives you complete control of your indexes, though your query may return unexpected data if you are not careful with item sorting. -When set to `attributes`, the `comparison` option instructs ElectroDB to apply attribute filters on each Sort Key composite attribute provided. In this way, the query comparison applies to the item's "attributes." +When set to `attributes`, the `compare` option instructs ElectroDB to apply attribute filters on each Sort Key composite attribute provided. In this way, the query comparison applies to the item's "attributes." ### consistent **Default Value:** _none_ From c1509640383c6ecc3c1bd72f44bffe13c118805c Mon Sep 17 00:00:00 2001 From: ty Date: Tue, 9 Jun 2026 23:16:29 -0400 Subject: [PATCH 02/17] test: extract shared offline fixtures (mock client, fixture entity) Lifts makeMockV2Client out of offline.audit-fixes.spec.js into test/fixtures/mock-client.js and adds a paging query handler plus a representative fixture entity (test/fixtures/entities.js) so upcoming guard tests and benchmark scenarios can share them. Co-Authored-By: Claude Fable 5 --- test/fixtures/entities.js | 96 ++++++++++++++++++++++++++ test/fixtures/mock-client.js | 111 +++++++++++++++++++++++++++++++ test/offline.audit-fixes.spec.js | 65 +----------------- 3 files changed, 208 insertions(+), 64 deletions(-) create mode 100644 test/fixtures/entities.js create mode 100644 test/fixtures/mock-client.js diff --git a/test/fixtures/entities.js b/test/fixtures/entities.js new file mode 100644 index 00000000..88459024 --- /dev/null +++ b/test/fixtures/entities.js @@ -0,0 +1,96 @@ +/** + * Shared entity fixtures for offline tests and benchmarks. A representative + * single-index entity exercising the common attribute types, plus helpers to + * produce from-DynamoDB-shaped stored items for feeding `parse`/ + * `formatResponse` without a database. + */ +const { Entity } = require("../../src/entity"); + +const table = "electro"; + +function makeFixtureModel({ + entity = "perfFixture", + service = "perfService", + withWatchers = false, +} = {}) { + const attributes = { + org: { type: "string" }, + id: { type: "string" }, + name: { type: "string" }, + count: { type: "number" }, + active: { type: "boolean" }, + tags: { type: "set", items: "string" }, + profile: { + type: "map", + properties: { + nickname: { type: "string" }, + age: { type: "number" }, + }, + }, + notes: { + type: "list", + items: { + type: "map", + properties: { + body: { type: "string" }, + }, + }, + }, + }; + if (withWatchers) { + attributes.displayName = { + type: "string", + watch: ["name"], + set: (_, item) => + typeof item.name === "string" ? item.name.toUpperCase() : undefined, + }; + } + return { + model: { entity, service, version: "1" }, + table, + attributes, + indexes: { + records: { + pk: { field: "pk", composite: ["org"] }, + sk: { field: "sk", composite: ["id"] }, + }, + }, + }; +} + +function makeFixtureEntity(options = {}) { + return new Entity(makeFixtureModel(options), { table }); +} + +function makeItemData(i) { + return { + org: "org1", + id: `id${String(i).padStart(8, "0")}`, + name: `name${i}`, + count: i, + active: i % 2 === 0, + tags: [`tag${i % 5}`, "common"], + profile: { nickname: `nick${i}`, age: 20 + (i % 50) }, + notes: [{ body: `note${i}` }], + }; +} + +// Builds the i-th item exactly as it would be stored in DynamoDB (keys, +// `__edb_e__`/`__edb_v__` identifiers, field names) by reusing the entity's +// own put-params formatting, then normalizing set attributes to plain arrays +// (the shape a v3 DocumentClient unmarshalls to). +function makeStoredItem(entity, i) { + const { Item } = entity.put(makeItemData(i)).params(); + if (Item.tags && Array.isArray(Item.tags.values)) { + Item.tags = [...Item.tags.values]; + } + return Item; +} + +module.exports = { + table, + makeFixtureModel, + makeFixtureEntity, + makeItemData, + makeStoredItem, +}; diff --git a/test/fixtures/mock-client.js b/test/fixtures/mock-client.js new file mode 100644 index 00000000..ad3e1700 --- /dev/null +++ b/test/fixtures/mock-client.js @@ -0,0 +1,111 @@ +/** + * Shared offline client mocks. These fixtures are used by both the offline + * test suites (test/offline.*.spec.js) and the benchmark scenarios + * (benchmark/scenarios/*.bench.js); they must stay dependency-free and + * synchronous so neither consumer needs network or DynamoDB access. + */ + +// A minimal v2-style DocumentClient mock. Non-transaction methods return an +// object with `.promise()`; transaction methods return a request-like object +// (`on`/`abort`/`promise`) so it works both directly (via `_exec`) and through +// the v2 wrapper. `handlers` maps a method name to a canned response (or a +// function of the params). Transaction handlers may return +// `{ cancellationReasons: [...] }` to simulate a canceled transaction. +function makeMockV2Client(handlers = {}) { + const calls = []; + const transactMethods = new Set(["transactWrite", "transactGet"]); + const methods = [ + "get", + "put", + "update", + "delete", + "batchWrite", + "batchGet", + "scan", + "query", + "transactWrite", + "transactGet", + ]; + const client = { + createSet: (value) => new Set([].concat(value)), + }; + for (const method of methods) { + client[method] = (params) => { + calls.push({ method, params }); + const handler = handlers[method]; + const value = typeof handler === "function" ? handler(params) : handler; + if (transactMethods.has(method)) { + const stored = {}; + return { + on: (event, cb) => { + stored[event] = cb; + }, + abort: () => {}, + promise: () => { + if (value && value.cancellationReasons) { + if (stored.extractError) { + stored.extractError({ + httpResponse: { + body: { + toString: () => + JSON.stringify({ + CancellationReasons: value.cancellationReasons, + }), + }, + }, + }); + } + return Promise.reject(new Error("TransactionCanceledException")); + } + return Promise.resolve(value === undefined ? {} : value); + }, + }; + } + return { + promise: () => Promise.resolve(value === undefined ? {} : value), + }; + }; + } + return { client, calls }; +} + +// Builds a `query`/`scan` handler that pages through `pages` responses of +// `perPage` items each, emitting a `LastEvaluatedKey` on every page but the +// last. `makeItem(i)` produces the i-th stored item (from-DynamoDB shape, +// including key fields; sort keys must be unique across items). Like DynamoDB, +// the handler resumes from the params' `ExclusiveStartKey` by locating the +// item whose pk/sk match, so it also honors cursors ElectroDB synthesizes +// mid-page (e.g. when a `count` limit lands inside a page). +function makePagingQueryHandler({ pages, perPage, makeItem }) { + const items = []; + const indexBySortKey = new Map(); + for (let i = 0; i < pages * perPage; i++) { + const item = makeItem(i); + items.push(item); + indexBySortKey.set(`${item.pk}|${item.sk}`, i); + } + return (params) => { + let start = 0; + const esk = params.ExclusiveStartKey; + if (esk !== undefined) { + const index = indexBySortKey.get(`${esk.pk}|${esk.sk}`); + if (index === undefined) { + throw new Error("Unknown ExclusiveStartKey provided to paging mock"); + } + start = index + 1; + } + const Items = items.slice(start, start + perPage); + const response = { Items, Count: Items.length }; + const lastReturned = start + Items.length - 1; + if (lastReturned < items.length - 1 && Items.length > 0) { + const lastItem = Items[Items.length - 1]; + response.LastEvaluatedKey = { pk: lastItem.pk, sk: lastItem.sk }; + } + return response; + }; +} + +module.exports = { + makeMockV2Client, + makePagingQueryHandler, +}; diff --git a/test/offline.audit-fixes.spec.js b/test/offline.audit-fixes.spec.js index 24823ad6..72f6e22e 100644 --- a/test/offline.audit-fixes.spec.js +++ b/test/offline.audit-fixes.spec.js @@ -13,73 +13,10 @@ const { const { DynamoDBSet } = require("../src/set"); const { TableIndex } = require("../src/types"); const { expect } = require("chai"); +const { makeMockV2Client } = require("./fixtures/mock-client"); const table = "electro"; -// A minimal v2-style DocumentClient mock. Non-transaction methods return an -// object with `.promise()`; transaction methods return a request-like object -// (`on`/`abort`/`promise`) so it works both directly (via `_exec`) and through -// the v2 wrapper. `handlers` maps a method name to a canned response (or a -// function of the params). Transaction handlers may return -// `{ cancellationReasons: [...] }` to simulate a canceled transaction. -function makeMockV2Client(handlers = {}) { - const calls = []; - const transactMethods = new Set(["transactWrite", "transactGet"]); - const methods = [ - "get", - "put", - "update", - "delete", - "batchWrite", - "batchGet", - "scan", - "query", - "transactWrite", - "transactGet", - ]; - const client = { - createSet: (value) => new Set([].concat(value)), - }; - for (const method of methods) { - client[method] = (params) => { - calls.push({ method, params }); - const handler = handlers[method]; - const value = typeof handler === "function" ? handler(params) : handler; - if (transactMethods.has(method)) { - const stored = {}; - return { - on: (event, cb) => { - stored[event] = cb; - }, - abort: () => {}, - promise: () => { - if (value && value.cancellationReasons) { - if (stored.extractError) { - stored.extractError({ - httpResponse: { - body: { - toString: () => - JSON.stringify({ - CancellationReasons: value.cancellationReasons, - }), - }, - }, - }); - } - return Promise.reject(new Error("TransactionCanceledException")); - } - return Promise.resolve(value === undefined ? {} : value); - }, - }; - } - return { - promise: () => Promise.resolve(value === undefined ? {} : value), - }; - }; - } - return { client, calls }; -} - // --------------------------------------------------------------------------- // B1: `unprocessed: "raw"` execution option is dropped due to a property typo // (`config.unproessed`) in `_normalizeExecutionOptions`. From 46cf5bca98dbd4e2b1875db841b4e602f1f9f2be Mon Sep 17 00:00:00 2001 From: ty Date: Tue, 9 Jun 2026 23:18:56 -0400 Subject: [PATCH 03/17] perf: accumulate query pages in place instead of rebuilding arrays MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit executeQuery rebuilt its full result accumulator on every page (results = [...results, ...items] and the collection/hydration equivalents), making auto-paging O(pages²) in copies. Push items in place instead; slicing for `count` and cursor derivation are untouched, so order, truncation, and resumability are identical. Guard tests pin multi-page union order, count truncation straddling a page boundary with cursor resume, and per-entity collection demixing. Co-Authored-By: Claude Fable 5 --- src/entity.js | 22 +++-- test/offline.perf-fixes.spec.js | 141 ++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 9 deletions(-) create mode 100644 test/offline.perf-fixes.spec.js diff --git a/src/entity.js b/src/entity.js index 0bdd109c..8910e7fa 100644 --- a/src/entity.js +++ b/src/entity.js @@ -753,12 +753,14 @@ class Entity { config, ); items = hydrated.data; - hydratedUnprocessed = hydratedUnprocessed.concat( - hydrated.unprocessed, - ); + for (const unprocessed of hydrated.unprocessed) { + hydratedUnprocessed.push(unprocessed); + } + } + const entityResults = (results[entity] = results[entity] || []); + for (const item of items) { + entityResults.push(item); } - results[entity] = results[entity] || []; - results[entity] = [...results[entity], ...items]; } } else if (Array.isArray(response.data)) { let prevCount = count; @@ -777,11 +779,13 @@ class Entity { config, ); items = hydrated.data; - hydratedUnprocessed = hydratedUnprocessed.concat( - hydrated.unprocessed, - ); + for (const unprocessed of hydrated.unprocessed) { + hydratedUnprocessed.push(unprocessed); + } + } + for (const item of items) { + results.push(item); } - results = [...results, ...items]; if (moreItemsThanRequired || count === config.count) { const lastItem = results[results.length - 1]; ExclusiveStartKey = this._fromCompositeToKeysByIndex({ diff --git a/test/offline.perf-fixes.spec.js b/test/offline.perf-fixes.spec.js new file mode 100644 index 00000000..0e2a7e5c --- /dev/null +++ b/test/offline.perf-fixes.spec.js @@ -0,0 +1,141 @@ +/** + * Behavioral guard tests for the performance-focused batch of audit fixes + * ("stage 2"). Each describe block pins user-visible behavior that the + * corresponding optimization must preserve; none of these rely on timing. + */ +const { expect } = require("chai"); +const { Entity } = require("../src/entity"); +const { Service } = require("../src/service"); +const { + makeMockV2Client, + makePagingQueryHandler, +} = require("./fixtures/mock-client"); +const { + table, + makeFixtureEntity, + makeStoredItem, + makeItemData, +} = require("./fixtures/entities"); + +// --------------------------------------------------------------------------- +// P1: executeQuery accumulated results by rebuilding the whole accumulator on +// every page (`results = [...results, ...items]`), making auto-paging +// O(pages²). These tests pin paging semantics: order, count truncation, +// cursor resumability, and collection demixing. +// --------------------------------------------------------------------------- +describe("P1: executeQuery result accumulation", () => { + const entity = makeFixtureEntity(); + + function makePagingClient({ pages, perPage }) { + const query = makePagingQueryHandler({ + pages, + perPage, + makeItem: (i) => makeStoredItem(entity, i), + }); + return makeMockV2Client({ query }); + } + + it("returns the in-order union of all pages with pages:'all'", async () => { + const pages = 5; + const perPage = 4; + const { client, calls } = makePagingClient({ pages, perPage }); + const { data, cursor } = await entity.query + .records({ org: "org1" }) + .go({ client, pages: "all" }); + expect(calls.length).to.equal(pages); + expect(cursor).to.equal(null); + expect(data.map((item) => item.id)).to.deep.equal( + Array.from({ length: pages * perPage }, (_, i) => makeItemData(i).id), + ); + // items keep full attribute formatting, not just identity + expect(data[0]).to.deep.equal(makeItemData(0)); + expect(data[data.length - 1]).to.deep.equal(makeItemData(19)); + }); + + it("truncates to exactly `count` mid-page and returns a resumable cursor", async () => { + const pages = 3; + const perPage = 4; + const count = 5; // straddles the first page boundary + const { client } = makePagingClient({ pages, perPage }); + const first = await entity.query + .records({ org: "org1" }) + .go({ client, count }); + expect(first.data.length).to.equal(count); + expect(first.data.map((item) => item.count)).to.deep.equal([0, 1, 2, 3, 4]); + expect(first.cursor).to.be.a("string"); + + // resuming from the returned cursor picks up at the very next item + const rest = await entity.query + .records({ org: "org1" }) + .go({ client, cursor: first.cursor, pages: "all" }); + expect(rest.data.map((item) => item.count)).to.deep.equal([ + 5, 6, 7, 8, 9, 10, 11, + ]); + expect(rest.cursor).to.equal(null); + }); + + describe("collection queries", () => { + function makeCollectionModel(name) { + return { + model: { entity: name, service: "perfService", version: "1" }, + table, + attributes: { + org: { type: "string" }, + id: { type: "string" }, + label: { type: "string" }, + }, + indexes: { + records: { + collection: "shared", + pk: { field: "pk", composite: ["org"] }, + sk: { field: "sk", composite: ["id"] }, + }, + }, + }; + } + const alpha = new Entity(makeCollectionModel("alpha"), { table }); + const beta = new Entity(makeCollectionModel("beta"), { table }); + const service = new Service({ alpha, beta }); + + it("accumulates per-entity arrays in order across pages", async () => { + const pages = 4; + const perPage = 3; + const query = makePagingQueryHandler({ + pages, + perPage, + makeItem: (i) => { + const owner = i % 2 === 0 ? alpha : beta; + const { Item } = owner + .put({ + org: "org1", + id: `id${String(i).padStart(4, "0")}`, + label: `label${i}`, + }) + .params(); + return Item; + }, + }); + const { client, calls } = makeMockV2Client({ query }); + const { data, cursor } = await service.collections + .shared({ org: "org1" }) + .go({ client, pages: "all" }); + expect(calls.length).to.equal(pages); + expect(cursor).to.equal(null); + const expectedIds = Array.from( + { length: pages * perPage }, + (_, i) => `id${String(i).padStart(4, "0")}`, + ); + expect(data.alpha.map((item) => item.id)).to.deep.equal( + expectedIds.filter((_, i) => i % 2 === 0), + ); + expect(data.beta.map((item) => item.id)).to.deep.equal( + expectedIds.filter((_, i) => i % 2 === 1), + ); + expect(data.alpha[0]).to.deep.equal({ + org: "org1", + id: "id0000", + label: "label0", + }); + }); + }); +}); From a365725fbe588c194d31610bdf51cdf0425edb5d Mon Sep 17 00:00:00 2001 From: ty Date: Tue, 9 Jun 2026 23:21:48 -0400 Subject: [PATCH 04/17] perf: skip doomed ModelBeta validation pass; check instance symbols first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validateModel ran the ModelBeta jsonschema pass for every model even though modern v1 models can never satisfy it (it requires a root `entity` string), enumerating and discarding errors on every Entity construction. Skip the beta pass when the model has no root entity string; every other path (valid beta, invalid beta-shaped, invalid v1, garbage) flows exactly as before, so thrown messages stay byte-identical — pinned by fixtures captured from the pre-fix implementation. getInstanceType similarly ran full schema validation before the cheap `_instance` symbol checks; symbols are now checked first and testModel only runs for the bare-model fallback. Co-Authored-By: Claude Fable 5 --- src/util.js | 12 ++-- src/validations.js | 9 ++- test/offline.perf-fixes.spec.js | 115 ++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 7 deletions(-) diff --git a/src/util.js b/src/util.js index 1e53218d..dd326bbb 100644 --- a/src/util.js +++ b/src/util.js @@ -15,20 +15,22 @@ function genericizeJSONPath(path = "") { } function getInstanceType(instance = {}) { - let [isModel, errors] = v.testModel(instance); + // check the cheap instance symbols before falling back to testModel, which + // runs full jsonschema validation just to be discarded for non-models if (!instance || Object.keys(instance).length === 0) { return ""; - } else if (isModel) { - return t.ElectroInstanceTypes.model; } else if (instance._instance === t.ElectroInstance.entity) { return t.ElectroInstanceTypes.entity; } else if (instance._instance === t.ElectroInstance.service) { return t.ElectroInstanceTypes.service; } else if (instance._instance === t.ElectroInstance.electro) { return t.ElectroInstanceTypes.electro; - } else { - return ""; } + let [isModel] = v.testModel(instance); + if (isModel) { + return t.ElectroInstanceTypes.model; + } + return ""; } function getModelVersion(model = {}) { diff --git a/src/validations.js b/src/validations.js index 54a5e30d..74b9c528 100644 --- a/src/validations.js +++ b/src/validations.js @@ -284,8 +284,13 @@ v.addSchema(Modelv1, "/Modelv1"); function validateModel(model = {}) { /** start beta/v1 condition **/ - let betaErrors = v.validate(model, ModelBeta).errors; - if (betaErrors.length) { + // ModelBeta requires a root `entity` string; without one that schema can + // only fail, so modern (v1-shaped) models skip straight to Modelv1 + // validation instead of enumerating-and-discarding beta errors on every + // construction. Any model that fails (or skips) the beta pass throws the + // Modelv1 errors, exactly as before. + const couldBeBeta = !!model && isStringHasLength(model.entity); + if (!couldBeBeta || v.validate(model, ModelBeta).errors.length) { /** end/v1 condition **/ let errors = v.validate(model, Modelv1).errors; if (errors.length) { diff --git a/test/offline.perf-fixes.spec.js b/test/offline.perf-fixes.spec.js index 0e2a7e5c..25e312d1 100644 --- a/test/offline.perf-fixes.spec.js +++ b/test/offline.perf-fixes.spec.js @@ -17,6 +17,121 @@ const { makeItemData, } = require("./fixtures/entities"); +// --------------------------------------------------------------------------- +// P5: validateModel ran the (always-failing) ModelBeta schema pass for every +// modern v1 model before validating Modelv1, and getInstanceType ran full +// jsonschema validation before the cheap `_instance` symbol checks. These +// tests pin the thrown messages byte-for-byte (captured from the pre-fix +// implementation) and the instance-type resolution order. +// --------------------------------------------------------------------------- +describe("P5: model validation short-circuits", () => { + const validations = require("../src/validations"); + const u = require("../src/util"); + const { ElectroInstance, ElectroInstanceTypes } = require("../src/types"); + + // captured verbatim from the pre-fix validateModel implementation + const invalidModelFixtures = [ + { + name: "empty object", + model: {}, + message: + 'instance.model is required, instance requires property "model", instance requires property "attributes", instance requires property "indexes" - For more detail on this error reference: https://electrodb.dev/en/reference/errors/#invalid-model', + }, + { + name: "v1 model missing indexes", + model: { + model: { entity: "e", service: "s", version: "1" }, + attributes: { id: { type: "string" } }, + }, + message: + 'instance requires property "indexes" - For more detail on this error reference: https://electrodb.dev/en/reference/errors/#invalid-model', + }, + { + name: "v1 model with non-function getter", + model: { + model: { entity: "e", service: "s", version: "1" }, + attributes: { id: { type: "string", get: "not-a-function" } }, + indexes: { main: { pk: { field: "pk", composite: ["id"] } } }, + }, + message: + "instance.attributes.id.get must be a function - For more detail on this error reference: https://electrodb.dev/en/reference/errors/#invalid-model", + }, + { + name: "beta-shaped model missing indexes (still throws v1 messages)", + model: { + entity: "e", + service: "s", + attributes: { id: { type: "string" } }, + }, + message: + 'instance.model is required, instance requires property "model", instance requires property "indexes" - For more detail on this error reference: https://electrodb.dev/en/reference/errors/#invalid-model', + }, + { + name: "non-object model namespace", + model: { + model: "nope", + attributes: { id: { type: "string" } }, + indexes: { main: { pk: { field: "pk", composite: ["id"] } } }, + }, + message: + "instance.model is not of a type(s) object - For more detail on this error reference: https://electrodb.dev/en/reference/errors/#invalid-model", + }, + ]; + + for (const { name, model, message } of invalidModelFixtures) { + it(`throws the exact pre-fix message for: ${name}`, () => { + let thrown; + try { + validations.model(model); + } catch (err) { + thrown = err; + } + expect(thrown, "expected validateModel to throw").to.not.equal(undefined); + expect(thrown.isElectroError).to.equal(true); + expect(thrown.message).to.equal(message); + }); + } + + it("accepts a valid v1 model and a valid beta model", () => { + const v1Model = { + model: { entity: "e", service: "s", version: "1" }, + attributes: { id: { type: "string" } }, + indexes: { main: { pk: { field: "pk", composite: ["id"] } } }, + }; + const betaModel = { + entity: "e", + service: "s", + version: "1", + attributes: { id: { type: "string" } }, + indexes: { main: { pk: { field: "pk", facets: ["id"] } } }, + }; + expect(() => validations.model(v1Model)).to.not.throw(); + expect(() => validations.model(betaModel)).to.not.throw(); + expect(() => new Entity(v1Model, { table })).to.not.throw(); + }); + + it("resolves instance types via symbols and bare models via validation", () => { + const entity = makeFixtureEntity(); + expect(u.getInstanceType(entity)).to.equal(ElectroInstanceTypes.entity); + expect(u.getInstanceType({ _instance: ElectroInstance.service })).to.equal( + ElectroInstanceTypes.service, + ); + expect(u.getInstanceType({ _instance: ElectroInstance.electro })).to.equal( + ElectroInstanceTypes.electro, + ); + expect( + u.getInstanceType({ + model: { entity: "e", service: "s", version: "1" }, + attributes: { id: { type: "string" } }, + indexes: { main: { pk: { field: "pk", composite: ["id"] } } }, + }), + ).to.equal(ElectroInstanceTypes.model); + expect(u.getInstanceType({})).to.equal(""); + expect(u.getInstanceType(undefined)).to.equal(""); + expect(u.getInstanceType({ anything: "else" })).to.equal(""); + }); +}); + // --------------------------------------------------------------------------- // P1: executeQuery accumulated results by rebuilding the whole accumulator on // every page (`results = [...results, ...items]`), making auto-paging From 643e6975a0d7ff9c8b8c8243294a58931f920192 Mon Sep 17 00:00:00 2001 From: ty Date: Tue, 9 Jun 2026 23:23:18 -0400 Subject: [PATCH 05/17] perf: allocate formatResponse's ElectroError only on the throw path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit formatResponse constructed an ElectroError (including stack capture) on every invocation when originalErr was unset — once per item across batchGet formatting, collection demixing, and transaction loops — and discarded it whenever formatting succeeded. Build it inside the catch instead: zero allocations on success, identical message/cause/code on failure. formatResponse is synchronous, so the stack captured in the catch is the same frame the eager capture rooted at. Guard tests pin the previously-untested wrapping contract: plain errors wrap with cause + exact message, ElectroErrors rethrow unwrapped, originalErr rethrows raw, and parse() surfaces wrapped errors. Co-Authored-By: Claude Fable 5 --- src/entity.js | 13 ++-- test/offline.perf-fixes.spec.js | 111 ++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 9 deletions(-) diff --git a/src/entity.js b/src/entity.js index 8910e7fa..1236660c 100644 --- a/src/entity.js +++ b/src/entity.js @@ -993,10 +993,6 @@ class Entity { } formatResponse(response, index, config = {}) { - let stackTrace; - if (!config.originalErr) { - stackTrace = new e.ElectroError(e.ErrorCodes.AWSError); - } try { let results = {}; if (validations.isFunction(config.parse)) { @@ -1062,13 +1058,12 @@ class Entity { return { data: results }; } catch (err) { - if ( - config.originalErr || - stackTrace === undefined || - err.isElectroError - ) { + if (config.originalErr || err.isElectroError) { throw err; } else { + // built only on the throw path; formatResponse is synchronous so the + // stack captured here matches what eager construction produced + const stackTrace = new e.ElectroError(e.ErrorCodes.AWSError); stackTrace.message = `Error thrown by DynamoDB client: "${err.message}" - For more detail on this error reference: https://electrodb.dev/en/reference/errors/#aws-error`; stackTrace.cause = err; throw stackTrace; diff --git a/test/offline.perf-fixes.spec.js b/test/offline.perf-fixes.spec.js index 25e312d1..323f50fe 100644 --- a/test/offline.perf-fixes.spec.js +++ b/test/offline.perf-fixes.spec.js @@ -132,6 +132,117 @@ describe("P5: model validation short-circuits", () => { }); }); +// --------------------------------------------------------------------------- +// P2: formatResponse eagerly allocated an ElectroError (with stack capture) +// on every call — once per item on batchGet/collection paths — only to +// discard it on success. The error is now built lazily on the throw path; +// these tests pin the wrapping contract, which had no prior coverage. +// --------------------------------------------------------------------------- +describe("P2: formatResponse error wrapping", () => { + const { ElectroError, ErrorCodes } = require("../src/errors"); + + function makeThrowingEntity(makeError) { + return new Entity( + { + model: { entity: "thrower", service: "perfService", version: "1" }, + table, + attributes: { + org: { type: "string" }, + id: { type: "string" }, + boom: { + type: "string", + get: () => { + throw makeError(); + }, + }, + }, + indexes: { + records: { + pk: { field: "pk", composite: ["org"] }, + sk: { field: "sk", composite: ["id"] }, + }, + }, + }, + { table }, + ); + } + + function makeGetClient(entity) { + const { Item } = entity + .put({ org: "org1", id: "id1", boom: "value" }) + .params(); + return makeMockV2Client({ get: { Item } }); + } + + it("wraps a plain error thrown during formatting in an ElectroError", async () => { + const original = new Error("boom"); + const entity = makeThrowingEntity(() => original); + const { client } = makeGetClient(entity); + let thrown; + try { + await entity.get({ org: "org1", id: "id1" }).go({ client }); + } catch (err) { + thrown = err; + } + expect(thrown, "expected go() to reject").to.not.equal(undefined); + expect(thrown.isElectroError).to.equal(true); + expect(thrown.code).to.equal(ErrorCodes.AWSError.code); + expect(thrown.cause).to.equal(original); + expect(thrown.message).to.equal( + 'Error thrown by DynamoDB client: "boom" - For more detail on this error reference: https://electrodb.dev/en/reference/errors/#aws-error', + ); + expect(thrown.stack).to.be.a("string").and.to.have.length.greaterThan(0); + }); + + it("rethrows an ElectroError unwrapped (same instance)", async () => { + const original = new ElectroError( + ErrorCodes.InvalidAttribute, + "already electro", + ); + const entity = makeThrowingEntity(() => original); + const { client } = makeGetClient(entity); + let thrown; + try { + await entity.get({ org: "org1", id: "id1" }).go({ client }); + } catch (err) { + thrown = err; + } + expect(thrown).to.equal(original); + }); + + it("rethrows the raw error when originalErr is set", async () => { + const original = new Error("boom"); + const entity = makeThrowingEntity(() => original); + const { client } = makeGetClient(entity); + let thrown; + try { + await entity + .get({ org: "org1", id: "id1" }) + .go({ client, originalErr: true }); + } catch (err) { + thrown = err; + } + expect(thrown).to.equal(original); + expect(thrown.isElectroError).to.equal(undefined); + }); + + it("wraps errors surfaced through the public parse() path", () => { + const original = new Error("parse boom"); + const entity = makeThrowingEntity(() => original); + const { Item } = entity + .put({ org: "org1", id: "id1", boom: "value" }) + .params(); + let thrown; + try { + entity.parse({ Item }); + } catch (err) { + thrown = err; + } + expect(thrown.isElectroError).to.equal(true); + expect(thrown.cause).to.equal(original); + }); +}); + // --------------------------------------------------------------------------- // P1: executeQuery accumulated results by rebuilding the whole accumulator on // every page (`results = [...results, ...items]`), making auto-paging From d2d6d17adb1c4357c16753783c957709cc5abdbe Mon Sep 17 00:00:00 2001 From: ty Date: Tue, 9 Jun 2026 23:25:30 -0400 Subject: [PATCH 06/17] perf: share the siblings snapshot per mutation pass; skip no-op path regex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _applyAttributeMutation rebuilt a full copy of the payload every time any attribute's getter/setter asked for its siblings — once per attribute, per pass, per item. The snapshot is now built lazily on first use and shared across the pass; the payload is never mutated within a pass (writes land on the separate `data` object), so the shared snapshot holds the same values the per-call copies did. genericizeJSONPath ran its [digits]→[*] regex on every attribute path lookup; it now returns bracket-free paths (the per-attribute common case) untouched. Guard tests pin sibling visibility (setters see original values, watchers fire once and see the watched setter's output, getters see siblings on parse) and bracketed list-path update resolution. Co-Authored-By: Claude Fable 5 --- src/schema.js | 6 +- src/util.js | 5 ++ test/offline.perf-fixes.spec.js | 129 ++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 1 deletion(-) diff --git a/src/schema.js b/src/schema.js index 67f99731..c4b5c734 100644 --- a/src/schema.js +++ b/src/schema.js @@ -1656,7 +1656,11 @@ class Schema { _applyAttributeMutation(method, include, avoid, payload) { let data = { ...payload }; - const getSiblings = () => ({ ...payload }); + // one read-only snapshot shared across the pass; `payload` is never + // mutated inside the loop (mutations land on `data`), so building it + // lazily on first use returns the same values a per-call copy would + let siblings; + const getSiblings = () => (siblings = siblings || { ...payload }); for (let path of Object.keys(include)) { // this.attributes[attribute] !== undefined | Attribute exists as actual attribute. If `includeKeys` is turned on for example this will include values that do not have a presence in the model and therefore will not have a `.get()` method // avoid[attribute] === undefined | Attribute shouldn't be in the avoided diff --git a/src/util.js b/src/util.js index dd326bbb..c1a8c03e 100644 --- a/src/util.js +++ b/src/util.js @@ -11,6 +11,11 @@ function parseJSONPath(path = "") { } function genericizeJSONPath(path = "") { + // the regex only rewrites `[digits]` segments; skip it for the common + // bracket-free attribute paths resolved on every getter/setter pass + if (path.indexOf("[") === -1) { + return path; + } return path.replace(/\[\d+\]/g, "[*]"); } diff --git a/test/offline.perf-fixes.spec.js b/test/offline.perf-fixes.spec.js index 323f50fe..5e6ec8bf 100644 --- a/test/offline.perf-fixes.spec.js +++ b/test/offline.perf-fixes.spec.js @@ -243,6 +243,135 @@ describe("P2: formatResponse error wrapping", () => { }); }); +// --------------------------------------------------------------------------- +// P4: each getter/setter pass copied the entire payload once per attribute +// (`getSiblings`) and re-ran a regex per path lookup. The snapshot is now +// shared per pass and the regex skipped for bracket-free paths. These +// tests pin the sibling-visibility semantics and bracketed-path +// resolution the optimizations must preserve. +// --------------------------------------------------------------------------- +describe("P4: attribute mutation passes", () => { + it("setters see original sibling values, not other setters' output", () => { + const entity = new Entity( + { + model: { entity: "siblings", service: "perfService", version: "1" }, + table, + attributes: { + org: { type: "string" }, + id: { type: "string" }, + a: { type: "string", set: (value, item) => `${value}:${item.b}` }, + b: { type: "string", set: (value, item) => `${value}:${item.a}` }, + }, + indexes: { + records: { + pk: { field: "pk", composite: ["org"] }, + sk: { field: "sk", composite: ["id"] }, + }, + }, + }, + { table }, + ); + const { Item } = entity + .put({ org: "org1", id: "id1", a: "1", b: "2" }) + .params(); + // each setter saw the *original* value of its sibling + expect(Item.a).to.equal("1:2"); + expect(Item.b).to.equal("2:1"); + }); + + it("watchers fire exactly once and see the watched attribute's set output", () => { + const counts = { name: 0, display: 0 }; + const entity = new Entity( + { + model: { entity: "watchers", service: "perfService", version: "1" }, + table, + attributes: { + org: { type: "string" }, + id: { type: "string" }, + name: { + type: "string", + set: (value) => { + counts.name++; + return `${value}!`; + }, + }, + display: { + type: "string", + watch: ["name"], + set: (_, item) => { + counts.display++; + return `D:${item.name}`; + }, + }, + }, + indexes: { + records: { + pk: { field: "pk", composite: ["org"] }, + sk: { field: "sk", composite: ["id"] }, + }, + }, + }, + { table }, + ); + const { Item } = entity + .put({ org: "org1", id: "id1", name: "joe" }) + .params(); + expect(counts).to.deep.equal({ name: 1, display: 1 }); + expect(Item.name).to.equal("joe!"); + // the watcher pass runs against the first pass's output + expect(Item.display).to.equal("D:joe!"); + }); + + it("getters see sibling values during retrieval formatting", () => { + const entity = new Entity( + { + model: { entity: "getters", service: "perfService", version: "1" }, + table, + attributes: { + org: { type: "string" }, + id: { type: "string" }, + label: { + type: "string", + get: (value, item) => `${value}@${item.org}`, + }, + }, + indexes: { + records: { + pk: { field: "pk", composite: ["org"] }, + sk: { field: "sk", composite: ["id"] }, + }, + }, + }, + { table }, + ); + const { Item } = entity + .put({ org: "org1", id: "id1", label: "x" }) + .params(); + const { data } = entity.parse({ Item }); + expect(data.label).to.equal("x@org1"); + }); + + it("resolves bracketed list paths through update validation", () => { + const entity = makeFixtureEntity(); + const params = entity + .update({ org: "org1", id: "id1" }) + .data((attr, op) => op.set(attr.notes[0].body, "edited")) + .params(); + expect(params.UpdateExpression).to.equal( + "SET #notes[0].#body = :body_u0, #org = :org_u0, #id = :id_u0, #__edb_e__ = :__edb_e___u0, #__edb_v__ = :__edb_v___u0", + ); + expect(params.ExpressionAttributeNames).to.deep.equal({ + "#notes": "notes", + "#body": "body", + "#org": "org", + "#id": "id", + "#__edb_e__": "__edb_e__", + "#__edb_v__": "__edb_v__", + }); + expect(params.ExpressionAttributeValues[":body_u0"]).to.equal("edited"); + }); +}); + // --------------------------------------------------------------------------- // P1: executeQuery accumulated results by rebuilding the whole accumulator on // every page (`results = [...results, ...items]`), making auto-paging From 61fd8f412791d5ab7963d06362475b69810f71cc Mon Sep 17 00:00:00 2001 From: ty Date: Tue, 9 Jun 2026 23:28:09 -0400 Subject: [PATCH 07/17] perf: build chain update machinery lazily; reads never construct it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every chain construction — including get/query/scan — eagerly built an AttributeOperationProxy, which defines a property per attribute and per operation and dominates chain-construction cost, despite read chains never using it. ChainState now exposes `updateProxy` as a cached lazy getter: the cache preserves the instance identity write clauses rely on to accumulate expression state, and `update`/the FilterExpressions stay eager since query paths read them. entity._params previously destructured updateProxy from state.query for every method, which would have triggered the getter on all reads; it is now read only inside the upsert case. Guard tests pin the accessor + identity stability, that plain read chains construct zero proxies while an update constructs exactly one, and byte-identical update/upsert/query params against fixtures captured from the eager implementation. Co-Authored-By: Claude Fable 5 --- src/clauses.js | 20 +++-- src/entity.js | 5 +- test/offline.perf-fixes.spec.js | 146 ++++++++++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 7 deletions(-) diff --git a/src/clauses.js b/src/clauses.js index 1c9c1507..23b46698 100644 --- a/src/clauses.js +++ b/src/clauses.js @@ -1392,6 +1392,11 @@ class ChainState { parentState = null, } = {}) { const update = new UpdateExpression({ prefix: "_u" }); + // AttributeOperationProxy defines a property per attribute and per + // operation, which makes it the dominant chain-construction cost; read + // chains (get/query/scan) never touch it, so it is built lazily and + // cached so write clauses keep accumulating into one instance + let updateProxy; this.parentState = parentState; this.error = null; this.attributes = attributes; @@ -1402,11 +1407,16 @@ class ChainState { method: "", facets: { ...compositeAttributes }, update, - updateProxy: new AttributeOperationProxy({ - builder: update, - attributes: attributes, - operations: UpdateOperations, - }), + get updateProxy() { + updateProxy = + updateProxy || + new AttributeOperationProxy({ + builder: update, + attributes: attributes, + operations: UpdateOperations, + }); + return updateProxy; + }, put: { data: {}, }, diff --git a/src/entity.js b/src/entity.js index 1236660c..9b6072c2 100644 --- a/src/entity.js +++ b/src/entity.js @@ -2118,7 +2118,6 @@ class Entity { update = {}, filter = {}, upsert, - updateProxy, } = state.query; let consolidatedQueryFacets = this._consolidateQueryFacets(keys.sk); let params = {}; @@ -2133,8 +2132,10 @@ class Entity { ); break; case MethodTypes.upsert: + // updateProxy is read inside the case (not destructured above) so + // read methods never trigger its lazy construction params = this._makeUpsertParams( - { update, upsert, updateProxy }, + { update, upsert, updateProxy: state.query.updateProxy }, keys.pk, ...keys.sk, ); diff --git a/test/offline.perf-fixes.spec.js b/test/offline.perf-fixes.spec.js index 5e6ec8bf..9683a96e 100644 --- a/test/offline.perf-fixes.spec.js +++ b/test/offline.perf-fixes.spec.js @@ -372,6 +372,152 @@ describe("P4: attribute mutation passes", () => { }); }); +// --------------------------------------------------------------------------- +// P3: every chain construction (including get/query/scan) eagerly built an +// AttributeOperationProxy — an Object.defineProperty per attribute and +// per operation — that read chains never use. It is now a cached lazy +// getter on ChainState's query object. These tests pin laziness, instance +// identity (write clauses accumulate into one builder), and exact params +// parity with the eager implementation (fixtures captured pre-change). +// --------------------------------------------------------------------------- +describe("P3: lazy update machinery on chain construction", () => { + const { ChainState } = require("../src/clauses"); + const { AttributeOperationProxy } = require("../src/operations"); + const entity = makeFixtureEntity(); + + it("exposes updateProxy as a cached lazy getter with stable identity", () => { + const state = new ChainState({ + index: "", + attributes: entity.model.schema.attributes, + hasSortKey: true, + options: {}, + }); + const descriptor = Object.getOwnPropertyDescriptor( + state.query, + "updateProxy", + ); + expect(descriptor.get, "updateProxy should be an accessor").to.be.a( + "function", + ); + const first = state.query.updateProxy; + expect(first).to.be.instanceOf(AttributeOperationProxy); + expect(state.query.updateProxy).to.equal(first); + }); + + it("never builds the proxy for read chains, builds once for writes", () => { + // AttributeOperationProxy's constructor always calls the static + // buildAttributes, so spying on it counts constructions + const original = AttributeOperationProxy.buildAttributes; + let constructions = 0; + AttributeOperationProxy.buildAttributes = function (...args) { + constructions++; + return original.apply(this, args); + }; + try { + entity.query.records({ org: "org1" }).params(); + entity.query.records({ org: "org1" }).gt({ id: "id1" }).params(); + entity.get({ org: "org1", id: "id1" }).params(); + entity.scan.params(); + expect(constructions, "read chains should not build the proxy").to.equal( + 0, + ); + entity.update({ org: "org1", id: "id1" }).set({ name: "x" }).params(); + expect(constructions).to.equal(1); + } finally { + AttributeOperationProxy.buildAttributes = original; + } + }); + + it("produces params identical to the eager implementation", () => { + // fixtures captured from the pre-change (eager) implementation + const update = entity + .update({ org: "org1", id: "id1" }) + .set({ name: "newname" }) + .add({ count: 5 }) + .append({ notes: [{ body: "extra" }] }) + .remove(["active"]) + .params(); + expect(update).to.deep.equal({ + UpdateExpression: + "SET #name = :name_u0, #notes = list_append(if_not_exists(#notes, :notes_default_value_u0), :notes_u0), #org = :org_u0, #id = :id_u0, #__edb_e__ = :__edb_e___u0, #__edb_v__ = :__edb_v___u0 REMOVE #active ADD #count :count_u0", + ExpressionAttributeNames: { + "#name": "name", + "#count": "count", + "#notes": "notes", + "#active": "active", + "#org": "org", + "#id": "id", + "#__edb_e__": "__edb_e__", + "#__edb_v__": "__edb_v__", + }, + ExpressionAttributeValues: { + ":name_u0": "newname", + ":count_u0": 5, + ":notes_u0": [{ body: "extra" }], + ":notes_default_value_u0": [], + ":org_u0": "org1", + ":id_u0": "id1", + ":__edb_e___u0": "perfFixture", + ":__edb_v___u0": "1", + }, + TableName: "electro", + Key: { + pk: "$perfservice#org_org1", + sk: "$perffixture_1#id_id1", + }, + }); + + const upsert = entity + .upsert({ org: "org1", id: "id1", name: "n", count: 1 }) + .params(); + expect(upsert).to.deep.equal({ + TableName: "electro", + UpdateExpression: + "SET #__edb_e__ = :__edb_e___u0, #__edb_v__ = :__edb_v___u0, #org = :org_u0, #id = :id_u0, #name = :name_u0, #count = :count_u0", + ExpressionAttributeNames: { + "#__edb_e__": "__edb_e__", + "#__edb_v__": "__edb_v__", + "#org": "org", + "#id": "id", + "#name": "name", + "#count": "count", + }, + ExpressionAttributeValues: { + ":__edb_e___u0": "perfFixture", + ":__edb_v___u0": "1", + ":org_u0": "org1", + ":id_u0": "id1", + ":name_u0": "n", + ":count_u0": 1, + }, + Key: { + pk: "$perfservice#org_org1", + sk: "$perffixture_1#id_id1", + }, + }); + + const query = entity.query + .records({ org: "org1" }) + .where((attr, op) => op.gt(attr.count, 10)) + .params(); + expect(query).to.deep.equal({ + KeyConditionExpression: "#pk = :pk and begins_with(#sk1, :sk1)", + TableName: "electro", + ExpressionAttributeNames: { + "#count": "count", + "#pk": "pk", + "#sk1": "sk", + }, + ExpressionAttributeValues: { + ":count0": 10, + ":pk": "$perfservice#org_org1", + ":sk1": "$perffixture_1#id_", + }, + FilterExpression: "#count > :count0", + }); + }); +}); + // --------------------------------------------------------------------------- // P1: executeQuery accumulated results by rebuilding the whole accumulator on // every page (`results = [...results, ...items]`), making auto-paging From bfa75f3793620c94a6746f42f97a043d966bbb3c Mon Sep 17 00:00:00 2001 From: ty Date: Tue, 9 Jun 2026 23:32:14 -0400 Subject: [PATCH 08/17] test: add tinybench benchmark harness with baseline compare Adds manual/local benchmarks (npm run benchmark / :json / :update / :compare) covering the hot paths touched by the perf fixes: entity construction, chain params building, parse/format at size, batchGet formatting, and multi-page query accumulation. Scenarios are dropped into benchmark/scenarios/*.bench.js with no registration list; results are normalized against a fixed reference task so the committed baseline.json is roughly machine-independent. Compare is advisory (exit 0); --strict is the one-flag hook for promoting it to CI later. Not wired into the test gate or any workflow. Co-Authored-By: Claude Fable 5 --- benchmark/README.md | 69 ++++++ benchmark/baseline.json | 78 ++++++ benchmark/run.js | 227 ++++++++++++++++++ benchmark/scenarios/batch-get-format.bench.js | 38 +++ .../scenarios/entity-construction.bench.js | 19 ++ benchmark/scenarios/params-chain.bench.js | 35 +++ benchmark/scenarios/parse-format.bench.js | 38 +++ benchmark/scenarios/query-pagination.bench.js | 44 ++++ package-lock.json | 14 ++ package.json | 5 + 10 files changed, 567 insertions(+) create mode 100644 benchmark/README.md create mode 100644 benchmark/baseline.json create mode 100644 benchmark/run.js create mode 100644 benchmark/scenarios/batch-get-format.bench.js create mode 100644 benchmark/scenarios/entity-construction.bench.js create mode 100644 benchmark/scenarios/params-chain.bench.js create mode 100644 benchmark/scenarios/parse-format.bench.js create mode 100644 benchmark/scenarios/query-pagination.bench.js diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 00000000..1c3e9ee6 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,69 @@ +# ElectroDB benchmarks + +Manual/local micro-benchmarks for ElectroDB's hot paths, built on +[tinybench](https://github.com/tinylibs/tinybench). They are **not** part of +the test suite or CI — nothing here gates a build. + +## Running + +```sh +npm run benchmark # run everything, print a table +npm run benchmark:json # machine-readable results (hz, mean, p99, rme) +npm run benchmark:update # rewrite baseline.json from a fresh run +npm run benchmark:compare # run + diff against baseline.json (advisory) +``` + +`benchmark:compare` flags a task as a regression when its normalized +throughput drops more than 20% below the baseline. It always exits 0 +(advisory); add `--strict` to exit 1 on regressions — that one flag is the +hook for promoting the comparison to CI later: + +```sh +node ./benchmark/run.js --compare --strict +``` + +You can also run a subset while iterating: + +```sh +node ./benchmark/run.js --filter pagination +``` + +## Baseline + normalization + +Raw ops/sec is machine-dependent, so the committed `baseline.json` stores each +task's throughput **normalized** to a fixed reference task +(`reference/parse-500-items`, a representative formatting workload) that runs +with every invocation: `normalized = task.hz / referenceHz`. Comparing +normalized values makes the baseline meaningful across reasonably similar +machines; for precise A/B work, regenerate the baseline on your own machine +first (`npm run benchmark:update`), apply your change, then +`npm run benchmark:compare`. + +## Adding a scenario + +Drop a file matching `benchmark/scenarios/*.bench.js` that exports an array of +entries — there is no registration list to edit: + +```js +const { makeFixtureEntity } = require("../../test/fixtures/entities"); + +const entity = makeFixtureEntity(); + +module.exports = [ + { + name: "my-scenario/some-task", // group/task; sizes get their own task names + fn: () => entity.query.records({ org: "org1" }).params(), + // opts: optional tinybench task options (beforeAll/beforeEach/...) + }, +]; +``` + +Conventions: + +- Do setup (entities, stored items, mock clients) at module scope or in + `opts.beforeAll` so it stays outside the timed region; `fn` should measure + only the operation under test. `fn` may be async. +- Express sizes as separate named tasks (`parse-format/1000-items`) rather + than parameters, so the baseline tracks each size independently. +- Reuse `test/fixtures/` (mock clients, fixture entities) rather than talking + to a real DynamoDB — benchmarks must run offline. diff --git a/benchmark/baseline.json b/benchmark/baseline.json new file mode 100644 index 00000000..d1f24a96 --- /dev/null +++ b/benchmark/baseline.json @@ -0,0 +1,78 @@ +{ + "schemaVersion": 1, + "generatedAt": "2026-06-10T03:31:53.192Z", + "node": "v24.3.0", + "referenceHz": 2327.262141538007, + "tasks": { + "reference/parse-500-items": { + "hz": 2327.262141538007, + "mean": 0.42968945446735735, + "normalized": 1 + }, + "batch-get-format/100-items": { + "hz": 4519.7695369513285, + "mean": 0.22125021902654782, + "normalized": 1.9420973066507965 + }, + "entity-construction/new-entity": { + "hz": 5731.994118974018, + "mean": 0.1744593555478023, + "normalized": 2.4629774259920465 + }, + "entity-construction/new-entity-watchers": { + "hz": 5468.781577339595, + "mean": 0.18285608702011302, + "normalized": 2.3498777725681843 + }, + "params-chain/get": { + "hz": 416715.25824700115, + "mean": 0.00239972014513389, + "normalized": 179.05815198437784 + }, + "params-chain/query": { + "hz": 302041.66219756944, + "mean": 0.0033108015388482624, + "normalized": 129.78411705608744 + }, + "params-chain/query-where": { + "hz": 119115.32935451993, + "mean": 0.008395225076561938, + "normalized": 51.18260088904326 + }, + "params-chain/update-set": { + "hz": 93418.97855690971, + "mean": 0.010704463005777912, + "normalized": 40.14114993301629 + }, + "params-chain/upsert": { + "hz": 82706.38805249817, + "mean": 0.01209096447743851, + "normalized": 35.538062763243495 + }, + "parse-format/100-items": { + "hz": 11449.031595110766, + "mean": 0.08734363179039907, + "normalized": 4.919528140282683 + }, + "parse-format/1000-items": { + "hz": 1159.505939157333, + "mean": 0.8624362896551843, + "normalized": 0.49822747444817517 + }, + "parse-format/1000-items-watchers": { + "hz": 794.5887420594664, + "mean": 1.258512670854278, + "normalized": 0.3414264031014358 + }, + "query-pagination/10-pages-x100": { + "hz": 1454.944405824759, + "mean": 0.6873114848901278, + "normalized": 0.6251742680191741 + }, + "query-pagination/100-pages-x100": { + "hz": 140.09983739361508, + "mean": 7.137767028169114, + "normalized": 0.060199422700627926 + } + } +} diff --git a/benchmark/run.js b/benchmark/run.js new file mode 100644 index 00000000..e78b64f0 --- /dev/null +++ b/benchmark/run.js @@ -0,0 +1,227 @@ +#!/usr/bin/env node +/** + * ElectroDB benchmark runner (manual/local — not wired into the test gate). + * + * npm run benchmark run all scenarios, print a table + * npm run benchmark:json print machine-readable results + * npm run benchmark:update write benchmark/baseline.json + * npm run benchmark:compare compare against baseline.json (advisory) + * --strict exit 1 when the compare finds regressions + * --filter only run tasks whose name includes substring + * + * Scenarios live in benchmark/scenarios/*.bench.js; each exports an array of + * `{ name, fn, opts? }` entries (see README.md). A fixed reference task runs + * with every invocation and task throughput is recorded relative to it + * (`normalized = hz / referenceHz`), which makes the committed baseline + * roughly machine-independent. + */ +const fs = require("fs"); +const path = require("path"); +const { Bench } = require("tinybench"); + +const SCENARIO_DIR = path.join(__dirname, "scenarios"); +const BASELINE_PATH = path.join(__dirname, "baseline.json"); +const REFERENCE_TASK = "reference/parse-500-items"; +const DEFAULT_TOLERANCE = 0.2; + +function parseArgs(argv) { + const args = { + json: false, + compare: false, + updateBaseline: false, + strict: false, + filter: null, + }; + for (let i = 2; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--json") args.json = true; + else if (arg === "--compare") args.compare = true; + else if (arg === "--update-baseline") args.updateBaseline = true; + else if (arg === "--strict") args.strict = true; + else if (arg === "--filter") args.filter = argv[++i]; + else { + console.error(`Unknown argument: ${arg}`); + process.exit(1); + } + } + return args; +} + +function makeReferenceTask() { + const { makeFixtureEntity, makeStoredItem } = require("../test/fixtures/entities"); + const entity = makeFixtureEntity(); + const Items = []; + for (let i = 0; i < 500; i++) { + Items.push(makeStoredItem(entity, i)); + } + return { name: REFERENCE_TASK, fn: () => entity.parse({ Items }) }; +} + +function loadScenarios() { + const entries = [makeReferenceTask()]; + const files = fs + .readdirSync(SCENARIO_DIR) + .filter((file) => file.endsWith(".bench.js")) + .sort(); + for (const file of files) { + const scenario = require(path.join(SCENARIO_DIR, file)); + if (!Array.isArray(scenario)) { + throw new Error(`${file} must export an array of {name, fn, opts?}`); + } + for (const entry of scenario) { + if (!entry || typeof entry.name !== "string" || typeof entry.fn !== "function") { + throw new Error(`${file} exported an entry without {name, fn}`); + } + entries.push(entry); + } + } + return entries; +} + +function collectResults(bench) { + const tasks = {}; + let failed = false; + for (const task of bench.tasks) { + if (!task.result || task.result.error) { + console.error(`Task "${task.name}" failed:`); + console.error(task.result && task.result.error); + failed = true; + continue; + } + tasks[task.name] = { + hz: task.result.hz, + mean: task.result.mean, + p99: task.result.p99, + rme: task.result.rme, + samples: task.result.samples.length, + }; + } + if (failed) { + process.exit(1); + } + const referenceHz = tasks[REFERENCE_TASK] ? tasks[REFERENCE_TASK].hz : null; + if (referenceHz) { + for (const name of Object.keys(tasks)) { + tasks[name].normalized = tasks[name].hz / referenceHz; + } + } + return { referenceHz, tasks }; +} + +function compare(baseline, current, { strict }) { + const tolerance = DEFAULT_TOLERANCE; + const rows = []; + const regressions = []; + for (const [name, base] of Object.entries(baseline.tasks)) { + if (name === REFERENCE_TASK) continue; + const task = current.tasks[name]; + if (!task) { + rows.push({ task: name, status: "removed" }); + continue; + } + const ratio = task.normalized / base.normalized; + const row = { + task: name, + "baseline (norm)": base.normalized.toFixed(4), + "current (norm)": task.normalized.toFixed(4), + ratio: ratio.toFixed(3), + status: + ratio < 1 - tolerance + ? "REGRESSION" + : ratio > 1 + tolerance + ? "improved" + : "ok", + }; + rows.push(row); + if (row.status === "REGRESSION") regressions.push(row); + } + for (const name of Object.keys(current.tasks)) { + if (name !== REFERENCE_TASK && !baseline.tasks[name]) { + rows.push({ task: name, status: "added (not in baseline)" }); + } + } + console.table(rows); + if (regressions.length) { + console.error( + `${regressions.length} regression(s) beyond ${tolerance * 100}% tolerance (vs baseline from ${baseline.generatedAt}, node ${baseline.node}).`, + ); + if (strict) process.exit(1); + } else { + console.log( + `No regressions beyond ${tolerance * 100}% tolerance (vs baseline from ${baseline.generatedAt}).`, + ); + } +} + +async function main() { + const args = parseArgs(process.argv); + let entries = loadScenarios(); + if (args.filter) { + entries = entries.filter( + (entry) => + entry.name === REFERENCE_TASK || entry.name.includes(args.filter), + ); + } + + const bench = new Bench({ time: 500, warmupTime: 100 }); + for (const { name, fn, opts } of entries) { + bench.add(name, fn, opts); + } + + await bench.warmup(); + await bench.run(); + + const current = collectResults(bench); + + if (args.json) { + console.log( + JSON.stringify( + { + generatedAt: new Date().toISOString(), + node: process.version, + referenceHz: current.referenceHz, + tasks: current.tasks, + }, + null, + 2, + ), + ); + } else { + console.table(bench.table()); + } + + if (args.updateBaseline) { + const baseline = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + node: process.version, + referenceHz: current.referenceHz, + tasks: {}, + }; + for (const [name, task] of Object.entries(current.tasks)) { + baseline.tasks[name] = { + hz: task.hz, + mean: task.mean, + normalized: task.normalized, + }; + } + fs.writeFileSync(BASELINE_PATH, `${JSON.stringify(baseline, null, 2)}\n`); + console.log(`Baseline written to ${path.relative(process.cwd(), BASELINE_PATH)}`); + } + + if (args.compare) { + if (!fs.existsSync(BASELINE_PATH)) { + console.error( + "No baseline.json found — run `npm run benchmark:update` first.", + ); + process.exit(1); + } + const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, "utf8")); + compare(baseline, current, { strict: args.strict }); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/benchmark/scenarios/batch-get-format.bench.js b/benchmark/scenarios/batch-get-format.bench.js new file mode 100644 index 00000000..b5384fd7 --- /dev/null +++ b/benchmark/scenarios/batch-get-format.bench.js @@ -0,0 +1,38 @@ +/** + * P2 — batchGet response formatting calls formatResponse once per item. + */ +const { makeMockV2Client } = require("../../test/fixtures/mock-client"); +const { + table, + makeFixtureEntity, + makeStoredItem, + makeItemData, +} = require("../../test/fixtures/entities"); + +const entity = makeFixtureEntity(); + +function makeBatch(count) { + const Responses = { [table]: [] }; + const keys = []; + for (let i = 0; i < count; i++) { + Responses[table].push(makeStoredItem(entity, i)); + const { org, id } = makeItemData(i); + keys.push({ org, id }); + } + const { client, calls } = makeMockV2Client({ + batchGet: { Responses, UnprocessedKeys: {} }, + }); + return { client, calls, keys }; +} + +const batch100 = makeBatch(100); + +module.exports = [ + { + name: "batch-get-format/100-items", + fn: async () => { + batch100.calls.length = 0; // keep the mock's call log from growing + return entity.get(batch100.keys).go({ client: batch100.client }); + }, + }, +]; diff --git a/benchmark/scenarios/entity-construction.bench.js b/benchmark/scenarios/entity-construction.bench.js new file mode 100644 index 00000000..acc964fe --- /dev/null +++ b/benchmark/scenarios/entity-construction.bench.js @@ -0,0 +1,19 @@ +/** + * P5 — Entity construction cost (model validation dominates). + */ +const { Entity } = require("../../src/entity"); +const { table, makeFixtureModel } = require("../../test/fixtures/entities"); + +const model = makeFixtureModel(); +const watcherModel = makeFixtureModel({ withWatchers: true }); + +module.exports = [ + { + name: "entity-construction/new-entity", + fn: () => new Entity(model, { table }), + }, + { + name: "entity-construction/new-entity-watchers", + fn: () => new Entity(watcherModel, { table }), + }, +]; diff --git a/benchmark/scenarios/params-chain.bench.js b/benchmark/scenarios/params-chain.bench.js new file mode 100644 index 00000000..e0d2fbc1 --- /dev/null +++ b/benchmark/scenarios/params-chain.bench.js @@ -0,0 +1,35 @@ +/** + * P3 — chain construction + params building. Read chains (get/query/scan) + * should never build update machinery; update/upsert chains pay for it once. + */ +const { makeFixtureEntity } = require("../../test/fixtures/entities"); + +const entity = makeFixtureEntity(); +const key = { org: "org1", id: "id1" }; + +module.exports = [ + { + name: "params-chain/get", + fn: () => entity.get(key).params(), + }, + { + name: "params-chain/query", + fn: () => entity.query.records({ org: "org1" }).params(), + }, + { + name: "params-chain/query-where", + fn: () => + entity.query + .records({ org: "org1" }) + .where((attr, op) => op.gt(attr.count, 10)) + .params(), + }, + { + name: "params-chain/update-set", + fn: () => entity.update(key).set({ name: "x" }).params(), + }, + { + name: "params-chain/upsert", + fn: () => entity.upsert({ ...key, name: "x", count: 1 }).params(), + }, +]; diff --git a/benchmark/scenarios/parse-format.bench.js b/benchmark/scenarios/parse-format.bench.js new file mode 100644 index 00000000..5ed0f236 --- /dev/null +++ b/benchmark/scenarios/parse-format.bench.js @@ -0,0 +1,38 @@ +/** + * P2 + P4 — response formatting: per-item getter passes, sibling snapshots, + * path lookups, and (formerly) per-call error allocation. + */ +const { + makeFixtureEntity, + makeStoredItem, +} = require("../../test/fixtures/entities"); + +const entity = makeFixtureEntity(); +const watcherEntity = makeFixtureEntity({ withWatchers: true }); + +function makeItems(owner, count) { + const Items = []; + for (let i = 0; i < count; i++) { + Items.push(makeStoredItem(owner, i)); + } + return Items; +} + +const items100 = makeItems(entity, 100); +const items1000 = makeItems(entity, 1000); +const watcherItems1000 = makeItems(watcherEntity, 1000); + +module.exports = [ + { + name: "parse-format/100-items", + fn: () => entity.parse({ Items: items100 }), + }, + { + name: "parse-format/1000-items", + fn: () => entity.parse({ Items: items1000 }), + }, + { + name: "parse-format/1000-items-watchers", + fn: () => watcherEntity.parse({ Items: watcherItems1000 }), + }, +]; diff --git a/benchmark/scenarios/query-pagination.bench.js b/benchmark/scenarios/query-pagination.bench.js new file mode 100644 index 00000000..d9459261 --- /dev/null +++ b/benchmark/scenarios/query-pagination.bench.js @@ -0,0 +1,44 @@ +/** + * P1 — auto-paging accumulation across many pages (`pages: "all"`). The + * O(pages²) → O(pages) change shows up most at high page counts. + */ +const { + makeMockV2Client, + makePagingQueryHandler, +} = require("../../test/fixtures/mock-client"); +const { + makeFixtureEntity, + makeStoredItem, +} = require("../../test/fixtures/entities"); + +const entity = makeFixtureEntity(); + +function makePagingClient({ pages, perPage }) { + const query = makePagingQueryHandler({ + pages, + perPage, + makeItem: (i) => makeStoredItem(entity, i), + }); + return makeMockV2Client({ query }); +} + +const mock10 = makePagingClient({ pages: 10, perPage: 100 }); +const mock100 = makePagingClient({ pages: 100, perPage: 100 }); + +function runAllPages(mock) { + mock.calls.length = 0; // keep the mock's call log from growing + return entity.query + .records({ org: "org1" }) + .go({ client: mock.client, pages: "all" }); +} + +module.exports = [ + { + name: "query-pagination/10-pages-x100", + fn: async () => runAllPages(mock10), + }, + { + name: "query-pagination/100-pages-x100", + fn: async () => runAllPages(mock100), + }, +]; diff --git a/package-lock.json b/package-lock.json index 9eaff02d..96930103 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,7 @@ "prettier-plugin-astro": "^0.12.0", "source-map-support": "^0.5.19", "sst": "^2.28.0", + "tinybench": "^2.9.0", "ts-node": "^10.9.1", "tsd": "^0.28.1", "typescript": "^5.2.2", @@ -29437,6 +29438,13 @@ "node": ">=0.6.0" } }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, "node_modules/tmpl": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.4.tgz", @@ -54336,6 +54344,12 @@ "process": "~0.11.0" } }, + "tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true + }, "tmpl": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.4.tgz", diff --git a/package.json b/package.json index 0bb0b450..f1829edb 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,10 @@ "local:stop": "npm run ddb:stop", "local:exec": "LOCAL_DYNAMO_ENDPOINT='http://localhost:8000' ts-node ./test/debug.ts", "local:debug": "npm run local:start && npm run local:exec", + "benchmark": "node ./benchmark/run.js", + "benchmark:json": "node ./benchmark/run.js --json", + "benchmark:compare": "node ./benchmark/run.js --compare", + "benchmark:update": "node ./benchmark/run.js --update-baseline", "format": "prettier -w src/**/*.js examples/**/* --log-level=error" }, "repository": { @@ -70,6 +74,7 @@ "prettier-plugin-astro": "^0.12.0", "source-map-support": "^0.5.19", "sst": "^2.28.0", + "tinybench": "^2.9.0", "ts-node": "^10.9.1", "tsd": "^0.28.1", "typescript": "^5.2.2", From 88c5f8d890dbe4175564016fd8a85d96f556faab Mon Sep 17 00:00:00 2001 From: ty Date: Tue, 9 Jun 2026 23:54:44 -0400 Subject: [PATCH 09/17] refactor: convert fixtures, guard spec, and benchmark harness to TypeScript Fixtures (test/fixtures/*.ts) gain typed exports; the guard spec and benchmark scenarios import them, while untyped src internals stay as require() per the existing ts_connected spec convention. The runner becomes benchmark/run.ts (scripts now use ts-node) and exports the ScenarioEntry type that scenario files default-export. No behavior change; the JS audit spec resolves the TS fixtures through ts-node's require hook in the existing mocha setup. Co-Authored-By: Claude Fable 5 --- benchmark/README.md | 17 ++- benchmark/{run.js => run.ts} | 140 +++++++++++++----- ...mat.bench.js => batch-get-format.bench.ts} | 17 ++- ....bench.js => entity-construction.bench.ts} | 8 +- ...s-chain.bench.js => params-chain.bench.ts} | 9 +- ...-format.bench.js => parse-format.bench.ts} | 14 +- ...ion.bench.js => query-pagination.bench.ts} | 24 ++- package.json | 8 +- test/fixtures/{entities.js => entities.ts} | 44 ++++-- .../{mock-client.js => mock-client.ts} | 81 +++++++--- ...xes.spec.js => offline.perf-fixes.spec.ts} | 79 ++++++---- 11 files changed, 298 insertions(+), 143 deletions(-) rename benchmark/{run.js => run.ts} (61%) rename benchmark/scenarios/{batch-get-format.bench.js => batch-get-format.bench.ts} (65%) rename benchmark/scenarios/{entity-construction.bench.js => entity-construction.bench.ts} (70%) rename benchmark/scenarios/{params-chain.bench.js => params-chain.bench.ts} (76%) rename benchmark/scenarios/{parse-format.bench.js => parse-format.bench.ts} (72%) rename benchmark/scenarios/{query-pagination.bench.js => query-pagination.bench.ts} (71%) rename test/fixtures/{entities.js => entities.ts} (70%) rename test/fixtures/{mock-client.js => mock-client.ts} (59%) rename test/{offline.perf-fixes.spec.js => offline.perf-fixes.spec.ts} (93%) diff --git a/benchmark/README.md b/benchmark/README.md index 1c3e9ee6..839c48ec 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -19,13 +19,13 @@ throughput drops more than 20% below the baseline. It always exits 0 hook for promoting the comparison to CI later: ```sh -node ./benchmark/run.js --compare --strict +npx ts-node ./benchmark/run.ts --compare --strict ``` You can also run a subset while iterating: ```sh -node ./benchmark/run.js --filter pagination +npx ts-node ./benchmark/run.ts --filter pagination ``` ## Baseline + normalization @@ -41,21 +41,24 @@ first (`npm run benchmark:update`), apply your change, then ## Adding a scenario -Drop a file matching `benchmark/scenarios/*.bench.js` that exports an array of -entries — there is no registration list to edit: +Drop a file matching `benchmark/scenarios/*.bench.ts` that default-exports an +array of entries — there is no registration list to edit: -```js -const { makeFixtureEntity } = require("../../test/fixtures/entities"); +```ts +import type { ScenarioEntry } from "../run"; +import { makeFixtureEntity } from "../../test/fixtures/entities"; const entity = makeFixtureEntity(); -module.exports = [ +const scenarios: ScenarioEntry[] = [ { name: "my-scenario/some-task", // group/task; sizes get their own task names fn: () => entity.query.records({ org: "org1" }).params(), // opts: optional tinybench task options (beforeAll/beforeEach/...) }, ]; + +export default scenarios; ``` Conventions: diff --git a/benchmark/run.js b/benchmark/run.ts similarity index 61% rename from benchmark/run.js rename to benchmark/run.ts index e78b64f0..e062314d 100644 --- a/benchmark/run.js +++ b/benchmark/run.ts @@ -1,31 +1,77 @@ -#!/usr/bin/env node +#!/usr/bin/env ts-node /** * ElectroDB benchmark runner (manual/local — not wired into the test gate). * - * npm run benchmark run all scenarios, print a table + * npm run benchmark run all scenarios, print a table * npm run benchmark:json print machine-readable results * npm run benchmark:update write benchmark/baseline.json * npm run benchmark:compare compare against baseline.json (advisory) * --strict exit 1 when the compare finds regressions * --filter only run tasks whose name includes substring * - * Scenarios live in benchmark/scenarios/*.bench.js; each exports an array of - * `{ name, fn, opts? }` entries (see README.md). A fixed reference task runs - * with every invocation and task throughput is recorded relative to it - * (`normalized = hz / referenceHz`), which makes the committed baseline + * Scenarios live in benchmark/scenarios/*.bench.ts; each default-exports an + * array of `{ name, fn, opts? }` entries (see README.md). A fixed reference + * task runs with every invocation and task throughput is recorded relative to + * it (`normalized = hz / referenceHz`), which makes the committed baseline * roughly machine-independent. */ -const fs = require("fs"); -const path = require("path"); -const { Bench } = require("tinybench"); +import * as fs from "fs"; +import * as path from "path"; +import { Bench } from "tinybench"; +import { makeFixtureEntity, makeStoredItem } from "../test/fixtures/entities"; + +export interface ScenarioEntry { + /** task name, conventionally `group/task` */ + name: string; + /** the operation under test; may be async */ + fn: () => unknown | Promise; + /** optional tinybench task options (beforeAll/beforeEach/...) */ + opts?: Record; +} + +interface TaskStats { + hz: number; + mean: number; + p99: number; + rme: number; + samples: number; + normalized?: number; +} + +interface RunResults { + referenceHz: number | null; + tasks: Record; +} + +interface BaselineTask { + hz: number; + mean: number; + normalized: number; +} + +interface BaselineFile { + schemaVersion: number; + generatedAt: string; + node: string; + referenceHz: number | null; + tasks: Record; +} + +interface CliArgs { + json: boolean; + compare: boolean; + updateBaseline: boolean; + strict: boolean; + filter: string | null; +} const SCENARIO_DIR = path.join(__dirname, "scenarios"); const BASELINE_PATH = path.join(__dirname, "baseline.json"); const REFERENCE_TASK = "reference/parse-500-items"; const DEFAULT_TOLERANCE = 0.2; -function parseArgs(argv) { - const args = { +function parseArgs(argv: string[]): CliArgs { + const args: CliArgs = { json: false, compare: false, updateBaseline: false, @@ -38,7 +84,7 @@ function parseArgs(argv) { else if (arg === "--compare") args.compare = true; else if (arg === "--update-baseline") args.updateBaseline = true; else if (arg === "--strict") args.strict = true; - else if (arg === "--filter") args.filter = argv[++i]; + else if (arg === "--filter") args.filter = argv[++i] ?? null; else { console.error(`Unknown argument: ${arg}`); process.exit(1); @@ -47,29 +93,33 @@ function parseArgs(argv) { return args; } -function makeReferenceTask() { - const { makeFixtureEntity, makeStoredItem } = require("../test/fixtures/entities"); +function makeReferenceTask(): ScenarioEntry { const entity = makeFixtureEntity(); - const Items = []; + const Items: Record[] = []; for (let i = 0; i < 500; i++) { Items.push(makeStoredItem(entity, i)); } return { name: REFERENCE_TASK, fn: () => entity.parse({ Items }) }; } -function loadScenarios() { - const entries = [makeReferenceTask()]; +function loadScenarios(): ScenarioEntry[] { + const entries: ScenarioEntry[] = [makeReferenceTask()]; const files = fs .readdirSync(SCENARIO_DIR) - .filter((file) => file.endsWith(".bench.js")) + .filter((file) => file.endsWith(".bench.ts") || file.endsWith(".bench.js")) .sort(); for (const file of files) { - const scenario = require(path.join(SCENARIO_DIR, file)); + const mod = require(path.join(SCENARIO_DIR, file)); + const scenario: unknown = mod.default ?? mod; if (!Array.isArray(scenario)) { throw new Error(`${file} must export an array of {name, fn, opts?}`); } for (const entry of scenario) { - if (!entry || typeof entry.name !== "string" || typeof entry.fn !== "function") { + if ( + !entry || + typeof entry.name !== "string" || + typeof entry.fn !== "function" + ) { throw new Error(`${file} exported an entry without {name, fn}`); } entries.push(entry); @@ -78,8 +128,8 @@ function loadScenarios() { return entries; } -function collectResults(bench) { - const tasks = {}; +function collectResults(bench: Bench): RunResults { + const tasks: Record = {}; let failed = false; for (const task of bench.tasks) { if (!task.result || task.result.error) { @@ -108,14 +158,18 @@ function collectResults(bench) { return { referenceHz, tasks }; } -function compare(baseline, current, { strict }) { +function compare( + baseline: BaselineFile, + current: RunResults, + { strict }: { strict: boolean }, +): void { const tolerance = DEFAULT_TOLERANCE; - const rows = []; - const regressions = []; + const rows: Record[] = []; + const regressions: Record[] = []; for (const [name, base] of Object.entries(baseline.tasks)) { if (name === REFERENCE_TASK) continue; const task = current.tasks[name]; - if (!task) { + if (!task || task.normalized === undefined) { rows.push({ task: name, status: "removed" }); continue; } @@ -129,8 +183,8 @@ function compare(baseline, current, { strict }) { ratio < 1 - tolerance ? "REGRESSION" : ratio > 1 + tolerance - ? "improved" - : "ok", + ? "improved" + : "ok", }; rows.push(row); if (row.status === "REGRESSION") regressions.push(row); @@ -143,23 +197,29 @@ function compare(baseline, current, { strict }) { console.table(rows); if (regressions.length) { console.error( - `${regressions.length} regression(s) beyond ${tolerance * 100}% tolerance (vs baseline from ${baseline.generatedAt}, node ${baseline.node}).`, + `${regressions.length} regression(s) beyond ${ + tolerance * 100 + }% tolerance (vs baseline from ${baseline.generatedAt}, node ${ + baseline.node + }).`, ); if (strict) process.exit(1); } else { console.log( - `No regressions beyond ${tolerance * 100}% tolerance (vs baseline from ${baseline.generatedAt}).`, + `No regressions beyond ${tolerance * 100}% tolerance (vs baseline from ${ + baseline.generatedAt + }).`, ); } } -async function main() { +async function main(): Promise { const args = parseArgs(process.argv); let entries = loadScenarios(); - if (args.filter) { + if (args.filter !== null) { + const filter = args.filter; entries = entries.filter( - (entry) => - entry.name === REFERENCE_TASK || entry.name.includes(args.filter), + (entry) => entry.name === REFERENCE_TASK || entry.name.includes(filter), ); } @@ -191,7 +251,7 @@ async function main() { } if (args.updateBaseline) { - const baseline = { + const baseline: BaselineFile = { schemaVersion: 1, generatedAt: new Date().toISOString(), node: process.version, @@ -202,11 +262,13 @@ async function main() { baseline.tasks[name] = { hz: task.hz, mean: task.mean, - normalized: task.normalized, + normalized: task.normalized ?? 0, }; } fs.writeFileSync(BASELINE_PATH, `${JSON.stringify(baseline, null, 2)}\n`); - console.log(`Baseline written to ${path.relative(process.cwd(), BASELINE_PATH)}`); + console.log( + `Baseline written to ${path.relative(process.cwd(), BASELINE_PATH)}`, + ); } if (args.compare) { @@ -216,7 +278,9 @@ async function main() { ); process.exit(1); } - const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, "utf8")); + const baseline: BaselineFile = JSON.parse( + fs.readFileSync(BASELINE_PATH, "utf8"), + ); compare(baseline, current, { strict: args.strict }); } } diff --git a/benchmark/scenarios/batch-get-format.bench.js b/benchmark/scenarios/batch-get-format.bench.ts similarity index 65% rename from benchmark/scenarios/batch-get-format.bench.js rename to benchmark/scenarios/batch-get-format.bench.ts index b5384fd7..0213e9cf 100644 --- a/benchmark/scenarios/batch-get-format.bench.js +++ b/benchmark/scenarios/batch-get-format.bench.ts @@ -1,19 +1,20 @@ /** * P2 — batchGet response formatting calls formatResponse once per item. */ -const { makeMockV2Client } = require("../../test/fixtures/mock-client"); -const { +import type { ScenarioEntry } from "../run"; +import { makeMockV2Client, StoredItem } from "../../test/fixtures/mock-client"; +import { table, makeFixtureEntity, makeStoredItem, makeItemData, -} = require("../../test/fixtures/entities"); +} from "../../test/fixtures/entities"; const entity = makeFixtureEntity(); -function makeBatch(count) { - const Responses = { [table]: [] }; - const keys = []; +function makeBatch(count: number) { + const Responses: Record = { [table]: [] }; + const keys: { org: string; id: string }[] = []; for (let i = 0; i < count; i++) { Responses[table].push(makeStoredItem(entity, i)); const { org, id } = makeItemData(i); @@ -27,7 +28,7 @@ function makeBatch(count) { const batch100 = makeBatch(100); -module.exports = [ +const scenarios: ScenarioEntry[] = [ { name: "batch-get-format/100-items", fn: async () => { @@ -36,3 +37,5 @@ module.exports = [ }, }, ]; + +export default scenarios; diff --git a/benchmark/scenarios/entity-construction.bench.js b/benchmark/scenarios/entity-construction.bench.ts similarity index 70% rename from benchmark/scenarios/entity-construction.bench.js rename to benchmark/scenarios/entity-construction.bench.ts index acc964fe..98cefd9f 100644 --- a/benchmark/scenarios/entity-construction.bench.js +++ b/benchmark/scenarios/entity-construction.bench.ts @@ -1,13 +1,15 @@ /** * P5 — Entity construction cost (model validation dominates). */ +import type { ScenarioEntry } from "../run"; +import { table, makeFixtureModel } from "../../test/fixtures/entities"; + const { Entity } = require("../../src/entity"); -const { table, makeFixtureModel } = require("../../test/fixtures/entities"); const model = makeFixtureModel(); const watcherModel = makeFixtureModel({ withWatchers: true }); -module.exports = [ +const scenarios: ScenarioEntry[] = [ { name: "entity-construction/new-entity", fn: () => new Entity(model, { table }), @@ -17,3 +19,5 @@ module.exports = [ fn: () => new Entity(watcherModel, { table }), }, ]; + +export default scenarios; diff --git a/benchmark/scenarios/params-chain.bench.js b/benchmark/scenarios/params-chain.bench.ts similarity index 76% rename from benchmark/scenarios/params-chain.bench.js rename to benchmark/scenarios/params-chain.bench.ts index e0d2fbc1..48901840 100644 --- a/benchmark/scenarios/params-chain.bench.js +++ b/benchmark/scenarios/params-chain.bench.ts @@ -2,12 +2,13 @@ * P3 — chain construction + params building. Read chains (get/query/scan) * should never build update machinery; update/upsert chains pay for it once. */ -const { makeFixtureEntity } = require("../../test/fixtures/entities"); +import type { ScenarioEntry } from "../run"; +import { makeFixtureEntity } from "../../test/fixtures/entities"; const entity = makeFixtureEntity(); const key = { org: "org1", id: "id1" }; -module.exports = [ +const scenarios: ScenarioEntry[] = [ { name: "params-chain/get", fn: () => entity.get(key).params(), @@ -21,7 +22,7 @@ module.exports = [ fn: () => entity.query .records({ org: "org1" }) - .where((attr, op) => op.gt(attr.count, 10)) + .where((attr: any, op: any) => op.gt(attr.count, 10)) .params(), }, { @@ -33,3 +34,5 @@ module.exports = [ fn: () => entity.upsert({ ...key, name: "x", count: 1 }).params(), }, ]; + +export default scenarios; diff --git a/benchmark/scenarios/parse-format.bench.js b/benchmark/scenarios/parse-format.bench.ts similarity index 72% rename from benchmark/scenarios/parse-format.bench.js rename to benchmark/scenarios/parse-format.bench.ts index 5ed0f236..92899a16 100644 --- a/benchmark/scenarios/parse-format.bench.js +++ b/benchmark/scenarios/parse-format.bench.ts @@ -2,16 +2,18 @@ * P2 + P4 — response formatting: per-item getter passes, sibling snapshots, * path lookups, and (formerly) per-call error allocation. */ -const { +import type { ScenarioEntry } from "../run"; +import type { StoredItem } from "../../test/fixtures/mock-client"; +import { makeFixtureEntity, makeStoredItem, -} = require("../../test/fixtures/entities"); +} from "../../test/fixtures/entities"; const entity = makeFixtureEntity(); const watcherEntity = makeFixtureEntity({ withWatchers: true }); -function makeItems(owner, count) { - const Items = []; +function makeItems(owner: any, count: number): StoredItem[] { + const Items: StoredItem[] = []; for (let i = 0; i < count; i++) { Items.push(makeStoredItem(owner, i)); } @@ -22,7 +24,7 @@ const items100 = makeItems(entity, 100); const items1000 = makeItems(entity, 1000); const watcherItems1000 = makeItems(watcherEntity, 1000); -module.exports = [ +const scenarios: ScenarioEntry[] = [ { name: "parse-format/100-items", fn: () => entity.parse({ Items: items100 }), @@ -36,3 +38,5 @@ module.exports = [ fn: () => watcherEntity.parse({ Items: watcherItems1000 }), }, ]; + +export default scenarios; diff --git a/benchmark/scenarios/query-pagination.bench.js b/benchmark/scenarios/query-pagination.bench.ts similarity index 71% rename from benchmark/scenarios/query-pagination.bench.js rename to benchmark/scenarios/query-pagination.bench.ts index d9459261..79a8fb39 100644 --- a/benchmark/scenarios/query-pagination.bench.js +++ b/benchmark/scenarios/query-pagination.bench.ts @@ -2,18 +2,26 @@ * P1 — auto-paging accumulation across many pages (`pages: "all"`). The * O(pages²) → O(pages) change shows up most at high page counts. */ -const { +import type { ScenarioEntry } from "../run"; +import { makeMockV2Client, makePagingQueryHandler, -} = require("../../test/fixtures/mock-client"); -const { + MockV2Client, +} from "../../test/fixtures/mock-client"; +import { makeFixtureEntity, makeStoredItem, -} = require("../../test/fixtures/entities"); +} from "../../test/fixtures/entities"; const entity = makeFixtureEntity(); -function makePagingClient({ pages, perPage }) { +function makePagingClient({ + pages, + perPage, +}: { + pages: number; + perPage: number; +}): MockV2Client { const query = makePagingQueryHandler({ pages, perPage, @@ -25,14 +33,14 @@ function makePagingClient({ pages, perPage }) { const mock10 = makePagingClient({ pages: 10, perPage: 100 }); const mock100 = makePagingClient({ pages: 100, perPage: 100 }); -function runAllPages(mock) { +function runAllPages(mock: MockV2Client) { mock.calls.length = 0; // keep the mock's call log from growing return entity.query .records({ org: "org1" }) .go({ client: mock.client, pages: "all" }); } -module.exports = [ +const scenarios: ScenarioEntry[] = [ { name: "query-pagination/10-pages-x100", fn: async () => runAllPages(mock10), @@ -42,3 +50,5 @@ module.exports = [ fn: async () => runAllPages(mock100), }, ]; + +export default scenarios; diff --git a/package.json b/package.json index f1829edb..225489df 100644 --- a/package.json +++ b/package.json @@ -35,10 +35,10 @@ "local:stop": "npm run ddb:stop", "local:exec": "LOCAL_DYNAMO_ENDPOINT='http://localhost:8000' ts-node ./test/debug.ts", "local:debug": "npm run local:start && npm run local:exec", - "benchmark": "node ./benchmark/run.js", - "benchmark:json": "node ./benchmark/run.js --json", - "benchmark:compare": "node ./benchmark/run.js --compare", - "benchmark:update": "node ./benchmark/run.js --update-baseline", + "benchmark": "ts-node ./benchmark/run.ts", + "benchmark:json": "ts-node ./benchmark/run.ts --json", + "benchmark:compare": "ts-node ./benchmark/run.ts --compare", + "benchmark:update": "ts-node ./benchmark/run.ts --update-baseline", "format": "prettier -w src/**/*.js examples/**/* --log-level=error" }, "repository": { diff --git a/test/fixtures/entities.js b/test/fixtures/entities.ts similarity index 70% rename from test/fixtures/entities.js rename to test/fixtures/entities.ts index 88459024..dc335d01 100644 --- a/test/fixtures/entities.js +++ b/test/fixtures/entities.ts @@ -4,16 +4,36 @@ * produce from-DynamoDB-shaped stored items for feeding `parse`/ * `formatResponse` without a database. */ +import type { StoredItem } from "./mock-client"; + +// internals are untyped; required (not imported) per repo test convention const { Entity } = require("../../src/entity"); -const table = "electro"; +export const table = "electro"; + +export interface FixtureModelOptions { + entity?: string; + service?: string; + withWatchers?: boolean; +} + +export interface FixtureItemData { + org: string; + id: string; + name: string; + count: number; + active: boolean; + tags: string[]; + profile: { nickname: string; age: number }; + notes: { body: string }[]; +} -function makeFixtureModel({ +export function makeFixtureModel({ entity = "perfFixture", service = "perfService", withWatchers = false, -} = {}) { - const attributes = { +}: FixtureModelOptions = {}): Record { + const attributes: Record = { org: { type: "string" }, id: { type: "string" }, name: { type: "string" }, @@ -41,7 +61,7 @@ function makeFixtureModel({ attributes.displayName = { type: "string", watch: ["name"], - set: (_, item) => + set: (_: unknown, item: Record) => typeof item.name === "string" ? item.name.toUpperCase() : undefined, }; } @@ -58,11 +78,11 @@ function makeFixtureModel({ }; } -function makeFixtureEntity(options = {}) { +export function makeFixtureEntity(options: FixtureModelOptions = {}): any { return new Entity(makeFixtureModel(options), { table }); } -function makeItemData(i) { +export function makeItemData(i: number): FixtureItemData { return { org: "org1", id: `id${String(i).padStart(8, "0")}`, @@ -79,18 +99,10 @@ function makeItemData(i) { // `__edb_e__`/`__edb_v__` identifiers, field names) by reusing the entity's // own put-params formatting, then normalizing set attributes to plain arrays // (the shape a v3 DocumentClient unmarshalls to). -function makeStoredItem(entity, i) { +export function makeStoredItem(entity: any, i: number): StoredItem { const { Item } = entity.put(makeItemData(i)).params(); if (Item.tags && Array.isArray(Item.tags.values)) { Item.tags = [...Item.tags.values]; } return Item; } - -module.exports = { - table, - makeFixtureModel, - makeFixtureEntity, - makeItemData, - makeStoredItem, -}; diff --git a/test/fixtures/mock-client.js b/test/fixtures/mock-client.ts similarity index 59% rename from test/fixtures/mock-client.js rename to test/fixtures/mock-client.ts index ad3e1700..4fdbdba8 100644 --- a/test/fixtures/mock-client.js +++ b/test/fixtures/mock-client.ts @@ -1,18 +1,38 @@ /** * Shared offline client mocks. These fixtures are used by both the offline - * test suites (test/offline.*.spec.js) and the benchmark scenarios - * (benchmark/scenarios/*.bench.js); they must stay dependency-free and + * test suites (test/offline.*.spec.*) and the benchmark scenarios + * (benchmark/scenarios/*.bench.ts); they must stay dependency-free and * synchronous so neither consumer needs network or DynamoDB access. */ +export type StoredItem = Record; + +export interface MockClientCall { + method: string; + params: Record; +} + +export type MockHandler = + | ((params: Record) => any) + | Record; + +export interface MockV2Client { + /** duck-typed v2 DocumentClient; hand to `.go({ client })` or the Entity config */ + client: any; + /** every call the entity made, in order */ + calls: MockClientCall[]; +} + // A minimal v2-style DocumentClient mock. Non-transaction methods return an // object with `.promise()`; transaction methods return a request-like object // (`on`/`abort`/`promise`) so it works both directly (via `_exec`) and through // the v2 wrapper. `handlers` maps a method name to a canned response (or a // function of the params). Transaction handlers may return // `{ cancellationReasons: [...] }` to simulate a canceled transaction. -function makeMockV2Client(handlers = {}) { - const calls = []; +export function makeMockV2Client( + handlers: Record = {}, +): MockV2Client { + const calls: MockClientCall[] = []; const transactMethods = new Set(["transactWrite", "transactGet"]); const methods = [ "get", @@ -26,18 +46,18 @@ function makeMockV2Client(handlers = {}) { "transactWrite", "transactGet", ]; - const client = { - createSet: (value) => new Set([].concat(value)), + const client: Record = { + createSet: (value: any) => new Set([].concat(value)), }; for (const method of methods) { - client[method] = (params) => { + client[method] = (params: Record) => { calls.push({ method, params }); const handler = handlers[method]; const value = typeof handler === "function" ? handler(params) : handler; if (transactMethods.has(method)) { - const stored = {}; + const stored: Record void> = {}; return { - on: (event, cb) => { + on: (event: string, cb: (input: any) => void) => { stored[event] = cb; }, abort: () => {}, @@ -69,22 +89,40 @@ function makeMockV2Client(handlers = {}) { return { client, calls }; } +export interface PagingQueryHandlerOptions { + pages: number; + perPage: number; + /** produces the i-th stored item (from-DynamoDB shape, including key fields) */ + makeItem: (i: number) => StoredItem; +} + +export interface PagingQueryResponse { + Items: StoredItem[]; + Count: number; + LastEvaluatedKey?: { pk: any; sk: any }; +} + // Builds a `query`/`scan` handler that pages through `pages` responses of // `perPage` items each, emitting a `LastEvaluatedKey` on every page but the -// last. `makeItem(i)` produces the i-th stored item (from-DynamoDB shape, -// including key fields; sort keys must be unique across items). Like DynamoDB, -// the handler resumes from the params' `ExclusiveStartKey` by locating the -// item whose pk/sk match, so it also honors cursors ElectroDB synthesizes -// mid-page (e.g. when a `count` limit lands inside a page). -function makePagingQueryHandler({ pages, perPage, makeItem }) { - const items = []; - const indexBySortKey = new Map(); +// last. Sort keys must be unique across items. Like DynamoDB, the handler +// resumes from the params' `ExclusiveStartKey` by locating the item whose +// pk/sk match, so it also honors cursors ElectroDB synthesizes mid-page +// (e.g. when a `count` limit lands inside a page). +export function makePagingQueryHandler({ + pages, + perPage, + makeItem, +}: PagingQueryHandlerOptions): ( + params: Record, +) => PagingQueryResponse { + const items: StoredItem[] = []; + const indexBySortKey = new Map(); for (let i = 0; i < pages * perPage; i++) { const item = makeItem(i); items.push(item); indexBySortKey.set(`${item.pk}|${item.sk}`, i); } - return (params) => { + return (params: Record) => { let start = 0; const esk = params.ExclusiveStartKey; if (esk !== undefined) { @@ -95,7 +133,7 @@ function makePagingQueryHandler({ pages, perPage, makeItem }) { start = index + 1; } const Items = items.slice(start, start + perPage); - const response = { Items, Count: Items.length }; + const response: PagingQueryResponse = { Items, Count: Items.length }; const lastReturned = start + Items.length - 1; if (lastReturned < items.length - 1 && Items.length > 0) { const lastItem = Items[Items.length - 1]; @@ -104,8 +142,3 @@ function makePagingQueryHandler({ pages, perPage, makeItem }) { return response; }; } - -module.exports = { - makeMockV2Client, - makePagingQueryHandler, -}; diff --git a/test/offline.perf-fixes.spec.js b/test/offline.perf-fixes.spec.ts similarity index 93% rename from test/offline.perf-fixes.spec.js rename to test/offline.perf-fixes.spec.ts index 9683a96e..701c710b 100644 --- a/test/offline.perf-fixes.spec.js +++ b/test/offline.perf-fixes.spec.ts @@ -3,19 +3,21 @@ * ("stage 2"). Each describe block pins user-visible behavior that the * corresponding optimization must preserve; none of these rely on timing. */ -const { expect } = require("chai"); -const { Entity } = require("../src/entity"); -const { Service } = require("../src/service"); -const { +import { expect } from "chai"; +import { makeMockV2Client, makePagingQueryHandler, -} = require("./fixtures/mock-client"); -const { +} from "./fixtures/mock-client"; +import { table, makeFixtureEntity, makeStoredItem, makeItemData, -} = require("./fixtures/entities"); +} from "./fixtures/entities"; + +// internals are untyped; required (not imported) per repo test convention +const { Entity } = require("../src/entity"); +const { Service } = require("../src/service"); // --------------------------------------------------------------------------- // P5: validateModel ran the (always-failing) ModelBeta schema pass for every @@ -80,7 +82,7 @@ describe("P5: model validation short-circuits", () => { for (const { name, model, message } of invalidModelFixtures) { it(`throws the exact pre-fix message for: ${name}`, () => { - let thrown; + let thrown: any; try { validations.model(model); } catch (err) { @@ -141,7 +143,7 @@ describe("P5: model validation short-circuits", () => { describe("P2: formatResponse error wrapping", () => { const { ElectroError, ErrorCodes } = require("../src/errors"); - function makeThrowingEntity(makeError) { + function makeThrowingEntity(makeError: () => Error) { return new Entity( { model: { entity: "thrower", service: "perfService", version: "1" }, @@ -167,7 +169,7 @@ describe("P2: formatResponse error wrapping", () => { ); } - function makeGetClient(entity) { + function makeGetClient(entity: any) { const { Item } = entity .put({ org: "org1", id: "id1", boom: "value" }) .params(); @@ -178,7 +180,7 @@ describe("P2: formatResponse error wrapping", () => { const original = new Error("boom"); const entity = makeThrowingEntity(() => original); const { client } = makeGetClient(entity); - let thrown; + let thrown: any; try { await entity.get({ org: "org1", id: "id1" }).go({ client }); } catch (err) { @@ -201,7 +203,7 @@ describe("P2: formatResponse error wrapping", () => { ); const entity = makeThrowingEntity(() => original); const { client } = makeGetClient(entity); - let thrown; + let thrown: any; try { await entity.get({ org: "org1", id: "id1" }).go({ client }); } catch (err) { @@ -214,7 +216,7 @@ describe("P2: formatResponse error wrapping", () => { const original = new Error("boom"); const entity = makeThrowingEntity(() => original); const { client } = makeGetClient(entity); - let thrown; + let thrown: any; try { await entity .get({ org: "org1", id: "id1" }) @@ -232,7 +234,7 @@ describe("P2: formatResponse error wrapping", () => { const { Item } = entity .put({ org: "org1", id: "id1", boom: "value" }) .params(); - let thrown; + let thrown: any; try { entity.parse({ Item }); } catch (err) { @@ -259,8 +261,14 @@ describe("P4: attribute mutation passes", () => { attributes: { org: { type: "string" }, id: { type: "string" }, - a: { type: "string", set: (value, item) => `${value}:${item.b}` }, - b: { type: "string", set: (value, item) => `${value}:${item.a}` }, + a: { + type: "string", + set: (value: any, item: any) => `${value}:${item.b}`, + }, + b: { + type: "string", + set: (value: any, item: any) => `${value}:${item.a}`, + }, }, indexes: { records: { @@ -290,7 +298,7 @@ describe("P4: attribute mutation passes", () => { id: { type: "string" }, name: { type: "string", - set: (value) => { + set: (value: any) => { counts.name++; return `${value}!`; }, @@ -298,7 +306,7 @@ describe("P4: attribute mutation passes", () => { display: { type: "string", watch: ["name"], - set: (_, item) => { + set: (_: any, item: any) => { counts.display++; return `D:${item.name}`; }, @@ -332,7 +340,7 @@ describe("P4: attribute mutation passes", () => { id: { type: "string" }, label: { type: "string", - get: (value, item) => `${value}@${item.org}`, + get: (value: any, item: any) => `${value}@${item.org}`, }, }, indexes: { @@ -355,7 +363,7 @@ describe("P4: attribute mutation passes", () => { const entity = makeFixtureEntity(); const params = entity .update({ org: "org1", id: "id1" }) - .data((attr, op) => op.set(attr.notes[0].body, "edited")) + .data((attr: any, op: any) => op.set(attr.notes[0].body, "edited")) .params(); expect(params.UpdateExpression).to.equal( "SET #notes[0].#body = :body_u0, #org = :org_u0, #id = :id_u0, #__edb_e__ = :__edb_e___u0, #__edb_v__ = :__edb_v___u0", @@ -396,7 +404,7 @@ describe("P3: lazy update machinery on chain construction", () => { state.query, "updateProxy", ); - expect(descriptor.get, "updateProxy should be an accessor").to.be.a( + expect(descriptor?.get, "updateProxy should be an accessor").to.be.a( "function", ); const first = state.query.updateProxy; @@ -409,7 +417,10 @@ describe("P3: lazy update machinery on chain construction", () => { // buildAttributes, so spying on it counts constructions const original = AttributeOperationProxy.buildAttributes; let constructions = 0; - AttributeOperationProxy.buildAttributes = function (...args) { + AttributeOperationProxy.buildAttributes = function ( + this: any, + ...args: any[] + ) { constructions++; return original.apply(this, args); }; @@ -498,7 +509,7 @@ describe("P3: lazy update machinery on chain construction", () => { const query = entity.query .records({ org: "org1" }) - .where((attr, op) => op.gt(attr.count, 10)) + .where((attr: any, op: any) => op.gt(attr.count, 10)) .params(); expect(query).to.deep.equal({ KeyConditionExpression: "#pk = :pk and begins_with(#sk1, :sk1)", @@ -527,7 +538,13 @@ describe("P3: lazy update machinery on chain construction", () => { describe("P1: executeQuery result accumulation", () => { const entity = makeFixtureEntity(); - function makePagingClient({ pages, perPage }) { + function makePagingClient({ + pages, + perPage, + }: { + pages: number; + perPage: number; + }) { const query = makePagingQueryHandler({ pages, perPage, @@ -545,7 +562,7 @@ describe("P1: executeQuery result accumulation", () => { .go({ client, pages: "all" }); expect(calls.length).to.equal(pages); expect(cursor).to.equal(null); - expect(data.map((item) => item.id)).to.deep.equal( + expect(data.map((item: any) => item.id)).to.deep.equal( Array.from({ length: pages * perPage }, (_, i) => makeItemData(i).id), ); // items keep full attribute formatting, not just identity @@ -562,21 +579,23 @@ describe("P1: executeQuery result accumulation", () => { .records({ org: "org1" }) .go({ client, count }); expect(first.data.length).to.equal(count); - expect(first.data.map((item) => item.count)).to.deep.equal([0, 1, 2, 3, 4]); + expect(first.data.map((item: any) => item.count)).to.deep.equal([ + 0, 1, 2, 3, 4, + ]); expect(first.cursor).to.be.a("string"); // resuming from the returned cursor picks up at the very next item const rest = await entity.query .records({ org: "org1" }) .go({ client, cursor: first.cursor, pages: "all" }); - expect(rest.data.map((item) => item.count)).to.deep.equal([ + expect(rest.data.map((item: any) => item.count)).to.deep.equal([ 5, 6, 7, 8, 9, 10, 11, ]); expect(rest.cursor).to.equal(null); }); describe("collection queries", () => { - function makeCollectionModel(name) { + function makeCollectionModel(name: string) { return { model: { entity: name, service: "perfService", version: "1" }, table, @@ -626,10 +645,10 @@ describe("P1: executeQuery result accumulation", () => { { length: pages * perPage }, (_, i) => `id${String(i).padStart(4, "0")}`, ); - expect(data.alpha.map((item) => item.id)).to.deep.equal( + expect(data.alpha.map((item: any) => item.id)).to.deep.equal( expectedIds.filter((_, i) => i % 2 === 0), ); - expect(data.beta.map((item) => item.id)).to.deep.equal( + expect(data.beta.map((item: any) => item.id)).to.deep.equal( expectedIds.filter((_, i) => i % 2 === 1), ); expect(data.alpha[0]).to.deep.equal({ From 2d56287d64fe3d6a7fd40e9f58abad0ca09dbee3 Mon Sep 17 00:00:00 2001 From: ty Date: Wed, 10 Jun 2026 00:00:26 -0400 Subject: [PATCH 10/17] feat(benchmark): statistical significance verdicts in compare MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compare previously used a blind 20% tolerance on normalized throughput — unable to distinguish a real 15% regression from noise or to confirm a real 4% change. Verdicts now apply two gates per task: statistical — the delta must beat a noise floor where the baseline and current 95% confidence intervals would overlap. With --runs > 1 (the default 3 for compare/update) the suite repeats and the interval comes from the t-based between-run spread of the normalized value, which captures GC phasing, JIT state, and thermal drift that within-run sampling underestimates — measured here: the pagination tasks swing ±20% between runs while their within-run margin reads ±1%, and single-run CIs produced false REGRESSIONs on identical code. With --runs 1 (quick iteration) it falls back to within-run tinybench rme combined with the reference task's, since normalized is a ratio. practical — a real delta must also exceed --threshold (default 5%) to be labeled REGRESSION/improved; smaller real changes report as "within threshold", sub-floor deltas as "~noise". Compare also reports the reference task's raw drift since baseline (non-uniform machine-condition changes make borderline verdicts suspect) and warns when noise floors exceed the threshold. The committed baseline moves to schemaVersion 2 (normalized + normalizedRme + samples, captured at 3 runs); filtered compares skip unmatched baseline tasks. Verified: same-code compares report no regressions; injected 30%/3% baseline shifts produce REGRESSION and within-threshold verdicts respectively; --strict exits 1 only on REGRESSION. Co-Authored-By: Claude Fable 5 --- benchmark/README.md | 75 ++++++++-- benchmark/baseline.json | 118 +++++++++------ benchmark/run.ts | 315 ++++++++++++++++++++++++++++++++++------ 3 files changed, 411 insertions(+), 97 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index 839c48ec..8b044f6b 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -13,20 +13,19 @@ npm run benchmark:update # rewrite baseline.json from a fresh run npm run benchmark:compare # run + diff against baseline.json (advisory) ``` -`benchmark:compare` flags a task as a regression when its normalized -throughput drops more than 20% below the baseline. It always exits 0 -(advisory); add `--strict` to exit 1 on regressions — that one flag is the -hook for promoting the comparison to CI later: +`benchmark:compare` always exits 0 (advisory); add `--strict` to exit 1 on +regressions — that one flag is the hook for promoting the comparison to CI +later: ```sh npx ts-node ./benchmark/run.ts --compare --strict ``` -You can also run a subset while iterating: - -```sh -npx ts-node ./benchmark/run.ts --filter pagination -``` +Other flags: `--filter ` runs a subset while iterating, +`--threshold ` sets the practical-significance bar (default 5), +`--time ` sets within-run sampling time per task (default 500), and +`--runs ` repeats the whole suite n times to derive error bars from the +between-run spread (default 3 for compare/update, 1 for plain runs). ## Baseline + normalization @@ -39,6 +38,64 @@ machines; for precise A/B work, regenerate the baseline on your own machine first (`npm run benchmark:update`), apply your change, then `npm run benchmark:compare`. +## Interpreting a compare: is the difference significant? + +Micro-benchmark deltas are meaningless without an error model, so every +compare applies **two gates** per task and prints a verdict: + +1. **Statistical — is the change real?** Both the baseline and the current + measurement carry a 95% confidence interval, and a change counts as _real_ + only when the two intervals do **not** overlap. The table prints this as + the task's **noise ±%** — if `|Δ%|` is below it, the verdict is `~noise` + no matter how the raw numbers look. Where the interval comes from matters: + - With `--runs > 1` (the default for compare/update) the suite executes + N times and the interval is a t-based CI of the normalized value's + **between-run spread**. This is the trustworthy estimate: it captures + GC phasing, JIT state, and thermal drift, which dominate for long + async tasks (the pagination scenarios can swing ±20% between runs + while their within-run margin reads ±1%). + - With `--runs 1` it falls back to tinybench's within-run `rme`, combined + with the reference task's (`√(rme_task² + rme_reference²)`, since a + normalized value is a ratio of two measurements). Fine for quick + iteration; too optimistic for decisions. +2. **Practical — is it big enough to care?** A real change must also exceed + `--threshold` (default 5%) to be labeled `REGRESSION` or `improved`; + otherwise it reports as `within threshold`. + +| Verdict | Meaning | +| ------------------ | -------------------------------------------------------------- | +| `REGRESSION` | Real (beats the noise floor) **and** ≥ threshold slower | +| `improved` | Real **and** ≥ threshold faster | +| `within threshold` | Real, but smaller than the threshold — judgment call | +| `~noise` | Indistinguishable from sampling error — not evidence of change | + +### Evaluating one of your own changes + +```sh +git stash # or check out the pre-change tree +npm run benchmark:update # capture the "before" baseline (3 runs) +git stash pop +npm run benchmark:compare # verdicts tell you if the change is real +``` + +Practical notes: + +- If a task you care about shows a high noise floor (the runner warns when it + exceeds the threshold), increase repetitions on **both** sides: + `--runs 5`. More runs shrink the between-run confidence interval, which is + the only honest way to detect small differences; `--time` only tightens + the (already optimistic) within-run estimate. +- The compare prints the reference task's raw throughput drift since the + baseline. Normalization absorbs uniform machine slowdown, but drift is + rarely uniform — when the drift is large the runner warns and borderline + verdicts deserve suspicion. +- The statistical gate assumes both sides came from the same machine under + broadly similar load. Close heavy apps, and re-run a suspicious result once + before believing it — two consistent verdicts beat one. +- The noise-floor test is deliberately conservative (CI overlap). A `~noise` + verdict doesn't prove there is no difference — only that this run can't + detect one at its current sensitivity. + ## Adding a scenario Drop a file matching `benchmark/scenarios/*.bench.ts` that default-exports an diff --git a/benchmark/baseline.json b/benchmark/baseline.json index d1f24a96..5ae94e4a 100644 --- a/benchmark/baseline.json +++ b/benchmark/baseline.json @@ -1,78 +1,106 @@ { - "schemaVersion": 1, - "generatedAt": "2026-06-10T03:31:53.192Z", + "schemaVersion": 2, + "generatedAt": "2026-06-10T04:11:58.119Z", "node": "v24.3.0", - "referenceHz": 2327.262141538007, + "referenceHz": 2424.4644886571928, "tasks": { "reference/parse-500-items": { - "hz": 2327.262141538007, - "mean": 0.42968945446735735, - "normalized": 1 + "hz": 2424.4644886571928, + "mean": 0.41253856308994147, + "normalized": 1, + "normalizedRme": 0, + "samples": 3638 }, "batch-get-format/100-items": { - "hz": 4519.7695369513285, - "mean": 0.22125021902654782, - "normalized": 1.9420973066507965 + "hz": 3844.0481109727643, + "mean": 0.26058448713831844, + "normalized": 1.5863465726981503, + "normalizedRme": 15.279148606132178, + "samples": 5770 }, "entity-construction/new-entity": { - "hz": 5731.994118974018, - "mean": 0.1744593555478023, - "normalized": 2.4629774259920465 + "hz": 6171.908033589301, + "mean": 0.16210953335278167, + "normalized": 2.5469529411935015, + "normalizedRme": 11.232955822414702, + "samples": 9260 }, "entity-construction/new-entity-watchers": { - "hz": 5468.781577339595, - "mean": 0.18285608702011302, - "normalized": 2.3498777725681843 + "hz": 5685.492287082828, + "mean": 0.17594353779412905, + "normalized": 2.3459484941148427, + "normalizedRme": 9.202963391510739, + "samples": 8535 }, "params-chain/get": { - "hz": 416715.25824700115, - "mean": 0.00239972014513389, - "normalized": 179.05815198437784 + "hz": 388292.19723028206, + "mean": 0.002576322384780131, + "normalized": 160.1758691120667, + "normalizedRme": 6.285730042215901, + "samples": 582975 }, "params-chain/query": { - "hz": 302041.66219756944, - "mean": 0.0033108015388482624, - "normalized": 129.78411705608744 + "hz": 260867.58213501098, + "mean": 0.00383364114333602, + "normalized": 107.62911073619341, + "normalizedRme": 6.596224300497242, + "samples": 391302 }, "params-chain/query-where": { - "hz": 119115.32935451993, - "mean": 0.008395225076561938, - "normalized": 51.18260088904326 + "hz": 114070.16974197427, + "mean": 0.008767453305424373, + "normalized": 47.064258493790426, + "normalizedRme": 7.115991189239105, + "samples": 171106 }, "params-chain/update-set": { - "hz": 93418.97855690971, - "mean": 0.010704463005777912, - "normalized": 40.14114993301629 + "hz": 88781.91955723667, + "mean": 0.011286994319514502, + "normalized": 36.646340920850015, + "normalizedRme": 17.661690693644044, + "samples": 133187 }, "params-chain/upsert": { - "hz": 82706.38805249817, - "mean": 0.01209096447743851, - "normalized": 35.538062763243495 + "hz": 76100.52166636738, + "mean": 0.013204382836770964, + "normalized": 31.413345966819623, + "normalizedRme": 23.45776790528955, + "samples": 114152 }, "parse-format/100-items": { - "hz": 11449.031595110766, - "mean": 0.08734363179039907, - "normalized": 4.919528140282683 + "hz": 12222.4801025672, + "mean": 0.08184534482640307, + "normalized": 5.043073744962243, + "normalizedRme": 8.92464931196236, + "samples": 18335 }, "parse-format/1000-items": { - "hz": 1159.505939157333, - "mean": 0.8624362896551843, - "normalized": 0.49822747444817517 + "hz": 1220.4660830788864, + "mean": 0.82037659733162, + "normalized": 0.5036812142834938, + "normalizedRme": 14.156044011327662, + "samples": 1832 }, "parse-format/1000-items-watchers": { - "hz": 794.5887420594664, - "mean": 1.258512670854278, - "normalized": 0.3414264031014358 + "hz": 853.6491345659579, + "mean": 1.1720107583020443, + "normalized": 0.35221239509147423, + "normalizedRme": 9.357872920197668, + "samples": 1282 }, "query-pagination/10-pages-x100": { - "hz": 1454.944405824759, - "mean": 0.6873114848901278, - "normalized": 0.6251742680191741 + "hz": 1566.1567517550523, + "mean": 0.6386070826937228, + "normalized": 0.6461657403932399, + "normalizedRme": 7.1360968374326, + "samples": 2350 }, "query-pagination/100-pages-x100": { - "hz": 140.09983739361508, - "mean": 7.137767028169114, - "normalized": 0.060199422700627926 + "hz": 139.20933634125876, + "mean": 7.226922047990283, + "normalized": 0.057446423042089835, + "normalizedRme": 24.407407433915417, + "samples": 210 } } } diff --git a/benchmark/run.ts b/benchmark/run.ts index e062314d..92c710d1 100644 --- a/benchmark/run.ts +++ b/benchmark/run.ts @@ -8,12 +8,28 @@ * npm run benchmark:compare compare against baseline.json (advisory) * --strict exit 1 when the compare finds regressions * --filter only run tasks whose name includes substring + * --threshold practical-significance threshold (default 5) + * --time within-run sampling time per task (default 500) + * --runs repeat the whole suite n times and derive + * error bars from the between-run spread + * (default 3 for compare/update, 1 otherwise) * * Scenarios live in benchmark/scenarios/*.bench.ts; each default-exports an * array of `{ name, fn, opts? }` entries (see README.md). A fixed reference * task runs with every invocation and task throughput is recorded relative to * it (`normalized = hz / referenceHz`), which makes the committed baseline * roughly machine-independent. + * + * Compare verdicts use two gates (see README "Interpreting a compare"): + * statistical — the change must exceed the noise floor derived from both + * sides' 95% confidence intervals. With --runs > 1 the interval comes + * from the between-run spread of the normalized value (which captures + * GC/JIT/thermal variance that within-run sampling underestimates, + * especially for long async tasks); with --runs 1 it falls back to the + * within-run tinybench rme combined with the reference task's. + * practical — the change must also exceed --threshold percent to be worth + * acting on. Real-but-small changes report as "within threshold", and + * changes inside the noise floor report as "~noise". */ import * as fs from "fs"; import * as path from "path"; @@ -33,9 +49,16 @@ interface TaskStats { hz: number; mean: number; p99: number; + /** tinybench's relative margin of error for the task itself (95%, percent) */ rme: number; samples: number; normalized?: number; + /** + * relative 95% margin of error of `normalized` (percent). `normalized` is a + * ratio of two independent estimates (task hz / reference hz), so its + * relative error combines both: sqrt(rme² + rmeReference²). + */ + normalizedRme?: number; } interface RunResults { @@ -47,6 +70,9 @@ interface BaselineTask { hz: number; mean: number; normalized: number; + /** relative 95% margin of error of `normalized` (percent) */ + normalizedRme: number; + samples: number; } interface BaselineFile { @@ -63,12 +89,21 @@ interface CliArgs { updateBaseline: boolean; strict: boolean; filter: string | null; + /** practical-significance threshold, percent */ + threshold: number; + /** tinybench sampling time per task, ms */ + time: number; + /** full-suite repetitions; >1 derives error bars from between-run spread */ + runs: number; } const SCENARIO_DIR = path.join(__dirname, "scenarios"); const BASELINE_PATH = path.join(__dirname, "baseline.json"); const REFERENCE_TASK = "reference/parse-500-items"; -const DEFAULT_TOLERANCE = 0.2; +const BASELINE_SCHEMA_VERSION = 2; +const DEFAULT_THRESHOLD_PCT = 5; +const DEFAULT_TIME_MS = 500; +const DEFAULT_DECISION_RUNS = 3; function parseArgs(argv: string[]): CliArgs { const args: CliArgs = { @@ -77,6 +112,9 @@ function parseArgs(argv: string[]): CliArgs { updateBaseline: false, strict: false, filter: null, + threshold: DEFAULT_THRESHOLD_PCT, + time: DEFAULT_TIME_MS, + runs: 0, // resolved after flags are read; see below }; for (let i = 2; i < argv.length; i++) { const arg = argv[i]; @@ -85,11 +123,25 @@ function parseArgs(argv: string[]): CliArgs { else if (arg === "--update-baseline") args.updateBaseline = true; else if (arg === "--strict") args.strict = true; else if (arg === "--filter") args.filter = argv[++i] ?? null; - else { + else if (arg === "--threshold" || arg === "--time" || arg === "--runs") { + const value = Number(argv[++i]); + if (!Number.isFinite(value) || value <= 0) { + console.error(`${arg} requires a positive number`); + process.exit(1); + } + if (arg === "--threshold") args.threshold = value; + else if (arg === "--time") args.time = value; + else args.runs = Math.floor(value); + } else { console.error(`Unknown argument: ${arg}`); process.exit(1); } } + if (args.runs === 0) { + // compare/update need trustworthy error bars, which only between-run + // spread provides; plain exploratory runs stay fast by default + args.runs = args.compare || args.updateBaseline ? DEFAULT_DECISION_RUNS : 1; + } return args; } @@ -149,70 +201,210 @@ function collectResults(bench: Bench): RunResults { if (failed) { process.exit(1); } - const referenceHz = tasks[REFERENCE_TASK] ? tasks[REFERENCE_TASK].hz : null; - if (referenceHz) { + const reference = tasks[REFERENCE_TASK]; + if (reference) { for (const name of Object.keys(tasks)) { - tasks[name].normalized = tasks[name].hz / referenceHz; + const task = tasks[name]; + task.normalized = task.hz / reference.hz; + task.normalizedRme = Math.sqrt( + task.rme * task.rme + reference.rme * reference.rme, + ); } } - return { referenceHz, tasks }; + return { referenceHz: reference ? reference.hz : null, tasks }; } +type Verdict = + | "REGRESSION" + | "improved" + | "within threshold" + | "~noise" + | "added (not in baseline)" + | "removed"; + +interface CompareRow { + task: string; + "baseline (norm)"?: string; + "current (norm)"?: string; + "delta %"?: string; + "noise +/-%"?: string; + verdict: Verdict; +} + +/** + * Two-gate comparison per task: + * + * 1. statistical -- is the change distinguishable from sampling error? Each + * run's normalized value carries a 95% confidence interval (normalizedRme). + * The change is significant only when the two intervals do not overlap, + * i.e. |delta| exceeds the "noise floor" (the sum of both CI half-widths, + * expressed in percent of the baseline value). This is conservative: when + * in doubt it says "~noise". + * 2. practical -- a real change must also exceed `threshold` percent before + * it is labeled REGRESSION/improved; real-but-small changes report as + * "within threshold". + */ function compare( baseline: BaselineFile, current: RunResults, - { strict }: { strict: boolean }, + { + strict, + threshold, + filter, + }: { strict: boolean; threshold: number; filter: string | null }, ): void { - const tolerance = DEFAULT_TOLERANCE; - const rows: Record[] = []; - const regressions: Record[] = []; + if (baseline.schemaVersion !== BASELINE_SCHEMA_VERSION) { + console.error( + `baseline.json has schemaVersion ${baseline.schemaVersion}; expected ${BASELINE_SCHEMA_VERSION}. Regenerate it with \`npm run benchmark:update\`.`, + ); + process.exit(1); + } + const rows: CompareRow[] = []; + const regressions: CompareRow[] = []; + let insensitive = 0; for (const [name, base] of Object.entries(baseline.tasks)) { if (name === REFERENCE_TASK) continue; + // a filtered run intentionally skips tasks; don't report those as removed + if (filter !== null && !name.includes(filter)) continue; const task = current.tasks[name]; - if (!task || task.normalized === undefined) { - rows.push({ task: name, status: "removed" }); + if ( + !task || + task.normalized === undefined || + task.normalizedRme === undefined + ) { + rows.push({ task: name, verdict: "removed" }); continue; } - const ratio = task.normalized / base.normalized; - const row = { + const deltaPct = (task.normalized / base.normalized - 1) * 100; + const baseHalfWidth = base.normalized * (base.normalizedRme / 100); + const currentHalfWidth = task.normalized * (task.normalizedRme / 100); + const noiseFloorPct = + ((baseHalfWidth + currentHalfWidth) / base.normalized) * 100; + const significant = Math.abs(deltaPct) > noiseFloorPct; + const material = Math.abs(deltaPct) >= threshold; + const verdict: Verdict = !significant + ? "~noise" + : !material + ? "within threshold" + : deltaPct < 0 + ? "REGRESSION" + : "improved"; + if (noiseFloorPct > threshold) { + insensitive++; + } + const row: CompareRow = { task: name, "baseline (norm)": base.normalized.toFixed(4), "current (norm)": task.normalized.toFixed(4), - ratio: ratio.toFixed(3), - status: - ratio < 1 - tolerance - ? "REGRESSION" - : ratio > 1 + tolerance - ? "improved" - : "ok", + "delta %": `${deltaPct >= 0 ? "+" : ""}${deltaPct.toFixed(1)}`, + "noise +/-%": noiseFloorPct.toFixed(1), + verdict, }; rows.push(row); - if (row.status === "REGRESSION") regressions.push(row); + if (verdict === "REGRESSION") regressions.push(row); } for (const name of Object.keys(current.tasks)) { if (name !== REFERENCE_TASK && !baseline.tasks[name]) { - rows.push({ task: name, status: "added (not in baseline)" }); + rows.push({ task: name, verdict: "added (not in baseline)" }); } } console.table(rows); + console.log( + `Gates: statistical (|delta| must beat the per-task noise floor, from both runs' 95% CIs) and practical (|delta| >= ${threshold}%). Baseline: ${baseline.generatedAt}, node ${baseline.node}.`, + ); + // Normalization absorbs *uniform* machine slowdown, but drift is rarely + // uniform (GC/async-heavy tasks throttle harder), so large reference drift + // means conditions changed between runs and borderline verdicts are suspect. + if (baseline.referenceHz && current.referenceHz) { + const driftPct = (current.referenceHz / baseline.referenceHz - 1) * 100; + console.log( + `Reference-task raw throughput drift since baseline: ${ + driftPct >= 0 ? "+" : "" + }${driftPct.toFixed(1)}%.`, + ); + if (Math.abs(driftPct) > 10) { + console.warn( + "Machine conditions differ noticeably from the baseline run (thermal throttling, background load, or a different machine). Treat borderline verdicts with suspicion; re-run when idle or regenerate the baseline.", + ); + } + } + if (insensitive > 0) { + console.warn( + `Note: ${insensitive} task(s) have a noise floor above ${threshold}% -- a change of that size could hide in noise there. Increase --runs on both sides (and regenerate the baseline) for more sensitivity.`, + ); + } if (regressions.length) { console.error( - `${regressions.length} regression(s) beyond ${ - tolerance * 100 - }% tolerance (vs baseline from ${baseline.generatedAt}, node ${ - baseline.node - }).`, + `${regressions.length} regression(s): ${regressions + .map((row) => row.task) + .join(", ")}.`, ); if (strict) process.exit(1); } else { - console.log( - `No regressions beyond ${tolerance * 100}% tolerance (vs baseline from ${ - baseline.generatedAt - }).`, - ); + console.log("No regressions."); } } +// two-sided 95% critical values of Student's t, indexed by degrees of freedom +const T95 = [ + 12.71, 4.3, 3.18, 2.78, 2.57, 2.45, 2.36, 2.31, 2.26, 2.23, 2.2, 2.18, 2.16, + 2.14, 2.13, +]; +function t95(df: number): number { + return df <= 0 ? Infinity : df <= T95.length ? T95[df - 1] : 1.96; +} + +/** + * Collapse N independent runs into one result. For N > 1, each task's + * normalized value is averaged across runs and its margin of error is + * computed from the BETWEEN-run spread (t-based 95% CI of the mean). This + * captures variance that within-run sampling cannot see — GC phasing, JIT + * state, thermal drift — which dominates for long async tasks. For N = 1 the + * within-run estimate from collectResults is kept as-is. + */ +function aggregateRuns(runs: RunResults[]): RunResults { + if (runs.length === 1) { + return runs[0]; + } + const tasks: Record = {}; + const names = Object.keys(runs[0].tasks); + for (const name of names) { + const perRun = runs + .map((run) => run.tasks[name]) + .filter((task): task is TaskStats => task !== undefined); + const n = perRun.length; + const avg = (values: number[]) => + values.reduce((sum, value) => sum + value, 0) / values.length; + const normalizedValues = perRun.map((task) => task.normalized ?? 0); + const normalizedMean = avg(normalizedValues); + const variance = + normalizedValues.reduce( + (sum, value) => sum + (value - normalizedMean) ** 2, + 0, + ) / + (n - 1); + const ciHalfWidth = t95(n - 1) * Math.sqrt(variance / n); + tasks[name] = { + hz: avg(perRun.map((task) => task.hz)), + mean: avg(perRun.map((task) => task.mean)), + p99: avg(perRun.map((task) => task.p99)), + rme: avg(perRun.map((task) => task.rme)), + samples: perRun.reduce((sum, task) => sum + task.samples, 0), + normalized: normalizedMean, + normalizedRme: + normalizedMean > 0 ? (ciHalfWidth / normalizedMean) * 100 : 0, + }; + } + return { + referenceHz: avg2(runs.map((run) => run.referenceHz ?? 0)), + tasks, + }; +} + +function avg2(values: number[]): number { + return values.reduce((sum, value) => sum + value, 0) / values.length; +} + async function main(): Promise { const args = parseArgs(process.argv); let entries = loadScenarios(); @@ -223,15 +415,35 @@ async function main(): Promise { ); } - const bench = new Bench({ time: 500, warmupTime: 100 }); - for (const { name, fn, opts } of entries) { - bench.add(name, fn, opts); + const allRuns: RunResults[] = []; + for (let run = 0; run < args.runs; run++) { + const bench = new Bench({ time: args.time, warmupTime: 100 }); + for (const { name, fn, opts } of entries) { + bench.add(name, fn, opts); + } + await bench.warmup(); + await bench.run(); + allRuns.push(collectResults(bench)); + if (args.runs > 1) { + console.error(`run ${run + 1}/${args.runs} complete`); + } + if (!args.json && run === args.runs - 1) { + console.table(bench.table()); + } } - await bench.warmup(); - await bench.run(); + const current = aggregateRuns(allRuns); - const current = collectResults(bench); + if (args.runs > 1 && !args.json) { + console.table( + Object.entries(current.tasks).map(([name, task]) => ({ + task: name, + "normalized (mean)": (task.normalized ?? 0).toFixed(4), + "±% (95% CI, between-run)": (task.normalizedRme ?? 0).toFixed(1), + runs: args.runs, + })), + ); + } if (args.json) { console.log( @@ -239,6 +451,7 @@ async function main(): Promise { { generatedAt: new Date().toISOString(), node: process.version, + runs: args.runs, referenceHz: current.referenceHz, tasks: current.tasks, }, @@ -246,29 +459,41 @@ async function main(): Promise { 2, ), ); - } else { - console.table(bench.table()); } if (args.updateBaseline) { const baseline: BaselineFile = { - schemaVersion: 1, + schemaVersion: BASELINE_SCHEMA_VERSION, generatedAt: new Date().toISOString(), node: process.version, referenceHz: current.referenceHz, tasks: {}, }; + let noisy = 0; for (const [name, task] of Object.entries(current.tasks)) { baseline.tasks[name] = { hz: task.hz, mean: task.mean, normalized: task.normalized ?? 0, + normalizedRme: task.normalizedRme ?? 0, + samples: task.samples, }; + if ( + name !== REFERENCE_TASK && + (task.normalizedRme ?? 0) > args.threshold + ) { + noisy++; + } } fs.writeFileSync(BASELINE_PATH, `${JSON.stringify(baseline, null, 2)}\n`); console.log( `Baseline written to ${path.relative(process.cwd(), BASELINE_PATH)}`, ); + if (noisy > 0) { + console.warn( + `Note: ${noisy} task(s) were captured with a margin of error above ${args.threshold}% — comparisons against them will have a high noise floor. Consider regenerating with more --runs.`, + ); + } } if (args.compare) { @@ -281,7 +506,11 @@ async function main(): Promise { const baseline: BaselineFile = JSON.parse( fs.readFileSync(BASELINE_PATH, "utf8"), ); - compare(baseline, current, { strict: args.strict }); + compare(baseline, current, { + strict: args.strict, + threshold: args.threshold, + filter: args.filter, + }); } } From 407b97ea898e632e922ec253b5cf8074ef45cfa7 Mon Sep 17 00:00:00 2001 From: ty Date: Wed, 10 Jun 2026 09:37:52 -0400 Subject: [PATCH 11/17] Adds benchmark to .npmignore --- .npmignore | 3 ++- src/schema.js | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.npmignore b/.npmignore index 0e9d3a76..9fcff87f 100644 --- a/.npmignore +++ b/.npmignore @@ -14,4 +14,5 @@ playground/ .idea/ www/ *.yml -z/ \ No newline at end of file +z/ +benchmark/ \ No newline at end of file diff --git a/src/schema.js b/src/schema.js index c4b5c734..9d5e1ec0 100644 --- a/src/schema.js +++ b/src/schema.js @@ -1662,7 +1662,7 @@ class Schema { let siblings; const getSiblings = () => (siblings = siblings || { ...payload }); for (let path of Object.keys(include)) { - // this.attributes[attribute] !== undefined | Attribute exists as actual attribute. If `includeKeys` is turned on for example this will include values that do not have a presence in the model and therefore will not have a `.get()` method + // this.attributes[attribute] !== undefined | Attribute exists as an actual attribute. If `includeKeys` is turned on for example this will include values that do not have a presence in the model and therefore will not have a `.get()` method // avoid[attribute] === undefined | Attribute shouldn't be in the avoided const attribute = this.getAttribute(path); if (attribute !== undefined && avoid[path] === undefined) { From 1644a1103deb2e8dd30132e2fda6cf89b9ab8132 Mon Sep 17 00:00:00 2001 From: ty Date: Wed, 10 Jun 2026 09:48:05 -0400 Subject: [PATCH 12/17] refactor(benchmark): consolidate duplicate average helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit aggregateRuns defined a loop-local avg closure and a second module-level avg2 doing the same thing; one module-level average() now serves both sites. Also drops the missing-task filter in the per-run aggregation — collectResults aborts the process on any task failure, so every run is guaranteed to contain every task. Co-Authored-By: Claude Fable 5 --- benchmark/run.ts | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/benchmark/run.ts b/benchmark/run.ts index 92c710d1..d7ad138b 100644 --- a/benchmark/run.ts +++ b/benchmark/run.ts @@ -354,6 +354,10 @@ function t95(df: number): number { return df <= 0 ? Infinity : df <= T95.length ? T95[df - 1] : 1.96; } +function average(values: number[]): number { + return values.reduce((sum, value) => sum + value, 0) / values.length; +} + /** * Collapse N independent runs into one result. For N > 1, each task's * normalized value is averaged across runs and its margin of error is @@ -366,17 +370,14 @@ function aggregateRuns(runs: RunResults[]): RunResults { if (runs.length === 1) { return runs[0]; } + // collectResults aborts the process on any task failure, so every run + // contains every task + const n = runs.length; const tasks: Record = {}; - const names = Object.keys(runs[0].tasks); - for (const name of names) { - const perRun = runs - .map((run) => run.tasks[name]) - .filter((task): task is TaskStats => task !== undefined); - const n = perRun.length; - const avg = (values: number[]) => - values.reduce((sum, value) => sum + value, 0) / values.length; + for (const name of Object.keys(runs[0].tasks)) { + const perRun = runs.map((run) => run.tasks[name]); const normalizedValues = perRun.map((task) => task.normalized ?? 0); - const normalizedMean = avg(normalizedValues); + const normalizedMean = average(normalizedValues); const variance = normalizedValues.reduce( (sum, value) => sum + (value - normalizedMean) ** 2, @@ -385,10 +386,10 @@ function aggregateRuns(runs: RunResults[]): RunResults { (n - 1); const ciHalfWidth = t95(n - 1) * Math.sqrt(variance / n); tasks[name] = { - hz: avg(perRun.map((task) => task.hz)), - mean: avg(perRun.map((task) => task.mean)), - p99: avg(perRun.map((task) => task.p99)), - rme: avg(perRun.map((task) => task.rme)), + hz: average(perRun.map((task) => task.hz)), + mean: average(perRun.map((task) => task.mean)), + p99: average(perRun.map((task) => task.p99)), + rme: average(perRun.map((task) => task.rme)), samples: perRun.reduce((sum, task) => sum + task.samples, 0), normalized: normalizedMean, normalizedRme: @@ -396,15 +397,11 @@ function aggregateRuns(runs: RunResults[]): RunResults { }; } return { - referenceHz: avg2(runs.map((run) => run.referenceHz ?? 0)), + referenceHz: average(runs.map((run) => run.referenceHz ?? 0)), tasks, }; } -function avg2(values: number[]): number { - return values.reduce((sum, value) => sum + value, 0) / values.length; -} - async function main(): Promise { const args = parseArgs(process.argv); let entries = loadScenarios(); From 26e7e3f4e605167cb1134346987a184bc489a9db Mon Sep 17 00:00:00 2001 From: ty Date: Wed, 10 Jun 2026 10:28:56 -0400 Subject: [PATCH 13/17] Clean up --- benchmark/run.ts | 46 +++++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/benchmark/run.ts b/benchmark/run.ts index d7ad138b..8caef16c 100644 --- a/benchmark/run.ts +++ b/benchmark/run.ts @@ -214,13 +214,25 @@ function collectResults(bench: Bench): RunResults { return { referenceHz: reference ? reference.hz : null, tasks }; } -type Verdict = - | "REGRESSION" - | "improved" - | "within threshold" - | "~noise" - | "added (not in baseline)" - | "removed"; +const Verdicts = { + regression: "regression", + improved: "improved", + within_threshold: "within_threshold", + noise: "noise", + added: "added", + removed: "removed", +} as const; + +type Verdict = keyof typeof Verdicts; + +const VerdictDisplayText: Record = { + regression: "REGRESSION", + improved: "improved", + within_threshold: "within threshold", + noise: "~noise", + added: "added (not in baseline)", + removed: "removed", +}; interface CompareRow { task: string; @@ -272,7 +284,7 @@ function compare( task.normalized === undefined || task.normalizedRme === undefined ) { - rows.push({ task: name, verdict: "removed" }); + rows.push({ task: name, verdict: Verdicts.removed }); continue; } const deltaPct = (task.normalized / base.normalized - 1) * 100; @@ -283,12 +295,12 @@ function compare( const significant = Math.abs(deltaPct) > noiseFloorPct; const material = Math.abs(deltaPct) >= threshold; const verdict: Verdict = !significant - ? "~noise" + ? Verdicts.noise : !material - ? "within threshold" + ? Verdicts.within_threshold : deltaPct < 0 - ? "REGRESSION" - : "improved"; + ? Verdicts.regression + : Verdicts.improved; if (noiseFloorPct > threshold) { insensitive++; } @@ -301,14 +313,18 @@ function compare( verdict, }; rows.push(row); - if (verdict === "REGRESSION") regressions.push(row); + if (verdict === Verdicts.regression) regressions.push(row); } for (const name of Object.keys(current.tasks)) { if (name !== REFERENCE_TASK && !baseline.tasks[name]) { - rows.push({ task: name, verdict: "added (not in baseline)" }); + rows.push({ task: name, verdict: Verdicts.added }); } } - console.table(rows); + const formatted = rows.map((row) => ({ + ...row, + verdict: VerdictDisplayText[row.verdict], + })) + console.table(formatted); console.log( `Gates: statistical (|delta| must beat the per-task noise floor, from both runs' 95% CIs) and practical (|delta| >= ${threshold}%). Baseline: ${baseline.generatedAt}, node ${baseline.node}.`, ); From e45c73704a1c1bd09d401df01ec4d4c7f11093fa Mon Sep 17 00:00:00 2001 From: ty Date: Wed, 10 Jun 2026 14:58:42 -0400 Subject: [PATCH 14/17] refactor(benchmark): spell out all abbreviated names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rme→relativeMarginOfErrorPercent, hz→operationsPerSecond, p99→percentile99, fn/opts→benchmarkFunction/options, CliArgs→ CommandLineArguments, lo/hi→lowerBound/upperBound, config knobs gain units (_MILLISECONDS/_PERCENT), scenario files rename to *.benchmark.ts. Field renames reach baseline.json and --json output, so the baseline schema bumps to v3 and the committed baseline is regenerated. External names that mirror third-party APIs (tinybench result fields, DynamoDB pk/sk) are kept at their access sites. --- benchmark/README.md | 63 +- benchmark/baseline.json | 145 ++-- benchmark/config.ts | 153 ++++ benchmark/run.ts | 820 ++++++++++++++---- ...bench.ts => batch-get-format.benchmark.ts} | 0 ...ch.ts => entity-construction.benchmark.ts} | 0 ...ain.bench.ts => params-chain.benchmark.ts} | 4 +- ...mat.bench.ts => parse-format.benchmark.ts} | 0 ...bench.ts => query-pagination.benchmark.ts} | 0 test/fixtures/mock-client.ts | 14 +- test/offline.perf-fixes.spec.ts | 32 +- 11 files changed, 956 insertions(+), 275 deletions(-) create mode 100644 benchmark/config.ts rename benchmark/scenarios/{batch-get-format.bench.ts => batch-get-format.benchmark.ts} (100%) rename benchmark/scenarios/{entity-construction.bench.ts => entity-construction.benchmark.ts} (100%) rename benchmark/scenarios/{params-chain.bench.ts => params-chain.benchmark.ts} (89%) rename benchmark/scenarios/{parse-format.bench.ts => parse-format.benchmark.ts} (100%) rename benchmark/scenarios/{query-pagination.bench.ts => query-pagination.benchmark.ts} (100%) diff --git a/benchmark/README.md b/benchmark/README.md index 8b044f6b..eec3dff3 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -8,7 +8,7 @@ the test suite or CI — nothing here gates a build. ```sh npm run benchmark # run everything, print a table -npm run benchmark:json # machine-readable results (hz, mean, p99, rme) +npm run benchmark:json # machine-readable results (operations per second, mean, 99th percentile, margin of error) npm run benchmark:update # rewrite baseline.json from a fresh run npm run benchmark:compare # run + diff against baseline.json (advisory) ``` @@ -22,17 +22,56 @@ npx ts-node ./benchmark/run.ts --compare --strict ``` Other flags: `--filter ` runs a subset while iterating, -`--threshold ` sets the practical-significance bar (default 5), -`--time ` sets within-run sampling time per task (default 500), and -`--runs ` repeats the whole suite n times to derive error bars from the -between-run spread (default 3 for compare/update, 1 for plain runs). +`--threshold ` sets the practical-significance bar, +`--time ` sets within-run sampling time per task, and +`--runs ` repeats the whole suite that many times to derive error bars from the +between-run spread. + +All defaults and tuning knobs — sampling (`runs`/`time`/warmup), the verdict +gates, the reference task, color bands, and lane widths — live in +[`config.ts`](./config.ts), each documented with how it is used and how the +knobs interact (the file header walks the whole pipeline). + +Human-readable output includes terminal visualizations (suppressed by +`--json`; colors respect `NO_COLOR`/non-TTY): plain runs print log-scale +throughput bars; multi-run invocations plot each run's normalized score as a +dot, with every task's lane sharing one percent scale centered on that +task's own mean (deviation-from-own-mean is dimensionless, so rows are +comparable across otherwise apples-and-oranges tasks — sorted noisiest +first for triage, with ├ ┤ marking the CI of the mean over the scatter); +and compare draws each task's baseline and current confidence intervals as +two lanes on a shared scale — the overlap test from the section below, made +literal. Color carries +meaning everywhere: green = good (improved, steady sampling), red = bad +(regression, jittery sampling), yellow = caution (real-but-small change, +moderate variance), cyan = neutral measurement, and magenta = abnormal — +the instrument itself is suspect (a task's noise floor exceeds the +threshold, or machine conditions drifted since the baseline). + +``` +~noise params-chain/get Δ -3.9% vs noise ±11.8% + baseline 160.1759 ±5.4% ··········├──────────────●───────────────┤·· + current 153.8809 ±6.1% ··├─────────────●────────────┤·············· +``` + +Each lane also prints its own CI width (`±`), which is the consistency +signal: a current lane much narrower than its baseline lane means the change +made the benchmark steadier run-to-run, not just faster. When the underlying +spreads differ by ≥2.5x the header calls it out (`· consistency ×N steadier` +/ `noisier`) — a deliberate hint rather than a gated verdict, since spread +estimates from a handful of runs are themselves noisy. Note that a raw CI +width shrinks with more `--runs` regardless of the code (`width = +t(n−1)·sd/√n` — sampling effort, not steadiness), so the baseline records +its run count and the hint converts both sides back to underlying spread +before comparing; compare also warns when the two sides used different +`--runs`. ## Baseline + normalization Raw ops/sec is machine-dependent, so the committed `baseline.json` stores each task's throughput **normalized** to a fixed reference task (`reference/parse-500-items`, a representative formatting workload) that runs -with every invocation: `normalized = task.hz / referenceHz`. Comparing +with every invocation: `normalized = task operations per second ÷ reference operations per second`. Comparing normalized values makes the baseline meaningful across reasonably similar machines; for precise A/B work, regenerate the baseline on your own machine first (`npm run benchmark:update`), apply your change, then @@ -54,8 +93,8 @@ compare applies **two gates** per task and prints a verdict: GC phasing, JIT state, and thermal drift, which dominate for long async tasks (the pagination scenarios can swing ±20% between runs while their within-run margin reads ±1%). - - With `--runs 1` it falls back to tinybench's within-run `rme`, combined - with the reference task's (`√(rme_task² + rme_reference²)`, since a + - With `--runs 1` it falls back to tinybench's within-run relative margin of error, combined + with the reference task's (combining both sides as `√(taskMargin² + referenceMargin²)`, since a normalized value is a ratio of two measurements). Fine for quick iteration; too optimistic for decisions. 2. **Practical — is it big enough to care?** A real change must also exceed @@ -73,7 +112,7 @@ compare applies **two gates** per task and prints a verdict: ```sh git stash # or check out the pre-change tree -npm run benchmark:update # capture the "before" baseline (3 runs) +npm run benchmark:update # capture the "before" baseline git stash pop npm run benchmark:compare # verdicts tell you if the change is real ``` @@ -81,8 +120,8 @@ npm run benchmark:compare # verdicts tell you if the change is real Practical notes: - If a task you care about shows a high noise floor (the runner warns when it - exceeds the threshold), increase repetitions on **both** sides: - `--runs 5`. More runs shrink the between-run confidence interval, which is + exceeds the threshold), increase repetitions on **both** sides, e.g. + `--runs 8`. More runs shrink the between-run confidence interval, which is the only honest way to detect small differences; `--time` only tightens the (already optimistic) within-run estimate. - The compare prints the reference task's raw throughput drift since the @@ -98,7 +137,7 @@ Practical notes: ## Adding a scenario -Drop a file matching `benchmark/scenarios/*.bench.ts` that default-exports an +Drop a file matching `benchmark/scenarios/*.benchmark.ts` that default-exports an array of entries — there is no registration list to edit: ```ts diff --git a/benchmark/baseline.json b/benchmark/baseline.json index 5ae94e4a..9e86d990 100644 --- a/benchmark/baseline.json +++ b/benchmark/baseline.json @@ -1,106 +1,107 @@ { - "schemaVersion": 2, - "generatedAt": "2026-06-10T04:11:58.119Z", + "schemaVersion": 3, + "generatedAt": "2026-06-10T18:25:03.362Z", "node": "v24.3.0", - "referenceHz": 2424.4644886571928, + "runs": 10, + "referenceOperationsPerSecond": 2433.0243439650476, "tasks": { "reference/parse-500-items": { - "hz": 2424.4644886571928, - "mean": 0.41253856308994147, + "operationsPerSecond": 2433.0243439650476, + "mean": 0.4111418215058819, "normalized": 1, - "normalizedRme": 0, - "samples": 3638 + "normalizedRelativeMarginOfErrorPercent": 0, + "samples": 12171 }, "batch-get-format/100-items": { - "hz": 3844.0481109727643, - "mean": 0.26058448713831844, - "normalized": 1.5863465726981503, - "normalizedRme": 15.279148606132178, - "samples": 5770 + "operationsPerSecond": 3858.3806990022845, + "mean": 0.2592887852712155, + "normalized": 1.585890381497777, + "normalizedRelativeMarginOfErrorPercent": 1.013746647150447, + "samples": 19320 }, "entity-construction/new-entity": { - "hz": 6171.908033589301, - "mean": 0.16210953335278167, - "normalized": 2.5469529411935015, - "normalizedRme": 11.232955822414702, - "samples": 9260 + "operationsPerSecond": 6254.40892557932, + "mean": 0.15991138163073665, + "normalized": 2.5710560328456022, + "normalizedRelativeMarginOfErrorPercent": 0.9698110978836614, + "samples": 31277 }, "entity-construction/new-entity-watchers": { - "hz": 5685.492287082828, - "mean": 0.17594353779412905, - "normalized": 2.3459484941148427, - "normalizedRme": 9.202963391510739, - "samples": 8535 + "operationsPerSecond": 5711.937004785308, + "mean": 0.17507553094332612, + "normalized": 2.348459472810711, + "normalizedRelativeMarginOfErrorPercent": 1.4770300388933606, + "samples": 28565 }, "params-chain/get": { - "hz": 388292.19723028206, - "mean": 0.002576322384780131, - "normalized": 160.1758691120667, - "normalizedRme": 6.285730042215901, - "samples": 582975 + "operationsPerSecond": 387973.2106890793, + "mean": 0.002577654861021239, + "normalized": 159.52233021131025, + "normalizedRelativeMarginOfErrorPercent": 1.7151408896518312, + "samples": 1939870 }, "params-chain/query": { - "hz": 260867.58213501098, - "mean": 0.00383364114333602, - "normalized": 107.62911073619341, - "normalizedRme": 6.596224300497242, - "samples": 391302 + "operationsPerSecond": 259533.33275631623, + "mean": 0.003855515012352204, + "normalized": 106.72685226798549, + "normalizedRelativeMarginOfErrorPercent": 2.7938466279878544, + "samples": 1297941 }, "params-chain/query-where": { - "hz": 114070.16974197427, - "mean": 0.008767453305424373, - "normalized": 47.064258493790426, - "normalizedRme": 7.115991189239105, - "samples": 171106 + "operationsPerSecond": 112780.48112728125, + "mean": 0.008868369193681808, + "normalized": 46.37406347031307, + "normalizedRelativeMarginOfErrorPercent": 2.0433268693388253, + "samples": 564575 }, "params-chain/update-set": { - "hz": 88781.91955723667, - "mean": 0.011286994319514502, - "normalized": 36.646340920850015, - "normalizedRme": 17.661690693644044, - "samples": 133187 + "operationsPerSecond": 90675.1380517179, + "mean": 0.01103189367465487, + "normalized": 37.283339936275425, + "normalizedRelativeMarginOfErrorPercent": 2.1245831960796413, + "samples": 453519 }, "params-chain/upsert": { - "hz": 76100.52166636738, - "mean": 0.013204382836770964, - "normalized": 31.413345966819623, - "normalizedRme": 23.45776790528955, - "samples": 114152 + "operationsPerSecond": 78631.54163923618, + "mean": 0.01272494054815596, + "normalized": 32.33392307722848, + "normalizedRelativeMarginOfErrorPercent": 2.6188127725487638, + "samples": 393253 }, "parse-format/100-items": { - "hz": 12222.4801025672, - "mean": 0.08184534482640307, - "normalized": 5.043073744962243, - "normalizedRme": 8.92464931196236, - "samples": 18335 + "operationsPerSecond": 12200.533019125965, + "mean": 0.08200320403389624, + "normalized": 5.017144207152649, + "normalizedRelativeMarginOfErrorPercent": 2.5964103316993548, + "samples": 61010 }, "parse-format/1000-items": { - "hz": 1220.4660830788864, - "mean": 0.82037659733162, - "normalized": 0.5036812142834938, - "normalizedRme": 14.156044011327662, - "samples": 1832 + "operationsPerSecond": 1226.9213915426685, + "mean": 0.8154303062335536, + "normalized": 0.50445033377815, + "normalizedRelativeMarginOfErrorPercent": 2.1773722652963046, + "samples": 6141 }, "parse-format/1000-items-watchers": { - "hz": 853.6491345659579, - "mean": 1.1720107583020443, - "normalized": 0.35221239509147423, - "normalizedRme": 9.357872920197668, - "samples": 1282 + "operationsPerSecond": 836.0174721181766, + "mean": 1.1979540983114136, + "normalized": 0.34373482855789816, + "normalizedRelativeMarginOfErrorPercent": 3.236479099054849, + "samples": 4184 }, "query-pagination/10-pages-x100": { - "hz": 1566.1567517550523, - "mean": 0.6386070826937228, - "normalized": 0.6461657403932399, - "normalizedRme": 7.1360968374326, - "samples": 2350 + "operationsPerSecond": 1516.132762922901, + "mean": 0.6604050249489395, + "normalized": 0.6234040932196916, + "normalizedRelativeMarginOfErrorPercent": 3.1251304614877977, + "samples": 7587 }, "query-pagination/100-pages-x100": { - "hz": 139.20933634125876, - "mean": 7.226922047990283, - "normalized": 0.057446423042089835, - "normalizedRme": 24.407407433915417, - "samples": 210 + "operationsPerSecond": 136.18571131973573, + "mean": 7.350136782834127, + "normalized": 0.056003780045826124, + "normalizedRelativeMarginOfErrorPercent": 3.1941200045888243, + "samples": 687 } } } diff --git a/benchmark/config.ts b/benchmark/config.ts new file mode 100644 index 00000000..f6dffea8 --- /dev/null +++ b/benchmark/config.ts @@ -0,0 +1,153 @@ +/** + * Tuning knobs for the benchmark runner, grouped by the pipeline stage they + * affect. How they play together, end to end: + * + * 1. sampling Each invocation repeats the whole suite `--runs` times + * (DEFAULT_DECISION_RUNS for compare/update, 1 otherwise); + * within each run tinybench warms a task for + * WARMUP_TIME_MILLISECONDS and then samples it for `--time` + * milliseconds (DEFAULT_SAMPLING_TIME_MILLISECONDS). + * RUNS buys narrower between-run confidence intervals — the + * error bars every decision is made with. TIME buys steadier + * within-run numbers — mostly cosmetic at runs > 1 (it feeds + * the stability colors and the runs=1 fallback interval). + * 2. normalize Every task's operations/second is divided by + * REFERENCE_TASK's, so stored scores are machine-portable + * ratios. + * 3. verdicts compare labels a task REGRESSION/improved only when |Δ| + * beats BOTH the measured noise floor (from the two + * confidence intervals — shrinks as runs grow) AND + * DEFAULT_THRESHOLD_PERCENT. CONSISTENCY_NOTE_RATIO and + * CONSISTENCY_MINIMUM_RUNS separately gate the + * spread-change hint. DRIFT_* only annotate machine + * conditions; they never change a verdict. + * 4. rendering STABILITY_* bands pick green/yellow/red for ± values, and + * the *_WIDTH knobs size the ASCII lanes and bars. + * + * Rules of thumb: to detect smaller effects raise --runs (not --time); to + * make verdicts stricter or looser change DEFAULT_THRESHOLD_PERCENT; + * everything else is presentation. + */ + +// --------------------------------------------------------------------------- +// sampling — how much data an invocation collects +// --------------------------------------------------------------------------- + +/** + * Whole-suite repetitions when `--runs` is not given to compare/update + * (plain/exploratory runs default to 1). This is the dominant cost AND the + * dominant sensitivity lever: the between-run 95% confidence interval + * shrinks roughly as t(n−1)/√n, so 5→10 runs is ~1.7x sharper, each + * doubling ~1.5x more. The consistency hint needs CONSISTENCY_MINIMUM_RUNS + * on both sides to fire at all. + */ +export const DEFAULT_DECISION_RUNS = 10; + +/** + * tinybench sampling window per task per run, milliseconds, when `--time` is + * not given. Total invocation cost ≈ runs × tasks × (sampling time + + * WARMUP_TIME_MILLISECONDS). Raising it tightens the within-run margin of + * error — which colors the throughput-bar stability and is the confidence + * interval fallback at runs=1 — but does NOT tighten the between-run + * interval that verdicts use; raise runs for that. + */ +export const DEFAULT_SAMPLING_TIME_MILLISECONDS = 500; + +/** + * tinybench per-task warmup before sampling starts, milliseconds, within + * every run. This warms a task immediately before ITS measurement; it does + * not prevent the first whole-suite run from being JIT-cold relative to + * later runs — that effect is what the ○ marker in the spread view shows. + */ +export const WARMUP_TIME_MILLISECONDS = 100; + +// --------------------------------------------------------------------------- +// normalization — what raw operations/second is divided by +// --------------------------------------------------------------------------- + +/** + * The yardstick task every score is normalized against + * (normalized = task speed ÷ reference speed). It runs in every invocation, + * is excluded from verdicts/spread rows, and its raw drift between baseline + * and compare is reported as the machine-condition indicator. Pick a + * workload representative of the library's hot path; changing it invalidates + * committed baselines (regenerate with benchmark:update). + */ +export const REFERENCE_TASK = "reference/parse-500-items"; + +// --------------------------------------------------------------------------- +// verdict gates — when compare is allowed to claim something +// --------------------------------------------------------------------------- + +/** + * Practical-significance gate, percent, when `--threshold` is not given. + * A delta must beat BOTH the statistical noise floor and this value to earn + * REGRESSION/improved; real-but-smaller deltas report "within threshold". + * It also drives the undersensitivity warnings: tasks whose noise floor + * exceeds it are flagged magenta (the run cannot detect changes this small). + */ +export const DEFAULT_THRESHOLD_PERCENT = 5; + +/** + * Underlying-spread ratio before compare calls out a consistency change + * (`consistency ×N steadier/noisier`). Spread estimates from a handful of + * runs are themselves noisy — ~2.5x is what an F-test needs to clear 95% + * confidence at 5 runs — so this stays conservative; lower it only if you + * also raise runs. The hint converts confidence-interval widths back to + * spread (spread ∝ width·√n/t(n−1)) so unequal --runs between baseline and + * current do not fake a change. + */ +export const CONSISTENCY_NOTE_RATIO = 2.5; + +/** + * Minimum --runs on BOTH sides before the consistency hint may fire. A + * spread estimated from 2 runs has one degree of freedom and swings wildly; + * 3+ keeps the hint from reacting to pure sampling luck. + */ +export const CONSISTENCY_MINIMUM_RUNS = 3; + +/** + * Reference-task raw-drift bands, percent: ≤ ACCEPTABLE green, ≤ CAUTION + * yellow, beyond magenta plus a warning. Drift means machine conditions + * changed between baseline and compare (thermal, load); normalization + * absorbs uniform drift but not GC/async-heavy skew, so large drift makes + * borderline verdicts suspect. Advisory only — never changes a verdict. + */ +export const DRIFT_ACCEPTABLE_PERCENT = 3; +export const DRIFT_CAUTION_PERCENT = 10; + +// --------------------------------------------------------------------------- +// rendering — colors and layout only; no effect on measurements or verdicts +// --------------------------------------------------------------------------- + +/** + * Variance bands, percent, for coloring ± values green/yellow/red across all + * views (throughput bars, spread rows, compare lanes) and per-dot deviation + * colors in the spread view. Purely presentational. + */ +export const STABILITY_STEADY_PERCENT = 3; +export const STABILITY_MODERATE_PERCENT = 10; + +/** column width for verdict labels; lane rows indent to clear it */ +export const VERDICT_LABEL_WIDTH = 17; + +/** character width of compare confidence-interval lanes */ +export const LANE_WIDTH = 44; + +/** character width of spread-view lanes; odd so the mean has an exact center */ +export const SPREAD_LANE_WIDTH = 45; + +/** character width of throughput bars */ +export const BAR_WIDTH = 28; + +// --------------------------------------------------------------------------- +// wiring — not tuning knobs +// --------------------------------------------------------------------------- + +/** + * Bump when baseline.json's shape changes incompatibly; compare refuses + * baselines with a different version and asks for benchmark:update. + * (Version 3: field names spelled out — operationsPerSecond, + * normalizedRelativeMarginOfErrorPercent, referenceOperationsPerSecond.) + */ +export const BASELINE_SCHEMA_VERSION = 3; diff --git a/benchmark/run.ts b/benchmark/run.ts index 8caef16c..742e8450 100644 --- a/benchmark/run.ts +++ b/benchmark/run.ts @@ -8,16 +8,18 @@ * npm run benchmark:compare compare against baseline.json (advisory) * --strict exit 1 when the compare finds regressions * --filter only run tasks whose name includes substring - * --threshold practical-significance threshold (default 5) - * --time within-run sampling time per task (default 500) - * --runs repeat the whole suite n times and derive - * error bars from the between-run spread - * (default 3 for compare/update, 1 otherwise) + * --threshold practical-significance threshold + * --time within-run sampling time per task + * --runs repeat the whole suite this many times and + * derive error bars from the between-run spread + * (defaults for all of these live in config.ts, with documentation on + * how the knobs interact) * - * Scenarios live in benchmark/scenarios/*.bench.ts; each default-exports an - * array of `{ name, fn, opts? }` entries (see README.md). A fixed reference + * Scenarios live in benchmark/scenarios/*.benchmark.ts; each default-exports an + * array of `{ name, fn, options? }` entries (see README.md). A fixed reference * task runs with every invocation and task throughput is recorded relative to - * it (`normalized = hz / referenceHz`), which makes the committed baseline + * it (`normalized = task operations/second ÷ reference operations/second`), + * which makes the committed baseline * roughly machine-independent. * * Compare verdicts use two gates (see README "Interpreting a compare"): @@ -26,7 +28,8 @@ * from the between-run spread of the normalized value (which captures * GC/JIT/thermal variance that within-run sampling underestimates, * especially for long async tasks); with --runs 1 it falls back to the - * within-run tinybench rme combined with the reference task's. + * within-run tinybench relative margin of error combined with the + * reference task's. * practical — the change must also exceed --threshold percent to be worth * acting on. Real-but-small changes report as "within threshold", and * changes inside the noise floor report as "~noise". @@ -35,6 +38,24 @@ import * as fs from "fs"; import * as path from "path"; import { Bench } from "tinybench"; import { makeFixtureEntity, makeStoredItem } from "../test/fixtures/entities"; +import { + BAR_WIDTH, + BASELINE_SCHEMA_VERSION, + CONSISTENCY_MINIMUM_RUNS, + CONSISTENCY_NOTE_RATIO, + DEFAULT_DECISION_RUNS, + DEFAULT_THRESHOLD_PERCENT, + DEFAULT_SAMPLING_TIME_MILLISECONDS, + DRIFT_CAUTION_PERCENT, + DRIFT_ACCEPTABLE_PERCENT, + LANE_WIDTH, + REFERENCE_TASK, + SPREAD_LANE_WIDTH, + STABILITY_MODERATE_PERCENT, + STABILITY_STEADY_PERCENT, + VERDICT_LABEL_WIDTH, + WARMUP_TIME_MILLISECONDS, +} from "./config"; export interface ScenarioEntry { /** task name, conventionally `group/task` */ @@ -42,36 +63,37 @@ export interface ScenarioEntry { /** the operation under test; may be async */ fn: () => unknown | Promise; /** optional tinybench task options (beforeAll/beforeEach/...) */ - opts?: Record; + options?: Record; } -interface TaskStats { - hz: number; +interface TaskStatistics { + operationsPerSecond: number; mean: number; - p99: number; + percentile99: number; /** tinybench's relative margin of error for the task itself (95%, percent) */ - rme: number; + relativeMarginOfErrorPercent: number; samples: number; normalized?: number; /** * relative 95% margin of error of `normalized` (percent). `normalized` is a - * ratio of two independent estimates (task hz / reference hz), so its - * relative error combines both: sqrt(rme² + rmeReference²). + * ratio of two independent estimates (task speed ÷ reference speed), so + * its relative error combines both sides' margins: + * sqrt(taskMargin² + referenceMargin²). */ - normalizedRme?: number; + normalizedRelativeMarginOfErrorPercent?: number; } interface RunResults { - referenceHz: number | null; - tasks: Record; + referenceOperationsPerSecond: number | null; + tasks: Record; } interface BaselineTask { - hz: number; + operationsPerSecond: number; mean: number; normalized: number; /** relative 95% margin of error of `normalized` (percent) */ - normalizedRme: number; + normalizedRelativeMarginOfErrorPercent: number; samples: number; } @@ -79,11 +101,13 @@ interface BaselineFile { schemaVersion: number; generatedAt: string; node: string; - referenceHz: number | null; + /** --runs the baseline was captured at; absent on older baselines */ + runs?: number; + referenceOperationsPerSecond: number | null; tasks: Record; } -interface CliArgs { +interface CommandLineArguments { json: boolean; compare: boolean; updateBaseline: boolean; @@ -91,58 +115,64 @@ interface CliArgs { filter: string | null; /** practical-significance threshold, percent */ threshold: number; - /** tinybench sampling time per task, ms */ + /** tinybench sampling time per task, milliseconds */ time: number; /** full-suite repetitions; >1 derives error bars from between-run spread */ runs: number; } -const SCENARIO_DIR = path.join(__dirname, "scenarios"); +const SCENARIO_DIRECTORY = path.join(__dirname, "scenarios"); const BASELINE_PATH = path.join(__dirname, "baseline.json"); -const REFERENCE_TASK = "reference/parse-500-items"; -const BASELINE_SCHEMA_VERSION = 2; -const DEFAULT_THRESHOLD_PCT = 5; -const DEFAULT_TIME_MS = 500; -const DEFAULT_DECISION_RUNS = 3; - -function parseArgs(argv: string[]): CliArgs { - const args: CliArgs = { + +function parseCommandLineArguments( + processArguments: string[], +): CommandLineArguments { + const commandLineArguments: CommandLineArguments = { json: false, compare: false, updateBaseline: false, strict: false, filter: null, - threshold: DEFAULT_THRESHOLD_PCT, - time: DEFAULT_TIME_MS, + threshold: DEFAULT_THRESHOLD_PERCENT, + time: DEFAULT_SAMPLING_TIME_MILLISECONDS, runs: 0, // resolved after flags are read; see below }; - for (let i = 2; i < argv.length; i++) { - const arg = argv[i]; - if (arg === "--json") args.json = true; - else if (arg === "--compare") args.compare = true; - else if (arg === "--update-baseline") args.updateBaseline = true; - else if (arg === "--strict") args.strict = true; - else if (arg === "--filter") args.filter = argv[++i] ?? null; - else if (arg === "--threshold" || arg === "--time" || arg === "--runs") { - const value = Number(argv[++i]); + for (let i = 2; i < processArguments.length; i++) { + const argument = processArguments[i]; + if (argument === "--json") commandLineArguments.json = true; + else if (argument === "--compare") commandLineArguments.compare = true; + else if (argument === "--update-baseline") + commandLineArguments.updateBaseline = true; + else if (argument === "--strict") commandLineArguments.strict = true; + else if (argument === "--filter") + commandLineArguments.filter = processArguments[++i] ?? null; + else if ( + argument === "--threshold" || + argument === "--time" || + argument === "--runs" + ) { + const value = Number(processArguments[++i]); if (!Number.isFinite(value) || value <= 0) { - console.error(`${arg} requires a positive number`); + console.error(`${argument} requires a positive number`); process.exit(1); } - if (arg === "--threshold") args.threshold = value; - else if (arg === "--time") args.time = value; - else args.runs = Math.floor(value); + if (argument === "--threshold") commandLineArguments.threshold = value; + else if (argument === "--time") commandLineArguments.time = value; + else commandLineArguments.runs = Math.floor(value); } else { - console.error(`Unknown argument: ${arg}`); + console.error(`Unknown argument: ${argument}`); process.exit(1); } } - if (args.runs === 0) { + if (commandLineArguments.runs === 0) { // compare/update need trustworthy error bars, which only between-run // spread provides; plain exploratory runs stay fast by default - args.runs = args.compare || args.updateBaseline ? DEFAULT_DECISION_RUNS : 1; + commandLineArguments.runs = + commandLineArguments.compare || commandLineArguments.updateBaseline + ? DEFAULT_DECISION_RUNS + : 1; } - return args; + return commandLineArguments; } function makeReferenceTask(): ScenarioEntry { @@ -151,20 +181,28 @@ function makeReferenceTask(): ScenarioEntry { for (let i = 0; i < 500; i++) { Items.push(makeStoredItem(entity, i)); } - return { name: REFERENCE_TASK, fn: () => entity.parse({ Items }) }; + return { + name: REFERENCE_TASK, + fn: () => entity.parse({ Items }), + }; } function loadScenarios(): ScenarioEntry[] { const entries: ScenarioEntry[] = [makeReferenceTask()]; const files = fs - .readdirSync(SCENARIO_DIR) - .filter((file) => file.endsWith(".bench.ts") || file.endsWith(".bench.js")) + .readdirSync(SCENARIO_DIRECTORY) + .filter( + (file) => + file.endsWith(".benchmark.ts") || file.endsWith(".benchmark.js"), + ) .sort(); for (const file of files) { - const mod = require(path.join(SCENARIO_DIR, file)); - const scenario: unknown = mod.default ?? mod; + const loadedModule = require(path.join(SCENARIO_DIRECTORY, file)); + const scenario: unknown = loadedModule.default ?? loadedModule; if (!Array.isArray(scenario)) { - throw new Error(`${file} must export an array of {name, fn, opts?}`); + throw new Error( + `${file} must export an array of {name, fn, options?}`, + ); } for (const entry of scenario) { if ( @@ -172,7 +210,9 @@ function loadScenarios(): ScenarioEntry[] { typeof entry.name !== "string" || typeof entry.fn !== "function" ) { - throw new Error(`${file} exported an entry without {name, fn}`); + throw new Error( + `${file} exported an entry without {name, fn}`, + ); } entries.push(entry); } @@ -180,10 +220,10 @@ function loadScenarios(): ScenarioEntry[] { return entries; } -function collectResults(bench: Bench): RunResults { - const tasks: Record = {}; +function collectResults(benchmarkSuite: Bench): RunResults { + const tasks: Record = {}; let failed = false; - for (const task of bench.tasks) { + for (const task of benchmarkSuite.tasks) { if (!task.result || task.result.error) { console.error(`Task "${task.name}" failed:`); console.error(task.result && task.result.error); @@ -191,10 +231,10 @@ function collectResults(bench: Bench): RunResults { continue; } tasks[task.name] = { - hz: task.result.hz, + operationsPerSecond: task.result.hz, mean: task.result.mean, - p99: task.result.p99, - rme: task.result.rme, + percentile99: task.result.p99, + relativeMarginOfErrorPercent: task.result.rme, samples: task.result.samples.length, }; } @@ -205,13 +245,21 @@ function collectResults(bench: Bench): RunResults { if (reference) { for (const name of Object.keys(tasks)) { const task = tasks[name]; - task.normalized = task.hz / reference.hz; - task.normalizedRme = Math.sqrt( - task.rme * task.rme + reference.rme * reference.rme, + task.normalized = + task.operationsPerSecond / reference.operationsPerSecond; + task.normalizedRelativeMarginOfErrorPercent = Math.sqrt( + task.relativeMarginOfErrorPercent * task.relativeMarginOfErrorPercent + + reference.relativeMarginOfErrorPercent * + reference.relativeMarginOfErrorPercent, ); } } - return { referenceHz: reference ? reference.hz : null, tasks }; + return { + referenceOperationsPerSecond: reference + ? reference.operationsPerSecond + : null, + tasks, + }; } const Verdicts = { @@ -234,20 +282,292 @@ const VerdictDisplayText: Record = { removed: "removed", }; -interface CompareRow { - task: string; - "baseline (norm)"?: string; - "current (norm)"?: string; - "delta %"?: string; - "noise +/-%"?: string; - verdict: Verdict; +// --------------------------------------------------------------------------- +// terminal rendering — ANSI colors, CI lanes, and throughput bars +// --------------------------------------------------------------------------- + +const useColor = + process.stdout.isTTY === true && process.env.NO_COLOR === undefined; + +function paint(code: number): (text: string) => string { + return (text) => (useColor ? `\u001b[${code}m${text}\u001b[0m` : text); +} + +const bold = paint(1); +const dim = paint(2); +const red = paint(31); +const green = paint(32); +const yellow = paint(33); +const magenta = paint(35); +const cyan = paint(36); + +// color semantics used across all output: +// green = good (improved / steady) red = bad (regression / jittery) +// yellow = caution (real-but-small / moderate variance) +// cyan = neutral measurement magenta = abnormal (instrument too +// blunt: noise floor above the threshold, or large machine drift) +const VerdictPaint: Record string> = { + regression: (text) => bold(red(text)), + improved: green, + within_threshold: yellow, + noise: dim, + added: cyan, + removed: dim, +}; + +// the lane/delta color for a verdict; ~noise stays a visible neutral so the +// current lane never disappears into the dim baseline lane +const VerdictEmphasisPaint: Record string> = { + regression: red, + improved: green, + within_threshold: yellow, + noise: cyan, + added: cyan, + removed: dim, +}; + +function stabilityPaint( + relativeMarginOfErrorPercent: number, +): (text: string) => string { + if (relativeMarginOfErrorPercent <= STABILITY_STEADY_PERCENT) return green; + if (relativeMarginOfErrorPercent <= STABILITY_MODERATE_PERCENT) return yellow; + return red; +} + +const LANE_INDENT = " ".repeat(VERDICT_LABEL_WIDTH + 1); + +function verdictLabel(verdict: Verdict): string { + return VerdictPaint[verdict]( + VerdictDisplayText[verdict].padEnd(VERDICT_LABEL_WIDTH), + ); +} + +// dim, single-line "how to read this" bullets under a section header +function explain(bullets: string[]): void { + for (const bullet of bullets) { + console.log(dim(` • ${bullet}`)); + } +} + +interface Interval { + lowerBound: number; + mean: number; + upperBound: number; +} + +function intervalOf( + mean: number, + relativeMarginOfErrorPercent: number, +): Interval { + const half = mean * (relativeMarginOfErrorPercent / 100); + return { + lowerBound: Math.max(0, mean - half), + mean, + upperBound: mean + half, + }; +} + +// one fixed-width lane — ····├────●────┤···· — the 95% CI drawn over a dotted +// track; two lanes rendered on the same scale make interval overlap (or the +// gap between them) directly visible in the terminal +function renderLane( + interval: Interval, + scaleLowerBound: number, + scaleUpperBound: number, + color: (text: string) => string, +): string { + const cells: string[] = new Array(LANE_WIDTH).fill("·"); + const span = scaleUpperBound - scaleLowerBound; + const at = (value: number) => + span <= 0 + ? 0 + : Math.max( + 0, + Math.min( + LANE_WIDTH - 1, + Math.round(((value - scaleLowerBound) / span) * (LANE_WIDTH - 1)), + ), + ); + const lowerBoundPosition = at(interval.lowerBound); + const upperBoundPosition = at(interval.upperBound); + if (upperBoundPosition - lowerBoundPosition < 2) { + // interval narrower than the lane resolution: render a point, not caps + cells[at(interval.mean)] = "●"; + } else { + for (let i = lowerBoundPosition; i <= upperBoundPosition; i++) { + cells[i] = "─"; + } + cells[lowerBoundPosition] = "├"; + cells[upperBoundPosition] = "┤"; + cells[at(interval.mean)] = "●"; + } + const track = cells.join(""); + return ( + dim(track.slice(0, lowerBoundPosition)) + + color(track.slice(lowerBoundPosition, upperBoundPosition + 1)) + + dim(track.slice(upperBoundPosition + 1)) + ); +} + +// log scale: tasks span ~140 to ~450k ops/sec, so linear bars would flatten +// everything but the fastest group +function renderThroughputBars(tasks: Record): void { + const names = Object.keys(tasks); + if (names.length === 0) { + return; + } + const nameWidth = Math.max(...names.map((name) => name.length)); + const logarithms = names.map((name) => + Math.log10(tasks[name].operationsPerSecond), + ); + const logarithmLowerBound = Math.min(...logarithms); + const logarithmUpperBound = Math.max(...logarithms); + console.log(""); + console.log("throughput — operations per second"); + explain([ + "longer bar = faster (log scale: small bar gaps are big speed gaps)", + "±% = sampling wobble — smaller is more trustworthy", + ]); + console.log( + ` • bar color = stability: ${green( + `steady ≤${STABILITY_STEADY_PERCENT}%`, + )} · ${yellow(`moderate ≤${STABILITY_MODERATE_PERCENT}%`)} · ${red( + "jittery above", + )}`, + ); + for (const name of names) { + const task = tasks[name]; + const fraction = + logarithmUpperBound > logarithmLowerBound + ? (Math.log10(task.operationsPerSecond) - logarithmLowerBound) / + (logarithmUpperBound - logarithmLowerBound) + : 1; + const filled = Math.max(1, Math.round(fraction * BAR_WIDTH)); + const stability = stabilityPaint(task.relativeMarginOfErrorPercent); + const bar = + stability("█".repeat(filled)) + dim("░".repeat(BAR_WIDTH - filled)); + const operationsPerSecondLabel = Math.round(task.operationsPerSecond) + .toLocaleString("en-US") + .padStart(9); + const relativeMarginOfErrorLabel = stability( + `±${task.relativeMarginOfErrorPercent.toFixed(1)}%`.padStart(7), + ); + console.log( + ` ${name.padEnd( + nameWidth, + )} ${bar} ${operationsPerSecondLabel} ${relativeMarginOfErrorLabel}`, + ); + } +} + +// terminal version of the "same scatter, shrinking bracket" picture: every +// lane shares ONE percent scale centered on each task's own mean, so rows +// are directly comparable even across apples-and-oranges tasks (deviation +// from your own mean is dimensionless). Dots = individual runs (colored by +// deviation band; ○ marks the first, usually-cold run), │ = the task's +// mean, ├ ┤ = the 95% CI of the mean — caps tighten with --runs while the +// scatter does not. Rows sort noisiest-first for triage. +function renderRunSpread(allRuns: RunResults[], current: RunResults): void { + const names = Object.keys(current.tasks).filter( + (name) => name !== REFERENCE_TASK, + ); + if (names.length === 0) { + return; + } + const rows = names.map((name) => { + const stats = current.tasks[name]; + const mean = stats.normalized ?? 0; + const values = allRuns.map((run) => run.tasks[name].normalized ?? 0); + const deviations = values.map((value) => + mean > 0 ? (value / mean - 1) * 100 : 0, + ); + return { + name, + mean, + relativeMarginOfErrorPercent: + stats.normalizedRelativeMarginOfErrorPercent ?? 0, + deviations, + }; + }); + rows.sort( + (a, b) => b.relativeMarginOfErrorPercent - a.relativeMarginOfErrorPercent, + ); + const scaleMaximum = Math.max( + 1, + ...rows.map((row) => + Math.max( + row.relativeMarginOfErrorPercent, + ...row.deviations.map(Math.abs), + ), + ), + ); + const nameWidth = Math.max(...names.map((name) => name.length)); + const center = Math.floor(SPREAD_LANE_WIDTH / 2); + const at = (percent: number) => + center + + Math.max( + -center, + Math.min(center, Math.round((percent / scaleMaximum) * center)), + ); + console.log(""); + console.log( + `normalized score per run — each dot is one of ${ + allRuns.length + } runs, every lane spans ±${scaleMaximum.toFixed(1)}% of its task's mean`, + ); + explain([ + "one shared scale: wider dot scatter = jitterier task — rows sorted noisiest first for triage", + ]); + console.log( + ` • dots = individual runs, colored by deviation from their mean: ${green( + `≤${STABILITY_STEADY_PERCENT}%`, + )} · ${yellow(`≤${STABILITY_MODERATE_PERCENT}%`)} · ${red( + "beyond", + )}; ${cyan("○")} = first run (often cold)`, + ); + explain([ + "│ = the mean · ├ ┤ = how precisely that mean is known (the ±% column) — more --runs pulls the caps inward", + "the dot scatter is the code's actual jitter — more runs measure it better but don't shrink it", + ]); + for (const row of rows) { + const cells: string[] = new Array(SPREAD_LANE_WIDTH).fill("·"); + const paints: ((text: string) => string)[] = new Array( + SPREAD_LANE_WIDTH, + ).fill(dim); + const capPaint = stabilityPaint(row.relativeMarginOfErrorPercent); + // the mean line stays un-dimmed so the lane has a visible anchor + cells[center] = "│"; + paints[center] = (text) => text; + if (at(row.relativeMarginOfErrorPercent) !== center) { + cells[at(-row.relativeMarginOfErrorPercent)] = "├"; + paints[at(-row.relativeMarginOfErrorPercent)] = capPaint; + cells[at(row.relativeMarginOfErrorPercent)] = "┤"; + paints[at(row.relativeMarginOfErrorPercent)] = capPaint; + } + row.deviations.forEach((deviation, runIndex) => { + const position = at(deviation); + // first run is marked distinctly: it is the usual cold-JIT outlier + cells[position] = runIndex === 0 ? "○" : "●"; + paints[position] = stabilityPaint(Math.abs(deviation)); + }); + const lane = cells.map((cell, index) => paints[index](cell)).join(""); + const normalized = row.mean.toFixed(4).padStart(10); + const confidenceIntervalLabel = capPaint( + `±${row.relativeMarginOfErrorPercent.toFixed(1)}%`.padStart(7), + ); + console.log( + ` ${row.name.padEnd( + nameWidth, + )} ${normalized} ${confidenceIntervalLabel} ${lane}`, + ); + } } /** * Two-gate comparison per task: * * 1. statistical -- is the change distinguishable from sampling error? Each - * run's normalized value carries a 95% confidence interval (normalizedRme). + * run's normalized value carries a 95% confidence interval (normalizedRelativeMarginOfErrorPercent). * The change is significant only when the two intervals do not overlap, * i.e. |delta| exceeds the "noise floor" (the sum of both CI half-widths, * expressed in percent of the baseline value). This is conservative: when @@ -263,7 +583,13 @@ function compare( strict, threshold, filter, - }: { strict: boolean; threshold: number; filter: string | null }, + runs, + }: { + strict: boolean; + threshold: number; + filter: string | null; + runs: number; + }, ): void { if (baseline.schemaVersion !== BASELINE_SCHEMA_VERSION) { console.error( @@ -271,9 +597,34 @@ function compare( ); process.exit(1); } - const rows: CompareRow[] = []; - const regressions: CompareRow[] = []; + const regressions: string[] = []; let insensitive = 0; + console.log(""); + console.log( + `compare vs baseline from ${baseline.generatedAt} (node ${baseline.node})`, + ); + explain([ + "each lane = 95% CI of normalized speed (\u25cf mean) \u2014 further right is faster/better", + "overlapping lanes \u2192 no provable change (~noise)", + `\u0394 = % vs baseline (positive is better); must beat noise \u00b1 and the ${threshold}% threshold for a verdict`, + "noise \u00b1 = smallest provable change \u2014 smaller is sharper (raise --runs to sharpen)", + "\u00b1 per lane = that side's CI width: current much narrower than baseline = consistency improved (hint, not a verdict)", + ]); + console.log( + ` \u2022 current lane color: ${red("regression")} \u00b7 ${green( + "improved", + )} \u00b7 ${yellow("within threshold")} \u00b7 ${cyan("~noise")}; ${magenta( + "magenta \u00b1", + )} = noise floor above threshold`, + ); + if (baseline.runs !== undefined && baseline.runs !== runs) { + console.log( + yellow( + ` \u2022 run counts differ (baseline ${baseline.runs}, current ${runs}) \u2014 \u00b1 widths partly reflect sampling effort; the consistency hint corrects for this, the raw \u00b1 columns do not`, + ), + ); + } + console.log(""); for (const [name, base] of Object.entries(baseline.tasks)) { if (name === REFERENCE_TASK) continue; // a filtered run intentionally skips tasks; don't report those as removed @@ -282,92 +633,209 @@ function compare( if ( !task || task.normalized === undefined || - task.normalizedRme === undefined + task.normalizedRelativeMarginOfErrorPercent === undefined ) { - rows.push({ task: name, verdict: Verdicts.removed }); + console.log(`${verdictLabel(Verdicts.removed)} ${name}`); continue; } - const deltaPct = (task.normalized / base.normalized - 1) * 100; - const baseHalfWidth = base.normalized * (base.normalizedRme / 100); - const currentHalfWidth = task.normalized * (task.normalizedRme / 100); - const noiseFloorPct = + const deltaPercent = (task.normalized / base.normalized - 1) * 100; + const baseHalfWidth = + base.normalized * (base.normalizedRelativeMarginOfErrorPercent / 100); + const currentHalfWidth = + task.normalized * (task.normalizedRelativeMarginOfErrorPercent / 100); + const noiseFloorPercent = ((baseHalfWidth + currentHalfWidth) / base.normalized) * 100; - const significant = Math.abs(deltaPct) > noiseFloorPct; - const material = Math.abs(deltaPct) >= threshold; + const significant = Math.abs(deltaPercent) > noiseFloorPercent; + const material = Math.abs(deltaPercent) >= threshold; const verdict: Verdict = !significant ? Verdicts.noise : !material ? Verdicts.within_threshold - : deltaPct < 0 + : deltaPercent < 0 ? Verdicts.regression : Verdicts.improved; - if (noiseFloorPct > threshold) { + if (noiseFloorPercent > threshold) { insensitive++; } - const row: CompareRow = { - task: name, - "baseline (norm)": base.normalized.toFixed(4), - "current (norm)": task.normalized.toFixed(4), - "delta %": `${deltaPct >= 0 ? "+" : ""}${deltaPct.toFixed(1)}`, - "noise +/-%": noiseFloorPct.toFixed(1), - verdict, - }; - rows.push(row); - if (verdict === Verdicts.regression) regressions.push(row); + if (verdict === Verdicts.regression) { + regressions.push(name); + } + const baseInterval = intervalOf( + base.normalized, + base.normalizedRelativeMarginOfErrorPercent, + ); + const currentInterval = intervalOf( + task.normalized, + task.normalizedRelativeMarginOfErrorPercent, + ); + const scaleLowerBound = Math.min( + baseInterval.lowerBound, + currentInterval.lowerBound, + ); + const scaleUpperBound = Math.max( + baseInterval.upperBound, + currentInterval.upperBound, + ); + const margin = + (scaleUpperBound - scaleLowerBound) * 0.05 || scaleUpperBound * 0.01 || 1; + const emphasis = VerdictEmphasisPaint[verdict]; + const deltaText = emphasis( + `\u0394 ${deltaPercent >= 0 ? "+" : ""}${deltaPercent.toFixed(1)}%`, + ); + // magenta marks the abnormal case: the comparison itself is too blunt to + // detect threshold-sized changes for this task + const noisePaint = noiseFloorPercent > threshold ? magenta : dim; + const noiseText = noisePaint( + `vs noise \u00b1${noiseFloorPercent.toFixed(1)}%`, + ); + // consistency hint: a much narrower/wider current CI than the baseline's + // means the change altered run-to-run variability, not just the mean. + // CI width = t(n-1)\u00b7sd/\u221an, so each side is converted back to its + // underlying spread (sd \u221d width\u00b7\u221an/t) before comparing \u2014 otherwise a + // higher --runs on one side fakes a consistency change. Spread estimates + // from a handful of runs are themselves noisy, so only call it out + // beyond a conservative ratio (\u2248 what an F-test needs at 5 runs). + // Requires the per-side run count, and CONSISTENCY_MINIMUM_RUNS runs (at 1 + // run the \u00b1 is a within-run margin, not a between-run spread; at 2 the + // spread estimate has a single degree of freedom and swings wildly). + let consistencyText = ""; + if ( + baseline.runs !== undefined && + baseline.runs >= CONSISTENCY_MINIMUM_RUNS && + runs >= CONSISTENCY_MINIMUM_RUNS && + base.normalizedRelativeMarginOfErrorPercent > 0 && + task.normalizedRelativeMarginOfErrorPercent > 0 + ) { + const confidenceIntervalWidthToSpread = ( + relativeMarginOfErrorPercent: number, + runCount: number, + ) => + (relativeMarginOfErrorPercent * Math.sqrt(runCount)) / + studentTCriticalValue95(runCount - 1); + const ratio = + confidenceIntervalWidthToSpread( + base.normalizedRelativeMarginOfErrorPercent, + baseline.runs, + ) / + confidenceIntervalWidthToSpread( + task.normalizedRelativeMarginOfErrorPercent, + runs, + ); + if (ratio >= CONSISTENCY_NOTE_RATIO) { + consistencyText = ` ${green( + `\u00b7 consistency \u00d7${ratio.toFixed(1)} steadier`, + )}`; + } else if (ratio <= 1 / CONSISTENCY_NOTE_RATIO) { + consistencyText = ` ${red( + `\u00b7 consistency \u00d7${(1 / ratio).toFixed(1)} noisier`, + )}`; + } + } + console.log( + `${verdictLabel( + verdict, + )} ${name} ${deltaText} ${noiseText}${consistencyText}`, + ); + console.log( + `${LANE_INDENT}baseline ${base.normalized + .toFixed(4) + .padStart(10)} ${stabilityPaint( + base.normalizedRelativeMarginOfErrorPercent, + )( + `\u00b1${base.normalizedRelativeMarginOfErrorPercent.toFixed( + 1, + )}%`.padStart(7), + )} ${renderLane( + baseInterval, + scaleLowerBound - margin, + scaleUpperBound + margin, + dim, + )}`, + ); + console.log( + `${LANE_INDENT}current ${task.normalized + .toFixed(4) + .padStart(10)} ${stabilityPaint( + task.normalizedRelativeMarginOfErrorPercent, + )( + `\u00b1${task.normalizedRelativeMarginOfErrorPercent.toFixed( + 1, + )}%`.padStart(7), + )} ${renderLane( + currentInterval, + scaleLowerBound - margin, + scaleUpperBound + margin, + emphasis, + )}`, + ); + console.log(""); } for (const name of Object.keys(current.tasks)) { if (name !== REFERENCE_TASK && !baseline.tasks[name]) { - rows.push({ task: name, verdict: Verdicts.added }); + console.log(`${verdictLabel(Verdicts.added)} ${name}`); } } - const formatted = rows.map((row) => ({ - ...row, - verdict: VerdictDisplayText[row.verdict], - })) - console.table(formatted); - console.log( - `Gates: statistical (|delta| must beat the per-task noise floor, from both runs' 95% CIs) and practical (|delta| >= ${threshold}%). Baseline: ${baseline.generatedAt}, node ${baseline.node}.`, - ); // Normalization absorbs *uniform* machine slowdown, but drift is rarely // uniform (GC/async-heavy tasks throttle harder), so large reference drift // means conditions changed between runs and borderline verdicts are suspect. - if (baseline.referenceHz && current.referenceHz) { - const driftPct = (current.referenceHz / baseline.referenceHz - 1) * 100; + if ( + baseline.referenceOperationsPerSecond && + current.referenceOperationsPerSecond + ) { + const driftPercent = + (current.referenceOperationsPerSecond / + baseline.referenceOperationsPerSecond - + 1) * + 100; + const driftPaint = + Math.abs(driftPercent) <= DRIFT_ACCEPTABLE_PERCENT + ? green + : Math.abs(driftPercent) <= DRIFT_CAUTION_PERCENT + ? yellow + : magenta; console.log( - `Reference-task raw throughput drift since baseline: ${ - driftPct >= 0 ? "+" : "" - }${driftPct.toFixed(1)}%.`, + `Reference-task raw throughput drift since baseline: ${driftPaint( + `${driftPercent >= 0 ? "+" : ""}${driftPercent.toFixed(1)}%`, + )}.`, ); - if (Math.abs(driftPct) > 10) { + if (Math.abs(driftPercent) > DRIFT_CAUTION_PERCENT) { console.warn( - "Machine conditions differ noticeably from the baseline run (thermal throttling, background load, or a different machine). Treat borderline verdicts with suspicion; re-run when idle or regenerate the baseline.", + magenta( + "Machine conditions differ noticeably from the baseline run (thermal throttling, background load, or a different machine). Treat borderline verdicts with suspicion; re-run when idle or regenerate the baseline.", + ), ); } } if (insensitive > 0) { console.warn( - `Note: ${insensitive} task(s) have a noise floor above ${threshold}% -- a change of that size could hide in noise there. Increase --runs on both sides (and regenerate the baseline) for more sensitivity.`, + magenta( + `Note: ${insensitive} task(s) have a noise floor above ${threshold}% -- a change of that size could hide in noise there. Increase --runs on both sides (and regenerate the baseline) for more sensitivity.`, + ), ); } if (regressions.length) { console.error( - `${regressions.length} regression(s): ${regressions - .map((row) => row.task) - .join(", ")}.`, + bold( + red(`${regressions.length} regression(s): ${regressions.join(", ")}.`), + ), ); if (strict) process.exit(1); } else { - console.log("No regressions."); + console.log(green("No regressions.")); } } // two-sided 95% critical values of Student's t, indexed by degrees of freedom -const T95 = [ +const STUDENT_T_CRITICAL_VALUES_95_TWO_SIDED = [ 12.71, 4.3, 3.18, 2.78, 2.57, 2.45, 2.36, 2.31, 2.26, 2.23, 2.2, 2.18, 2.16, 2.14, 2.13, ]; -function t95(df: number): number { - return df <= 0 ? Infinity : df <= T95.length ? T95[df - 1] : 1.96; +function studentTCriticalValue95(degreesOfFreedom: number): number { + return degreesOfFreedom <= 0 + ? Infinity + : degreesOfFreedom <= STUDENT_T_CRITICAL_VALUES_95_TWO_SIDED.length + ? STUDENT_T_CRITICAL_VALUES_95_TWO_SIDED[degreesOfFreedom - 1] + : 1.96; } function average(values: number[]): number { @@ -388,8 +856,8 @@ function aggregateRuns(runs: RunResults[]): RunResults { } // collectResults aborts the process on any task failure, so every run // contains every task - const n = runs.length; - const tasks: Record = {}; + const runCount = runs.length; + const tasks: Record = {}; for (const name of Object.keys(runs[0].tasks)) { const perRun = runs.map((run) => run.tasks[name]); const normalizedValues = perRun.map((task) => task.normalized ?? 0); @@ -399,73 +867,81 @@ function aggregateRuns(runs: RunResults[]): RunResults { (sum, value) => sum + (value - normalizedMean) ** 2, 0, ) / - (n - 1); - const ciHalfWidth = t95(n - 1) * Math.sqrt(variance / n); + (runCount - 1); + const confidenceIntervalHalfWidth = + studentTCriticalValue95(runCount - 1) * Math.sqrt(variance / runCount); tasks[name] = { - hz: average(perRun.map((task) => task.hz)), + operationsPerSecond: average( + perRun.map((task) => task.operationsPerSecond), + ), mean: average(perRun.map((task) => task.mean)), - p99: average(perRun.map((task) => task.p99)), - rme: average(perRun.map((task) => task.rme)), + percentile99: average(perRun.map((task) => task.percentile99)), + relativeMarginOfErrorPercent: average( + perRun.map((task) => task.relativeMarginOfErrorPercent), + ), samples: perRun.reduce((sum, task) => sum + task.samples, 0), normalized: normalizedMean, - normalizedRme: - normalizedMean > 0 ? (ciHalfWidth / normalizedMean) * 100 : 0, + normalizedRelativeMarginOfErrorPercent: + normalizedMean > 0 + ? (confidenceIntervalHalfWidth / normalizedMean) * 100 + : 0, }; } return { - referenceHz: average(runs.map((run) => run.referenceHz ?? 0)), + referenceOperationsPerSecond: average( + runs.map((run) => run.referenceOperationsPerSecond ?? 0), + ), tasks, }; } async function main(): Promise { - const args = parseArgs(process.argv); + const commandLineArguments = parseCommandLineArguments(process.argv); let entries = loadScenarios(); - if (args.filter !== null) { - const filter = args.filter; + if (commandLineArguments.filter !== null) { + const filter = commandLineArguments.filter; entries = entries.filter( (entry) => entry.name === REFERENCE_TASK || entry.name.includes(filter), ); } const allRuns: RunResults[] = []; - for (let run = 0; run < args.runs; run++) { - const bench = new Bench({ time: args.time, warmupTime: 100 }); - for (const { name, fn, opts } of entries) { - bench.add(name, fn, opts); + for (let run = 0; run < commandLineArguments.runs; run++) { + const benchmarkSuite = new Bench({ + time: commandLineArguments.time, + warmupTime: WARMUP_TIME_MILLISECONDS, + }); + for (const { name, fn, options } of entries) { + benchmarkSuite.add(name, fn, options); } - await bench.warmup(); - await bench.run(); - allRuns.push(collectResults(bench)); - if (args.runs > 1) { - console.error(`run ${run + 1}/${args.runs} complete`); + await benchmarkSuite.warmup(); + await benchmarkSuite.run(); + allRuns.push(collectResults(benchmarkSuite)); + if (commandLineArguments.runs > 1) { + console.error(`run ${run + 1}/${commandLineArguments.runs} complete`); } - if (!args.json && run === args.runs - 1) { - console.table(bench.table()); + if (!commandLineArguments.json && run === commandLineArguments.runs - 1) { + console.table(benchmarkSuite.table()); } } const current = aggregateRuns(allRuns); - if (args.runs > 1 && !args.json) { - console.table( - Object.entries(current.tasks).map(([name, task]) => ({ - task: name, - "normalized (mean)": (task.normalized ?? 0).toFixed(4), - "±% (95% CI, between-run)": (task.normalizedRme ?? 0).toFixed(1), - runs: args.runs, - })), - ); + if (!commandLineArguments.json) { + renderThroughputBars(current.tasks); + if (commandLineArguments.runs > 1) { + renderRunSpread(allRuns, current); + } } - if (args.json) { + if (commandLineArguments.json) { console.log( JSON.stringify( { generatedAt: new Date().toISOString(), node: process.version, - runs: args.runs, - referenceHz: current.referenceHz, + runs: commandLineArguments.runs, + referenceOperationsPerSecond: current.referenceOperationsPerSecond, tasks: current.tasks, }, null, @@ -474,26 +950,29 @@ async function main(): Promise { ); } - if (args.updateBaseline) { + if (commandLineArguments.updateBaseline) { const baseline: BaselineFile = { schemaVersion: BASELINE_SCHEMA_VERSION, generatedAt: new Date().toISOString(), node: process.version, - referenceHz: current.referenceHz, + runs: commandLineArguments.runs, + referenceOperationsPerSecond: current.referenceOperationsPerSecond, tasks: {}, }; let noisy = 0; for (const [name, task] of Object.entries(current.tasks)) { baseline.tasks[name] = { - hz: task.hz, + operationsPerSecond: task.operationsPerSecond, mean: task.mean, normalized: task.normalized ?? 0, - normalizedRme: task.normalizedRme ?? 0, + normalizedRelativeMarginOfErrorPercent: + task.normalizedRelativeMarginOfErrorPercent ?? 0, samples: task.samples, }; if ( name !== REFERENCE_TASK && - (task.normalizedRme ?? 0) > args.threshold + (task.normalizedRelativeMarginOfErrorPercent ?? 0) > + commandLineArguments.threshold ) { noisy++; } @@ -504,12 +983,12 @@ async function main(): Promise { ); if (noisy > 0) { console.warn( - `Note: ${noisy} task(s) were captured with a margin of error above ${args.threshold}% — comparisons against them will have a high noise floor. Consider regenerating with more --runs.`, + `Note: ${noisy} task(s) were captured with a margin of error above ${commandLineArguments.threshold}% — comparisons against them will have a high noise floor. Consider regenerating with more --runs.`, ); } } - if (args.compare) { + if (commandLineArguments.compare) { if (!fs.existsSync(BASELINE_PATH)) { console.error( "No baseline.json found — run `npm run benchmark:update` first.", @@ -520,9 +999,10 @@ async function main(): Promise { fs.readFileSync(BASELINE_PATH, "utf8"), ); compare(baseline, current, { - strict: args.strict, - threshold: args.threshold, - filter: args.filter, + strict: commandLineArguments.strict, + threshold: commandLineArguments.threshold, + filter: commandLineArguments.filter, + runs: commandLineArguments.runs, }); } } diff --git a/benchmark/scenarios/batch-get-format.bench.ts b/benchmark/scenarios/batch-get-format.benchmark.ts similarity index 100% rename from benchmark/scenarios/batch-get-format.bench.ts rename to benchmark/scenarios/batch-get-format.benchmark.ts diff --git a/benchmark/scenarios/entity-construction.bench.ts b/benchmark/scenarios/entity-construction.benchmark.ts similarity index 100% rename from benchmark/scenarios/entity-construction.bench.ts rename to benchmark/scenarios/entity-construction.benchmark.ts diff --git a/benchmark/scenarios/params-chain.bench.ts b/benchmark/scenarios/params-chain.benchmark.ts similarity index 89% rename from benchmark/scenarios/params-chain.bench.ts rename to benchmark/scenarios/params-chain.benchmark.ts index 48901840..f56db2d5 100644 --- a/benchmark/scenarios/params-chain.bench.ts +++ b/benchmark/scenarios/params-chain.benchmark.ts @@ -22,7 +22,9 @@ const scenarios: ScenarioEntry[] = [ fn: () => entity.query .records({ org: "org1" }) - .where((attr: any, op: any) => op.gt(attr.count, 10)) + .where((attributes: any, operations: any) => + operations.gt(attributes.count, 10), + ) .params(), }, { diff --git a/benchmark/scenarios/parse-format.bench.ts b/benchmark/scenarios/parse-format.benchmark.ts similarity index 100% rename from benchmark/scenarios/parse-format.bench.ts rename to benchmark/scenarios/parse-format.benchmark.ts diff --git a/benchmark/scenarios/query-pagination.bench.ts b/benchmark/scenarios/query-pagination.benchmark.ts similarity index 100% rename from benchmark/scenarios/query-pagination.bench.ts rename to benchmark/scenarios/query-pagination.benchmark.ts diff --git a/test/fixtures/mock-client.ts b/test/fixtures/mock-client.ts index 4fdbdba8..cc0d6663 100644 --- a/test/fixtures/mock-client.ts +++ b/test/fixtures/mock-client.ts @@ -1,7 +1,7 @@ /** * Shared offline client mocks. These fixtures are used by both the offline * test suites (test/offline.*.spec.*) and the benchmark scenarios - * (benchmark/scenarios/*.bench.ts); they must stay dependency-free and + * (benchmark/scenarios/*.benchmark.ts); they must stay dependency-free and * synchronous so neither consumer needs network or DynamoDB access. */ @@ -57,8 +57,8 @@ export function makeMockV2Client( if (transactMethods.has(method)) { const stored: Record void> = {}; return { - on: (event: string, cb: (input: any) => void) => { - stored[event] = cb; + on: (event: string, callback: (input: any) => void) => { + stored[event] = callback; }, abort: () => {}, promise: () => { @@ -124,9 +124,11 @@ export function makePagingQueryHandler({ } return (params: Record) => { let start = 0; - const esk = params.ExclusiveStartKey; - if (esk !== undefined) { - const index = indexBySortKey.get(`${esk.pk}|${esk.sk}`); + const exclusiveStartKey = params.ExclusiveStartKey; + if (exclusiveStartKey !== undefined) { + const index = indexBySortKey.get( + `${exclusiveStartKey.pk}|${exclusiveStartKey.sk}`, + ); if (index === undefined) { throw new Error("Unknown ExclusiveStartKey provided to paging mock"); } diff --git a/test/offline.perf-fixes.spec.ts b/test/offline.perf-fixes.spec.ts index 701c710b..f4dc45e6 100644 --- a/test/offline.perf-fixes.spec.ts +++ b/test/offline.perf-fixes.spec.ts @@ -28,7 +28,7 @@ const { Service } = require("../src/service"); // --------------------------------------------------------------------------- describe("P5: model validation short-circuits", () => { const validations = require("../src/validations"); - const u = require("../src/util"); + const util = require("../src/util"); const { ElectroInstance, ElectroInstanceTypes } = require("../src/types"); // captured verbatim from the pre-fix validateModel implementation @@ -114,23 +114,23 @@ describe("P5: model validation short-circuits", () => { it("resolves instance types via symbols and bare models via validation", () => { const entity = makeFixtureEntity(); - expect(u.getInstanceType(entity)).to.equal(ElectroInstanceTypes.entity); - expect(u.getInstanceType({ _instance: ElectroInstance.service })).to.equal( - ElectroInstanceTypes.service, - ); - expect(u.getInstanceType({ _instance: ElectroInstance.electro })).to.equal( - ElectroInstanceTypes.electro, - ); + expect(util.getInstanceType(entity)).to.equal(ElectroInstanceTypes.entity); + expect( + util.getInstanceType({ _instance: ElectroInstance.service }), + ).to.equal(ElectroInstanceTypes.service); + expect( + util.getInstanceType({ _instance: ElectroInstance.electro }), + ).to.equal(ElectroInstanceTypes.electro); expect( - u.getInstanceType({ + util.getInstanceType({ model: { entity: "e", service: "s", version: "1" }, attributes: { id: { type: "string" } }, indexes: { main: { pk: { field: "pk", composite: ["id"] } } }, }), ).to.equal(ElectroInstanceTypes.model); - expect(u.getInstanceType({})).to.equal(""); - expect(u.getInstanceType(undefined)).to.equal(""); - expect(u.getInstanceType({ anything: "else" })).to.equal(""); + expect(util.getInstanceType({})).to.equal(""); + expect(util.getInstanceType(undefined)).to.equal(""); + expect(util.getInstanceType({ anything: "else" })).to.equal(""); }); }); @@ -363,7 +363,9 @@ describe("P4: attribute mutation passes", () => { const entity = makeFixtureEntity(); const params = entity .update({ org: "org1", id: "id1" }) - .data((attr: any, op: any) => op.set(attr.notes[0].body, "edited")) + .data((attributes: any, operations: any) => + operations.set(attributes.notes[0].body, "edited"), + ) .params(); expect(params.UpdateExpression).to.equal( "SET #notes[0].#body = :body_u0, #org = :org_u0, #id = :id_u0, #__edb_e__ = :__edb_e___u0, #__edb_v__ = :__edb_v___u0", @@ -509,7 +511,9 @@ describe("P3: lazy update machinery on chain construction", () => { const query = entity.query .records({ org: "org1" }) - .where((attr: any, op: any) => op.gt(attr.count, 10)) + .where((attributes: any, operations: any) => + operations.gt(attributes.count, 10), + ) .params(); expect(query).to.deep.equal({ KeyConditionExpression: "#pk = :pk and begins_with(#sk1, :sk1)", From 682eb64a45ffa1ec03e1276ea55494d84eebdf54 Mon Sep 17 00:00:00 2001 From: ty Date: Wed, 10 Jun 2026 15:36:16 -0400 Subject: [PATCH 15/17] Clean up --- benchmark/run.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/run.ts b/benchmark/run.ts index 742e8450..87f95f4b 100644 --- a/benchmark/run.ts +++ b/benchmark/run.ts @@ -287,7 +287,7 @@ const VerdictDisplayText: Record = { // --------------------------------------------------------------------------- const useColor = - process.stdout.isTTY === true && process.env.NO_COLOR === undefined; + process.stdout.isTTY && process.env.NO_COLOR === undefined; function paint(code: number): (text: string) => string { return (text) => (useColor ? `\u001b[${code}m${text}\u001b[0m` : text); From ef2f3005ae1bbaf5492a230631a168385c0c8efb Mon Sep 17 00:00:00 2001 From: Ty Walch Date: Sat, 13 Jun 2026 09:36:04 -0400 Subject: [PATCH 16/17] Clean up --- benchmark/baseline.json | 107 ------------------ .../scenarios/batch-get-format.benchmark.ts | 3 - .../entity-construction.benchmark.ts | 3 - benchmark/scenarios/params-chain.benchmark.ts | 4 - benchmark/scenarios/parse-format.benchmark.ts | 4 - .../scenarios/query-pagination.benchmark.ts | 4 - 6 files changed, 125 deletions(-) delete mode 100644 benchmark/baseline.json diff --git a/benchmark/baseline.json b/benchmark/baseline.json deleted file mode 100644 index 9e86d990..00000000 --- a/benchmark/baseline.json +++ /dev/null @@ -1,107 +0,0 @@ -{ - "schemaVersion": 3, - "generatedAt": "2026-06-10T18:25:03.362Z", - "node": "v24.3.0", - "runs": 10, - "referenceOperationsPerSecond": 2433.0243439650476, - "tasks": { - "reference/parse-500-items": { - "operationsPerSecond": 2433.0243439650476, - "mean": 0.4111418215058819, - "normalized": 1, - "normalizedRelativeMarginOfErrorPercent": 0, - "samples": 12171 - }, - "batch-get-format/100-items": { - "operationsPerSecond": 3858.3806990022845, - "mean": 0.2592887852712155, - "normalized": 1.585890381497777, - "normalizedRelativeMarginOfErrorPercent": 1.013746647150447, - "samples": 19320 - }, - "entity-construction/new-entity": { - "operationsPerSecond": 6254.40892557932, - "mean": 0.15991138163073665, - "normalized": 2.5710560328456022, - "normalizedRelativeMarginOfErrorPercent": 0.9698110978836614, - "samples": 31277 - }, - "entity-construction/new-entity-watchers": { - "operationsPerSecond": 5711.937004785308, - "mean": 0.17507553094332612, - "normalized": 2.348459472810711, - "normalizedRelativeMarginOfErrorPercent": 1.4770300388933606, - "samples": 28565 - }, - "params-chain/get": { - "operationsPerSecond": 387973.2106890793, - "mean": 0.002577654861021239, - "normalized": 159.52233021131025, - "normalizedRelativeMarginOfErrorPercent": 1.7151408896518312, - "samples": 1939870 - }, - "params-chain/query": { - "operationsPerSecond": 259533.33275631623, - "mean": 0.003855515012352204, - "normalized": 106.72685226798549, - "normalizedRelativeMarginOfErrorPercent": 2.7938466279878544, - "samples": 1297941 - }, - "params-chain/query-where": { - "operationsPerSecond": 112780.48112728125, - "mean": 0.008868369193681808, - "normalized": 46.37406347031307, - "normalizedRelativeMarginOfErrorPercent": 2.0433268693388253, - "samples": 564575 - }, - "params-chain/update-set": { - "operationsPerSecond": 90675.1380517179, - "mean": 0.01103189367465487, - "normalized": 37.283339936275425, - "normalizedRelativeMarginOfErrorPercent": 2.1245831960796413, - "samples": 453519 - }, - "params-chain/upsert": { - "operationsPerSecond": 78631.54163923618, - "mean": 0.01272494054815596, - "normalized": 32.33392307722848, - "normalizedRelativeMarginOfErrorPercent": 2.6188127725487638, - "samples": 393253 - }, - "parse-format/100-items": { - "operationsPerSecond": 12200.533019125965, - "mean": 0.08200320403389624, - "normalized": 5.017144207152649, - "normalizedRelativeMarginOfErrorPercent": 2.5964103316993548, - "samples": 61010 - }, - "parse-format/1000-items": { - "operationsPerSecond": 1226.9213915426685, - "mean": 0.8154303062335536, - "normalized": 0.50445033377815, - "normalizedRelativeMarginOfErrorPercent": 2.1773722652963046, - "samples": 6141 - }, - "parse-format/1000-items-watchers": { - "operationsPerSecond": 836.0174721181766, - "mean": 1.1979540983114136, - "normalized": 0.34373482855789816, - "normalizedRelativeMarginOfErrorPercent": 3.236479099054849, - "samples": 4184 - }, - "query-pagination/10-pages-x100": { - "operationsPerSecond": 1516.132762922901, - "mean": 0.6604050249489395, - "normalized": 0.6234040932196916, - "normalizedRelativeMarginOfErrorPercent": 3.1251304614877977, - "samples": 7587 - }, - "query-pagination/100-pages-x100": { - "operationsPerSecond": 136.18571131973573, - "mean": 7.350136782834127, - "normalized": 0.056003780045826124, - "normalizedRelativeMarginOfErrorPercent": 3.1941200045888243, - "samples": 687 - } - } -} diff --git a/benchmark/scenarios/batch-get-format.benchmark.ts b/benchmark/scenarios/batch-get-format.benchmark.ts index 0213e9cf..2e5cbb5a 100644 --- a/benchmark/scenarios/batch-get-format.benchmark.ts +++ b/benchmark/scenarios/batch-get-format.benchmark.ts @@ -1,6 +1,3 @@ -/** - * P2 — batchGet response formatting calls formatResponse once per item. - */ import type { ScenarioEntry } from "../run"; import { makeMockV2Client, StoredItem } from "../../test/fixtures/mock-client"; import { diff --git a/benchmark/scenarios/entity-construction.benchmark.ts b/benchmark/scenarios/entity-construction.benchmark.ts index 98cefd9f..2705fb33 100644 --- a/benchmark/scenarios/entity-construction.benchmark.ts +++ b/benchmark/scenarios/entity-construction.benchmark.ts @@ -1,6 +1,3 @@ -/** - * P5 — Entity construction cost (model validation dominates). - */ import type { ScenarioEntry } from "../run"; import { table, makeFixtureModel } from "../../test/fixtures/entities"; diff --git a/benchmark/scenarios/params-chain.benchmark.ts b/benchmark/scenarios/params-chain.benchmark.ts index f56db2d5..ed32311c 100644 --- a/benchmark/scenarios/params-chain.benchmark.ts +++ b/benchmark/scenarios/params-chain.benchmark.ts @@ -1,7 +1,3 @@ -/** - * P3 — chain construction + params building. Read chains (get/query/scan) - * should never build update machinery; update/upsert chains pay for it once. - */ import type { ScenarioEntry } from "../run"; import { makeFixtureEntity } from "../../test/fixtures/entities"; diff --git a/benchmark/scenarios/parse-format.benchmark.ts b/benchmark/scenarios/parse-format.benchmark.ts index 92899a16..d298206b 100644 --- a/benchmark/scenarios/parse-format.benchmark.ts +++ b/benchmark/scenarios/parse-format.benchmark.ts @@ -1,7 +1,3 @@ -/** - * P2 + P4 — response formatting: per-item getter passes, sibling snapshots, - * path lookups, and (formerly) per-call error allocation. - */ import type { ScenarioEntry } from "../run"; import type { StoredItem } from "../../test/fixtures/mock-client"; import { diff --git a/benchmark/scenarios/query-pagination.benchmark.ts b/benchmark/scenarios/query-pagination.benchmark.ts index 79a8fb39..60dd913a 100644 --- a/benchmark/scenarios/query-pagination.benchmark.ts +++ b/benchmark/scenarios/query-pagination.benchmark.ts @@ -1,7 +1,3 @@ -/** - * P1 — auto-paging accumulation across many pages (`pages: "all"`). The - * O(pages²) → O(pages) change shows up most at high page counts. - */ import type { ScenarioEntry } from "../run"; import { makeMockV2Client, From 60abea3c5faee5b1805179eba0086aa93eaa7b8a Mon Sep 17 00:00:00 2001 From: Ty Walch Date: Sat, 13 Jun 2026 09:38:17 -0400 Subject: [PATCH 17/17] ignore baseline.json --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 48e0444d..48e0b513 100644 --- a/.gitignore +++ b/.gitignore @@ -109,4 +109,6 @@ dist .tern-port playground/bundle.js -test/debug.ts \ No newline at end of file +test/debug.ts + +benchmark/baseline.json \ No newline at end of file