From 686b35d9eb9b88da2e7a32669f7b682e7357a640 Mon Sep 17 00:00:00 2001 From: George Moon Date: Sat, 31 Oct 2026 18:38:22 -0400 Subject: [PATCH] chore(deps): adopt @forkzero/s3-website-test-kit 0.4.0 (closes #201) Kit 0.4.0 drops the in-package Artillery processor (which accumulated per-worker partial metrics, kit #9) in favor of Artillery's own aggregated `-o` reports; s3-website-perf-compare now diffs two of those. - Bump devDependency ^0.2.1 -> ^0.4.0. - artillery-local: capture Artillery's own report (-o artillery-report- .json) and drop the now-dead `env TEST_ENVIRONMENT=local`. - Add opt-in `perf-compare` target: A/B load two framework examples (BASELINE vs CANDIDATE) and diff via `s3-website-perf-compare compare`. Not in all/functional-tests (two load runs ~2 GB S3 egress). - Document perf-compare in docs/performance.md. Conformance config is unchanged in 0.4.0; `make conformance-local` (the CI gate) verified green against the live bucket with the new kit. Co-Authored-By: Claude Opus 4.8 (1M context) --- Makefile | 32 ++++++++++++++++++++++++++++++-- docs/performance.md | 9 ++++++++- package-lock.json | 8 ++++---- package.json | 2 +- 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 6ccc317..a80d1f9 100644 --- a/Makefile +++ b/Makefile @@ -47,8 +47,36 @@ RUN_LOCAL := bash scripts/with-local-server.sh artillery-local: @echo "Load-testing $(EXAMPLE) on :8080..." @$(RUN_LOCAL) $(EXAMPLE) 8080 \ - env TEST_ENVIRONMENT=local npx artillery run --target http://localhost:8080 \ - --config $(TEST_KIT)/configs/load-test.yml $(TEST_KIT)/scenarios/core/load-test.yml + npx artillery run --target http://localhost:8080 \ + --config $(TEST_KIT)/configs/load-test.yml \ + -o artillery-report-$(basename $(notdir $(EXAMPLE))).json \ + $(TEST_KIT)/scenarios/core/load-test.yml + +# Opt-in A/B load comparison of two framework examples, diffed by the kit's +# perf-compare over Artillery's own `-o` reports (test-kit >= 0.4.0). Manual +# only — deliberately NOT in `all`/`functional-tests`: two full load runs are +# ~2 GB of S3 egress. +# make perf-compare BASELINE=examples/express-basic.ts CANDIDATE=examples/hono-basic.ts +BASELINE ?= examples/express-basic.ts +CANDIDATE ?= examples/hono-basic.ts + +.PHONY: perf-compare +perf-compare: + @echo "Baseline: $(BASELINE) on :8080" + @$(RUN_LOCAL) $(BASELINE) 8080 \ + npx artillery run --target http://localhost:8080 \ + --config $(TEST_KIT)/configs/load-test.yml \ + -o artillery-report-baseline.json \ + $(TEST_KIT)/scenarios/core/load-test.yml + @echo "Candidate: $(CANDIDATE) on :8081" + @$(RUN_LOCAL) $(CANDIDATE) 8081 \ + npx artillery run --target http://localhost:8081 \ + --config $(TEST_KIT)/configs/load-test.yml \ + -o artillery-report-candidate.json \ + $(TEST_KIT)/scenarios/core/load-test.yml + @npx s3-website-perf-compare compare \ + --baseline artillery-report-baseline.json --baseline-label $(basename $(notdir $(BASELINE))) \ + --candidate artillery-report-candidate.json --candidate-label $(basename $(notdir $(CANDIDATE))) # Conformance gate: assert the HTTP contract (status, content-type, # content-length) with the test kit's `expect`-enabled config. Unlike the load diff --git a/docs/performance.md b/docs/performance.md index a6997ae..f6d61fa 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -175,10 +175,17 @@ Run the kit against a `tsx` example server on your local `src/` (no build, no Docker image): ```bash -make artillery-local # load test +make artillery-local # load test (writes artillery-report-.json) make conformance-local # HTTP-contract gate ``` +A/B load comparison between two framework examples (opt-in; two load runs are +~2 GB of S3 egress, so it is not part of `make all`): + +```bash +make perf-compare BASELINE=examples/express-basic.ts CANDIDATE=examples/hono-basic.ts +``` + ### Container image testing The published container image is built and load/conformance-tested in diff --git a/package-lock.json b/package-lock.json index 4955b04..a69f4e2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "devDependencies": { "@aws-sdk/client-s3": "^3.832.0", "@biomejs/biome": "^2.0.4", - "@forkzero/s3-website-test-kit": "^0.2.1", + "@forkzero/s3-website-test-kit": "^0.4.0", "@hono/node-server": "^2.0.0", "@types/debug": "^4.1.12", "@types/express": "^5.0.3", @@ -2021,9 +2021,9 @@ } }, "node_modules/@forkzero/s3-website-test-kit": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/@forkzero/s3-website-test-kit/-/s3-website-test-kit-0.2.1.tgz", - "integrity": "sha512-/VDfJoomJxDX0FAJbNt0CzoYt1WNc536p2Ia4m0wC4b0i+BpH+2AujYdVSZMpT14+R12VnCmR23KckReg135CA==", + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@forkzero/s3-website-test-kit/-/s3-website-test-kit-0.4.0.tgz", + "integrity": "sha512-UziHuWnDFfniSl7+TFz6XxLG78TEma39kT3KLhs6BnOx47LC6Vy+O5HvrOkhvf9aRtFMkpqaNKRE6kkjNE5ulg==", "dev": true, "license": "Apache-2.0", "bin": { diff --git a/package.json b/package.json index e49fdb3..181cac1 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "devDependencies": { "@aws-sdk/client-s3": "^3.832.0", "@biomejs/biome": "^2.0.4", - "@forkzero/s3-website-test-kit": "^0.2.1", + "@forkzero/s3-website-test-kit": "^0.4.0", "@hono/node-server": "^2.0.0", "@types/debug": "^4.1.12", "@types/express": "^5.0.3",