diff --git a/Makefile b/Makefile index 6ccc317..a80d1f9 100644 --- a/Makefile +++ b/Makefile @@ -47,8 +47,36 @@ RUN_LOCAL := bash scripts/with-local-server.sh artillery-local: @echo "Load-testing $(EXAMPLE) on :8080..." @$(RUN_LOCAL) $(EXAMPLE) 8080 \ - env TEST_ENVIRONMENT=local npx artillery run --target http://localhost:8080 \ - --config $(TEST_KIT)/configs/load-test.yml $(TEST_KIT)/scenarios/core/load-test.yml + npx artillery run --target http://localhost:8080 \ + --config $(TEST_KIT)/configs/load-test.yml \ + -o artillery-report-$(basename $(notdir $(EXAMPLE))).json \ + $(TEST_KIT)/scenarios/core/load-test.yml + +# Opt-in A/B load comparison of two framework examples, diffed by the kit's +# perf-compare over Artillery's own `-o` reports (test-kit >= 0.4.0). Manual +# only — deliberately NOT in `all`/`functional-tests`: two full load runs are +# ~2 GB of S3 egress. +# make perf-compare BASELINE=examples/express-basic.ts CANDIDATE=examples/hono-basic.ts +BASELINE ?= examples/express-basic.ts +CANDIDATE ?= examples/hono-basic.ts + +.PHONY: perf-compare +perf-compare: + @echo "Baseline: $(BASELINE) on :8080" + @$(RUN_LOCAL) $(BASELINE) 8080 \ + npx artillery run --target http://localhost:8080 \ + --config $(TEST_KIT)/configs/load-test.yml \ + -o artillery-report-baseline.json \ + $(TEST_KIT)/scenarios/core/load-test.yml + @echo "Candidate: $(CANDIDATE) on :8081" + @$(RUN_LOCAL) $(CANDIDATE) 8081 \ + npx artillery run --target http://localhost:8081 \ + --config $(TEST_KIT)/configs/load-test.yml \ + -o artillery-report-candidate.json \ + $(TEST_KIT)/scenarios/core/load-test.yml + @npx s3-website-perf-compare compare \ + --baseline artillery-report-baseline.json --baseline-label $(basename $(notdir $(BASELINE))) \ + --candidate artillery-report-candidate.json --candidate-label $(basename $(notdir $(CANDIDATE))) # Conformance gate: assert the HTTP contract (status, content-type, # content-length) with the test kit's `expect`-enabled config. Unlike the load diff --git a/docs/performance.md b/docs/performance.md index a6997ae..f6d61fa 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -175,10 +175,17 @@ Run the kit against a `tsx` example server on your local `src/` (no build, no Docker image): ```bash -make artillery-local # load test +make artillery-local # load test (writes artillery-report-.json) make conformance-local # HTTP-contract gate ``` +A/B load comparison between two framework examples (opt-in; two load runs are +~2 GB of S3 egress, so it is not part of `make all`): + +```bash +make perf-compare BASELINE=examples/express-basic.ts CANDIDATE=examples/hono-basic.ts +``` + ### Container image testing The published container image is built and load/conformance-tested in diff --git a/package-lock.json b/package-lock.json index 4955b04..a69f4e2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "devDependencies": { "@aws-sdk/client-s3": "^3.832.0", "@biomejs/biome": "^2.0.4", - "@forkzero/s3-website-test-kit": "^0.2.1", + "@forkzero/s3-website-test-kit": "^0.4.0", "@hono/node-server": "^2.0.0", "@types/debug": "^4.1.12", "@types/express": "^5.0.3", @@ -2021,9 +2021,9 @@ } }, "node_modules/@forkzero/s3-website-test-kit": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/@forkzero/s3-website-test-kit/-/s3-website-test-kit-0.2.1.tgz", - "integrity": "sha512-/VDfJoomJxDX0FAJbNt0CzoYt1WNc536p2Ia4m0wC4b0i+BpH+2AujYdVSZMpT14+R12VnCmR23KckReg135CA==", + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@forkzero/s3-website-test-kit/-/s3-website-test-kit-0.4.0.tgz", + "integrity": "sha512-UziHuWnDFfniSl7+TFz6XxLG78TEma39kT3KLhs6BnOx47LC6Vy+O5HvrOkhvf9aRtFMkpqaNKRE6kkjNE5ulg==", "dev": true, "license": "Apache-2.0", "bin": { diff --git a/package.json b/package.json index e49fdb3..181cac1 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "devDependencies": { "@aws-sdk/client-s3": "^3.832.0", "@biomejs/biome": "^2.0.4", - "@forkzero/s3-website-test-kit": "^0.2.1", + "@forkzero/s3-website-test-kit": "^0.4.0", "@hono/node-server": "^2.0.0", "@types/debug": "^4.1.12", "@types/express": "^5.0.3",