From f338269677f38f237378b39803314a12815fa62b Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Fri, 19 Jun 2026 11:05:28 +1000 Subject: [PATCH 1/3] chore: wire native pytest-benchmark regression gating; fix broken benchmark targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `make benchmark`/`make benchmark-full` pointed at a `benchmarks.cli` package that does not exist in this repo (a leftover from the pyredis-cache-pro prototype), so both failed on invocation. tests/benchmarks/ was also orphaned: its files (benchmark_*.py / Benchmark* classes) never matched default pytest discovery, so the only pytest-benchmark suite in the repo ran nowhere. - Makefile: `make benchmark` runs tests/benchmarks/ with --benchmark-autosave; new `make benchmark-compare` fails on >10% median regression vs the last saved baseline. Median over mean for outlier-robustness; 10% sits above the ~6% run-to-run median noise measured on this suite. Native --benchmark-save/--benchmark-compare-fail โ€” no custom stats code. - tests/benchmarks/conftest.py: no-op override of the root autouse Redis-isolation fixture (serializer benchmarks need no backend), mirroring tests/unit/conftest.py. - Baselines live in .benchmarks/ (already gitignored, per-machine), so regression comparison is a local developer tool; wall-clock benchmarks deliberately do not gate CI. - Fix doc drift still pointing at the removed benchmarks.cli: CONTRIBUTING.md, docs/getting-started.md, tests/performance/README.md. --- CONTRIBUTING.md | 7 ++++++- Makefile | 37 +++++++++++++++++++++++++----------- docs/getting-started.md | 4 ++-- tests/benchmarks/conftest.py | 20 +++++++++++++++++++ tests/performance/README.md | 2 +- 5 files changed, 55 insertions(+), 15 deletions(-) create mode 100644 tests/benchmarks/conftest.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f7ec90c..bbd41fb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -246,9 +246,14 @@ make build-pgo # Profile-Guided Optimization (5-8% faster) ### Running Benchmarks ```bash -make benchmark-quick # Quick performance check +make benchmark # Run serializer benchmarks + save a local baseline +make benchmark-compare # Re-run and fail on >10% median regression vs that baseline ``` +Baselines are written to `.benchmarks/` (gitignored, per-machine), so regression +comparison is a local developer tool โ€” wall-clock benchmarks deliberately do not +gate CI. + ### Formatting Code ```bash diff --git a/Makefile b/Makefile index 6c43f1f..bc04a03 100644 --- a/Makefile +++ b/Makefile @@ -353,17 +353,32 @@ rust-bench: ## Run Rust benchmarks # ๐Ÿ“Š BENCHMARKS # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• -benchmark: setup-logs ## Run quick benchmarks - @echo "$(BLUE)Running quick benchmarks...$(RESET)" - @echo "$(YELLOW)Logging to $(LOG_BENCHMARK_DIR)/quick_$(TIMESTAMP).log$(RESET)" - @uv run python -m benchmarks.cli quick 2>&1 | tee $(LOG_BENCHMARK_DIR)/quick_$(TIMESTAMP).log - @echo "$(GREEN)โœ“ Benchmarks completed$(RESET)" - -benchmark-full: setup-logs ## Run comprehensive benchmarks - @echo "$(BLUE)Running comprehensive benchmarks...$(RESET)" - @echo "$(YELLOW)Logging to $(LOG_BENCHMARK_DIR)/full_$(TIMESTAMP).log$(RESET)" - @uv run python -m benchmarks.cli performance --comprehensive 2>&1 | tee $(LOG_BENCHMARK_DIR)/full_$(TIMESTAMP).log - @echo "$(GREEN)โœ“ Comprehensive benchmarks completed$(RESET)" +# Serializer micro-benchmarks live in tests/benchmarks/ under benchmark_*.py / +# Benchmark* names, deliberately kept OUT of the default `make test` discovery +# (wall-clock numbers are noisy and must not gate unit CI). The -o overrides +# make them collectable only here; addopts is reset to drop the +# --doctest-modules/--markdown-docs/--verbose noise. pytest-benchmark writes +# baselines to .benchmarks/ (gitignored, per-machine). +BENCHMARK_PYTEST := uv run pytest tests/benchmarks/ \ + -o addopts="" \ + -o python_files="benchmark_*.py" \ + -o python_classes="Benchmark*" \ + --benchmark-only --benchmark-disable-gc + +benchmark: setup-logs ## Run serializer benchmarks + save a local baseline + @echo "$(BLUE)Running serializer benchmarks (saving baseline)...$(RESET)" + @echo "$(YELLOW)Logging to $(LOG_BENCHMARK_DIR)/bench_$(TIMESTAMP).log$(RESET)" + @$(BENCHMARK_PYTEST) --benchmark-autosave 2>&1 | tee $(LOG_BENCHMARK_DIR)/bench_$(TIMESTAMP).log + @echo "$(GREEN)โœ“ Saved to .benchmarks/ (gitignored, per-machine)$(RESET)" + +benchmark-compare: setup-logs ## Run benchmarks, fail on >10% median regression vs last saved baseline + @echo "$(BLUE)Comparing against last saved baseline (run 'make benchmark' first)...$(RESET)" + @if ! $(BENCHMARK_PYTEST) --benchmark-compare --benchmark-compare-fail=median:10% 2>&1 | tee $(LOG_BENCHMARK_DIR)/bench_compare_$(TIMESTAMP).log; then \ + echo "$(YELLOW)โŒ benchmark-compare failed: median regression >10% vs baseline, or no baseline yet (run 'make benchmark' first).$(RESET)"; \ + echo "$(YELLOW) Threshold is tunable โ€” set it above your machine's run-to-run median noise (~6% here).$(RESET)"; \ + exit 1; \ + fi + @echo "$(GREEN)โœ“ No median regression beyond 10% vs baseline$(RESET)" # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• # ๐Ÿ”ง BUILD & RELEASE diff --git a/docs/getting-started.md b/docs/getting-started.md index b5ce0bb..7bcd495 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -469,8 +469,8 @@ print("Caching working correctly!") # Run the built-in performance tests pytest tests/performance/ -v -# Or run benchmarks -uv run python -m benchmarks.cli quick --rust-only +# Or run the serializer benchmarks (saves a local baseline) +make benchmark ``` --- diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py new file mode 100644 index 0000000..60a7e53 --- /dev/null +++ b/tests/benchmarks/conftest.py @@ -0,0 +1,20 @@ +"""Benchmark suite configuration - no Redis required. + +These benchmarks instantiate serializers directly and never touch a backend, +so the root conftest's autouse Redis-isolation fixture (which would spawn a +local redis-server binary when REDIS_URL is unset) is overridden to a no-op +here, the same way tests/unit/conftest.py does for unit tests. +""" + +import pytest + + +@pytest.fixture(autouse=True) +def setup_di_for_redis_isolation(): + """Override root conftest's Redis isolation - benchmarks need no backend. + + Shadows the parent autouse fixture by having the same name (nearest + conftest wins), so serializer benchmarks run without Redis. + """ + # No-op for benchmarks - just yield without Redis setup + yield diff --git a/tests/performance/README.md b/tests/performance/README.md index 8e2b1b9..7133228 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -93,7 +93,7 @@ Provides rigorous performance measurement tools: ### Quick Check (Basic Tests) ```bash -make test-performance-quick +uv run pytest tests/performance/ -m "performance and not slow" -q ``` ### Full Suite (All Tests) From b15640012e71f32526bc13f89a3cd8feb0f65d7d Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Fri, 19 Jun 2026 13:08:35 +1000 Subject: [PATCH 2/3] chore: annotate benchmark conftest fixture return type Add `-> Iterator[None]` to the setup_di_for_redis_isolation override, matching the existing precedent for the same fixture in tests/unit/test_wrapper_lock_bare_key.py. Addresses a PR #188 review note. --- tests/benchmarks/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py index 60a7e53..4d4584d 100644 --- a/tests/benchmarks/conftest.py +++ b/tests/benchmarks/conftest.py @@ -6,11 +6,13 @@ here, the same way tests/unit/conftest.py does for unit tests. """ +from collections.abc import Iterator + import pytest @pytest.fixture(autouse=True) -def setup_di_for_redis_isolation(): +def setup_di_for_redis_isolation() -> Iterator[None]: """Override root conftest's Redis isolation - benchmarks need no backend. Shadows the parent autouse fixture by having the same name (nearest From 3bd321bb2cea21dac9d7705ac998efed2d5da5b5 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Fri, 19 Jun 2026 22:06:56 +1000 Subject: [PATCH 3/3] =?UTF-8?q?chore:=20benchmark=20robustness=20=E2=80=94?= =?UTF-8?q?=20measurement=20integrity,=20GIL=20scaling,=20unified=20runner?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds on the benchmark harness (#188) with the robust-systems layer the pyredis-cache-pro prototype had but the rewrite lacked, and consolidates the two perf folders into one. Folder consolidation: move tests/benchmarks/benchmark_serializer_integrity.py into tests/performance/test_serializer_microbench.py (Test* classes); delete tests/benchmarks/. Guard-vs-tracker is now separated by mechanism, not folder โ€” pytest-benchmark tests are selected with --benchmark-only and skipped in normal runs via the --benchmark-skip default (pyproject addopts), dropping the prior -o python_files/python_classes discovery hack. A file-scoped Redis no-op override lives in the microbench file (tests/performance/ has Redis-dependent tests). Drops the orphaned Blake3OverheadAnalysis print-only methods. Measurement integrity (measurement_env.py + conftest.py): stable system fingerprint (hash over deterministic fields only), environment pre-flight check (thermal-under- load / CPU / memory / loadavg), and a timer self-calibration gate (test_measurement_calibration.py), surfaced as a warn-only session banner. Reuses stats_utils; supersedes the duplicate fingerprint fixture in test_statistical_rigor.py. GIL scaling (gil_benchmark.py): thread-scaling of StandardSerializer.serialize under the current interpreter (speedup + efficiency%). The no-GIL arm is interpreter-driven and currently ecosystem-blocked (PyO3<3.14; orjson/numpy/pandas/pyarrow lack free- threaded wheels) โ€” no code change needed once a free-threaded cachekit installs. Unified runner (Makefile): `make perf` runs the battery (env+calibration โ†’ serializer benchmarks โ†’ GIL scaling); `make perf-compare` gates on >10% median regression. Docs updated (CONTRIBUTING.md, tests/performance/README.md). --- CONTRIBUTING.md | 13 +- Makefile | 30 ++- pyproject.toml | 1 + tests/benchmarks/conftest.py | 22 --- tests/performance/README.md | 13 ++ tests/performance/conftest.py | 42 ++++ tests/performance/gil_benchmark.py | 88 +++++++++ tests/performance/measurement_env.py | 143 ++++++++++++++ .../test_measurement_calibration.py | 24 +++ .../test_serializer_microbench.py} | 180 ++---------------- tests/performance/test_statistical_rigor.py | 19 -- 11 files changed, 363 insertions(+), 212 deletions(-) delete mode 100644 tests/benchmarks/conftest.py create mode 100644 tests/performance/conftest.py create mode 100644 tests/performance/gil_benchmark.py create mode 100644 tests/performance/measurement_env.py create mode 100644 tests/performance/test_measurement_calibration.py rename tests/{benchmarks/benchmark_serializer_integrity.py => performance/test_serializer_microbench.py} (56%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bbd41fb..0db693d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -246,13 +246,20 @@ make build-pgo # Profile-Guided Optimization (5-8% faster) ### Running Benchmarks ```bash -make benchmark # Run serializer benchmarks + save a local baseline +make perf # Full battery: env fingerprint + timer calibration, serializer benchmarks, GIL scaling +make perf-compare # Regression gate: fail on >10% median serializer regression vs baseline + +make benchmark # Serializer benchmarks only + save a local baseline make benchmark-compare # Re-run and fail on >10% median regression vs that baseline +make benchmark-gil # Serializer thread-scaling under the current interpreter (GIL) ``` +All perf tests live in one folder, `tests/performance/`; the pytest-benchmark ones are +selected by `--benchmark-only` and skipped elsewhere via the `--benchmark-skip` default. Baselines are written to `.benchmarks/` (gitignored, per-machine), so regression -comparison is a local developer tool โ€” wall-clock benchmarks deliberately do not -gate CI. +comparison is a local developer tool โ€” wall-clock benchmarks deliberately do not gate +CI. `make perf` first prints a system fingerprint + environment verdict and +self-calibrates the timer, so numbers come with the context needed to trust them. ### Formatting Code diff --git a/Makefile b/Makefile index bc04a03..5e25569 100644 --- a/Makefile +++ b/Makefile @@ -353,16 +353,13 @@ rust-bench: ## Run Rust benchmarks # ๐Ÿ“Š BENCHMARKS # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• -# Serializer micro-benchmarks live in tests/benchmarks/ under benchmark_*.py / -# Benchmark* names, deliberately kept OUT of the default `make test` discovery -# (wall-clock numbers are noisy and must not gate unit CI). The -o overrides -# make them collectable only here; addopts is reset to drop the -# --doctest-modules/--markdown-docs/--verbose noise. pytest-benchmark writes -# baselines to .benchmarks/ (gitignored, per-machine). -BENCHMARK_PYTEST := uv run pytest tests/benchmarks/ \ +# Serializer micro-benchmarks live in tests/performance/test_serializer_microbench.py +# and use the pytest-benchmark fixture. They are skipped in the normal suite via the +# --benchmark-skip default (pyproject addopts) and selected here with --benchmark-only. +# addopts is reset to drop the --doctest-modules/--markdown-docs/--verbose noise. +# pytest-benchmark writes baselines to .benchmarks/ (gitignored, per-machine). +BENCHMARK_PYTEST := uv run pytest tests/performance/ \ -o addopts="" \ - -o python_files="benchmark_*.py" \ - -o python_classes="Benchmark*" \ --benchmark-only --benchmark-disable-gc benchmark: setup-logs ## Run serializer benchmarks + save a local baseline @@ -380,6 +377,21 @@ benchmark-compare: setup-logs ## Run benchmarks, fail on >10% median regression fi @echo "$(GREEN)โœ“ No median regression beyond 10% vs baseline$(RESET)" +benchmark-gil: setup-logs ## Serializer thread-scaling under the current interpreter (GIL) + @echo "$(BLUE)GIL thread-scaling (current interpreter)...$(RESET)" + @uv run python tests/performance/gil_benchmark.py 2>&1 | tee $(LOG_BENCHMARK_DIR)/gil_$(TIMESTAMP).log + @echo "$(GREEN)โœ“ GIL arm complete (no-GIL arm: run under a free-threaded interpreter once cachekit installs there)$(RESET)" + +perf: setup-logs ## Unified benchmark battery: env + calibration, serializer benchmarks, GIL scaling (informational) + @echo "$(BLUE)Measurement environment + timer calibration...$(RESET)" + @uv run pytest tests/performance/test_measurement_calibration.py -m performance -q -s 2>&1 | tee $(LOG_BENCHMARK_DIR)/perf_$(TIMESTAMP).log + @$(MAKE) --no-print-directory benchmark + @$(MAKE) --no-print-directory benchmark-gil + @echo "$(GREEN)โœ“ perf battery complete (informational; gate with 'make perf-compare')$(RESET)" + +perf-compare: benchmark-compare ## Perf regression gate: fail on >10% median serializer regression + @echo "$(GREEN)โœ“ perf-compare passed$(RESET)" + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• # ๐Ÿ”ง BUILD & RELEASE # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• diff --git a/pyproject.toml b/pyproject.toml index 3dbb215..8300ec4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -156,6 +156,7 @@ addopts = [ "--doctest-modules", # Validate docstring examples "--doctest-continue-on-failure", # Report all doctest failures, not just first "--markdown-docs", # Validate markdown documentation examples + "--benchmark-skip", # pytest-benchmark tests run only via --benchmark-only (make benchmark) ] markers = [ "asyncio: Async tests using pytest-asyncio", diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py deleted file mode 100644 index 4d4584d..0000000 --- a/tests/benchmarks/conftest.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Benchmark suite configuration - no Redis required. - -These benchmarks instantiate serializers directly and never touch a backend, -so the root conftest's autouse Redis-isolation fixture (which would spawn a -local redis-server binary when REDIS_URL is unset) is overridden to a no-op -here, the same way tests/unit/conftest.py does for unit tests. -""" - -from collections.abc import Iterator - -import pytest - - -@pytest.fixture(autouse=True) -def setup_di_for_redis_isolation() -> Iterator[None]: - """Override root conftest's Redis isolation - benchmarks need no backend. - - Shadows the parent autouse fixture by having the same name (nearest - conftest wins), so serializer benchmarks run without Redis. - """ - # No-op for benchmarks - just yield without Redis setup - yield diff --git a/tests/performance/README.md b/tests/performance/README.md index 7133228..011c7b5 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -4,6 +4,19 @@ **Philosophy**: Test what users ACTUALLY experience, not idealized microbenchmarks. +**One folder, two kinds.** Everything perf lives here. Assertion-based regression +*guards* (via `stats_utils.py`) assert fixed targets and fail on regression; baseline- +tracked *benchmarks* (pytest-benchmark, in `test_serializer_microbench.py`) measure a +distribution and compare against a saved baseline. The two are separated by mechanism, +not folders: pytest-benchmark tests are selected with `--benchmark-only` and skipped +elsewhere via the `--benchmark-skip` default in `pyproject.toml`. + +**Measurement integrity.** `measurement_env.py` provides a reproducible system +fingerprint, an environment pre-flight check (throttling / load), and a timer +self-calibration (`test_measurement_calibration.py`) โ€” run before trusting any number. +`gil_benchmark.py` reports serializer thread-scaling under the current interpreter. +Run the whole battery with `make perf`; gate regressions with `make perf-compare`. + --- ## Test Organization diff --git a/tests/performance/conftest.py b/tests/performance/conftest.py new file mode 100644 index 0000000..00060eb --- /dev/null +++ b/tests/performance/conftest.py @@ -0,0 +1,42 @@ +"""Performance-suite fixtures: surface measurement integrity at session start. + +The autouse fixture prints the system fingerprint (+ stable hash) and an environment +pre-flight verdict once per perf session. It is WARN-ONLY: a non-trustworthy +environment is reported, never gated โ€” wall-clock benchmarks deliberately do not fail +the build (see .github/workflows/ci.yml). Tests that want a hard gate can request the +`measurement_env` fixture and inspect the returned verdict. +""" + +from __future__ import annotations + +from collections.abc import Iterator + +import pytest + +from .measurement_env import ( + EnvVerdict, + check_measurement_environment, + fingerprint_hash, + system_fingerprint, +) + + +@pytest.fixture(scope="session", autouse=True) +def measurement_env() -> Iterator[EnvVerdict]: + """Print the fingerprint + environment verdict once; yield it for optional gating.""" + fp = system_fingerprint() + verdict = check_measurement_environment() + + print(f"\n{'=' * 70}") + print(f"Measurement environment [fingerprint {fingerprint_hash(fp)}]") + print(f"{'=' * 70}") + for key, value in fp.items(): + print(f" {key:<20} {value}") + if verdict.trustworthy: + print(" environment โœ“ clean (no throttling / load warnings)") + else: + for warning in verdict.warnings: + print(f" environment โš  {warning}") + print(f"{'=' * 70}\n") + + yield verdict diff --git a/tests/performance/gil_benchmark.py b/tests/performance/gil_benchmark.py new file mode 100644 index 0000000..516312c --- /dev/null +++ b/tests/performance/gil_benchmark.py @@ -0,0 +1,88 @@ +"""GIL / free-threading thread-scaling benchmark for the cachekit serializer. + +Runs an identical CPU-bound serialize workload across 1, 2, 4, 8 threads under the +*current* interpreter and reports speedup + parallel efficiency. Under a stock +(GIL-enabled) build, efficiency reveals how much the Rust ByteStorage path already +runs GIL-free; under a free-threaded build it shows the parallel ceiling. + +Free-threaded comparison is interpreter-driven, not code-driven: run this same script +under a free-threaded interpreter and compare the efficiency column. Today that is +blocked for cachekit โ€” PyO3 < 3.14 has no free-threaded support, and orjson / +numpy / pandas / pyarrow lack free-threaded wheels โ€” so the no-GIL arm is reported as +unavailable. Once a free-threaded cachekit installs, no change here is needed: the +same run produces the no-GIL numbers. + +Run: uv run python tests/performance/gil_benchmark.py +""" + +from __future__ import annotations + +import sys +import time +from concurrent.futures import ThreadPoolExecutor + +from cachekit.serializers import StandardSerializer + +THREAD_COUNTS = (1, 2, 4, 8) +OPS_PER_THREAD = 2000 + + +def gil_enabled() -> bool: + """True on a stock GIL build; matches src/cachekit/hiredis_compat.py's check.""" + return bool(getattr(sys, "_is_gil_enabled", lambda: True)()) + + +def _payload() -> dict[str, dict[str, object]]: + """Medium nested dict โ€” exercises msgpack encode + Rust ByteStorage (LZ4 + checksum).""" + return {f"key_{i}": {"value": f"data_{i}", "count": i, "vals": list(range(20))} for i in range(200)} + + +def _work(serializer: StandardSerializer, data: object, ops: int) -> None: + for _ in range(ops): + serializer.serialize(data) + + +def run_threads(n_threads: int, ops_total: int) -> float: + """Run ops_total serialize calls spread across n_threads; return wall seconds. + + Setup (serializer + payload) happens before the clock starts, so the measurement + captures only the concurrent serialize work. + """ + serializer = StandardSerializer() + data = _payload() + per = ops_total // n_threads + start = time.perf_counter() + with ThreadPoolExecutor(max_workers=n_threads) as pool: + futures = [pool.submit(_work, serializer, data, per) for _ in range(n_threads)] + for future in futures: + future.result() + return time.perf_counter() - start + + +def main() -> None: + gil = gil_enabled() + label = "GIL ENABLED (stock)" if gil else "FREE-THREADED (no-GIL)" + ops_total = OPS_PER_THREAD * max(THREAD_COUNTS) # fixed total work across all thread counts + + print("\nGIL thread-scaling: StandardSerializer.serialize") + print(f" interpreter: {sys.version.split()[0]} | {label}") + print(f" total ops: {ops_total} (held constant across thread counts)\n") + + run_threads(1, max(THREAD_COUNTS)) # warmup + + single = run_threads(1, ops_total) + print(f" {'threads':>7} {'wall (s)':>9} {'speedup':>8} {'efficiency':>11}") + for n_threads in THREAD_COUNTS: + wall = run_threads(n_threads, ops_total) + speedup = single / wall + efficiency = (speedup / n_threads) * 100 + print(f" {n_threads:>7} {wall:>9.4f} {speedup:>7.2f}x {efficiency:>10.0f}%") + + if gil: + print("\n No-GIL arm: run this under a free-threaded interpreter to compare the") + print(" efficiency column. Blocked today for cachekit (PyO3 < 3.14; orjson /") + print(" numpy / pandas / pyarrow lack free-threaded wheels).") + + +if __name__ == "__main__": + main() diff --git a/tests/performance/measurement_env.py b/tests/performance/measurement_env.py new file mode 100644 index 0000000..2843f6b --- /dev/null +++ b/tests/performance/measurement_env.py @@ -0,0 +1,143 @@ +"""Measurement integrity for the perf suite: reproducible fingerprint, environment +pre-flight gating, and a timer self-calibration check. + +Benchmark numbers are only trustworthy if you can tell (a) which machine produced +them โ€” `system_fingerprint` + `fingerprint_hash`; (b) that the machine was not +throttled or loaded while measuring โ€” `check_measurement_environment`; and (c) that +the timer itself measures what you think โ€” `calibrate_timer`. `conftest.py` wires +the first two into a session banner; `test_measurement_calibration.py` asserts the +third. Salvaged (minimally) from the pyredis-cache-pro prototype's measurement layer. +""" + +from __future__ import annotations + +import hashlib +import os +import platform +import statistics +import sys +import time +from dataclasses import dataclass, field + +import psutil + +from .stats_utils import coefficient_of_variation + + +def system_fingerprint() -> dict[str, str | int | float]: + """Capture machine characteristics for cross-run comparability.""" + vm = psutil.virtual_memory() + return { + "cpu_model": platform.processor() or "unknown", + "cpu_cores_physical": psutil.cpu_count(logical=False) or 0, + "cpu_cores_logical": psutil.cpu_count(logical=True) or 0, + "ram_total_gb": round(vm.total / (1024**3), 1), + "platform": platform.system(), + "platform_machine": platform.machine(), + "python_version": platform.python_version(), + "python_minor": ".".join(platform.python_version_tuple()[:2]), + "python_impl": platform.python_implementation(), + "gil_enabled": int(getattr(sys, "_is_gil_enabled", lambda: True)()), + } + + +# Deterministic fields only: two runs on the same machine + interpreter build hash +# identically, so results are compared only when they share a hash. Excludes the full +# patch version (3.13.x bumps do not change the hardware) and anything volatile +# (current freq, load) โ€” the prototype's stable-fingerprint trick. python_minor and +# gil_enabled ARE included: 3.12-vs-3.13 and GIL-vs-free-threaded change performance. +_HASH_FIELDS = ( + "cpu_model", + "cpu_cores_physical", + "cpu_cores_logical", + "ram_total_gb", + "platform", + "platform_machine", + "python_minor", + "python_impl", + "gil_enabled", +) + + +def fingerprint_hash(fp: dict[str, str | int | float]) -> str: + """Stable 12-char hash over the deterministic fingerprint fields.""" + payload = "|".join(f"{k}={fp.get(k)}" for k in _HASH_FIELDS) + return hashlib.sha256(payload.encode()).hexdigest()[:12] + + +@dataclass +class EnvVerdict: + """Result of the pre-flight environment check.""" + + trustworthy: bool + warnings: list[str] = field(default_factory=list) + + +def check_measurement_environment() -> EnvVerdict: + """Pre-flight: is this machine fit to produce trustworthy timings right now? + + Warns (does not raise) on thermal throttling or load โ€” a high-variance run on a + hot or busy box yields numbers you cannot compare. Thresholds match the prototype: + cpu_freq < 85% of max, CPU > 80%, memory > 85%, load average > 1.5x cores. + """ + warnings: list[str] = [] + + cpu_pct = psutil.cpu_percent(interval=0.1) + if cpu_pct > 80.0: + warnings.append(f"high CPU load: {cpu_pct:.0f}%") + + # Thermal throttling only matters when the CPU is actually working: at idle the + # current frequency sits far below the boost max (healthy, not throttling), so a + # bare current/max ratio cries wolf on every idle machine. Flag a depressed + # frequency only UNDER load. + try: + freq = psutil.cpu_freq() + if freq and freq.max and cpu_pct > 50.0 and (freq.current / freq.max) < 0.85: + warnings.append( + f"CPU may be throttled under load: {freq.current:.0f}/{freq.max:.0f} MHz ({freq.current / freq.max:.0%} of max)" + ) + except (AttributeError, NotImplementedError, OSError): + pass # cpu_freq() is unavailable on some platforms / VMs + + mem_pct = psutil.virtual_memory().percent + if mem_pct > 85.0: + warnings.append(f"high memory usage: {mem_pct:.0f}%") + + if hasattr(os, "getloadavg"): + cores = psutil.cpu_count(logical=True) or 1 + load1 = os.getloadavg()[0] + if (load1 / cores) > 1.5: + warnings.append(f"high load average: {load1:.1f} over {cores} cores") + + return EnvVerdict(trustworthy=not warnings, warnings=warnings) + + +def calibrate_timer(sleep_ms: float = 1.0, samples: int = 50) -> tuple[bool, str]: + """Self-check the measurement instrument: time a known sleep, confirm sanity. + + A 1 ms sleep that measures ~1 ms means the `perf_counter_ns` loop captures the + operation and nothing else. If it measured 10-30 ms (the classic timing-scope- + pollution bug, where setup leaks into the timed region) or near-zero, the timer is + lying and no benchmark on this machine can be trusted. The window is deliberately + generous on the high side to absorb OS scheduling overshoot โ€” it catches only gross + errors, not precision. Reuses `coefficient_of_variation` for the stability report; + see `validate_measurement_accuracy` for the per-operation (symmetric) analog. + """ + expected_ns = sleep_ms * 1_000_000.0 + time.sleep(sleep_ms / 1000.0) # one untimed warmup sleep + measured: list[float] = [] + for _ in range(samples): + start = time.perf_counter_ns() + time.sleep(sleep_ms / 1000.0) + measured.append(float(time.perf_counter_ns() - start)) + + median_ns = statistics.median(measured) + cv = coefficient_of_variation(measured) + trustworthy = (expected_ns * 0.8) <= median_ns <= (expected_ns * 3.0) + status = "โœ“" if trustworthy else "โœ—" + message = ( + f"{status} timer calibration: {sleep_ms} ms sleep measured median " + f"{median_ns / 1_000_000:.3f} ms (CV {cv:.2f}); " + f"honest window [{sleep_ms * 0.8:.2f}, {sleep_ms * 3.0:.2f}] ms" + ) + return trustworthy, message diff --git a/tests/performance/test_measurement_calibration.py b/tests/performance/test_measurement_calibration.py new file mode 100644 index 0000000..8bc8e31 --- /dev/null +++ b/tests/performance/test_measurement_calibration.py @@ -0,0 +1,24 @@ +"""The measurement instrument must be honest before any benchmark is trusted. + +This is the perf suite's self-check: if the timer cannot accurately measure a known +1 ms sleep, every other number it produces is suspect. Marked `performance` (not +`slow`), so it stays out of the CI memory-invariant gate and runs locally / on demand. +""" + +from __future__ import annotations + +import pytest + +from .measurement_env import calibrate_timer + + +@pytest.mark.performance +def test_timer_is_honest() -> None: + """A known 1 ms sleep must measure within a sane window. + + Catches timing-scope pollution (setup leaking into the timed region) that would + inflate a 1 ms operation to 10-30 ms, and under-counting that would report near-zero. + """ + trustworthy, message = calibrate_timer(sleep_ms=1.0) + print(f"\n{message}") + assert trustworthy, message diff --git a/tests/benchmarks/benchmark_serializer_integrity.py b/tests/performance/test_serializer_microbench.py similarity index 56% rename from tests/benchmarks/benchmark_serializer_integrity.py rename to tests/performance/test_serializer_microbench.py index cda78a3..7f7faeb 100644 --- a/tests/benchmarks/benchmark_serializer_integrity.py +++ b/tests/performance/test_serializer_microbench.py @@ -1,16 +1,14 @@ -"""Detailed performance benchmarks for serializer integrity checking. - -Measures: -- Serialize latency (with Blake3 checksum) -- Deserialize latency (with Blake3 validation) -- Blake3 hashing cost breakdown -- Overhead as percentage of total operation time -- Data size impact on overhead +"""Serializer micro-benchmarks (pytest-benchmark, baseline-tracked). + +Distribution-level latency for serialize / deserialize / roundtrip across JSON +(OrjsonSerializer) and DataFrame (ArrowSerializer) payloads, with the serializers' +built-in integrity checksums. Selected by ``--benchmark-only`` (``make benchmark``); +skipped in the normal suite via the ``--benchmark-skip`` default in pyproject.toml. """ from __future__ import annotations -import time +from collections.abc import Iterator import pandas as pd import pytest @@ -18,8 +16,18 @@ from cachekit.serializers import ArrowSerializer, OrjsonSerializer -class BenchmarkOrjsonIntegrity: - """Benchmark OrjsonSerializer with Blake3 integrity checking.""" +@pytest.fixture(autouse=True) +def setup_di_for_redis_isolation() -> Iterator[None]: + """Serializer benchmarks need no backend โ€” override the root autouse Redis fixture. + + File-scoped (not a folder conftest): tests/performance/ also holds Redis-dependent + tests, so the override must stay local to this file. + """ + yield + + +class TestOrjsonSerializerBench: + """Benchmark OrjsonSerializer with integrity checking.""" @pytest.mark.benchmark def test_serialize_small_json(self, benchmark): @@ -105,8 +113,8 @@ def roundtrip(): assert result == data -class BenchmarkArrowIntegrity: - """Benchmark ArrowSerializer with Blake3 integrity checking.""" +class TestArrowSerializerBench: + """Benchmark ArrowSerializer with integrity checking.""" @pytest.mark.benchmark def test_serialize_small_dataframe(self, benchmark): @@ -227,149 +235,3 @@ def roundtrip(): result = benchmark(roundtrip) assert len(result) == 10000 - - -class Blake3OverheadAnalysis: - """Analyze Blake3 hashing cost independent of serialization.""" - - def test_blake3_cost_breakdown(self): - """Measure Blake3 hashing cost for various data sizes.""" - import blake3 - - print("\n=== Blake3 Hashing Cost Breakdown ===\n") - - test_sizes = [ - (100, "100 bytes (tiny)"), - (1_000, "1 KB (small)"), - (10_000, "10 KB (medium)"), - (100_000, "100 KB (large)"), - (1_000_000, "1 MB (very large)"), - ] - - for size, label in test_sizes: - data = b"x" * size - iterations = 100 - - start = time.perf_counter() - for _ in range(iterations): - blake3.blake3(data).digest() - elapsed = time.perf_counter() - start - - avg_micros = (elapsed / iterations) * 1_000_000 - overhead_pct = (32 / size) * 100 - - print(f"{label:25} | {avg_micros:8.2f}ยตs | {overhead_pct:6.2f}% size overhead") - - def test_orjson_serialize_breakdown(self): - """Break down OrjsonSerializer serialize cost.""" - import orjson - - from cachekit.serializers import OrjsonSerializer - - print("\n=== OrjsonSerializer Serialize Breakdown ===\n") - - serializer = OrjsonSerializer() - - # Small data - small_data = {"key": "value", "number": 42} - iterations = 1000 - - start = time.perf_counter() - for _ in range(iterations): - serializer.serialize(small_data) - elapsed_with_checksum = time.perf_counter() - start - - # Measure orjson only (approximate) - start = time.perf_counter() - for _ in range(iterations): - orjson.dumps(small_data) - elapsed_orjson = time.perf_counter() - start - - checksum_cost = (elapsed_with_checksum - elapsed_orjson) / iterations * 1_000_000 - total_cost = elapsed_with_checksum / iterations * 1_000_000 - - print("Small JSON (58 bytes):") - print(f" orjson only: {(elapsed_orjson / iterations * 1_000_000):8.2f}ยตs") - print(f" Blake3 checksum: {checksum_cost:8.2f}ยตs") - print(f" Total (with envelope): {total_cost:8.2f}ยตs") - print(f" Checksum overhead: {(checksum_cost / total_cost * 100):6.2f}%") - - # Large data - large_data = {f"key_{i}": {"value": f"data_{i}", "count": i} for i in range(1000)} - iterations = 100 - - start = time.perf_counter() - for _ in range(iterations): - serializer.serialize(large_data) - elapsed_with_checksum = time.perf_counter() - start - - start = time.perf_counter() - for _ in range(iterations): - orjson.dumps(large_data) - elapsed_orjson = time.perf_counter() - start - - checksum_cost = (elapsed_with_checksum - elapsed_orjson) / iterations * 1_000_000 - total_cost = elapsed_with_checksum / iterations * 1_000_000 - - print("\nLarge JSON (~30 KB):") - print(f" orjson only: {(elapsed_orjson / iterations * 1_000_000):8.2f}ยตs") - print(f" Blake3 checksum: {checksum_cost:8.2f}ยตs") - print(f" Total (with envelope): {total_cost:8.2f}ยตs") - print(f" Checksum overhead: {(checksum_cost / total_cost * 100):6.2f}%") - - def test_arrow_serialize_breakdown(self): - """Break down ArrowSerializer serialize cost.""" - import pyarrow as pa - - from cachekit.serializers import ArrowSerializer - - print("\n=== ArrowSerializer Serialize Breakdown ===\n") - - serializer = ArrowSerializer() - - # Small DataFrame - df_small = pd.DataFrame({"a": range(10), "b": range(10, 20)}) - iterations = 100 - - start = time.perf_counter() - for _ in range(iterations): - serializer.serialize(df_small) - elapsed_with_checksum = time.perf_counter() - start - - # Approximate Arrow IPC cost (simple bench) - table = pa.Table.from_pandas(df_small) - sink = pa.BufferOutputStream() - start = time.perf_counter() - for _ in range(iterations): - writer = pa.ipc.new_stream(sink, table.schema) - writer.write(table) - elapsed_arrow = time.perf_counter() - start - - checksum_cost = (elapsed_with_checksum - elapsed_arrow) / iterations * 1_000_000 - total_cost = elapsed_with_checksum / iterations * 1_000_000 - - print("Small DataFrame (10 rows, 2 cols):") - print(f" Arrow IPC only: {(elapsed_arrow / iterations * 1_000_000):8.2f}ยตs") - print(f" Blake3 checksum: {checksum_cost:8.2f}ยตs") - print(f" Total (with envelope): {total_cost:8.2f}ยตs") - if total_cost > 0: - print(f" Checksum overhead: {(checksum_cost / total_cost * 100):6.2f}%") - - # Large DataFrame - df_large = pd.DataFrame( - { - "a": range(10000), - "b": range(10000, 20000), - "c": [f"row_{i}" for i in range(10000)], - } - ) - iterations = 10 - - start = time.perf_counter() - for _ in range(iterations): - serializer.serialize(df_large) - elapsed_with_checksum = time.perf_counter() - start - - print("\nLarge DataFrame (10K rows, 3 cols):") - print(f" Total (with checksum): {(elapsed_with_checksum / iterations * 1_000_000):8.2f}ยตs") - print(f" Per-row cost: {((elapsed_with_checksum / iterations) / 10000 * 1_000_000):8.2f}ยตs") diff --git a/tests/performance/test_statistical_rigor.py b/tests/performance/test_statistical_rigor.py index e4702e9..59bb449 100644 --- a/tests/performance/test_statistical_rigor.py +++ b/tests/performance/test_statistical_rigor.py @@ -10,8 +10,6 @@ from __future__ import annotations -import platform - import pytest from cachekit.l1_cache import L1Cache @@ -19,23 +17,6 @@ from .stats_utils import benchmark_with_gc_handling -@pytest.fixture(scope="session", autouse=True) -def system_fingerprint(): - """Capture system info for benchmark context and reproducibility.""" - fingerprint = { - "python_version": platform.python_version(), - "platform": platform.platform(), - "processor": platform.processor(), - } - print(f"\n{'=' * 70}") - print("System Fingerprint (for benchmark context)") - print(f"{'=' * 70}") - for key, value in fingerprint.items(): - print(f" {key:<20} {value}") - print(f"{'=' * 70}\n") - return fingerprint - - @pytest.mark.performance def test_l1_cache_hit_statistically_rigorous() -> None: """L1 cache hit latency with statistical rigor.