chore: benchmark suite — fix harness, one-folder consolidation, measurement integrity + GIL + unified runner

CONTRIBUTING.md

@@ -246,9 +246,21 @@ make build-pgo  # Profile-Guided Optimization (5-8% faster)
 ### Running Benchmarks
 
 ```bash
-make benchmark-quick  # Quick performance check
+make perf               # Full battery: env fingerprint + timer calibration, serializer benchmarks, GIL scaling
+make perf-compare       # Regression gate: fail on >10% median serializer regression vs baseline
+
+make benchmark          # Serializer benchmarks only + save a local baseline
+make benchmark-compare  # Re-run and fail on >10% median regression vs that baseline
+make benchmark-gil      # Serializer thread-scaling under the current interpreter (GIL)
 ```
 
+All perf tests live in one folder, `tests/performance/`; the pytest-benchmark ones are
+selected by `--benchmark-only` and skipped elsewhere via the `--benchmark-skip` default.
+Baselines are written to `.benchmarks/` (gitignored, per-machine), so regression
+comparison is a local developer tool — wall-clock benchmarks deliberately do not gate
+CI. `make perf` first prints a system fingerprint + environment verdict and
+self-calibrates the timer, so numbers come with the context needed to trust them.
+
 ### Formatting Code
 
 ```bash

Makefile

@@ -353,17 +353,44 @@ rust-bench: ## Run Rust benchmarks
 # 📊 BENCHMARKS
 # ═══════════════════════════════════════════════════════════════════════════════
 
-benchmark: setup-logs ## Run quick benchmarks
-	@echo "$(BLUE)Running quick benchmarks...$(RESET)"
-	@echo "$(YELLOW)Logging to $(LOG_BENCHMARK_DIR)/quick_$(TIMESTAMP).log$(RESET)"
-	@uv run python -m benchmarks.cli quick 2>&1 | tee $(LOG_BENCHMARK_DIR)/quick_$(TIMESTAMP).log
-	@echo "$(GREEN)✓ Benchmarks completed$(RESET)"
-
-benchmark-full: setup-logs ## Run comprehensive benchmarks
-	@echo "$(BLUE)Running comprehensive benchmarks...$(RESET)"
-	@echo "$(YELLOW)Logging to $(LOG_BENCHMARK_DIR)/full_$(TIMESTAMP).log$(RESET)"
-	@uv run python -m benchmarks.cli performance --comprehensive 2>&1 | tee $(LOG_BENCHMARK_DIR)/full_$(TIMESTAMP).log
-	@echo "$(GREEN)✓ Comprehensive benchmarks completed$(RESET)"
+# Serializer micro-benchmarks live in tests/performance/test_serializer_microbench.py
+# and use the pytest-benchmark fixture. They are skipped in the normal suite via the
+# --benchmark-skip default (pyproject addopts) and selected here with --benchmark-only.
+# addopts is reset to drop the --doctest-modules/--markdown-docs/--verbose noise.
+# pytest-benchmark writes baselines to .benchmarks/ (gitignored, per-machine).
+BENCHMARK_PYTEST := uv run pytest tests/performance/ \
+    -o addopts="" \
+    --benchmark-only --benchmark-disable-gc
+
+benchmark: setup-logs ## Run serializer benchmarks + save a local baseline
+	@echo "$(BLUE)Running serializer benchmarks (saving baseline)...$(RESET)"
+	@echo "$(YELLOW)Logging to $(LOG_BENCHMARK_DIR)/bench_$(TIMESTAMP).log$(RESET)"
+	@$(BENCHMARK_PYTEST) --benchmark-autosave 2>&1 | tee $(LOG_BENCHMARK_DIR)/bench_$(TIMESTAMP).log
+	@echo "$(GREEN)✓ Saved to .benchmarks/ (gitignored, per-machine)$(RESET)"
+
+benchmark-compare: setup-logs ## Run benchmarks, fail on >10% median regression vs last saved baseline
+	@echo "$(BLUE)Comparing against last saved baseline (run 'make benchmark' first)...$(RESET)"
+	@if ! $(BENCHMARK_PYTEST) --benchmark-compare --benchmark-compare-fail=median:10% 2>&1 | tee $(LOG_BENCHMARK_DIR)/bench_compare_$(TIMESTAMP).log; then \
+		echo "$(YELLOW)❌ benchmark-compare failed: median regression >10% vs baseline, or no baseline yet (run 'make benchmark' first).$(RESET)"; \
+		echo "$(YELLOW)   Threshold is tunable — set it above your machine's run-to-run median noise (~6% here).$(RESET)"; \
+		exit 1; \
+	fi
+	@echo "$(GREEN)✓ No median regression beyond 10% vs baseline$(RESET)"
+
+benchmark-gil: setup-logs ## Serializer thread-scaling under the current interpreter (GIL)
+	@echo "$(BLUE)GIL thread-scaling (current interpreter)...$(RESET)"
+	@uv run python tests/performance/gil_benchmark.py 2>&1 | tee $(LOG_BENCHMARK_DIR)/gil_$(TIMESTAMP).log
+	@echo "$(GREEN)✓ GIL arm complete (no-GIL arm: run under a free-threaded interpreter once cachekit installs there)$(RESET)"
+
+perf: setup-logs ## Unified benchmark battery: env + calibration, serializer benchmarks, GIL scaling (informational)
+	@echo "$(BLUE)Measurement environment + timer calibration...$(RESET)"
+	@uv run pytest tests/performance/test_measurement_calibration.py -m performance -q -s 2>&1 | tee $(LOG_BENCHMARK_DIR)/perf_$(TIMESTAMP).log
+	@$(MAKE) --no-print-directory benchmark
+	@$(MAKE) --no-print-directory benchmark-gil
+	@echo "$(GREEN)✓ perf battery complete (informational; gate with 'make perf-compare')$(RESET)"
+
+perf-compare: benchmark-compare ## Perf regression gate: fail on >10% median serializer regression
+	@echo "$(GREEN)✓ perf-compare passed$(RESET)"
 
 # ═══════════════════════════════════════════════════════════════════════════════
 # 🔧 BUILD & RELEASE

docs/getting-started.md

@@ -469,8 +469,8 @@ print("Caching working correctly!")
 # Run the built-in performance tests
 pytest tests/performance/ -v
 
-# Or run benchmarks
-uv run python -m benchmarks.cli quick --rust-only
+# Or run the serializer benchmarks (saves a local baseline)
+make benchmark
 ```
 
 ---

pyproject.toml

@@ -156,6 +156,7 @@ addopts = [
     "--doctest-modules",              # Validate docstring examples
     "--doctest-continue-on-failure",  # Report all doctest failures, not just first
     "--markdown-docs",                # Validate markdown documentation examples
+    "--benchmark-skip",               # pytest-benchmark tests run only via --benchmark-only (make benchmark)
 ]
 markers = [
     "asyncio: Async tests using pytest-asyncio",

tests/performance/README.md

@@ -4,6 +4,19 @@
 
 **Philosophy**: Test what users ACTUALLY experience, not idealized microbenchmarks.
 
+**One folder, two kinds.** Everything perf lives here. Assertion-based regression
+*guards* (via `stats_utils.py`) assert fixed targets and fail on regression; baseline-
+tracked *benchmarks* (pytest-benchmark, in `test_serializer_microbench.py`) measure a
+distribution and compare against a saved baseline. The two are separated by mechanism,
+not folders: pytest-benchmark tests are selected with `--benchmark-only` and skipped
+elsewhere via the `--benchmark-skip` default in `pyproject.toml`.
+
+**Measurement integrity.** `measurement_env.py` provides a reproducible system
+fingerprint, an environment pre-flight check (throttling / load), and a timer
+self-calibration (`test_measurement_calibration.py`) — run before trusting any number.
+`gil_benchmark.py` reports serializer thread-scaling under the current interpreter.
+Run the whole battery with `make perf`; gate regressions with `make perf-compare`.
+
 ---
 
 ## Test Organization
@@ -93,7 +106,7 @@ Provides rigorous performance measurement tools:
 
 ### Quick Check (Basic Tests)
 ```bash
-make test-performance-quick
+uv run pytest tests/performance/ -m "performance and not slow" -q
 ```
 
 ### Full Suite (All Tests)

tests/performance/conftest.py

@@ -0,0 +1,42 @@
+"""Performance-suite fixtures: surface measurement integrity at session start.
+
+The autouse fixture prints the system fingerprint (+ stable hash) and an environment
+pre-flight verdict once per perf session. It is WARN-ONLY: a non-trustworthy
+environment is reported, never gated — wall-clock benchmarks deliberately do not fail
+the build (see .github/workflows/ci.yml). Tests that want a hard gate can request the
+`measurement_env` fixture and inspect the returned verdict.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+import pytest
+
+from .measurement_env import (
+    EnvVerdict,
+    check_measurement_environment,
+    fingerprint_hash,
+    system_fingerprint,
+)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def measurement_env() -> Iterator[EnvVerdict]:
+    """Print the fingerprint + environment verdict once; yield it for optional gating."""
+    fp = system_fingerprint()
+    verdict = check_measurement_environment()
+
+    print(f"\n{'=' * 70}")
+    print(f"Measurement environment  [fingerprint {fingerprint_hash(fp)}]")
+    print(f"{'=' * 70}")
+    for key, value in fp.items():
+        print(f"  {key:<20} {value}")
+    if verdict.trustworthy:
+        print("  environment          ✓ clean (no throttling / load warnings)")
+    else:
+        for warning in verdict.warnings:
+            print(f"  environment          ⚠ {warning}")
+    print(f"{'=' * 70}\n")
+
+    yield verdict

tests/performance/gil_benchmark.py

@@ -0,0 +1,88 @@
+"""GIL / free-threading thread-scaling benchmark for the cachekit serializer.
+
+Runs an identical CPU-bound serialize workload across 1, 2, 4, 8 threads under the
+*current* interpreter and reports speedup + parallel efficiency. Under a stock
+(GIL-enabled) build, efficiency reveals how much the Rust ByteStorage path already
+runs GIL-free; under a free-threaded build it shows the parallel ceiling.
+
+Free-threaded comparison is interpreter-driven, not code-driven: run this same script
+under a free-threaded interpreter and compare the efficiency column. Today that is
+blocked for cachekit — PyO3 < 3.14 has no free-threaded support, and orjson /
+numpy / pandas / pyarrow lack free-threaded wheels — so the no-GIL arm is reported as
+unavailable. Once a free-threaded cachekit installs, no change here is needed: the
+same run produces the no-GIL numbers.
+
+Run:  uv run python tests/performance/gil_benchmark.py
+"""
+
+from __future__ import annotations
+
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+from cachekit.serializers import StandardSerializer
+
+THREAD_COUNTS = (1, 2, 4, 8)
+OPS_PER_THREAD = 2000
+
+
+def gil_enabled() -> bool:
+    """True on a stock GIL build; matches src/cachekit/hiredis_compat.py's check."""
+    return bool(getattr(sys, "_is_gil_enabled", lambda: True)())
+
+
+def _payload() -> dict[str, dict[str, object]]:
+    """Medium nested dict — exercises msgpack encode + Rust ByteStorage (LZ4 + checksum)."""
+    return {f"key_{i}": {"value": f"data_{i}", "count": i, "vals": list(range(20))} for i in range(200)}
+
+
+def _work(serializer: StandardSerializer, data: object, ops: int) -> None:
+    for _ in range(ops):
+        serializer.serialize(data)
+
+
+def run_threads(n_threads: int, ops_total: int) -> float:
+    """Run ops_total serialize calls spread across n_threads; return wall seconds.
+
+    Setup (serializer + payload) happens before the clock starts, so the measurement
+    captures only the concurrent serialize work.
+    """
+    serializer = StandardSerializer()
+    data = _payload()
+    per = ops_total // n_threads
+    start = time.perf_counter()
+    with ThreadPoolExecutor(max_workers=n_threads) as pool:
+        futures = [pool.submit(_work, serializer, data, per) for _ in range(n_threads)]
+        for future in futures:
+            future.result()
+    return time.perf_counter() - start
+
+
+def main() -> None:
+    gil = gil_enabled()
+    label = "GIL ENABLED (stock)" if gil else "FREE-THREADED (no-GIL)"
+    ops_total = OPS_PER_THREAD * max(THREAD_COUNTS)  # fixed total work across all thread counts
+
+    print("\nGIL thread-scaling: StandardSerializer.serialize")
+    print(f"  interpreter: {sys.version.split()[0]}  |  {label}")
+    print(f"  total ops:   {ops_total} (held constant across thread counts)\n")
+
+    run_threads(1, max(THREAD_COUNTS))  # warmup
+
+    single = run_threads(1, ops_total)
+    print(f"  {'threads':>7}  {'wall (s)':>9}  {'speedup':>8}  {'efficiency':>11}")
+    for n_threads in THREAD_COUNTS:
+        wall = run_threads(n_threads, ops_total)
+        speedup = single / wall
+        efficiency = (speedup / n_threads) * 100
+        print(f"  {n_threads:>7}  {wall:>9.4f}  {speedup:>7.2f}x  {efficiency:>10.0f}%")
+
+    if gil:
+        print("\n  No-GIL arm: run this under a free-threaded interpreter to compare the")
+        print("  efficiency column. Blocked today for cachekit (PyO3 < 3.14; orjson /")
+        print("  numpy / pandas / pyarrow lack free-threaded wheels).")
+
+
+if __name__ == "__main__":
+    main()