From 3e3514ae415e936f10a3ca1ad02569acb65feb45 Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 17:22:06 -0500 Subject: [PATCH 01/12] feat: add labeled corpus quality evaluation --- logsight/benchmarking.py | 214 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 logsight/benchmarking.py diff --git a/logsight/benchmarking.py b/logsight/benchmarking.py new file mode 100644 index 0000000..55c3176 --- /dev/null +++ b/logsight/benchmarking.py @@ -0,0 +1,214 @@ +"""Reproducible labeled-corpus evaluation for LogSight detectors.""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from dataclasses import dataclass +from math import isfinite +from pathlib import Path +from typing import Any + +from logsight.analyzer import detect_anomalies +from logsight.parser import LogEntry, LogLevel + +_MAX_CORPUS_BYTES = 5_000_000 +_MAX_RECORDS = 10_000 +_MAX_MESSAGE_CHARS = 4_000 + + +@dataclass(frozen=True, slots=True) +class LabeledLog: + """One synthetic or approved de-identified benchmark record.""" + + identifier: str + level: LogLevel + message: str + is_anomaly: bool + incident_id: str | None = None + + +@dataclass(frozen=True, slots=True) +class AlertMetrics: + """Entry- and incident-level quality and alert-volume measurements.""" + + total: int + true_positives: int + false_positives: int + false_negatives: int + true_negatives: int + precision: float + recall: float + f1: float + incident_recall: float + alert_count: int + alerts_per_1000_events: float + false_alerts_per_1000_events: float + alerts_per_true_incident: float + + +def load_labeled_corpus(path: str | Path) -> tuple[LabeledLog, ...]: + """Load a bounded JSONL corpus and enforce the published label contract.""" + + corpus_path = Path(path) + if not corpus_path.is_file(): + raise FileNotFoundError(f"benchmark corpus was not found: {corpus_path}") + if corpus_path.stat().st_size > _MAX_CORPUS_BYTES: + raise ValueError(f"benchmark corpus exceeds {_MAX_CORPUS_BYTES} bytes") + + records: list[LabeledLog] = [] + identifiers: set[str] = set() + for line_number, line in enumerate(corpus_path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + if len(records) >= _MAX_RECORDS: + raise ValueError(f"benchmark corpus exceeds {_MAX_RECORDS} records") + try: + raw = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"invalid JSON on benchmark line {line_number}") from exc + record = _parse_record(raw, line_number) + if record.identifier in identifiers: + raise ValueError(f"duplicate benchmark record id: {record.identifier}") + identifiers.add(record.identifier) + records.append(record) + + if not records: + raise ValueError("benchmark corpus must contain at least one record") + return tuple(records) + + +def evaluate_labeled_corpus( + corpus: Sequence[LabeledLog], *, zscore_threshold: float = 2.5 +) -> AlertMetrics: + """Evaluate the built-in detector against immutable labels in a corpus.""" + + if not corpus: + raise ValueError("benchmark corpus must contain at least one record") + entries = [ + LogEntry( + raw=f"{record.level.value} {record.message}", + level=record.level, + message=record.message, + ) + for record in corpus + ] + alerts = {id(entry) for entry in detect_anomalies(entries, zscore_threshold).anomalies} + + true_positives = sum( + 1 for entry, record in zip(entries, corpus, strict=True) if id(entry) in alerts and record.is_anomaly + ) + false_positives = sum( + 1 + for entry, record in zip(entries, corpus, strict=True) + if id(entry) in alerts and not record.is_anomaly + ) + false_negatives = sum( + 1 + for entry, record in zip(entries, corpus, strict=True) + if id(entry) not in alerts and record.is_anomaly + ) + true_negatives = len(corpus) - true_positives - false_positives - false_negatives + + precision = _ratio(true_positives, true_positives + false_positives) + recall = _ratio(true_positives, true_positives + false_negatives) + f1 = _ratio(2 * precision * recall, precision + recall) + true_incidents = {record.incident_id for record in corpus if record.is_anomaly} + detected_incidents = { + record.incident_id + for entry, record in zip(entries, corpus, strict=True) + if record.is_anomaly and id(entry) in alerts + } + alert_count = true_positives + false_positives + return AlertMetrics( + total=len(corpus), + true_positives=true_positives, + false_positives=false_positives, + false_negatives=false_negatives, + true_negatives=true_negatives, + precision=precision, + recall=recall, + f1=f1, + incident_recall=_ratio(len(detected_incidents), len(true_incidents)), + alert_count=alert_count, + alerts_per_1000_events=1000 * _ratio(alert_count, len(corpus)), + false_alerts_per_1000_events=1000 * _ratio(false_positives, len(corpus)), + alerts_per_true_incident=_ratio(alert_count, len(true_incidents)), + ) + + +def quality_gate_failures( + metrics: AlertMetrics, + *, + min_precision: float | None = None, + min_recall: float | None = None, + max_alerts_per_1000_events: float | None = None, +) -> tuple[str, ...]: + """Return explicit quality-gate failures without choosing deployment policy.""" + + _validate_fraction("minimum precision", min_precision) + _validate_fraction("minimum recall", min_recall) + if max_alerts_per_1000_events is not None and ( + not isfinite(max_alerts_per_1000_events) or max_alerts_per_1000_events < 0 + ): + raise ValueError("maximum alerts per 1000 events must be finite and non-negative") + + failures: list[str] = [] + if min_precision is not None and metrics.precision < min_precision: + failures.append(f"precision {metrics.precision:.4f} is below {min_precision:.4f}") + if min_recall is not None and metrics.recall < min_recall: + failures.append(f"recall {metrics.recall:.4f} is below {min_recall:.4f}") + if ( + max_alerts_per_1000_events is not None + and metrics.alerts_per_1000_events > max_alerts_per_1000_events + ): + failures.append( + "alerts per 1000 events " + f"{metrics.alerts_per_1000_events:.4f} exceeds {max_alerts_per_1000_events:.4f}" + ) + return tuple(failures) + + +def _parse_record(raw: Any, line_number: int) -> LabeledLog: + if not isinstance(raw, dict): + raise ValueError(f"benchmark line {line_number} must be a JSON object") + identifier = _required_string(raw, "id", line_number) + message = _required_string(raw, "message", line_number) + if len(message) > _MAX_MESSAGE_CHARS: + raise ValueError(f"benchmark line {line_number} message exceeds {_MAX_MESSAGE_CHARS} characters") + level_name = _required_string(raw, "level", line_number).upper() + try: + level = LogLevel(level_name) + except ValueError as exc: + raise ValueError(f"benchmark line {line_number} has an invalid level") from exc + label = raw.get("is_anomaly") + if type(label) is not bool: + raise ValueError(f"benchmark line {line_number} requires boolean is_anomaly") + incident_id = raw.get("incident_id") + if incident_id is not None and (not isinstance(incident_id, str) or not incident_id.strip()): + raise ValueError(f"benchmark line {line_number} incident_id must be a non-empty string or null") + if label and incident_id is None: + raise ValueError(f"benchmark line {line_number} anomalous records require incident_id") + return LabeledLog( + identifier=identifier, + level=level, + message=message, + is_anomaly=label, + incident_id=incident_id, + ) + + +def _required_string(raw: dict[str, Any], field: str, line_number: int) -> str: + value = raw.get(field) + if not isinstance(value, str) or not value.strip() or len(value) > 256: + raise ValueError(f"benchmark line {line_number} requires a bounded {field}") + return value.strip() + + +def _ratio(numerator: int | float, denominator: int | float) -> float: + return numerator / denominator if denominator else 0.0 + + +def _validate_fraction(name: str, value: float | None) -> None: + if value is not None and (not isfinite(value) or not 0 <= value <= 1): + raise ValueError(f"{name} must be a finite value between zero and one") From 1f1da76bf0552e2c1ed50748c509e17f82f278c3 Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 17:22:24 -0500 Subject: [PATCH 02/12] feat: add benchmark quality gate CLI --- logsight/cli.py | 67 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/logsight/cli.py b/logsight/cli.py index fe93291..76cdf0e 100644 --- a/logsight/cli.py +++ b/logsight/cli.py @@ -11,6 +11,7 @@ from rich.table import Table from logsight.analyzer import AnomalyReport, detect_anomalies, error_rate_spike_details +from logsight.benchmarking import evaluate_labeled_corpus, load_labeled_corpus, quality_gate_failures from logsight.investigator import IncidentInvestigator, RunbookRepository from logsight.parser import parse_file, parse_lines from logsight.reasoning import EvidenceExplanation, explain_report @@ -229,6 +230,72 @@ def investigate_cmd( console.print(f" excerpt : {escape(citation.excerpt)}") +@main.command("benchmark") +@click.option( + "--corpus", + type=click.Path(exists=True, dir_okay=False, readable=True, path_type=Path), + default=Path("benchmarks/corpus/v1/events.jsonl"), + show_default=True, + help="Versioned JSONL corpus with approved anomaly labels.", +) +@click.option( + "--threshold", + default=2.5, + type=click.FloatRange(min=0), + show_default=True, + help="Z-score threshold supplied to the deterministic detector.", +) +@click.option( + "--min-precision", + type=click.FloatRange(min=0, max=1), + help="Fail when precision is below this deployment-defined gate.", +) +@click.option( + "--min-recall", + type=click.FloatRange(min=0, max=1), + help="Fail when recall is below this deployment-defined gate.", +) +@click.option( + "--max-alerts-per-1000-events", + type=click.FloatRange(min=0), + help="Fail when alert volume exceeds this deployment-defined gate.", +) +def benchmark_cmd( + corpus: Path, + threshold: float, + min_precision: float | None, + min_recall: float | None, + max_alerts_per_1000_events: float | None, +) -> None: + """Evaluate anomaly quality and alert burden against a labeled corpus.""" + + try: + metrics = evaluate_labeled_corpus( + load_labeled_corpus(corpus), + zscore_threshold=threshold, + ) + failures = quality_gate_failures( + metrics, + min_precision=min_precision, + min_recall=min_recall, + max_alerts_per_1000_events=max_alerts_per_1000_events, + ) + except (OSError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + + console.print("[bold]Labeled benchmark results[/bold]") + console.print(f" events : [cyan]{metrics.total}[/cyan]") + console.print(f" precision / recall / F1 : {metrics.precision:.4f} / " + f"{metrics.recall:.4f} / {metrics.f1:.4f}") + console.print(f" incident recall : {metrics.incident_recall:.4f}") + console.print(f" alerts : {metrics.alert_count}") + console.print(f" alerts per 1,000 events : {metrics.alerts_per_1000_events:.4f}") + console.print(f" false alerts per 1,000 events: {metrics.false_alerts_per_1000_events:.4f}") + console.print(f" alerts per true incident : {metrics.alerts_per_true_incident:.4f}") + if failures: + raise click.ClickException("benchmark quality gate failed: " + "; ".join(failures)) + + @main.command("health") def health_cmd() -> None: """Verify that LogSight-AI is installed and operational.""" From 8558474e31be4c00ffcaf37dd7c8871c4bf3ad16 Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 17:22:56 -0500 Subject: [PATCH 03/12] docs: add versioned labeled benchmark corpus --- benchmarks/corpus/v1/events.jsonl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 benchmarks/corpus/v1/events.jsonl diff --git a/benchmarks/corpus/v1/events.jsonl b/benchmarks/corpus/v1/events.jsonl new file mode 100644 index 0000000..a7c066e --- /dev/null +++ b/benchmarks/corpus/v1/events.jsonl @@ -0,0 +1,16 @@ +{"id":"steady-001","level":"INFO","message":"checkout request completed within normal latency","is_anomaly":false,"incident_id":null} +{"id":"steady-002","level":"INFO","message":"catalog request completed within normal latency","is_anomaly":false,"incident_id":null} +{"id":"steady-003","level":"WARNING","message":"cache refresh deferred until next scheduled cycle","is_anomaly":false,"incident_id":null} +{"id":"steady-004","level":"INFO","message":"worker heartbeat received from primary consumer","is_anomaly":false,"incident_id":null} +{"id":"steady-005","level":"INFO","message":"inventory request completed within normal latency","is_anomaly":false,"incident_id":null} +{"id":"steady-006","level":"WARNING","message":"connection retry completed before request timeout","is_anomaly":false,"incident_id":null} +{"id":"steady-007","level":"INFO","message":"payment request completed within normal latency","is_anomaly":false,"incident_id":null} +{"id":"steady-008","level":"INFO","message":"queue consumer processed the expected message batch","is_anomaly":false,"incident_id":null} +{"id":"steady-009","level":"INFO","message":"session request completed within normal latency","is_anomaly":false,"incident_id":null} +{"id":"steady-010","level":"WARNING","message":"cache entry refreshed after normal expiry interval","is_anomaly":false,"incident_id":null} +{"id":"steady-011","level":"INFO","message":"profile request completed within normal latency","is_anomaly":false,"incident_id":null} +{"id":"steady-012","level":"INFO","message":"worker heartbeat received from secondary consumer","is_anomaly":false,"incident_id":null} +{"id":"db-001","level":"ERROR","message":"database connection pool timeout during checkout","is_anomaly":true,"incident_id":"database-timeout"} +{"id":"db-002","level":"ERROR","message":"database connection pool timeout during payment","is_anomaly":true,"incident_id":"database-timeout"} +{"id":"queue-001","level":"ERROR","message":"queue consumer lag exceeded the service objective","is_anomaly":true,"incident_id":"queue-backlog"} +{"id":"queue-002","level":"CRITICAL","message":"queue consumer stopped acknowledging new messages","is_anomaly":true,"incident_id":"queue-backlog"} From e321a459bd8f286f5cb6e579fe7fc74135e5936b Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 17:22:58 -0500 Subject: [PATCH 04/12] docs: add versioned labeled benchmark corpus --- benchmarks/corpus/v1/manifest.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 benchmarks/corpus/v1/manifest.json diff --git a/benchmarks/corpus/v1/manifest.json b/benchmarks/corpus/v1/manifest.json new file mode 100644 index 0000000..edb9fa0 --- /dev/null +++ b/benchmarks/corpus/v1/manifest.json @@ -0,0 +1,12 @@ +{ + "schema_version": "1.0", + "corpus_version": "v1", + "record_count": 16, + "purpose": "Deterministic quality and alert-burden regression evaluation.", + "data_classification": "Synthetic examples only; no customer or production log data.", + "label_definition": { + "is_anomaly": "True when a record belongs to the documented incident scenario.", + "incident_id": "Stable grouping identifier required for every anomalous record." + }, + "maintenance": "Changes require review, a version bump, and updated expected metrics." +} From d425c90a3af5ae8a868cb060b3b0aa7c781f5a90 Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 17:23:00 -0500 Subject: [PATCH 05/12] docs: add versioned labeled benchmark corpus --- docs/LABELED_BENCHMARKS.md | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 docs/LABELED_BENCHMARKS.md diff --git a/docs/LABELED_BENCHMARKS.md b/docs/LABELED_BENCHMARKS.md new file mode 100644 index 0000000..efc0a6b --- /dev/null +++ b/docs/LABELED_BENCHMARKS.md @@ -0,0 +1,46 @@ +# Labeled benchmark corpus + +The versioned corpus at `benchmarks/corpus/v1/events.jsonl` measures the deterministic baseline detector against reviewed, synthetic labels. It contains no customer, credential, or production log data. + +## Record contract + +Each non-empty JSONL record requires: + +```json +{ + "id": "db-001", + "level": "ERROR", + "message": "database connection pool timeout during checkout", + "is_anomaly": true, + "incident_id": "database-timeout" +} +``` + +Anomalous records must have an `incident_id`; normal records use `null`. The manifest records the corpus schema, version, data classification, label meaning, and record count. Corpus changes need review, a version increment, and re-baselined results. + +## Measurements + +`logsight benchmark` invokes the repository's deterministic anomaly detector and reports: + +- entry-level precision, recall, and F1; +- incident recall: the share of labeled incident groups with at least one alert; +- total alerts; +- alerts per 1,000 events; +- false alerts per 1,000 events; +- alerts per true incident. + +An alert-volume metric is not an availability or customer-impact metric. It quantifies detector output for this fixed corpus only. The corpus is a regression signal, not a claim that the detector is calibrated for every production workload. + +## Run and gate + +```bash +logsight benchmark +logsight benchmark \ + --min-precision 0.95 \ + --min-recall 0.90 \ + --max-alerts-per-1000-events 250 +``` + +Gates are intentionally supplied by the deployment or release process: the project does not hard-code a universal production threshold. A failed gate returns a non-zero exit code after printing measured values. + +Before promotion, use a representative, approved, de-identified corpus with an immutable version; evaluate both false-negative risk and alert fatigue; record the corpus version, detector threshold, and metrics with the release; and obtain human approval for any change to the label taxonomy or gate. From c27ac7f5fe1a3a1b4f577e10f07fd14c65f22fce Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 17:23:29 -0500 Subject: [PATCH 06/12] test: cover labeled benchmark evaluation --- tests/test_benchmarking.py | 150 +++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 tests/test_benchmarking.py diff --git a/tests/test_benchmarking.py b/tests/test_benchmarking.py new file mode 100644 index 0000000..9babb78 --- /dev/null +++ b/tests/test_benchmarking.py @@ -0,0 +1,150 @@ +"""Tests for labeled corpus benchmarking and quality gates.""" + +from __future__ import annotations + +import json + +import pytest + +from logsight.benchmarking import ( + LabeledLog, + evaluate_labeled_corpus, + load_labeled_corpus, + quality_gate_failures, +) +from logsight.parser import LogLevel + + +def _record( + identifier: str, + *, + level: str = "INFO", + is_anomaly: bool = False, + incident_id: str | None = None, +) -> dict[str, object]: + return { + "id": identifier, + "level": level, + "message": "steady service request completed normally", + "is_anomaly": is_anomaly, + "incident_id": incident_id, + } + + +def test_loads_valid_jsonl_corpus(tmp_path) -> None: + corpus_file = tmp_path / "corpus.jsonl" + corpus_file.write_text( + "\n".join( + [ + json.dumps(_record("normal-1")), + json.dumps( + _record( + "incident-1", + level="ERROR", + is_anomaly=True, + incident_id="database-timeout", + ) + ), + ] + ), + encoding="utf-8", + ) + + corpus = load_labeled_corpus(corpus_file) + + assert len(corpus) == 2 + assert corpus[1].level is LogLevel.ERROR + assert corpus[1].incident_id == "database-timeout" + + +def test_calculates_precision_recall_and_alert_burden() -> None: + corpus = ( + LabeledLog("true-positive", LogLevel.ERROR, "same length message", True, "incident-a"), + LabeledLog("false-positive", LogLevel.ERROR, "same length message", False), + LabeledLog("false-negative", LogLevel.WARNING, "same length message", True, "incident-b"), + LabeledLog("true-negative", LogLevel.INFO, "same length message", False), + ) + + metrics = evaluate_labeled_corpus(corpus) + + assert metrics.true_positives == 1 + assert metrics.false_positives == 1 + assert metrics.false_negatives == 1 + assert metrics.true_negatives == 1 + assert metrics.precision == 0.5 + assert metrics.recall == 0.5 + assert metrics.f1 == 0.5 + assert metrics.incident_recall == 0.5 + assert metrics.alert_count == 2 + assert metrics.alerts_per_1000_events == 500 + assert metrics.false_alerts_per_1000_events == 250 + assert metrics.alerts_per_true_incident == 1 + + +def test_quality_gate_failures_are_explicit() -> None: + metrics = evaluate_labeled_corpus( + ( + LabeledLog("alert", LogLevel.ERROR, "same length message", True, "incident-a"), + LabeledLog("normal", LogLevel.INFO, "same length message", False), + ) + ) + + assert quality_gate_failures(metrics, min_precision=1, min_recall=1) == () + failures = quality_gate_failures( + metrics, + min_precision=1, + min_recall=1, + max_alerts_per_1000_events=400, + ) + + assert failures == ("alerts per 1000 events 500.0000 exceeds 400.0000",) + + +@pytest.mark.parametrize( + ("records", "message"), + [ + ([], "at least one"), + ([{"id": "bad"}], "bounded message"), + ([_record("duplicate"), _record("duplicate")], "duplicate"), + ([_record("missing-incident", is_anomaly=True)], "require incident_id"), + ([_record("invalid-label", is_anomaly=1)], "boolean"), + ], +) +def test_rejects_invalid_corpus_records(tmp_path, records, message: str) -> None: + corpus_file = tmp_path / "corpus.jsonl" + corpus_file.write_text("\n".join(json.dumps(record) for record in records), encoding="utf-8") + + with pytest.raises(ValueError, match=message): + load_labeled_corpus(corpus_file) + + +def test_rejects_missing_or_malformed_corpus(tmp_path) -> None: + with pytest.raises(FileNotFoundError): + load_labeled_corpus(tmp_path / "missing.jsonl") + + malformed = tmp_path / "malformed.jsonl" + malformed.write_text("{not json}", encoding="utf-8") + with pytest.raises(ValueError, match="invalid JSON"): + load_labeled_corpus(malformed) + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"min_precision": -0.1}, "minimum precision"), + ({"min_recall": 1.1}, "minimum recall"), + ({"max_alerts_per_1000_events": -1}, "maximum alerts"), + ], +) +def test_rejects_invalid_quality_gates(kwargs, message: str) -> None: + metrics = evaluate_labeled_corpus( + (LabeledLog("normal", LogLevel.INFO, "same length message", False),) + ) + + with pytest.raises(ValueError, match=message): + quality_gate_failures(metrics, **kwargs) + + +def test_rejects_empty_in_memory_corpus() -> None: + with pytest.raises(ValueError, match="at least one"): + evaluate_labeled_corpus(()) From 077f3a402ea8d7526bd614d9331f14c4dfec2cb9 Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 17:23:42 -0500 Subject: [PATCH 07/12] test: exercise benchmark command --- tests/test_cli.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 6d6e8bf..4c66e29 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -123,3 +123,46 @@ def test_rejects_invalid_runbook_revision(self, tmp_path): assert result.exit_code == 1 assert "7-64 character identifier" in result.output + + + +class TestBenchmarkCommand: + def test_reports_quality_and_alert_burden_metrics(self, tmp_path): + corpus = tmp_path / "corpus.jsonl" + corpus.write_text( + "\n".join( + [ + '{"id":"normal","level":"INFO","message":"same length message","is_anomaly":false,"incident_id":null}', + '{"id":"alert","level":"ERROR","message":"same length message","is_anomaly":true,"incident_id":"incident-a"}', + ] + ), + encoding="utf-8", + ) + + result = CliRunner().invoke(main, ["benchmark", "--corpus", str(corpus)]) + + assert result.exit_code == 0 + assert "Labeled benchmark results" in result.output + assert "precision / recall / F1" in result.output + assert "alerts per 1,000 events" in result.output + + def test_fails_when_a_quality_gate_is_not_met(self, tmp_path): + corpus = tmp_path / "corpus.jsonl" + corpus.write_text( + '{"id":"normal","level":"INFO","message":"same length message","is_anomaly":false,"incident_id":null}', + encoding="utf-8", + ) + + result = CliRunner().invoke( + main, + [ + "benchmark", + "--corpus", + str(corpus), + "--min-recall", + "0.1", + ], + ) + + assert result.exit_code == 1 + assert "benchmark quality gate failed" in result.output From 58796a2587e211ff50a3069335a0974ea25e602c Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 17:23:51 -0500 Subject: [PATCH 08/12] docs: complete labeled benchmark roadmap item --- README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 71c48ae..98e39ec 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ flowchart LR - `logsight.redis_control.RedisAdmissionController` — atomic Redis deduplication and per-service rate-limit admission control. - `logsight.distributed.CeleryInferenceDispatcher` — bounded JSON Celery dispatch for asynchronous embedding work. - `logsight.investigator.IncidentInvestigator` — retrieval-grounded, version-pinned incident runbook investigator with evidence citations. +- `logsight.benchmarking` — versioned labeled-corpus evaluator for precision, recall, incident coverage, and alert burden. - `logsight.streaming.InMemoryStream` — bounded local async stream for tests and development. - `logsight.streaming.KafkaStream` — optional Kafka/Redpanda producer using `aiokafka`. - `logsight.storage.ClickHouseStore` and `QdrantStore` — HTTP adapters for durable analytical and semantic storage. @@ -95,6 +96,21 @@ pip install -e ".[api,kafka,observability,ai,parsing,redis]" The optional integrations are deliberately separated so a developer can use the original local-first CLI without downloading heavyweight ML or infrastructure clients. +### Labeled benchmark corpus + +The synthetic, versioned benchmark corpus provides a reproducible regression signal for +precision, recall, incident coverage, and alert burden. Run it directly or apply +deployment-owned quality gates: + +```bash +logsight benchmark +logsight benchmark --min-precision 0.95 --min-recall 0.90 \ + --max-alerts-per-1000-events 250 +``` + +The gate values are not universal production claims: choose them from a reviewed, +representative corpus and promotion policy. See [labeled benchmark controls](docs/LABELED_BENCHMARKS.md). + ### Incident investigator The incident investigator retrieves only approved local Markdown runbooks and returns @@ -266,7 +282,7 @@ Pull requests retain the existing formatting, linting, strict typing, unit/integ - [x] Celery/Ray distributed inference workers - [x] Helm chart with HPA/PDB/network policies - [x] RAG incident investigator with versioned runbooks/commits -- [ ] Labeled benchmark corpus with precision/recall and alert-burden metrics +- [x] Labeled benchmark corpus with precision/recall and alert-burden metrics ## Documentation @@ -278,6 +294,7 @@ Pull requests retain the existing formatting, linting, strict typing, unit/integ - [Incident investigator](docs/INCIDENT_INVESTIGATOR.md) - [Deployment and rollback checklist](docs/DEPLOYMENT.md) - [Benchmark methodology](docs/BENCHMARKING.md) +- [Labeled benchmark corpus](docs/LABELED_BENCHMARKS.md) - [Mathematical foundations](docs/MATHEMATICAL_FOUNDATIONS.md) - [Security policy](SECURITY.md) From 7a607a91de8d4682bebcf3aceadcd84acccc079c Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 17:23:57 -0500 Subject: [PATCH 09/12] docs: distinguish runtime and detection benchmarks --- docs/BENCHMARKING.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/BENCHMARKING.md b/docs/BENCHMARKING.md index cb894b7..5dc6a63 100644 --- a/docs/BENCHMARKING.md +++ b/docs/BENCHMARKING.md @@ -3,3 +3,10 @@ Run `pytest tests/test_benchmarks.py --benchmark-only --benchmark-json=benchmarks/latest.json`. The JSON includes distribution statistics such as min, max, mean, median, standard deviation, rounds, and operations per second. CI retains it as an artifact. Compare two runs with `pytest-benchmark compare benchmarks/baseline.json benchmarks/latest.json`. Use the same Python version, hardware class, power profile, input size, and warm-up settings. A microbenchmark is not an end-to-end service load test; user-concurrency, GPU, model-quality, and network metrics are not applicable to the current local CLI architecture. + +## Detection-quality benchmark + +The performance benchmark above measures runtime only. For labeled detection quality and alert +burden, run `logsight benchmark` against the versioned synthetic corpus. See +[Labeled benchmark corpus](LABELED_BENCHMARKS.md) for labels, metric definitions, and +promotion-gate guidance. From 54769e2442966bc9dadb34bbae90d195e2c579e7 Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 17:25:20 -0500 Subject: [PATCH 10/12] style: format benchmark evaluator --- logsight/benchmarking.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/logsight/benchmarking.py b/logsight/benchmarking.py index 55c3176..a95c3d9 100644 --- a/logsight/benchmarking.py +++ b/logsight/benchmarking.py @@ -58,7 +58,9 @@ def load_labeled_corpus(path: str | Path) -> tuple[LabeledLog, ...]: records: list[LabeledLog] = [] identifiers: set[str] = set() - for line_number, line in enumerate(corpus_path.read_text(encoding="utf-8").splitlines(), start=1): + for line_number, line in enumerate( + corpus_path.read_text(encoding="utf-8").splitlines(), start=1 + ): if not line.strip(): continue if len(records) >= _MAX_RECORDS: @@ -96,7 +98,9 @@ def evaluate_labeled_corpus( alerts = {id(entry) for entry in detect_anomalies(entries, zscore_threshold).anomalies} true_positives = sum( - 1 for entry, record in zip(entries, corpus, strict=True) if id(entry) in alerts and record.is_anomaly + 1 + for entry, record in zip(entries, corpus, strict=True) + if id(entry) in alerts and record.is_anomaly ) false_positives = sum( 1 @@ -175,7 +179,9 @@ def _parse_record(raw: Any, line_number: int) -> LabeledLog: identifier = _required_string(raw, "id", line_number) message = _required_string(raw, "message", line_number) if len(message) > _MAX_MESSAGE_CHARS: - raise ValueError(f"benchmark line {line_number} message exceeds {_MAX_MESSAGE_CHARS} characters") + raise ValueError( + f"benchmark line {line_number} message exceeds {_MAX_MESSAGE_CHARS} characters" + ) level_name = _required_string(raw, "level", line_number).upper() try: level = LogLevel(level_name) @@ -186,7 +192,9 @@ def _parse_record(raw: Any, line_number: int) -> LabeledLog: raise ValueError(f"benchmark line {line_number} requires boolean is_anomaly") incident_id = raw.get("incident_id") if incident_id is not None and (not isinstance(incident_id, str) or not incident_id.strip()): - raise ValueError(f"benchmark line {line_number} incident_id must be a non-empty string or null") + raise ValueError( + f"benchmark line {line_number} incident_id must be a non-empty string or null" + ) if label and incident_id is None: raise ValueError(f"benchmark line {line_number} anomalous records require incident_id") return LabeledLog( From f95b2c4ea36fa851ead4199c3f203383f348a968 Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 17:25:26 -0500 Subject: [PATCH 11/12] style: format benchmark command --- logsight/cli.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/logsight/cli.py b/logsight/cli.py index 76cdf0e..9dc26b8 100644 --- a/logsight/cli.py +++ b/logsight/cli.py @@ -11,7 +11,11 @@ from rich.table import Table from logsight.analyzer import AnomalyReport, detect_anomalies, error_rate_spike_details -from logsight.benchmarking import evaluate_labeled_corpus, load_labeled_corpus, quality_gate_failures +from logsight.benchmarking import ( + evaluate_labeled_corpus, + load_labeled_corpus, + quality_gate_failures, +) from logsight.investigator import IncidentInvestigator, RunbookRepository from logsight.parser import parse_file, parse_lines from logsight.reasoning import EvidenceExplanation, explain_report @@ -285,8 +289,10 @@ def benchmark_cmd( console.print("[bold]Labeled benchmark results[/bold]") console.print(f" events : [cyan]{metrics.total}[/cyan]") - console.print(f" precision / recall / F1 : {metrics.precision:.4f} / " - f"{metrics.recall:.4f} / {metrics.f1:.4f}") + console.print( + f" precision / recall / F1 : {metrics.precision:.4f} / " + f"{metrics.recall:.4f} / {metrics.f1:.4f}" + ) console.print(f" incident recall : {metrics.incident_recall:.4f}") console.print(f" alerts : {metrics.alert_count}") console.print(f" alerts per 1,000 events : {metrics.alerts_per_1000_events:.4f}") From f53d058480d4ed18fb12489b6a2c245231f90acb Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 17:44:22 -0500 Subject: [PATCH 12/12] style: format benchmark CLI tests --- tests/test_cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 4c66e29..39bb414 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -125,7 +125,6 @@ def test_rejects_invalid_runbook_revision(self, tmp_path): assert "7-64 character identifier" in result.output - class TestBenchmarkCommand: def test_reports_quality_and_alert_burden_metrics(self, tmp_path): corpus = tmp_path / "corpus.jsonl"