Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ flowchart LR
- `logsight.redis_control.RedisAdmissionController` — atomic Redis deduplication and per-service rate-limit admission control.
- `logsight.distributed.CeleryInferenceDispatcher` — bounded JSON Celery dispatch for asynchronous embedding work.
- `logsight.investigator.IncidentInvestigator` — retrieval-grounded, version-pinned incident runbook investigator with evidence citations.
- `logsight.benchmarking` — versioned labeled-corpus evaluator for precision, recall, incident coverage, and alert burden.
- `logsight.streaming.InMemoryStream` — bounded local async stream for tests and development.
- `logsight.streaming.KafkaStream` — optional Kafka/Redpanda producer using `aiokafka`.
- `logsight.storage.ClickHouseStore` and `QdrantStore` — HTTP adapters for durable analytical and semantic storage.
Expand Down Expand Up @@ -95,6 +96,21 @@ pip install -e ".[api,kafka,observability,ai,parsing,redis]"

The optional integrations are deliberately separated so a developer can use the original local-first CLI without downloading heavyweight ML or infrastructure clients.

### Labeled benchmark corpus

The synthetic, versioned benchmark corpus provides a reproducible regression signal for
precision, recall, incident coverage, and alert burden. Run it directly or apply
deployment-owned quality gates:

```bash
logsight benchmark
logsight benchmark --min-precision 0.95 --min-recall 0.90 \
--max-alerts-per-1000-events 250
```

The gate values are not universal production claims: choose them from a reviewed,
representative corpus and promotion policy. See [labeled benchmark controls](docs/LABELED_BENCHMARKS.md).

### Incident investigator

The incident investigator retrieves only approved local Markdown runbooks and returns
Expand Down Expand Up @@ -266,7 +282,7 @@ Pull requests retain the existing formatting, linting, strict typing, unit/integ
- [x] Celery/Ray distributed inference workers
- [x] Helm chart with HPA/PDB/network policies
- [x] RAG incident investigator with versioned runbooks/commits
- [ ] Labeled benchmark corpus with precision/recall and alert-burden metrics
- [x] Labeled benchmark corpus with precision/recall and alert-burden metrics

## Documentation

Expand All @@ -278,6 +294,7 @@ Pull requests retain the existing formatting, linting, strict typing, unit/integ
- [Incident investigator](docs/INCIDENT_INVESTIGATOR.md)
- [Deployment and rollback checklist](docs/DEPLOYMENT.md)
- [Benchmark methodology](docs/BENCHMARKING.md)
- [Labeled benchmark corpus](docs/LABELED_BENCHMARKS.md)
- [Mathematical foundations](docs/MATHEMATICAL_FOUNDATIONS.md)
- [Security policy](SECURITY.md)

Expand Down
16 changes: 16 additions & 0 deletions benchmarks/corpus/v1/events.jsonl
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{"id":"steady-001","level":"INFO","message":"checkout request completed within normal latency","is_anomaly":false,"incident_id":null}
{"id":"steady-002","level":"INFO","message":"catalog request completed within normal latency","is_anomaly":false,"incident_id":null}
{"id":"steady-003","level":"WARNING","message":"cache refresh deferred until next scheduled cycle","is_anomaly":false,"incident_id":null}
{"id":"steady-004","level":"INFO","message":"worker heartbeat received from primary consumer","is_anomaly":false,"incident_id":null}
{"id":"steady-005","level":"INFO","message":"inventory request completed within normal latency","is_anomaly":false,"incident_id":null}
{"id":"steady-006","level":"WARNING","message":"connection retry completed before request timeout","is_anomaly":false,"incident_id":null}
{"id":"steady-007","level":"INFO","message":"payment request completed within normal latency","is_anomaly":false,"incident_id":null}
{"id":"steady-008","level":"INFO","message":"queue consumer processed the expected message batch","is_anomaly":false,"incident_id":null}
{"id":"steady-009","level":"INFO","message":"session request completed within normal latency","is_anomaly":false,"incident_id":null}
{"id":"steady-010","level":"WARNING","message":"cache entry refreshed after normal expiry interval","is_anomaly":false,"incident_id":null}
{"id":"steady-011","level":"INFO","message":"profile request completed within normal latency","is_anomaly":false,"incident_id":null}
{"id":"steady-012","level":"INFO","message":"worker heartbeat received from secondary consumer","is_anomaly":false,"incident_id":null}
{"id":"db-001","level":"ERROR","message":"database connection pool timeout during checkout","is_anomaly":true,"incident_id":"database-timeout"}
{"id":"db-002","level":"ERROR","message":"database connection pool timeout during payment","is_anomaly":true,"incident_id":"database-timeout"}
{"id":"queue-001","level":"ERROR","message":"queue consumer lag exceeded the service objective","is_anomaly":true,"incident_id":"queue-backlog"}
{"id":"queue-002","level":"CRITICAL","message":"queue consumer stopped acknowledging new messages","is_anomaly":true,"incident_id":"queue-backlog"}
12 changes: 12 additions & 0 deletions benchmarks/corpus/v1/manifest.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{
"schema_version": "1.0",
"corpus_version": "v1",
"record_count": 16,
"purpose": "Deterministic quality and alert-burden regression evaluation.",
"data_classification": "Synthetic examples only; no customer or production log data.",
"label_definition": {
"is_anomaly": "True when a record belongs to the documented incident scenario.",
"incident_id": "Stable grouping identifier required for every anomalous record."
},
"maintenance": "Changes require review, a version bump, and updated expected metrics."
}
7 changes: 7 additions & 0 deletions docs/BENCHMARKING.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,10 @@
Run `pytest tests/test_benchmarks.py --benchmark-only --benchmark-json=benchmarks/latest.json`. The JSON includes distribution statistics such as min, max, mean, median, standard deviation, rounds, and operations per second. CI retains it as an artifact.

Compare two runs with `pytest-benchmark compare benchmarks/baseline.json benchmarks/latest.json`. Use the same Python version, hardware class, power profile, input size, and warm-up settings. A microbenchmark is not an end-to-end service load test; user-concurrency, GPU, model-quality, and network metrics are not applicable to the current local CLI architecture.

## Detection-quality benchmark

The performance benchmark above measures runtime only. For labeled detection quality and alert
burden, run `logsight benchmark` against the versioned synthetic corpus. See
[Labeled benchmark corpus](LABELED_BENCHMARKS.md) for labels, metric definitions, and
promotion-gate guidance.
46 changes: 46 additions & 0 deletions docs/LABELED_BENCHMARKS.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
# Labeled benchmark corpus

The versioned corpus at `benchmarks/corpus/v1/events.jsonl` measures the deterministic baseline detector against reviewed, synthetic labels. It contains no customer, credential, or production log data.

## Record contract

Each non-empty JSONL record requires:

```json
{
"id": "db-001",
"level": "ERROR",
"message": "database connection pool timeout during checkout",
"is_anomaly": true,
"incident_id": "database-timeout"
}
```

Anomalous records must have an `incident_id`; normal records use `null`. The manifest records the corpus schema, version, data classification, label meaning, and record count. Corpus changes need review, a version increment, and re-baselined results.

## Measurements

`logsight benchmark` invokes the repository's deterministic anomaly detector and reports:

- entry-level precision, recall, and F1;
- incident recall: the share of labeled incident groups with at least one alert;
- total alerts;
- alerts per 1,000 events;
- false alerts per 1,000 events;
- alerts per true incident.

An alert-volume metric is not an availability or customer-impact metric. It quantifies detector output for this fixed corpus only. The corpus is a regression signal, not a claim that the detector is calibrated for every production workload.

## Run and gate

```bash
logsight benchmark
logsight benchmark \
--min-precision 0.95 \
--min-recall 0.90 \
--max-alerts-per-1000-events 250
```

Gates are intentionally supplied by the deployment or release process: the project does not hard-code a universal production threshold. A failed gate returns a non-zero exit code after printing measured values.

Before promotion, use a representative, approved, de-identified corpus with an immutable version; evaluate both false-negative risk and alert fatigue; record the corpus version, detector threshold, and metrics with the release; and obtain human approval for any change to the label taxonomy or gate.
222 changes: 222 additions & 0 deletions logsight/benchmarking.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,222 @@
"""Reproducible labeled-corpus evaluation for LogSight detectors."""

from __future__ import annotations

import json
from collections.abc import Sequence
from dataclasses import dataclass
from math import isfinite
from pathlib import Path
from typing import Any

from logsight.analyzer import detect_anomalies
from logsight.parser import LogEntry, LogLevel

_MAX_CORPUS_BYTES = 5_000_000
_MAX_RECORDS = 10_000
_MAX_MESSAGE_CHARS = 4_000


@dataclass(frozen=True, slots=True)
class LabeledLog:
"""One synthetic or approved de-identified benchmark record."""

identifier: str
level: LogLevel
message: str
is_anomaly: bool
incident_id: str | None = None


@dataclass(frozen=True, slots=True)
class AlertMetrics:
"""Entry- and incident-level quality and alert-volume measurements."""

total: int
true_positives: int
false_positives: int
false_negatives: int
true_negatives: int
precision: float
recall: float
f1: float
incident_recall: float
alert_count: int
alerts_per_1000_events: float
false_alerts_per_1000_events: float
alerts_per_true_incident: float


def load_labeled_corpus(path: str | Path) -> tuple[LabeledLog, ...]:
"""Load a bounded JSONL corpus and enforce the published label contract."""

corpus_path = Path(path)
if not corpus_path.is_file():
raise FileNotFoundError(f"benchmark corpus was not found: {corpus_path}")
if corpus_path.stat().st_size > _MAX_CORPUS_BYTES:
raise ValueError(f"benchmark corpus exceeds {_MAX_CORPUS_BYTES} bytes")

records: list[LabeledLog] = []
identifiers: set[str] = set()
for line_number, line in enumerate(
corpus_path.read_text(encoding="utf-8").splitlines(), start=1
):
if not line.strip():
continue
if len(records) >= _MAX_RECORDS:
raise ValueError(f"benchmark corpus exceeds {_MAX_RECORDS} records")
try:
raw = json.loads(line)
except json.JSONDecodeError as exc:
raise ValueError(f"invalid JSON on benchmark line {line_number}") from exc
record = _parse_record(raw, line_number)
if record.identifier in identifiers:
raise ValueError(f"duplicate benchmark record id: {record.identifier}")
identifiers.add(record.identifier)
records.append(record)

if not records:
raise ValueError("benchmark corpus must contain at least one record")
return tuple(records)


def evaluate_labeled_corpus(
corpus: Sequence[LabeledLog], *, zscore_threshold: float = 2.5
) -> AlertMetrics:
"""Evaluate the built-in detector against immutable labels in a corpus."""

if not corpus:
raise ValueError("benchmark corpus must contain at least one record")
entries = [
LogEntry(
raw=f"{record.level.value} {record.message}",
level=record.level,
message=record.message,
)
for record in corpus
]
alerts = {id(entry) for entry in detect_anomalies(entries, zscore_threshold).anomalies}

true_positives = sum(
1
for entry, record in zip(entries, corpus, strict=True)
if id(entry) in alerts and record.is_anomaly
)
false_positives = sum(
1
for entry, record in zip(entries, corpus, strict=True)
if id(entry) in alerts and not record.is_anomaly
)
false_negatives = sum(
1
for entry, record in zip(entries, corpus, strict=True)
if id(entry) not in alerts and record.is_anomaly
)
true_negatives = len(corpus) - true_positives - false_positives - false_negatives

precision = _ratio(true_positives, true_positives + false_positives)
recall = _ratio(true_positives, true_positives + false_negatives)
f1 = _ratio(2 * precision * recall, precision + recall)
true_incidents = {record.incident_id for record in corpus if record.is_anomaly}
detected_incidents = {
record.incident_id
for entry, record in zip(entries, corpus, strict=True)
if record.is_anomaly and id(entry) in alerts
}
alert_count = true_positives + false_positives
return AlertMetrics(
total=len(corpus),
true_positives=true_positives,
false_positives=false_positives,
false_negatives=false_negatives,
true_negatives=true_negatives,
precision=precision,
recall=recall,
f1=f1,
incident_recall=_ratio(len(detected_incidents), len(true_incidents)),
alert_count=alert_count,
alerts_per_1000_events=1000 * _ratio(alert_count, len(corpus)),
false_alerts_per_1000_events=1000 * _ratio(false_positives, len(corpus)),
alerts_per_true_incident=_ratio(alert_count, len(true_incidents)),
)


def quality_gate_failures(
metrics: AlertMetrics,
*,
min_precision: float | None = None,
min_recall: float | None = None,
max_alerts_per_1000_events: float | None = None,
) -> tuple[str, ...]:
"""Return explicit quality-gate failures without choosing deployment policy."""

_validate_fraction("minimum precision", min_precision)
_validate_fraction("minimum recall", min_recall)
if max_alerts_per_1000_events is not None and (
not isfinite(max_alerts_per_1000_events) or max_alerts_per_1000_events < 0
):
raise ValueError("maximum alerts per 1000 events must be finite and non-negative")

failures: list[str] = []
if min_precision is not None and metrics.precision < min_precision:
failures.append(f"precision {metrics.precision:.4f} is below {min_precision:.4f}")
if min_recall is not None and metrics.recall < min_recall:
failures.append(f"recall {metrics.recall:.4f} is below {min_recall:.4f}")
if (
max_alerts_per_1000_events is not None
and metrics.alerts_per_1000_events > max_alerts_per_1000_events
):
failures.append(
"alerts per 1000 events "
f"{metrics.alerts_per_1000_events:.4f} exceeds {max_alerts_per_1000_events:.4f}"
)
return tuple(failures)


def _parse_record(raw: Any, line_number: int) -> LabeledLog:
if not isinstance(raw, dict):
raise ValueError(f"benchmark line {line_number} must be a JSON object")
identifier = _required_string(raw, "id", line_number)
message = _required_string(raw, "message", line_number)
if len(message) > _MAX_MESSAGE_CHARS:
raise ValueError(
f"benchmark line {line_number} message exceeds {_MAX_MESSAGE_CHARS} characters"
)
level_name = _required_string(raw, "level", line_number).upper()
try:
level = LogLevel(level_name)
except ValueError as exc:
raise ValueError(f"benchmark line {line_number} has an invalid level") from exc
label = raw.get("is_anomaly")
if type(label) is not bool:
raise ValueError(f"benchmark line {line_number} requires boolean is_anomaly")
incident_id = raw.get("incident_id")
if incident_id is not None and (not isinstance(incident_id, str) or not incident_id.strip()):
raise ValueError(
f"benchmark line {line_number} incident_id must be a non-empty string or null"
)
if label and incident_id is None:
raise ValueError(f"benchmark line {line_number} anomalous records require incident_id")
return LabeledLog(
identifier=identifier,
level=level,
message=message,
is_anomaly=label,
incident_id=incident_id,
)


def _required_string(raw: dict[str, Any], field: str, line_number: int) -> str:
value = raw.get(field)
if not isinstance(value, str) or not value.strip() or len(value) > 256:
raise ValueError(f"benchmark line {line_number} requires a bounded {field}")
return value.strip()


def _ratio(numerator: int | float, denominator: int | float) -> float:
return numerator / denominator if denominator else 0.0


def _validate_fraction(name: str, value: float | None) -> None:
if value is not None and (not isfinite(value) or not 0 <= value <= 1):
raise ValueError(f"{name} must be a finite value between zero and one")
Loading
Loading