diff --git a/README.md b/README.md index 208c81b..e4604bb 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ flowchart LR - `logsight.storage.ClickHouseStore` and `QdrantStore` — HTTP adapters for durable analytical and semantic storage. - `logsight.api` — optional FastAPI gateway with `/health`, `/metrics`, `/v1/logs`, `/v1/logs/batch`, and `/v1/logs/recent`. - `logsight.observability.span()` — no-op-safe OpenTelemetry instrumentation helper. +- `logsight.observability.correlate_with_active_span()` — attaches active trace/span IDs to normalized events without overwriting supplied correlation fields. ## Enterprise local stack @@ -79,6 +80,7 @@ Services exposed locally: | ClickHouse | 8123 | Log analytics/storage | | Qdrant | 6333 | Vector similarity search | | Redis | 6379 | Cache/dedup/rate limiting foundation | +| OTel Collector | 4317 / 4318 | OTLP trace and log intake | | Prometheus | 9090 | Metrics collection | | Grafana | 3000 | Dashboards | @@ -90,6 +92,18 @@ pip install -e ".[api,kafka,observability,ai,parsing,redis]" The optional integrations are deliberately separated so a developer can use the original local-first CLI without downloading heavyweight ML or infrastructure clients. +### OpenTelemetry Collector and trace–log correlation + +The enterprise reference stack includes an OTLP Collector. Set +`LOGSIGHT_OTEL_ENABLED=true` and `OTEL_EXPORTER_OTLP_ENDPOINT` to export ingestion +spans. The API preserves trace/span IDs found in log content and supplies missing +IDs from its active ingestion span, making each normalized event joinable to its +trace. The supplied Collector configuration uses a debug exporter for local +validation only; configure authenticated production backends before promotion. + +See [Collector deployment and correlation](docs/OTEL_COLLECTOR.md) for validation +and production-promotion guidance. + ### Redis admission controls Redis admission controls are disabled unless `LOGSIGHT_REDIS_URL` is configured. Once @@ -203,7 +217,7 @@ Pull requests retain the existing formatting, linting, strict typing, unit/integ - [x] LogBERT anomaly-model adapter - [x] ONNX Runtime inference worker - [x] Redis-backed deduplication/rate limiting -- [ ] OTel Collector deployment and trace-log correlation pipeline +- [x] OTel Collector deployment and trace-log correlation pipeline - [ ] Celery/Ray distributed inference workers - [ ] Helm chart with HPA/PDB/network policies - [ ] RAG incident investigator with versioned runbooks/commits @@ -213,6 +227,7 @@ Pull requests retain the existing formatting, linting, strict typing, unit/integ - [Production audit](docs/AUDIT.md) - [Architecture](docs/architecture.md) +- [Collector correlation](docs/OTEL_COLLECTOR.md) - [Deployment and rollback checklist](docs/DEPLOYMENT.md) - [Benchmark methodology](docs/BENCHMARKING.md) - [Mathematical foundations](docs/MATHEMATICAL_FOUNDATIONS.md) diff --git a/deploy/otel/collector.yml b/deploy/otel/collector.yml new file mode 100644 index 0000000..edaa3b1 --- /dev/null +++ b/deploy/otel/collector.yml @@ -0,0 +1,57 @@ +# Reference OpenTelemetry Collector pipeline for LogSight-AI. +# The debug exporters make local validation observable; replace them with +# authenticated, deployment-owned trace/log/metric backends before production. +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 256 + spike_limit_mib: 64 + batch: + send_batch_size: 1024 + timeout: 5s + attributes/log-correlation: + actions: + - key: service.name + action: insert + value: logsight-ai + +connectors: + spanmetrics: + dimensions: + - name: service.name + - name: http.route + +exporters: + debug: + verbosity: basic + +service: + extensions: [health_check] + telemetry: + metrics: + address: 0.0.0.0:8888 + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [spanmetrics, debug] + metrics: + receivers: [otlp, spanmetrics] + processors: [memory_limiter, batch] + exporters: [debug] + logs: + receivers: [otlp] + processors: [memory_limiter, attributes/log-correlation, batch] + exporters: [debug] diff --git a/deploy/prometheus.yml b/deploy/prometheus.yml index e528995..c27438d 100644 --- a/deploy/prometheus.yml +++ b/deploy/prometheus.yml @@ -7,3 +7,7 @@ scrape_configs: metrics_path: /metrics static_configs: - targets: ["logsight-api:8000"] + + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8888"] diff --git a/docker-compose.enterprise.yml b/docker-compose.enterprise.yml index d6b9810..7b15269 100644 --- a/docker-compose.enterprise.yml +++ b/docker-compose.enterprise.yml @@ -3,7 +3,7 @@ services: build: context: . args: - INSTALL_EXTRAS: api,redis + INSTALL_EXTRAS: api,redis,observability image: logsight-ai:enterprise entrypoint: [] command: ["uvicorn", "logsight.api:app", "--host", "0.0.0.0", "--port", "8000"] @@ -13,6 +13,9 @@ services: LOGSIGHT_RATE_LIMIT: "120" LOGSIGHT_RATE_WINDOW_SECONDS: "60" LOGSIGHT_DEDUP_TTL_SECONDS: "300" + LOGSIGHT_OTEL_ENABLED: "true" + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 + OTEL_SERVICE_NAME: logsight-api KAFKA_BOOTSTRAP_SERVERS: kafka:9092 CLICKHOUSE_URL: http://clickhouse:8123 QDRANT_URL: http://qdrant:6333 @@ -23,6 +26,23 @@ services: - clickhouse - qdrant - redis + - otel-collector + read_only: true + security_opt: + - no-new-privileges:true + cap_drop: [ALL] + tmpfs: + - /tmp:size=32m,mode=1777 + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.120.0 + command: ["--config=/etc/otelcol-contrib/collector.yml"] + volumes: + - ./deploy/otel/collector.yml:/etc/otelcol-contrib/collector.yml:ro + ports: + - "4317:4317" + - "4318:4318" + - "8888:8888" read_only: true security_opt: - no-new-privileges:true diff --git a/docs/OTEL_COLLECTOR.md b/docs/OTEL_COLLECTOR.md new file mode 100644 index 0000000..780930c --- /dev/null +++ b/docs/OTEL_COLLECTOR.md @@ -0,0 +1,37 @@ +# OpenTelemetry Collector and Trace–Log Correlation + +## Reference pipeline + +The enterprise Compose stack starts an OpenTelemetry Collector with OTLP/gRPC on +port 4317 and OTLP/HTTP on port 4318. Its reference configuration is +`deploy/otel/collector.yml`. + +When `LOGSIGHT_OTEL_ENABLED=true`, the API creates an OTLP tracer provider and +exports bounded ingestion spans to the Collector. Every normalized event keeps an +existing `trace_id` and `span_id` parsed from the log line; if either value is +absent, the API fills it from the active ingestion span. The API response and the +recent-event endpoint therefore carry identifiers that can be joined to exported +traces. + +## Local validation + +```bash +docker compose -f docker-compose.enterprise.yml up --build +curl -sS http://localhost:8000/v1/logs \ + -H 'content-type: application/json' \ + -d '{"line":"ERROR checkout timeout","service":"checkout"}' +``` + +The response event contains 32-hex-character `trace_id` and 16-hex-character +`span_id` values. The Collector's `debug` exporter prints the matching span, +and Prometheus scrapes its own pipeline metrics on port 8888. + +## Production promotion + +The debug exporters intentionally do not persist telemetry. Before a production +deployment, replace them with authenticated, TLS-protected exporters to the +organization's trace, log, and metric backends. Restrict OTLP listeners to trusted +workloads, set exporter retry/queue limits for the expected load, and propagate +W3C trace context at ingress. Trace IDs are correlation identifiers, not secrets; +treat log payloads and resource attributes according to their data-classification +policy. diff --git a/logsight/api.py b/logsight/api.py index 963fb27..8600413 100644 --- a/logsight/api.py +++ b/logsight/api.py @@ -8,6 +8,7 @@ from typing import Any from .enterprise import EnrichedLog, enrich, score_event +from .observability import configure_tracing, correlate_with_active_span, span from .parser import parse_line from .redis_control import RedisAdmissionController, build_redis_controller from .templates import TemplateParser, build_template_parser @@ -43,29 +44,34 @@ class BatchPayload(BaseModel): # type: ignore[misc] async def _process(item: LogPayload) -> tuple[EnrichedLog, Any]: - event = enrich( - parse_line(item.line), - service=item.service, - source=item.source, - host=item.host, - ) - if _admission_controller is not None: - decision = await _admission_controller.admit(event, item.service) - if not decision.accepted: - status_code = 409 if decision.reason == "duplicate" else 429 - raise HTTPException(status_code=status_code, detail=decision.reason) - event.template = _template_parser.parse(event.message) - _recent.append(event) - result = score_event(event, list(_recent)) - _metrics["logsight_ingested_total"] += 1 - if result.score >= 0.5: - _metrics["logsight_anomalies_total"] += 1 - return event, result + with span("logsight.ingest", service=item.service, source=item.source) as _: + event = correlate_with_active_span( + enrich( + parse_line(item.line), + service=item.service, + source=item.source, + host=item.host, + ) + ) + if _admission_controller is not None: + decision = await _admission_controller.admit(event, item.service) + if not decision.accepted: + status_code = 409 if decision.reason == "duplicate" else 429 + raise HTTPException(status_code=status_code, detail=decision.reason) + event.template = _template_parser.parse(event.message) + _recent.append(event) + result = score_event(event, list(_recent)) + _metrics["logsight_ingested_total"] += 1 + if result.score >= 0.5: + _metrics["logsight_anomalies_total"] += 1 + return event, result def create_app() -> Any: if FastAPI is None: raise RuntimeError("Install the 'api' extra to run the LogSight API") + if os.getenv("LOGSIGHT_OTEL_ENABLED", "").lower() == "true": + configure_tracing("logsight-api") app = FastAPI(title="LogSight-AI Enterprise API", version="0.2.0") @app.get("/health") diff --git a/logsight/observability.py b/logsight/observability.py index 8ffd594..50c6389 100644 --- a/logsight/observability.py +++ b/logsight/observability.py @@ -1,4 +1,4 @@ -"""Optional OpenTelemetry instrumentation helpers.""" +"""Optional OpenTelemetry instrumentation and correlation helpers.""" from __future__ import annotations @@ -6,18 +6,56 @@ from contextlib import contextmanager from typing import Any +from .enterprise import EnrichedLog + try: from opentelemetry import trace except ImportError: # pragma: no cover trace = None # type: ignore[assignment] +def configure_tracing(service_name: str) -> None: + """Configure an OTLP tracer provider for an explicitly enabled deployment.""" + + if trace is None: + raise RuntimeError("Install the observability extra to enable OpenTelemetry tracing") + try: + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk.resources import SERVICE_NAME, Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + except ImportError as exc: + raise RuntimeError( + "Install the observability extra to enable OpenTelemetry tracing" + ) from exc + + provider = TracerProvider(resource=Resource.create({SERVICE_NAME: service_name})) + provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) + trace.set_tracer_provider(provider) + + @contextmanager def span(name: str, **attributes: str | int | float) -> Iterator[Any]: """Create an OTel span when installed, otherwise remain a no-op.""" + if trace is None: yield None return tracer = trace.get_tracer("logsight-ai") with tracer.start_as_current_span(name, attributes=attributes) as current: yield current + + +def correlate_with_active_span(event: EnrichedLog) -> EnrichedLog: + """Fill missing event correlation fields from the current valid OTel span.""" + + if trace is None: + return event + context = trace.get_current_span().get_span_context() + if not context.is_valid: + return event + if event.trace_id is None: + event.trace_id = f"{context.trace_id:032x}" + if event.span_id is None: + event.span_id = f"{context.span_id:016x}" + return event diff --git a/tests/test_otel_collector.py b/tests/test_otel_collector.py new file mode 100644 index 0000000..92dc2be --- /dev/null +++ b/tests/test_otel_collector.py @@ -0,0 +1,29 @@ +"""Contract checks for the OpenTelemetry Collector reference deployment.""" + +from pathlib import Path + +from logsight.enterprise import enrich +from logsight.observability import correlate_with_active_span +from logsight.parser import parse_line + + +def test_collector_configuration_exposes_otlp_signal_pipelines() -> None: + configuration = Path("deploy/otel/collector.yml").read_text(encoding="utf-8") + + assert "grpc:" in configuration + assert "http:" in configuration + assert "traces:" in configuration + assert "logs:" in configuration + assert "spanmetrics:" in configuration + assert "memory_limiter" in configuration + + +def test_correlation_never_overwrites_log_supplied_identifiers() -> None: + event = enrich(parse_line("ERROR request failed")) + event.trace_id = "upstream" + event.span_id = "child" + + correlated = correlate_with_active_span(event) + + assert correlated.trace_id == "upstream" + assert correlated.span_id == "child"