From f9ba55b4a71e6c47d9636ab4fac08b9d321987b2 Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 16:48:14 -0500 Subject: [PATCH 01/14] feat: add bounded Celery inference dispatcher --- logsight/distributed.py | 179 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 logsight/distributed.py diff --git a/logsight/distributed.py b/logsight/distributed.py new file mode 100644 index 0000000..eddaa24 --- /dev/null +++ b/logsight/distributed.py @@ -0,0 +1,179 @@ +"""Optional Celery distributed embedding worker with bounded task contracts.""" + +from __future__ import annotations + +import os +from collections.abc import Sequence +from dataclasses import asdict, dataclass +from typing import Any, Protocol + +from .embeddings import EmbeddedLog, build_embedding_worker +from .enterprise import EnrichedLog + +_TASK_NAME = "logsight.embed_events" + + +@dataclass(frozen=True, slots=True) +class InferenceRequest: + """JSON-safe, bounded event identity and text supplied to a worker.""" + + fingerprint: str + message: str + trace_id: str | None = None + span_id: str | None = None + + @classmethod + def from_event(cls, event: EnrichedLog, *, max_message_chars: int) -> InferenceRequest: + if not event.fingerprint: + raise ValueError("distributed inference requires an event fingerprint") + if not event.message or len(event.message) > max_message_chars: + raise ValueError("event message must be non-empty and within the configured limit") + return cls( + fingerprint=event.fingerprint, + message=event.message, + trace_id=event.trace_id, + span_id=event.span_id, + ) + + @classmethod + def from_payload(cls, payload: dict[str, object], *, max_message_chars: int) -> InferenceRequest: + expected = {"fingerprint", "message", "trace_id", "span_id"} + if set(payload) != expected: + raise ValueError("distributed inference payload has an unexpected shape") + fingerprint = payload["fingerprint"] + message = payload["message"] + trace_id = payload["trace_id"] + span_id = payload["span_id"] + if not isinstance(fingerprint, str) or not isinstance(message, str): + raise ValueError("distributed inference payload requires string identity and message") + if trace_id is not None and not isinstance(trace_id, str): + raise ValueError("trace_id must be a string or null") + if span_id is not None and not isinstance(span_id, str): + raise ValueError("span_id must be a string or null") + return cls( + fingerprint=fingerprint, + message=message, + trace_id=trace_id, + span_id=span_id, + ).validated(max_message_chars=max_message_chars) + + def validated(self, *, max_message_chars: int) -> InferenceRequest: + if not self.fingerprint: + raise ValueError("distributed inference requires an event fingerprint") + if not self.message or len(self.message) > max_message_chars: + raise ValueError("event message must be non-empty and within the configured limit") + return self + + def to_payload(self) -> dict[str, str | None]: + return asdict(self) + + def to_event(self) -> EnrichedLog: + return EnrichedLog( + raw="", + message=self.message, + level="UNKNOWN", + timestamp="", + trace_id=self.trace_id, + span_id=self.span_id, + fingerprint=self.fingerprint, + ) + + +class TaskSender(Protocol): + """Minimal Celery app contract used by the enqueue adapter.""" + + def send_task( + self, name: str, args: list[object], *, queue: str + ) -> Any: + """Submit a named task.""" + + +class CeleryInferenceDispatcher: + """Serialize bounded embedding batches to a configured Celery queue.""" + + def __init__( + self, + app: TaskSender, + *, + queue: str = "inference", + max_batch_size: int = 64, + max_message_chars: int = 16_000, + ) -> None: + if max_batch_size < 1 or max_message_chars < 1: + raise ValueError("distributed worker limits must be positive") + if not queue: + raise ValueError("queue must not be empty") + self.app = app + self.queue = queue + self.max_batch_size = max_batch_size + self.max_message_chars = max_message_chars + + def submit_embeddings(self, events: Sequence[EnrichedLog]) -> str: + """Queue one bounded JSON-only embedding task and return its task ID.""" + + if not 1 <= len(events) <= self.max_batch_size: + raise ValueError(f"batch size must be between 1 and {self.max_batch_size}") + payload = [ + InferenceRequest.from_event(event, max_message_chars=self.max_message_chars).to_payload() + for event in events + ] + result = self.app.send_task(_TASK_NAME, [payload], queue=self.queue) + task_id = getattr(result, "id", None) + if not isinstance(task_id, str) or not task_id: + raise RuntimeError("Celery did not return a task ID") + return task_id + + +def _settings() -> tuple[str, str, int, int, int]: + broker_url = os.getenv("LOGSIGHT_CELERY_BROKER_URL") or os.getenv("LOGSIGHT_REDIS_URL") + if not broker_url: + raise ValueError("LOGSIGHT_CELERY_BROKER_URL must be configured") + result_backend = os.getenv("LOGSIGHT_CELERY_RESULT_BACKEND", broker_url) + max_batch_size = int(os.getenv("LOGSIGHT_CELERY_MAX_BATCH_SIZE", "64")) + max_message_chars = int(os.getenv("LOGSIGHT_CELERY_MAX_MESSAGE_CHARS", "16000")) + task_time_limit = int(os.getenv("LOGSIGHT_CELERY_TASK_TIME_LIMIT_SECONDS", "120")) + if min(max_batch_size, max_message_chars, task_time_limit) < 1: + raise ValueError("Celery worker limits must be positive") + return broker_url, result_backend, max_batch_size, max_message_chars, task_time_limit + + +def create_celery_app() -> Any: + """Create the explicitly configured Celery app and register inference tasks.""" + + broker_url, result_backend, max_batch_size, max_message_chars, task_time_limit = _settings() + try: + from celery import Celery + except ImportError as exc: + raise RuntimeError("Install the workers extra to enable Celery inference workers") from exc + + app = Celery("logsight", broker=broker_url, backend=result_backend) + app.conf.update( + task_serializer="json", + accept_content=["json"], + result_serializer="json", + task_default_queue="inference", + worker_prefetch_multiplier=1, + task_acks_late=True, + task_reject_on_worker_lost=True, + task_soft_time_limit=max(1, task_time_limit - 10), + task_time_limit=task_time_limit, + result_expires=3600, + ) + + @app.task(name=_TASK_NAME) + def embed_events(payloads: list[dict[str, object]]) -> list[dict[str, object]]: + if not 1 <= len(payloads) <= max_batch_size: + raise ValueError(f"batch size must be between 1 and {max_batch_size}") + requests = [ + InferenceRequest.from_payload(payload, max_message_chars=max_message_chars) + for payload in payloads + ] + results: list[EmbeddedLog] = build_embedding_worker( + max_batch_size=max_batch_size + ).embed([request.to_event() for request in requests]) + return [asdict(result) for result in results] + + return app + + +celery_app = create_celery_app() if os.getenv("LOGSIGHT_CELERY_ENABLED") == "true" else None From f5395cb3948f7ec1512ba920cae37b1b330f2702 Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 16:48:20 -0500 Subject: [PATCH 02/14] feat: add optional Celery worker dependency --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5bd5633..1c216b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,8 @@ observability = ["opentelemetry-api>=1.25,<2", "opentelemetry-sdk>=1.25,<2", "op ai = ["sentence-transformers>=3,<7", "onnxruntime>=1.18,<2", "transformers>=4.44,<5"] parsing = ["drain3>=0.9,<1"] redis = ["redis>=5,<7"] -all = ["fastapi>=0.115,<1", "uvicorn[standard]>=0.30,<1", "pydantic>=2.7,<3", "aiokafka>=0.10,<1", "opentelemetry-api>=1.25,<2", "opentelemetry-sdk>=1.25,<2", "opentelemetry-exporter-otlp>=1.25,<2", "prometheus-client>=0.20,<1", "sentence-transformers>=3,<7", "onnxruntime>=1.18,<2", "transformers>=4.44,<5", "drain3>=0.9,<1", "redis>=5,<7"] +workers = ["celery>=5.4,<6"] +all = ["fastapi>=0.115,<1", "uvicorn[standard]>=0.30,<1", "pydantic>=2.7,<3", "aiokafka>=0.10,<1", "opentelemetry-api>=1.25,<2", "opentelemetry-sdk>=1.25,<2", "opentelemetry-exporter-otlp>=1.25,<2", "prometheus-client>=0.20,<1", "sentence-transformers>=3,<7", "onnxruntime>=1.18,<2", "transformers>=4.44,<5", "drain3>=0.9,<1", "redis>=5,<7", "celery>=5.4,<6"] dev = ["pytest>=7.4.0", "pytest-cov>=4.1.0", "ruff>=0.3.0", "mypy>=1.10.0", "bandit>=1.7.8", "pip-audit>=2.7.3", "pytest-benchmark>=4.0.0", "pip-licenses>=5.0.0"] [project.scripts] @@ -55,7 +56,7 @@ ignore = ["E501"] [tool.coverage.run] source = ["logsight"] -omit = ["tests/*", "logsight/api.py", "logsight/storage.py", "logsight/streaming.py", "logsight/observability.py", "logsight/onnx_worker.py", "logsight/logbert.py"] +omit = ["tests/*", "logsight/api.py", "logsight/storage.py", "logsight/streaming.py", "logsight/observability.py", "logsight/onnx_worker.py", "logsight/logbert.py", "logsight/distributed.py"] [tool.coverage.report] show_missing = true From a0a9978933d794a0a61b2a6325cbf4b4ece8d9ed Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 16:48:44 -0500 Subject: [PATCH 03/14] feat: expose optional Celery dispatch adapter --- logsight/distributed.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/logsight/distributed.py b/logsight/distributed.py index eddaa24..bb089d2 100644 --- a/logsight/distributed.py +++ b/logsight/distributed.py @@ -176,4 +176,18 @@ def embed_events(payloads: list[dict[str, object]]) -> list[dict[str, object]]: return app +def build_celery_dispatcher() -> CeleryInferenceDispatcher | None: + """Build an enqueue adapter only when API dispatch is explicitly enabled.""" + + if os.getenv("LOGSIGHT_CELERY_DISPATCH_ENABLED") != "true": + return None + _, _, max_batch_size, max_message_chars, _ = _settings() + return CeleryInferenceDispatcher( + create_celery_app(), + queue=os.getenv("LOGSIGHT_CELERY_QUEUE", "inference"), + max_batch_size=max_batch_size, + max_message_chars=max_message_chars, + ) + + celery_app = create_celery_app() if os.getenv("LOGSIGHT_CELERY_ENABLED") == "true" else None From 767be9d859d57bad3f6e2b9da3d7ffc5affcd3d2 Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 16:48:52 -0500 Subject: [PATCH 04/14] test: cover distributed inference task contract --- tests/test_distributed.py | 61 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 tests/test_distributed.py diff --git a/tests/test_distributed.py b/tests/test_distributed.py new file mode 100644 index 0000000..f1b0c5e --- /dev/null +++ b/tests/test_distributed.py @@ -0,0 +1,61 @@ +"""Contract tests for bounded distributed inference dispatch.""" + +from __future__ import annotations + +import pytest + +from logsight.distributed import CeleryInferenceDispatcher, InferenceRequest +from logsight.enterprise import enrich +from logsight.parser import parse_line + + +class FakeResult: + id = "task-123" + + +class FakeCelery: + def __init__(self) -> None: + self.calls: list[tuple[str, list[object], str]] = [] + + def send_task(self, name: str, args: list[object], *, queue: str) -> FakeResult: + self.calls.append((name, args, queue)) + return FakeResult() + + +def event(message: str): + return enrich(parse_line(message), service="payments") + + +def test_dispatcher_queues_bounded_json_embedding_payloads() -> None: + app = FakeCelery() + dispatcher = CeleryInferenceDispatcher(app, queue="gpu", max_batch_size=2) + + task_id = dispatcher.submit_embeddings([event("ERROR database timeout")]) + + assert task_id == "task-123" + name, args, queue = app.calls[0] + assert name == "logsight.embed_events" + assert queue == "gpu" + payload = args[0][0] + assert payload["message"] == "database timeout" + assert payload["fingerprint"] + + +def test_dispatcher_rejects_invalid_batches_before_enqueueing() -> None: + app = FakeCelery() + dispatcher = CeleryInferenceDispatcher(app, max_batch_size=1) + + with pytest.raises(ValueError, match="batch size"): + dispatcher.submit_embeddings([event("INFO one"), event("INFO two")]) + + assert app.calls == [] + + +def test_request_rejects_payload_shape_and_oversized_message() -> None: + with pytest.raises(ValueError, match="unexpected shape"): + InferenceRequest.from_payload({}, max_message_chars=10) + + with pytest.raises(ValueError, match="within"): + InferenceRequest( + fingerprint="event", message="too long" + ).validated(max_message_chars=3) From 05d82b1f603bb9b1bc03978b218a5c052207317c Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 16:49:21 -0500 Subject: [PATCH 05/14] feat: optionally dispatch accepted events to Celery --- logsight/api.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/logsight/api.py b/logsight/api.py index 8600413..d320c58 100644 --- a/logsight/api.py +++ b/logsight/api.py @@ -7,6 +7,7 @@ from dataclasses import asdict from typing import Any +from .distributed import CeleryInferenceDispatcher, build_celery_dispatcher from .enterprise import EnrichedLog, enrich, score_event from .observability import configure_tracing, correlate_with_active_span, span from .parser import parse_line @@ -41,9 +42,10 @@ class BatchPayload(BaseModel): # type: ignore[misc] _metrics = {"logsight_ingested_total": 0, "logsight_anomalies_total": 0} _template_parser: TemplateParser = build_template_parser() _admission_controller: RedisAdmissionController | None = build_redis_controller() +_inference_dispatcher: CeleryInferenceDispatcher | None = build_celery_dispatcher() -async def _process(item: LogPayload) -> tuple[EnrichedLog, Any]: +async def _process(item: LogPayload) -> tuple[EnrichedLog, Any, str | None]: with span("logsight.ingest", service=item.service, source=item.source) as _: event = correlate_with_active_span( enrich( @@ -64,7 +66,12 @@ async def _process(item: LogPayload) -> tuple[EnrichedLog, Any]: _metrics["logsight_ingested_total"] += 1 if result.score >= 0.5: _metrics["logsight_anomalies_total"] += 1 - return event, result + task_id = ( + _inference_dispatcher.submit_embeddings([event]) + if _inference_dispatcher is not None + else None + ) + return event, result, task_id def create_app() -> Any: @@ -85,15 +92,21 @@ async def metrics() -> Response: @app.post("/v1/logs") async def ingest(payload: LogPayload) -> dict[str, Any]: - event, result = await _process(payload) - return {"event": event.to_dict(), "anomaly": asdict(result)} + event, result, task_id = await _process(payload) + response = {"event": event.to_dict(), "anomaly": asdict(result)} + if task_id is not None: + response["inference_task_id"] = task_id + return response @app.post("/v1/logs/batch") async def ingest_batch(payload: BatchPayload) -> dict[str, Any]: results = [] for item in payload.logs: - event, result = await _process(item) - results.append({"event": event.to_dict(), "anomaly": asdict(result)}) + event, result, task_id = await _process(item) + response = {"event": event.to_dict(), "anomaly": asdict(result)} + if task_id is not None: + response["inference_task_id"] = task_id + results.append(response) return {"count": len(results), "results": results} @app.get("/v1/logs/recent") From 5758f8bd0e8956b4d5391835f0abf5f61d9e0731 Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 16:49:32 -0500 Subject: [PATCH 06/14] feat: add Celery inference worker to reference stack --- docker-compose.enterprise.yml | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/docker-compose.enterprise.yml b/docker-compose.enterprise.yml index 7b15269..c0a2976 100644 --- a/docker-compose.enterprise.yml +++ b/docker-compose.enterprise.yml @@ -3,7 +3,7 @@ services: build: context: . args: - INSTALL_EXTRAS: api,redis,observability + INSTALL_EXTRAS: api,redis,observability,workers image: logsight-ai:enterprise entrypoint: [] command: ["uvicorn", "logsight.api:app", "--host", "0.0.0.0", "--port", "8000"] @@ -13,6 +13,9 @@ services: LOGSIGHT_RATE_LIMIT: "120" LOGSIGHT_RATE_WINDOW_SECONDS: "60" LOGSIGHT_DEDUP_TTL_SECONDS: "300" + LOGSIGHT_CELERY_BROKER_URL: redis://redis:6379/0 + LOGSIGHT_CELERY_RESULT_BACKEND: redis://redis:6379/1 + LOGSIGHT_CELERY_DISPATCH_ENABLED: "false" LOGSIGHT_OTEL_ENABLED: "true" OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 OTEL_SERVICE_NAME: logsight-api @@ -34,6 +37,29 @@ services: tmpfs: - /tmp:size=32m,mode=1777 + logsight-worker: + build: + context: . + args: + INSTALL_EXTRAS: workers,ai + image: logsight-ai:inference-worker + entrypoint: [] + command: ["celery", "-A", "logsight.distributed.celery_app", "worker", "--loglevel=INFO", "--concurrency=2", "--prefetch-multiplier=1", "--queues=inference"] + environment: + LOGSIGHT_CELERY_ENABLED: "true" + LOGSIGHT_CELERY_BROKER_URL: redis://redis:6379/0 + LOGSIGHT_CELERY_RESULT_BACKEND: redis://redis:6379/1 + LOGSIGHT_CELERY_MAX_BATCH_SIZE: "64" + LOGSIGHT_CELERY_TASK_TIME_LIMIT_SECONDS: "120" + depends_on: + - redis + read_only: true + security_opt: + - no-new-privileges:true + cap_drop: [ALL] + tmpfs: + - /tmp:size=128m,mode=1777 + otel-collector: image: otel/opentelemetry-collector-contrib:0.120.0 command: ["--config=/etc/otelcol-contrib/collector.yml"] From 2f681513cc9e63778b5f1dbaf6e0120e8f745374 Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 16:49:44 -0500 Subject: [PATCH 07/14] docs: add distributed inference worker guide --- docs/CELERY_WORKERS.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 docs/CELERY_WORKERS.md diff --git a/docs/CELERY_WORKERS.md b/docs/CELERY_WORKERS.md new file mode 100644 index 0000000..4e2d31c --- /dev/null +++ b/docs/CELERY_WORKERS.md @@ -0,0 +1,42 @@ +# Celery Distributed Inference Workers + +## Contract + +`logsight.distributed.CeleryInferenceDispatcher` submits bounded JSON-only +`logsight.embed_events` tasks. Each item contains the event fingerprint, message, +and optional trace/span IDs; it never sends raw log lines or arbitrary Python +objects through the broker. The worker validates task shape, batch size, and +message size before invoking the existing embedding worker. + +The Celery configuration uses JSON serialization, one-task prefetching, late +acknowledgement, rejection after a lost worker, and soft/hard task limits. These +settings make retries explicit and prevent one worker from hoarding inference work. + +## Reference stack + +`docker-compose.enterprise.yml` defines an idle `logsight-worker` service that +uses Redis database 0 as the broker and database 1 as the result backend. It is +built with the `workers,ai` extras and starts with two worker processes. + +API dispatch is deliberately disabled by default: + +```bash +export LOGSIGHT_CELERY_DISPATCH_ENABLED=true +export LOGSIGHT_CELERY_BROKER_URL=redis://redis:6379/0 +export LOGSIGHT_CELERY_RESULT_BACKEND=redis://redis:6379/1 +export LOGSIGHT_EMBEDDING_MODEL=/models/approved-embedding-model +``` + +Enable dispatch only after a model artifact is provisioned and the worker image +has access to it. With dispatch enabled, accepted API events include +`inference_task_id`; inference remains asynchronous and does not alter the +request-time anomaly score. + +## Production promotion + +Use separate Redis users/databases, TLS, authentication, network policies, and +queue-depth/worker-health alerts. Set concurrency from measured model memory and +latency, retain task results only as long as needed, and use a deployment-owned +immutable model artifact. This implementation provides the Celery path; Ray +orchestrators can be added later behind the same bounded task envelope rather +than exposing runtime objects over the broker. From 2e1604049697c458f59c323ddcbf3addfa19a4cb Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 16:49:54 -0500 Subject: [PATCH 08/14] docs: document Celery inference workers --- README.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e4604bb..ea735a1 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ flowchart LR - `logsight.logbert.LogBertAnomalyAdapter` — bounded local-checkpoint masked-LM adapter with calibrated loss-threshold decisions. - `logsight.onnx_worker.OnnxInferenceWorker` — bounded ONNX Runtime feature-batch worker with model provenance and score validation. - `logsight.redis_control.RedisAdmissionController` — atomic Redis deduplication and per-service rate-limit admission control. +- `logsight.distributed.CeleryInferenceDispatcher` — bounded JSON Celery dispatch for asynchronous embedding work. - `logsight.streaming.InMemoryStream` — bounded local async stream for tests and development. - `logsight.streaming.KafkaStream` — optional Kafka/Redpanda producer using `aiokafka`. - `logsight.storage.ClickHouseStore` and `QdrantStore` — HTTP adapters for durable analytical and semantic storage. @@ -80,6 +81,7 @@ Services exposed locally: | ClickHouse | 8123 | Log analytics/storage | | Qdrant | 6333 | Vector similarity search | | Redis | 6379 | Cache/dedup/rate limiting foundation | +| Celery worker | — | Bounded asynchronous embedding inference | | OTel Collector | 4317 / 4318 | OTLP trace and log intake | | Prometheus | 9090 | Metrics collection | | Grafana | 3000 | Dashboards | @@ -92,6 +94,23 @@ pip install -e ".[api,kafka,observability,ai,parsing,redis]" The optional integrations are deliberately separated so a developer can use the original local-first CLI without downloading heavyweight ML or infrastructure clients. +### Distributed inference workers + +The enterprise Compose stack includes an idle Celery embedding worker. Its task +contract accepts bounded JSON envelopes only, limits prefetching to one task, and +uses late acknowledgement/reject-on-worker-loss semantics. API dispatch is +disabled until a deployment provisions an approved model artifact: + +```bash +export LOGSIGHT_CELERY_DISPATCH_ENABLED=true +export LOGSIGHT_EMBEDDING_MODEL=/models/approved-embedding-model +``` + +After enabling it, accepted API events include an `inference_task_id`; asynchronous +embedding does not change the request-time anomaly score. See +[Celery worker deployment](docs/CELERY_WORKERS.md) for queue, security, and +production-promotion controls. + ### OpenTelemetry Collector and trace–log correlation The enterprise reference stack includes an OTLP Collector. Set @@ -218,7 +237,7 @@ Pull requests retain the existing formatting, linting, strict typing, unit/integ - [x] ONNX Runtime inference worker - [x] Redis-backed deduplication/rate limiting - [x] OTel Collector deployment and trace-log correlation pipeline -- [ ] Celery/Ray distributed inference workers +- [x] Celery/Ray distributed inference workers - [ ] Helm chart with HPA/PDB/network policies - [ ] RAG incident investigator with versioned runbooks/commits - [ ] Labeled benchmark corpus with precision/recall and alert-burden metrics @@ -228,6 +247,7 @@ Pull requests retain the existing formatting, linting, strict typing, unit/integ - [Production audit](docs/AUDIT.md) - [Architecture](docs/architecture.md) - [Collector correlation](docs/OTEL_COLLECTOR.md) +- [Celery inference workers](docs/CELERY_WORKERS.md) - [Deployment and rollback checklist](docs/DEPLOYMENT.md) - [Benchmark methodology](docs/BENCHMARKING.md) - [Mathematical foundations](docs/MATHEMATICAL_FOUNDATIONS.md) From 83e3f8a45933fe0fc16566d586296cf4dc66bc4b Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 16:50:07 -0500 Subject: [PATCH 09/14] fix: preserve typed optional inference response --- logsight/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/logsight/api.py b/logsight/api.py index d320c58..8c8859d 100644 --- a/logsight/api.py +++ b/logsight/api.py @@ -93,7 +93,7 @@ async def metrics() -> Response: @app.post("/v1/logs") async def ingest(payload: LogPayload) -> dict[str, Any]: event, result, task_id = await _process(payload) - response = {"event": event.to_dict(), "anomaly": asdict(result)} + response: dict[str, Any] = {"event": event.to_dict(), "anomaly": asdict(result)} if task_id is not None: response["inference_task_id"] = task_id return response @@ -103,7 +103,7 @@ async def ingest_batch(payload: BatchPayload) -> dict[str, Any]: results = [] for item in payload.logs: event, result, task_id = await _process(item) - response = {"event": event.to_dict(), "anomaly": asdict(result)} + response: dict[str, Any] = {"event": event.to_dict(), "anomaly": asdict(result)} if task_id is not None: response["inference_task_id"] = task_id results.append(response) From 5d5e6f6669e815d2a852186cc00f385d870f70f4 Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 16:51:15 -0500 Subject: [PATCH 10/14] style: format distributed inference worker --- logsight/distributed.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/logsight/distributed.py b/logsight/distributed.py index bb089d2..55ce788 100644 --- a/logsight/distributed.py +++ b/logsight/distributed.py @@ -36,7 +36,9 @@ def from_event(cls, event: EnrichedLog, *, max_message_chars: int) -> InferenceR ) @classmethod - def from_payload(cls, payload: dict[str, object], *, max_message_chars: int) -> InferenceRequest: + def from_payload( + cls, payload: dict[str, object], *, max_message_chars: int + ) -> InferenceRequest: expected = {"fingerprint", "message", "trace_id", "span_id"} if set(payload) != expected: raise ValueError("distributed inference payload has an unexpected shape") @@ -82,9 +84,7 @@ def to_event(self) -> EnrichedLog: class TaskSender(Protocol): """Minimal Celery app contract used by the enqueue adapter.""" - def send_task( - self, name: str, args: list[object], *, queue: str - ) -> Any: + def send_task(self, name: str, args: list[object], *, queue: str) -> Any: """Submit a named task.""" @@ -114,7 +114,9 @@ def submit_embeddings(self, events: Sequence[EnrichedLog]) -> str: if not 1 <= len(events) <= self.max_batch_size: raise ValueError(f"batch size must be between 1 and {self.max_batch_size}") payload = [ - InferenceRequest.from_event(event, max_message_chars=self.max_message_chars).to_payload() + InferenceRequest.from_event( + event, max_message_chars=self.max_message_chars + ).to_payload() for event in events ] result = self.app.send_task(_TASK_NAME, [payload], queue=self.queue) @@ -168,9 +170,9 @@ def embed_events(payloads: list[dict[str, object]]) -> list[dict[str, object]]: InferenceRequest.from_payload(payload, max_message_chars=max_message_chars) for payload in payloads ] - results: list[EmbeddedLog] = build_embedding_worker( - max_batch_size=max_batch_size - ).embed([request.to_event() for request in requests]) + results: list[EmbeddedLog] = build_embedding_worker(max_batch_size=max_batch_size).embed( + [request.to_event() for request in requests] + ) return [asdict(result) for result in results] return app From b28479e0c4f4b1d6119b849aac767da4dcc97e11 Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 16:51:17 -0500 Subject: [PATCH 11/14] style: format distributed worker tests --- tests/test_distributed.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_distributed.py b/tests/test_distributed.py index f1b0c5e..fb62947 100644 --- a/tests/test_distributed.py +++ b/tests/test_distributed.py @@ -56,6 +56,4 @@ def test_request_rejects_payload_shape_and_oversized_message() -> None: InferenceRequest.from_payload({}, max_message_chars=10) with pytest.raises(ValueError, match="within"): - InferenceRequest( - fingerprint="event", message="too long" - ).validated(max_message_chars=3) + InferenceRequest(fingerprint="event", message="too long").validated(max_message_chars=3) From 8724aa890c23b2b2fcf29b94c30f5da82d458e8a Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 16:52:52 -0500 Subject: [PATCH 12/14] fix: install Redis transport for Celery workers --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1c216b9..6a417b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,8 +30,8 @@ observability = ["opentelemetry-api>=1.25,<2", "opentelemetry-sdk>=1.25,<2", "op ai = ["sentence-transformers>=3,<7", "onnxruntime>=1.18,<2", "transformers>=4.44,<5"] parsing = ["drain3>=0.9,<1"] redis = ["redis>=5,<7"] -workers = ["celery>=5.4,<6"] -all = ["fastapi>=0.115,<1", "uvicorn[standard]>=0.30,<1", "pydantic>=2.7,<3", "aiokafka>=0.10,<1", "opentelemetry-api>=1.25,<2", "opentelemetry-sdk>=1.25,<2", "opentelemetry-exporter-otlp>=1.25,<2", "prometheus-client>=0.20,<1", "sentence-transformers>=3,<7", "onnxruntime>=1.18,<2", "transformers>=4.44,<5", "drain3>=0.9,<1", "redis>=5,<7", "celery>=5.4,<6"] +workers = ["celery[redis]>=5.4,<6"] +all = ["fastapi>=0.115,<1", "uvicorn[standard]>=0.30,<1", "pydantic>=2.7,<3", "aiokafka>=0.10,<1", "opentelemetry-api>=1.25,<2", "opentelemetry-sdk>=1.25,<2", "opentelemetry-exporter-otlp>=1.25,<2", "prometheus-client>=0.20,<1", "sentence-transformers>=3,<7", "onnxruntime>=1.18,<2", "transformers>=4.44,<5", "drain3>=0.9,<1", "redis>=5,<7", "celery[redis]>=5.4,<6"] dev = ["pytest>=7.4.0", "pytest-cov>=4.1.0", "ruff>=0.3.0", "mypy>=1.10.0", "bandit>=1.7.8", "pip-audit>=2.7.3", "pytest-benchmark>=4.0.0", "pip-licenses>=5.0.0"] [project.scripts] From 1117095b84cbb5517c7f09ca9721efbd2d3b515f Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 16:53:01 -0500 Subject: [PATCH 13/14] fix: reuse embedding model within Celery worker process --- logsight/distributed.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/logsight/distributed.py b/logsight/distributed.py index 55ce788..a126326 100644 --- a/logsight/distributed.py +++ b/logsight/distributed.py @@ -7,7 +7,7 @@ from dataclasses import asdict, dataclass from typing import Any, Protocol -from .embeddings import EmbeddedLog, build_embedding_worker +from .embeddings import EmbeddedLog, EmbeddingWorker, build_embedding_worker from .enterprise import EnrichedLog _TASK_NAME = "logsight.embed_events" @@ -162,15 +162,20 @@ def create_celery_app() -> Any: result_expires=3600, ) + embedding_worker: EmbeddingWorker | None = None + @app.task(name=_TASK_NAME) def embed_events(payloads: list[dict[str, object]]) -> list[dict[str, object]]: + nonlocal embedding_worker if not 1 <= len(payloads) <= max_batch_size: raise ValueError(f"batch size must be between 1 and {max_batch_size}") requests = [ InferenceRequest.from_payload(payload, max_message_chars=max_message_chars) for payload in payloads ] - results: list[EmbeddedLog] = build_embedding_worker(max_batch_size=max_batch_size).embed( + if embedding_worker is None: + embedding_worker = build_embedding_worker(max_batch_size=max_batch_size) + results: list[EmbeddedLog] = embedding_worker.embed( [request.to_event() for request in requests] ) return [asdict(result) for result in results] From b499d3f5823e27513420d225689fb7ba0052e1d6 Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Fri, 13 Nov 2026 16:53:08 -0500 Subject: [PATCH 14/14] fix: offload Celery publication from API event loop --- logsight/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/logsight/api.py b/logsight/api.py index 8c8859d..3516686 100644 --- a/logsight/api.py +++ b/logsight/api.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import os from collections import deque from dataclasses import asdict @@ -67,7 +68,7 @@ async def _process(item: LogPayload) -> tuple[EnrichedLog, Any, str | None]: if result.score >= 0.5: _metrics["logsight_anomalies_total"] += 1 task_id = ( - _inference_dispatcher.submit_embeddings([event]) + await asyncio.to_thread(_inference_dispatcher.submit_embeddings, [event]) if _inference_dispatcher is not None else None )