From 53dce3719ef02d25e3ba77f252e49791683c259c Mon Sep 17 00:00:00 2001 From: Tom Sapletta Date: Wed, 26 Aug 2026 10:43:46 +0200 Subject: [PATCH 1/2] feat(llm): route NFO analysis through public SubLLM --- .github/workflows/ci.yml | 2 +- nfo/configure.py | 5 ++- nfo/llm.py | 72 +++++++++++++++++++++++----------- project/ticket-001/README.md | 21 ++++++++++ project/ticket-001/intent.json | 20 ++++++++++ pyproject.toml | 9 +++-- tests/test_llm.py | 41 ++++++++++++++++++- 7 files changed, 140 insertions(+), 30 deletions(-) create mode 100644 project/ticket-001/README.md create mode 100644 project/ticket-001/intent.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a7f109..ac63aa2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 diff --git a/nfo/configure.py b/nfo/configure.py index 765e48a..5fcad18 100644 --- a/nfo/configure.py +++ b/nfo/configure.py @@ -306,7 +306,8 @@ def configure( env_prefix: Prefix for environment variable overrides. environment: Environment tag (auto-detected if None and env tagging enabled). version: App version tag (auto-detected if None and env tagging enabled). - llm_model: litellm model for LLM-powered log analysis (e.g. "gpt-4o-mini"). + llm_model: Legacy LiteLLM model override. The default SubLLM path uses + centrally governed direct Z.AI GLM 5.3. Wraps sinks with LLMSink. Requires: pip install nfo[llm] detect_injection: Enable prompt injection detection in log args. meta_policy: :class:`~nfo.meta.ThresholdPolicy` for binary metadata @@ -344,7 +345,7 @@ def configure( configure( sinks=["sqlite:app.db"], environment="prod", - llm_model="gpt-4o-mini", + llm_model="glm-5.3", detect_injection=True, ) diff --git a/nfo/llm.py b/nfo/llm.py index f16529b..fe3da09 100644 --- a/nfo/llm.py +++ b/nfo/llm.py @@ -1,24 +1,24 @@ """ -LLM-powered log analysis via litellm. +LLM-powered log analysis via the public SubLLM policy boundary. Provides: - LLMSink: analyzes ERROR/EXCEPTION logs through LLM and appends root-cause suggestions directly to the log entry. - PromptInjectionDetector: scans log args for prompt injection patterns. -Requires: pip install nfo[llm] (installs litellm) +Requires: pip install nfo[llm] (installs subactor-subllm) """ from __future__ import annotations +import os import re import threading -from typing import Any, Callable, Dict, List, Optional +from collections.abc import Callable from nfo.models import LogEntry from nfo.sinks import Sink - # --------------------------------------------------------------------------- # Prompt injection detection # --------------------------------------------------------------------------- @@ -37,7 +37,7 @@ ] -def detect_prompt_injection(text: str) -> Optional[str]: +def detect_prompt_injection(text: str) -> str | None: """ Scan text for common prompt injection patterns. @@ -52,9 +52,9 @@ def detect_prompt_injection(text: str) -> Optional[str]: return None -def scan_entry_for_injection(entry: LogEntry) -> Optional[str]: +def scan_entry_for_injection(entry: LogEntry) -> str | None: """Scan a LogEntry's args/kwargs for prompt injection attempts.""" - texts_to_scan: List[str] = [] + texts_to_scan: list[str] = [] for arg in (entry.args or ()): if isinstance(arg, str): @@ -76,7 +76,7 @@ def scan_entry_for_injection(entry: LogEntry) -> Optional[str]: # --------------------------------------------------------------------------- -# LLM Sink — analyzes error logs via litellm +# LLM Sink — analyzes error logs via SubLLM # --------------------------------------------------------------------------- _DEFAULT_SYSTEM_PROMPT = ( @@ -93,10 +93,11 @@ class LLMSink(Sink): The LLM response is stored in entry.llm_analysis and also forwarded to an optional delegate sink (e.g. SQLiteSink) for persistence. - Uses litellm for model-agnostic LLM calls (OpenAI, Anthropic, Ollama, etc.). + Uses public SubLLM routing by default. Provider and model selection are + centrally governed; direct Z.AI GLM 5.3 is the preferred route. Args: - model: litellm model string (e.g. "gpt-4o-mini", "ollama/llama3"). + model: Legacy LiteLLM model string. Ignored by the default SubLLM path. delegate: Optional sink to forward the enriched entry to. system_prompt: Custom system prompt for analysis. analyze_levels: Log levels to analyze (default: ERROR only). @@ -107,12 +108,12 @@ class LLMSink(Sink): def __init__( self, - model: str = "gpt-4o-mini", + model: str = "glm-5.3", *, - delegate: Optional[Sink] = None, + delegate: Sink | None = None, system_prompt: str = _DEFAULT_SYSTEM_PROMPT, - analyze_levels: Optional[List[str]] = None, - on_analysis: Optional[Callable[[LogEntry, str], None]] = None, + analyze_levels: list[str] | None = None, + on_analysis: Callable[[LogEntry, str], None] | None = None, async_mode: bool = True, detect_injection: bool = True, ) -> None: @@ -136,7 +137,7 @@ def _build_user_prompt(self, entry: LogEntry) -> str: parts.append(f"Exception: {entry.exception_type}: {entry.exception}") if entry.traceback: tb_lines = entry.traceback.strip().split("\n") - parts.append(f"Traceback (last 10 lines):\n" + "\n".join(tb_lines[-10:])) + parts.append("Traceback (last 10 lines):\n" + "\n".join(tb_lines[-10:])) if entry.environment: parts.append(f"Environment: {entry.environment}") if entry.version: @@ -144,24 +145,51 @@ def _build_user_prompt(self, entry: LogEntry) -> str: return "\n".join(parts) def _analyze(self, entry: LogEntry) -> str: - """Call LLM via litellm and return analysis text.""" + """Call the centrally governed SubLLM route and return analysis text.""" + messages = [ + {"role": "system", "content": self.system_prompt}, + {"role": "user", "content": self._build_user_prompt(entry)}, + ] + + if os.environ.get("NFO_USE_LEGACY_LITELLM", "").strip().lower() in { + "1", + "true", + "yes", + "on", + }: + return self._analyze_legacy(messages) + + try: + from subllm import complete + + response = complete( + "semcod-nfo", + "analyze", + messages, + timeout_seconds=30, + ) + return response.content.strip() + except ImportError: + return "[nfo] subactor-subllm not installed. Run: pip install nfo[llm]" + except Exception as e: + return f"[nfo] SubLLM analysis failed: {type(e).__name__}: {e}" + + def _analyze_legacy(self, messages: list[dict[str, str]]) -> str: + """Use LiteLLM only when the operator explicitly enables legacy mode.""" try: from litellm import completion response = completion( model=self.model, - messages=[ - {"role": "system", "content": self.system_prompt}, - {"role": "user", "content": self._build_user_prompt(entry)}, - ], + messages=messages, max_tokens=200, temperature=0.3, ) return response.choices[0].message.content.strip() except ImportError: - return "[nfo] litellm not installed. Run: pip install nfo[llm]" + return "[nfo] litellm not installed. Run: pip install nfo[llm-legacy]" except Exception as e: - return f"[nfo] LLM analysis failed: {type(e).__name__}: {e}" + return f"[nfo] legacy LLM analysis failed: {type(e).__name__}: {e}" def _process(self, entry: LogEntry) -> None: """Analyze entry and enrich it.""" diff --git a/project/ticket-001/README.md b/project/ticket-001/README.md new file mode 100644 index 0000000..100c5a8 --- /dev/null +++ b/project/ticket-001/README.md @@ -0,0 +1,21 @@ +# Ticket 001: Route NFO analysis through SubLLM + +- **ID**: ticket-001 +- **Owner**: founder +- **Status**: ACTIVE +- **Created**: 2026-08-26 + +## Goal and scope + +Replace the default LiteLLM log-analysis transport with the public +`subactor-subllm` API and the exact `semcod-nfo/analyze` route. Keep LiteLLM +only as an explicit operator-selected compatibility mode. + +## Acceptance criteria + +- [x] Production analysis uses public SubLLM and central provider policy. +- [x] Direct Z.AI GLM 5.3 is the policy-owned default. +- [x] A failed SubLLM request is not replayed to another paid provider. +- [x] Legacy LiteLLM requires explicit opt-in and a separate extra. +- [x] CI covers the supported Python 3.11 through 3.13 range. +- [x] Tests cover routing and fail-closed behavior. diff --git a/project/ticket-001/intent.json b/project/ticket-001/intent.json new file mode 100644 index 0000000..908d7ed --- /dev/null +++ b/project/ticket-001/intent.json @@ -0,0 +1,20 @@ +{ + "schema": "new-project.intent/v3", + "ticket": "ticket-001", + "summary": "Route NFO log analysis through public SubLLM", + "workstream": "runtime", + "classification": {"kind": "FEATURE", "priority": "P1", "origin": "founder"}, + "allowedPaths": [ + "pyproject.toml", + ".github/workflows/ci.yml", + "nfo/llm.py", + "nfo/configure.py", + "tests/test_llm.py", + "project/ticket-001/**" + ], + "forbiddenPaths": [".env", "**/.env", "**/*secret*"], + "stacks": ["python", "subllm", "zai", "logging"], + "dependsOn": [], + "conflictsWith": [], + "integrationTicket": null +} diff --git a/pyproject.toml b/pyproject.toml index da7ca7f..8a47298 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,11 +8,11 @@ version = "0.2.23" description = "Automatic function logging system with decorators, supporting multiple output sinks (SQLite, CSV, Markdown, Prometheus) and LLM-powered analysis for DevOps observability." readme = "README.md" license = "Apache-2.0" -requires-python = ">=3.9" +requires-python = ">=3.11" authors = [ {name = "Tom Sapletta", email = "tom@sapletta.com"}, ] -keywords = ["logging", "decorator", "sqlite", "csv", "markdown", "auto-logging", "llm", "litellm", "prometheus", "grafana", "webhook", "devops"] +keywords = ["logging", "decorator", "sqlite", "csv", "markdown", "auto-logging", "llm", "subllm", "prometheus", "grafana", "webhook", "devops"] classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", @@ -20,6 +20,9 @@ classifiers = [ [project.optional-dependencies] llm = [ + "subactor-subllm>=1.4.2,<2.0", +] +llm-legacy = [ "litellm>=1.0", ] prometheus = [ @@ -50,7 +53,7 @@ grpc = [ "grpcio-tools>=1.60.0", ] all = [ - "litellm>=1.0", + "subactor-subllm>=1.4.2,<2.0", "prometheus_client>=0.20.0", "grpcio>=1.60.0", "click>=8.0", diff --git a/tests/test_llm.py b/tests/test_llm.py index bce5d2b..de6337b 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -1,12 +1,12 @@ """Tests for nfo.llm (LLM analysis, prompt injection detection).""" -import pytest +import sys +from types import ModuleType, SimpleNamespace from nfo.llm import ( LLMSink, detect_prompt_injection, scan_entry_for_injection, - _DEFAULT_SYSTEM_PROMPT, ) from nfo.models import LogEntry from nfo.sinks import Sink @@ -98,6 +98,43 @@ def test_scan_entry_extra_message(self): class TestLLMSink: + def test_analysis_uses_public_subllm_route(self, monkeypatch): + observed = {} + module = ModuleType("subllm") + + def complete(application, function, messages, **kwargs): + observed.update( + application=application, + function=function, + messages=messages, + kwargs=kwargs, + ) + return SimpleNamespace(content="root cause") + + module.complete = complete + monkeypatch.setitem(sys.modules, "subllm", module) + sink = LLMSink(model="ignored-by-policy", async_mode=False) + + assert sink._analyze(_make_entry()) == "root cause" + assert observed["application"] == "semcod-nfo" + assert observed["function"] == "analyze" + assert observed["kwargs"] == {"timeout_seconds": 30} + assert observed["messages"][0]["role"] == "system" + + def test_subllm_failure_does_not_replay_to_legacy_provider(self, monkeypatch): + module = ModuleType("subllm") + + def complete(*_args, **_kwargs): + raise RuntimeError("zai unavailable") + + module.complete = complete + monkeypatch.setitem(sys.modules, "subllm", module) + monkeypatch.delenv("NFO_USE_LEGACY_LITELLM", raising=False) + + analysis = LLMSink(async_mode=False)._analyze(_make_entry()) + + assert analysis == "[nfo] SubLLM analysis failed: RuntimeError: zai unavailable" + def test_delegates_to_sink(self): mem = MemorySink() llm_sink = LLMSink( From 7b899d1f67d2900ef720acba0092464daaeeecdc Mon Sep 17 00:00:00 2001 From: Tom Sapletta Date: Wed, 26 Aug 2026 10:44:44 +0200 Subject: [PATCH 2/2] fix(ci): install NFO test extras --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac63aa2..28293bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e . || pip install -r requirements.txt || true + pip install -e '.[llm,dev]' - name: Run tests run: |