From 4a47bb834bd0a640b2c793f63e90293c5ebd7590 Mon Sep 17 00:00:00 2001 From: Somebody Else Date: Sat, 6 Jun 2026 20:06:30 +0300 Subject: [PATCH 1/5] add otel --- README.md | 49 ++++ docker-compose.otel.yml | 17 ++ observability/otel-collector-config.yaml | 29 +++ .../signoz-dashboard-prompt-orchestrator.yaml | 51 +++++ prompt_orchestrator/__init__.py | 3 + prompt_orchestrator/llm/summary_llm.py | 60 +++-- .../orchestrator/orchestrator.py | 155 +++++++------ prompt_orchestrator/telemetry.py | 209 ++++++++++++++++++ pyproject.toml | 5 + 9 files changed, 497 insertions(+), 81 deletions(-) create mode 100644 docker-compose.otel.yml create mode 100644 observability/otel-collector-config.yaml create mode 100644 observability/signoz-dashboard-prompt-orchestrator.yaml create mode 100644 prompt_orchestrator/telemetry.py diff --git a/README.md b/README.md index 546d548..f5f9f71 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,55 @@ For development and tests: pip install -e .[dev] ``` +Install with optional OpenTelemetry support: + +```bash +pip install -e .[otel] +``` + +## Optional OpenTelemetry + SigNoz + +OpenTelemetry is optional. If not installed or not enabled, PromptOrchestrator works as before. + +Enable OTel from environment: + +```bash +ENABLE_OTEL=true +OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 +OTEL_SERVICE_NAME=prompt-orchestrator +OTEL_SERVICE_NAMESPACE=prompt-stack +OTEL_DEPLOYMENT_ENVIRONMENT=dev +``` + +Bring up OTel Collector + SigNoz (2 additional containers): + +```bash +docker compose -f docker-compose.otel.yml up -d +``` + +Files used: + +- `docker-compose.otel.yml` +- `observability/otel-collector-config.yaml` + +Default endpoints: + +- SigNoz UI: `http://localhost:8080` +- OTLP gRPC ingest: `http://localhost:4317` +- OTLP HTTP ingest: `http://localhost:4318` + +Exposed telemetry (when enabled): + +- traces: `prompt_orchestrator.build_for_request` +- metrics: build requests/errors/latency, prompt token+char volume, RAG chunk count, warnings count, safety events, summary-call latency +- logs: summary/build error events exported through OTLP logs pipeline + +Dashboard template blueprint: + +- `observability/signoz-dashboard-prompt-orchestrator.yaml` + +Use it as a panel/query blueprint in SigNoz to create a dashboard for prompt build latency, token pressure, RAG payload size, safety events, summary latency, logs, and traces. + ## Configuration Models - `PromptConfig`: static prompt structure diff --git a/docker-compose.otel.yml b/docker-compose.otel.yml new file mode 100644 index 0000000..394da8f --- /dev/null +++ b/docker-compose.otel.yml @@ -0,0 +1,17 @@ +services: + otel-collector: + image: otel/opentelemetry-collector-contrib:0.103.0 + command: ["--config=/etc/otelcol/config.yaml"] + volumes: + - ./observability/otel-collector-config.yaml:/etc/otelcol/config.yaml:ro + ports: + - "4317:4317" + - "4318:4318" + depends_on: + - signoz + + signoz: + image: signoz/signoz-standalone:0.69.0 + ports: + - "8080:8080" + - "3301:3301" diff --git a/observability/otel-collector-config.yaml b/observability/otel-collector-config.yaml new file mode 100644 index 0000000..4df54f6 --- /dev/null +++ b/observability/otel-collector-config.yaml @@ -0,0 +1,29 @@ +receivers: + otlp: + protocols: + grpc: + http: + +processors: + batch: + +exporters: + otlp: + endpoint: signoz:4317 + tls: + insecure: true + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlp] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [otlp] + logs: + receivers: [otlp] + processors: [batch] + exporters: [otlp] diff --git a/observability/signoz-dashboard-prompt-orchestrator.yaml b/observability/signoz-dashboard-prompt-orchestrator.yaml new file mode 100644 index 0000000..451fb7b --- /dev/null +++ b/observability/signoz-dashboard-prompt-orchestrator.yaml @@ -0,0 +1,51 @@ +dashboard: + title: PromptOrchestrator Overview + description: SigNoz dashboard blueprint for PromptOrchestrator metrics exported through OpenTelemetry. + variables: + - name: env + description: deployment.environment resource attribute + default: dev + - name: service + description: service.name resource attribute + default: prompt-orchestrator + panels: + - title: Build Requests + type: time_series + unit: req/s + query: sum(rate(prompt_build_requests_total{service_name="$service",deployment_environment="$env"}[5m])) + - title: Build Errors + type: time_series + unit: err/s + query: sum(rate(prompt_errors_total{service_name="$service",deployment_environment="$env"}[5m])) by (operation) + - title: Build Latency P95 + type: time_series + unit: ms + query: histogram_quantile(0.95, sum(rate(prompt_build_latency_ms_bucket{service_name="$service",deployment_environment="$env"}[5m])) by (le)) + - title: Prompt Tokens P50 + type: time_series + unit: tokens + query: histogram_quantile(0.50, sum(rate(prompt_total_tokens_bucket{service_name="$service",deployment_environment="$env"}[5m])) by (le)) + - title: Prompt Chars P50 + type: time_series + unit: chars + query: histogram_quantile(0.50, sum(rate(prompt_total_chars_bucket{service_name="$service",deployment_environment="$env"}[5m])) by (le)) + - title: RAG Chunks per Build + type: time_series + unit: chunks + query: histogram_quantile(0.50, sum(rate(prompt_rag_chunks_count_bucket{service_name="$service",deployment_environment="$env"}[5m])) by (le)) + - title: Warnings per Build + type: time_series + unit: warnings + query: histogram_quantile(0.95, sum(rate(prompt_warnings_count_bucket{service_name="$service",deployment_environment="$env"}[5m])) by (le)) + - title: Safety Events + type: bar + unit: events/s + query: sum(rate(prompt_safety_events_total{service_name="$service",deployment_environment="$env"}[5m])) by (severity) + - title: Summary Latency P95 + type: time_series + unit: ms + query: histogram_quantile(0.95, sum(rate(prompt_summary_latency_ms_bucket{service_name="$service",deployment_environment="$env"}[5m])) by (le, provider)) + logs: + default_query: service.name = "$service" AND deployment.environment = "$env" + traces: + span_filter: service.name = "$service" and name = "prompt_orchestrator.build_for_request" diff --git a/prompt_orchestrator/__init__.py b/prompt_orchestrator/__init__.py index acc59ce..5d40bbe 100644 --- a/prompt_orchestrator/__init__.py +++ b/prompt_orchestrator/__init__.py @@ -18,6 +18,7 @@ from .context.manager import PromptContextManager from .orchestrator.factory import PromptOrchestratorFactory from .orchestrator.orchestrator import OrchestratedPrompt, PromptOrchestrator +from .telemetry import init_telemetry, shutdown_telemetry from .tokenization import TokenCounter __all__ = [ @@ -47,4 +48,6 @@ "SummaryLLM", "SummaryLLMConfig", "TokenCounter", + "init_telemetry", + "shutdown_telemetry", ] diff --git a/prompt_orchestrator/llm/summary_llm.py b/prompt_orchestrator/llm/summary_llm.py index 8e062c8..d221bca 100644 --- a/prompt_orchestrator/llm/summary_llm.py +++ b/prompt_orchestrator/llm/summary_llm.py @@ -1,5 +1,6 @@ from __future__ import annotations +import time from typing import TYPE_CHECKING, Literal from pydantic import BaseModel, Field @@ -7,6 +8,7 @@ from .base_client import SummaryLLMClient from .ollama_client import OllamaConfig, OllamaSummaryClient from .openai_client import OpenAIConfig, OpenAISummaryClient +from ..telemetry import telemetry if TYPE_CHECKING: from ..context.state import Message @@ -35,24 +37,46 @@ def __init__( self.client = OllamaSummaryClient(config=self.config.ollama) def summarize(self, history: list[Message], prev_summary: str | None = None) -> str: + started = time.perf_counter() base = prev_summary.strip() + "\n\n" if prev_summary else "" transcript = "\n".join(f"{msg.role}: {msg.content}" for msg in history[-30:]) - if self.client is None: - # Fallback deterministic summarization without external LLM. - compact = " ".join(line.strip() for line in transcript.splitlines() if line.strip()) - return (base + compact)[:1200] - - prompt = ( - "Summarize the dialogue in a compact, factual format.\n" - "Keep constraints, decisions, open tasks and user preferences.\n" - "Avoid speculation and keep under 180 words.\n\n" - f"Previous summary:\n{prev_summary or 'None'}\n\n" - f"Dialogue:\n{transcript}" - ) - return self.client.generate( - prompt=prompt, - model=self.config.model, - max_tokens=self.config.max_tokens, - temperature=self.config.temperature, - ).strip() + try: + if self.client is None: + # Fallback deterministic summarization without external LLM. + compact = " ".join(line.strip() for line in transcript.splitlines() if line.strip()) + result = (base + compact)[:1200] + telemetry.record_summary_call( + duration_ms=(time.perf_counter() - started) * 1000.0, + provider="none", + status="ok", + ) + return result + + prompt = ( + "Summarize the dialogue in a compact, factual format.\n" + "Keep constraints, decisions, open tasks and user preferences.\n" + "Avoid speculation and keep under 180 words.\n\n" + f"Previous summary:\n{prev_summary or 'None'}\n\n" + f"Dialogue:\n{transcript}" + ) + result = self.client.generate( + prompt=prompt, + model=self.config.model, + max_tokens=self.config.max_tokens, + temperature=self.config.temperature, + ).strip() + telemetry.record_summary_call( + duration_ms=(time.perf_counter() - started) * 1000.0, + provider=self.config.provider, + status="ok", + ) + return result + except Exception as exc: + telemetry.record_error("summary", type(exc).__name__) + telemetry.record_summary_call( + duration_ms=(time.perf_counter() - started) * 1000.0, + provider=self.config.provider, + status="error", + ) + raise diff --git a/prompt_orchestrator/orchestrator/orchestrator.py b/prompt_orchestrator/orchestrator/orchestrator.py index 2a7e782..b014a10 100644 --- a/prompt_orchestrator/orchestrator/orchestrator.py +++ b/prompt_orchestrator/orchestrator/orchestrator.py @@ -1,5 +1,7 @@ from __future__ import annotations +import time + from pydantic import BaseModel from ..analyzer.analyzer import PromptAnalyzer @@ -13,6 +15,7 @@ from ..rag.base import RAGProvider from ..safety.engine import PromptSafetyEngine from ..safety.report import SafetyReport +from ..telemetry import init_telemetry, telemetry class OrchestratedPrompt(BaseModel): @@ -36,6 +39,7 @@ def __init__( analyzer: PromptAnalyzer | None = None, safety_engine: PromptSafetyEngine | None = None, ) -> None: + init_telemetry(service_name="prompt-orchestrator") self.config_store = config_store self.config = config_store.get_prompt() if config_store else config self.context_manager = context_manager @@ -58,69 +62,94 @@ def build_for_request( user_message: str, use_rag: bool | None = None, ) -> OrchestratedPrompt: - state = self.context_manager.load_state(session_id) - - if use_rag is None: - use_rag = self.settings.use_rag_default - if use_rag: - chunks = self.rag_provider.retrieve( - query=user_message, - limit=self.settings.rag_limit, - ) - state = self.context_manager.set_rag_chunks(state, chunks) - else: - state = self.context_manager.set_rag_chunks(state, []) - - sections = self.builder.build_sections( - config=self.config, - state=state, - user_message=user_message, - include_headers=self.settings.debug_mode, - ) + started = time.perf_counter() + with telemetry.span("prompt_orchestrator.build_for_request", {"session.id": session_id}): + try: + state = self.context_manager.load_state(session_id) - fit_payload = self.context_manager.ensure_fits_limit( - { - "static": sections["static"], - "summary": sections["summary"], - "recent": sections["recent"], - "user": sections["user"], - "rag": sections["rag"], - } - ) + if use_rag is None: + use_rag = self.settings.use_rag_default + if use_rag: + chunks = self.rag_provider.retrieve( + query=user_message, + limit=self.settings.rag_limit, + ) + state = self.context_manager.set_rag_chunks(state, chunks) + else: + state = self.context_manager.set_rag_chunks(state, []) - prompt = "\n\n".join( - [ - str(fit_payload["static"]), - str(fit_payload["summary"]), - str(fit_payload["recent"]), - str(fit_payload["rag"]), - ] - ) + sections = self.builder.build_sections( + config=self.config, + state=state, + user_message=user_message, + include_headers=self.settings.debug_mode, + ) - safety = self.safety_engine.ensure_safe( - prompt=prompt, - auto_rewrite=self.settings.safety_auto_rewrite, - ) - final_prompt = safety.sanitized_prompt or prompt - - stats = self.analyzer.analyze_sections( - { - "static": str(fit_payload["static"]), - "summary": str(fit_payload["summary"]), - "recent": str(fit_payload["recent"]), - "rag": str(fit_payload["rag"]), - } - ) - severity_to_score = {"none": 1.0, "low": 0.85, "medium": 0.5, "high": 0.1} - stats.safety_score = severity_to_score.get(safety.severity, 0.1) - - state = self.context_manager.update_state(state=state, user_message=user_message) - - return OrchestratedPrompt( - prompt=final_prompt, - state=state, - stats=stats, - safety=safety, - sections=sections, - fitted_sections={key: str(value) for key, value in fit_payload.items()}, - ) + fit_payload = self.context_manager.ensure_fits_limit( + { + "static": sections["static"], + "summary": sections["summary"], + "recent": sections["recent"], + "user": sections["user"], + "rag": sections["rag"], + } + ) + + prompt = "\n\n".join( + [ + str(fit_payload["static"]), + str(fit_payload["summary"]), + str(fit_payload["recent"]), + str(fit_payload["rag"]), + ] + ) + + safety = self.safety_engine.ensure_safe( + prompt=prompt, + auto_rewrite=self.settings.safety_auto_rewrite, + ) + final_prompt = safety.sanitized_prompt or prompt + + stats = self.analyzer.analyze_sections( + { + "static": str(fit_payload["static"]), + "summary": str(fit_payload["summary"]), + "recent": str(fit_payload["recent"]), + "rag": str(fit_payload["rag"]), + } + ) + severity_to_score = {"none": 1.0, "low": 0.85, "medium": 0.5, "high": 0.1} + stats.safety_score = severity_to_score.get(safety.severity, 0.1) + + state = self.context_manager.update_state(state=state, user_message=user_message) + + telemetry.record_build( + duration_ms=(time.perf_counter() - started) * 1000.0, + total_tokens=stats.total_tokens, + total_chars=stats.total_chars, + rag_chunks=len(state.rag_chunks), + warnings_count=len(stats.warnings), + safety_severity=safety.severity, + status="ok", + ) + + return OrchestratedPrompt( + prompt=final_prompt, + state=state, + stats=stats, + safety=safety, + sections=sections, + fitted_sections={key: str(value) for key, value in fit_payload.items()}, + ) + except Exception as exc: + telemetry.record_error("build_for_request", type(exc).__name__) + telemetry.record_build( + duration_ms=(time.perf_counter() - started) * 1000.0, + total_tokens=0, + total_chars=0, + rag_chunks=0, + warnings_count=0, + safety_severity="unknown", + status="error", + ) + raise diff --git a/prompt_orchestrator/telemetry.py b/prompt_orchestrator/telemetry.py new file mode 100644 index 0000000..f7537c5 --- /dev/null +++ b/prompt_orchestrator/telemetry.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +import atexit +import logging +import os +import time +from contextlib import contextmanager +from typing import Any + + +TRUE_VALUES = {"1", "true", "yes", "on"} + + +class PromptTelemetry: + """Optional OpenTelemetry bridge for PromptOrchestrator.""" + + def __init__(self) -> None: + self._initialized = False + self._enabled = False + self._init_error: str | None = None + + self._tracer: Any = None + self._meter: Any = None + self._tracer_provider: Any = None + self._meter_provider: Any = None + self._log_provider: Any = None + self._otlp_logger: logging.Logger | None = None + + self._build_requests: Any = None + self._errors: Any = None + self._latency_ms: Any = None + self._prompt_tokens: Any = None + self._prompt_chars: Any = None + self._rag_chunks: Any = None + self._warnings_count: Any = None + self._safety_events: Any = None + self._summary_calls: Any = None + self._summary_latency_ms: Any = None + + def initialize(self, service_name: str = "prompt-orchestrator") -> None: + if self._initialized: + return + + self._initialized = True + enabled_raw = os.getenv("ENABLE_OTEL", "false").strip().lower() + if enabled_raw not in TRUE_VALUES: + return + + try: + from opentelemetry import metrics, trace + from opentelemetry._logs import set_logger_provider + from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter + from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( + OTLPMetricExporter, + ) + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, + ) + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + except Exception as exc: # pragma: no cover - optional dependency path + self._init_error = str(exc) + return + + endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317") + namespace = os.getenv("OTEL_SERVICE_NAMESPACE", "prompt-stack") + environment = os.getenv("OTEL_DEPLOYMENT_ENVIRONMENT", "dev") + version = os.getenv("OTEL_SERVICE_VERSION", "unknown") + + resource = Resource.create( + { + "service.name": os.getenv("OTEL_SERVICE_NAME", service_name), + "service.namespace": namespace, + "service.version": version, + "deployment.environment": environment, + } + ) + + self._tracer_provider = TracerProvider(resource=resource) + self._tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint))) + trace.set_tracer_provider(self._tracer_provider) + self._tracer = trace.get_tracer("prompt-orchestrator") + + metric_reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=endpoint)) + self._meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader]) + metrics.set_meter_provider(self._meter_provider) + self._meter = metrics.get_meter("prompt-orchestrator") + + self._build_requests = self._meter.create_counter("prompt_build_requests_total") + self._errors = self._meter.create_counter("prompt_errors_total") + self._latency_ms = self._meter.create_histogram("prompt_build_latency_ms", unit="ms") + self._prompt_tokens = self._meter.create_histogram("prompt_total_tokens") + self._prompt_chars = self._meter.create_histogram("prompt_total_chars") + self._rag_chunks = self._meter.create_histogram("prompt_rag_chunks_count") + self._warnings_count = self._meter.create_histogram("prompt_warnings_count") + self._safety_events = self._meter.create_counter("prompt_safety_events_total") + self._summary_calls = self._meter.create_counter("prompt_summary_calls_total") + self._summary_latency_ms = self._meter.create_histogram("prompt_summary_latency_ms", unit="ms") + + self._log_provider = LoggerProvider(resource=resource) + self._log_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint))) + set_logger_provider(self._log_provider) + + self._otlp_logger = logging.getLogger("prompt-orchestrator.otel") + self._otlp_logger.setLevel(logging.INFO) + self._otlp_logger.propagate = False + self._otlp_logger.handlers.clear() + self._otlp_logger.addHandler(LoggingHandler(level=logging.INFO, logger_provider=self._log_provider)) + + self._enabled = True + atexit.register(self.shutdown) + + def shutdown(self) -> None: + if not self._initialized: + return + if self._meter_provider is not None: + try: + self._meter_provider.shutdown() + except Exception: + pass + if self._tracer_provider is not None: + try: + self._tracer_provider.shutdown() + except Exception: + pass + if self._log_provider is not None: + try: + self._log_provider.shutdown() + except Exception: + pass + + @contextmanager + def span(self, name: str, attributes: dict[str, Any] | None = None): + if not self._enabled or self._tracer is None: + yield None + return + with self._tracer.start_as_current_span(name, attributes=attributes or {}) as span: + yield span + + def record_build( + self, + *, + duration_ms: float, + total_tokens: int, + total_chars: int, + rag_chunks: int, + warnings_count: int, + safety_severity: str, + status: str, + ) -> None: + if not self._enabled: + return + attrs = {"operation": "build_for_request", "status": status} + self._build_requests.add(1, attrs) + self._latency_ms.record(duration_ms, attrs) + self._prompt_tokens.record(max(total_tokens, 0), attrs) + self._prompt_chars.record(max(total_chars, 0), attrs) + self._rag_chunks.record(max(rag_chunks, 0), attrs) + self._warnings_count.record(max(warnings_count, 0), attrs) + self._safety_events.add(1, {"severity": safety_severity or "unknown", "status": status}) + + def record_summary_call(self, *, duration_ms: float, provider: str, status: str) -> None: + if not self._enabled: + return + attrs = {"operation": "summary", "provider": provider, "status": status} + self._summary_calls.add(1, attrs) + self._summary_latency_ms.record(duration_ms, attrs) + + def record_error(self, operation: str, error_type: str) -> None: + if not self._enabled: + return + self._errors.add(1, {"operation": operation, "error.type": error_type}) + self.emit_log(level_name="ERROR", message=f"prompt.error operation={operation} error_type={error_type}") + + def emit_log(self, *, level_name: str, message: str) -> None: + if not self._enabled or self._otlp_logger is None: + return + + level = logging.INFO + normalized = level_name.strip().upper() + if normalized == "DEBUG": + level = logging.DEBUG + elif normalized in {"WARN", "WARNING"}: + level = logging.WARNING + elif normalized == "ERROR": + level = logging.ERROR + elif normalized == "CRITICAL": + level = logging.CRITICAL + self._otlp_logger.log(level, message) + + +telemetry = PromptTelemetry() + + +def init_telemetry(service_name: str = "prompt-orchestrator") -> None: + telemetry.initialize(service_name=service_name) + + +def shutdown_telemetry() -> None: + telemetry.shutdown() + + +def monotonic_ms() -> float: + return time.perf_counter() * 1000.0 diff --git a/pyproject.toml b/pyproject.toml index e7047b6..e8ea08c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,11 @@ Issues = "https://github.com/VeryComplexAndLongName/PromptOrchestrator/issues" Changelog = "https://github.com/VeryComplexAndLongName/PromptOrchestrator/blob/main/CHANGES.md" [project.optional-dependencies] +otel = [ + "opentelemetry-api>=1.27.0", + "opentelemetry-sdk>=1.27.0", + "opentelemetry-exporter-otlp>=1.27.0", +] dev = [ "pytest>=8.2.0", ] From bcfeca2f434499d4e1ebe272f81d5429ae3f1b95 Mon Sep 17 00:00:00 2001 From: Somebody Else Date: Sat, 6 Jun 2026 21:55:43 +0300 Subject: [PATCH 2/5] add docker-compose for otel --- docker-compose.otel.yml | 3 +- ...-dashboard-prompt-orchestrator.import.json | 86 +++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 observability/signoz-dashboard-prompt-orchestrator.import.json diff --git a/docker-compose.otel.yml b/docker-compose.otel.yml index 394da8f..9984917 100644 --- a/docker-compose.otel.yml +++ b/docker-compose.otel.yml @@ -11,7 +11,8 @@ services: - signoz signoz: - image: signoz/signoz-standalone:0.69.0 + image: signoz/signoz-standalone:latest + privileged: true ports: - "8080:8080" - "3301:3301" diff --git a/observability/signoz-dashboard-prompt-orchestrator.import.json b/observability/signoz-dashboard-prompt-orchestrator.import.json new file mode 100644 index 0000000..08f7350 --- /dev/null +++ b/observability/signoz-dashboard-prompt-orchestrator.import.json @@ -0,0 +1,86 @@ +{ + "title": "PromptOrchestrator Overview", + "description": "Importable dashboard for PromptOrchestrator OTel metrics.", + "tags": ["signoz", "otel", "prompt-orchestrator"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "panels": [ + { + "title": "Build Requests", + "type": "timeseries", + "targets": [ + { + "expr": "sum(rate(prompt_build_requests_total{service_name=~\"$service\",deployment_environment=~\"$env\"}[5m]))", + "legendFormat": "build req/s" + } + ] + }, + { + "title": "Build Errors", + "type": "timeseries", + "targets": [ + { + "expr": "sum(rate(prompt_errors_total{service_name=~\"$service\",deployment_environment=~\"$env\"}[5m])) by (operation)", + "legendFormat": "{{operation}}" + } + ] + }, + { + "title": "Build Latency P95", + "type": "timeseries", + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(prompt_build_latency_ms_bucket{service_name=~\"$service\",deployment_environment=~\"$env\"}[5m])) by (le))", + "legendFormat": "p95" + } + ] + }, + { + "title": "Prompt Tokens P50", + "type": "timeseries", + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(prompt_total_tokens_bucket{service_name=~\"$service\",deployment_environment=~\"$env\"}[5m])) by (le))", + "legendFormat": "p50" + } + ] + }, + { + "title": "Safety Events", + "type": "barchart", + "targets": [ + { + "expr": "sum(rate(prompt_safety_events_total{service_name=~\"$service\",deployment_environment=~\"$env\"}[5m])) by (severity)", + "legendFormat": "{{severity}}" + } + ] + }, + { + "title": "Summary Latency P95", + "type": "timeseries", + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(prompt_summary_latency_ms_bucket{service_name=~\"$service\",deployment_environment=~\"$env\"}[5m])) by (le, provider))", + "legendFormat": "{{provider}} p95" + } + ] + } + ], + "templating": { + "list": [ + { + "name": "env", + "type": "custom", + "query": "dev,stage,prod", + "current": { "text": "dev", "value": "dev" } + }, + { + "name": "service", + "type": "custom", + "query": "prompt-orchestrator", + "current": { "text": "prompt-orchestrator", "value": "prompt-orchestrator" } + } + ] + } +} From 777fa7b1696a6bf74362d7308bdc19e8a3d0f70c Mon Sep 17 00:00:00 2001 From: Somebody Else Date: Sat, 6 Jun 2026 22:12:20 +0300 Subject: [PATCH 3/5] import fix order --- prompt_orchestrator/__init__.py | 16 ++++----- prompt_orchestrator/cache/__init__.py | 2 +- prompt_orchestrator/context/__init__.py | 2 +- prompt_orchestrator/llm/summary_llm.py | 2 +- prompt_orchestrator/telemetry.py | 45 ++++++++++++++++++------- 5 files changed, 44 insertions(+), 23 deletions(-) diff --git a/prompt_orchestrator/__init__.py b/prompt_orchestrator/__init__.py index 5d40bbe..9b3b03a 100644 --- a/prompt_orchestrator/__init__.py +++ b/prompt_orchestrator/__init__.py @@ -1,23 +1,23 @@ """Prompt orchestration package.""" +from .analyzer.analyzer import PromptAnalyzer +from .builder.builder import PromptBuilder +from .cache.base import CacheBackend, NoCacheBackend +from .cache.local_ttl import LocalTTLCacheBackend from .config.config_store import ConfigStore from .config.module_config import ModuleConfig from .config.prompt_config import PromptConfig from .config.settings import OrchestratorSettings +from .context.manager import PromptContextManager from .context.state import DocChunk, Message, PromptContextState -from .cache.base import CacheBackend, NoCacheBackend -from .cache.local_ttl import LocalTTLCacheBackend -from .rag.base import RAGProvider -from .rag.no_rag import NoRAGProvider from .llm.ollama_client import OllamaConfig, OllamaSummaryClient from .llm.openai_client import OpenAIConfig, OpenAISummaryClient from .llm.summary_llm import SummaryLLM, SummaryLLMConfig -from .safety.engine import PromptSafetyEngine -from .analyzer.analyzer import PromptAnalyzer -from .builder.builder import PromptBuilder -from .context.manager import PromptContextManager from .orchestrator.factory import PromptOrchestratorFactory from .orchestrator.orchestrator import OrchestratedPrompt, PromptOrchestrator +from .rag.base import RAGProvider +from .rag.no_rag import NoRAGProvider +from .safety.engine import PromptSafetyEngine from .telemetry import init_telemetry, shutdown_telemetry from .tokenization import TokenCounter diff --git a/prompt_orchestrator/cache/__init__.py b/prompt_orchestrator/cache/__init__.py index ebf57c6..4f89a04 100644 --- a/prompt_orchestrator/cache/__init__.py +++ b/prompt_orchestrator/cache/__init__.py @@ -1,7 +1,7 @@ from .base import CacheBackend, NoCacheBackend +from .cornet_cache import CornetCacheBackend from .local_ttl import LocalTTLCacheBackend from .redis_cache import RedisCacheBackend -from .cornet_cache import CornetCacheBackend __all__ = [ "CacheBackend", diff --git a/prompt_orchestrator/context/__init__.py b/prompt_orchestrator/context/__init__.py index bee2c95..b5409be 100644 --- a/prompt_orchestrator/context/__init__.py +++ b/prompt_orchestrator/context/__init__.py @@ -1,4 +1,4 @@ -from .state import DocChunk, Message, PromptContextState from .manager import PromptContextManager +from .state import DocChunk, Message, PromptContextState __all__ = ["DocChunk", "Message", "PromptContextManager", "PromptContextState"] diff --git a/prompt_orchestrator/llm/summary_llm.py b/prompt_orchestrator/llm/summary_llm.py index d221bca..6f8f6d2 100644 --- a/prompt_orchestrator/llm/summary_llm.py +++ b/prompt_orchestrator/llm/summary_llm.py @@ -5,10 +5,10 @@ from pydantic import BaseModel, Field +from ..telemetry import telemetry from .base_client import SummaryLLMClient from .ollama_client import OllamaConfig, OllamaSummaryClient from .openai_client import OpenAIConfig, OpenAISummaryClient -from ..telemetry import telemetry if TYPE_CHECKING: from ..context.state import Message diff --git a/prompt_orchestrator/telemetry.py b/prompt_orchestrator/telemetry.py index f7537c5..b116541 100644 --- a/prompt_orchestrator/telemetry.py +++ b/prompt_orchestrator/telemetry.py @@ -7,7 +7,6 @@ from contextlib import contextmanager from typing import Any - TRUE_VALUES = {"1", "true", "yes", "on"} @@ -56,10 +55,10 @@ def initialize(self, service_name: str = "prompt-orchestrator") -> None: from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( OTLPSpanExporter, ) - from opentelemetry.sdk.metrics import MeterProvider - from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor @@ -68,6 +67,8 @@ def initialize(self, service_name: str = "prompt-orchestrator") -> None: return endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317") + grpc_endpoint = endpoint.replace("http://", "").replace("https://", "") + insecure = not endpoint.startswith("https://") namespace = os.getenv("OTEL_SERVICE_NAMESPACE", "prompt-stack") environment = os.getenv("OTEL_DEPLOYMENT_ENVIRONMENT", "dev") version = os.getenv("OTEL_SERVICE_VERSION", "unknown") @@ -81,14 +82,26 @@ def initialize(self, service_name: str = "prompt-orchestrator") -> None: } ) - self._tracer_provider = TracerProvider(resource=resource) - self._tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint))) - trace.set_tracer_provider(self._tracer_provider) + current_tracer_provider = trace.get_tracer_provider() + if isinstance(current_tracer_provider, TracerProvider): + self._tracer_provider = current_tracer_provider + else: + self._tracer_provider = TracerProvider(resource=resource) + self._tracer_provider.add_span_processor( + BatchSpanProcessor(OTLPSpanExporter(endpoint=grpc_endpoint, insecure=insecure)) + ) + trace.set_tracer_provider(self._tracer_provider) self._tracer = trace.get_tracer("prompt-orchestrator") - metric_reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=endpoint)) - self._meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader]) - metrics.set_meter_provider(self._meter_provider) + current_meter_provider = metrics.get_meter_provider() + if isinstance(current_meter_provider, MeterProvider): + self._meter_provider = current_meter_provider + else: + metric_reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=grpc_endpoint, insecure=insecure) + ) + self._meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader]) + metrics.set_meter_provider(self._meter_provider) self._meter = metrics.get_meter("prompt-orchestrator") self._build_requests = self._meter.create_counter("prompt_build_requests_total") @@ -102,9 +115,17 @@ def initialize(self, service_name: str = "prompt-orchestrator") -> None: self._summary_calls = self._meter.create_counter("prompt_summary_calls_total") self._summary_latency_ms = self._meter.create_histogram("prompt_summary_latency_ms", unit="ms") - self._log_provider = LoggerProvider(resource=resource) - self._log_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint))) - set_logger_provider(self._log_provider) + from opentelemetry._logs import get_logger_provider + + current_log_provider = get_logger_provider() + if isinstance(current_log_provider, LoggerProvider): + self._log_provider = current_log_provider + else: + self._log_provider = LoggerProvider(resource=resource) + self._log_provider.add_log_record_processor( + BatchLogRecordProcessor(OTLPLogExporter(endpoint=grpc_endpoint, insecure=insecure)) + ) + set_logger_provider(self._log_provider) self._otlp_logger = logging.getLogger("prompt-orchestrator.otel") self._otlp_logger.setLevel(logging.INFO) From 682f266c1bc9c087c3483d303a6229755ab490c7 Mon Sep 17 00:00:00 2001 From: Somebody Else Date: Sat, 6 Jun 2026 22:23:57 +0300 Subject: [PATCH 4/5] fix bugs --- docker-compose.otel.yml | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/docker-compose.otel.yml b/docker-compose.otel.yml index 9984917..6429267 100644 --- a/docker-compose.otel.yml +++ b/docker-compose.otel.yml @@ -9,6 +9,7 @@ services: - "4318:4318" depends_on: - signoz + restart: unless-stopped signoz: image: signoz/signoz-standalone:latest @@ -16,3 +17,43 @@ services: ports: - "8080:8080" - "3301:3301" + restart: unless-stopped + + telemetrygen-traces: + image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest + command: + - "traces" + - "--otlp-endpoint=otel-collector:4317" + - "--otlp-insecure" + - "--duration=inf" + - "--rate=2" + - "--service=seed-traces" + depends_on: + - otel-collector + restart: unless-stopped + + telemetrygen-metrics: + image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest + command: + - "metrics" + - "--otlp-endpoint=otel-collector:4317" + - "--otlp-insecure" + - "--duration=inf" + - "--rate=2" + - "--service=seed-metrics" + depends_on: + - otel-collector + restart: unless-stopped + + telemetrygen-logs: + image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest + command: + - "logs" + - "--otlp-endpoint=otel-collector:4317" + - "--otlp-insecure" + - "--duration=inf" + - "--rate=2" + - "--service=seed-logs" + depends_on: + - otel-collector + restart: unless-stopped From d0394f72f86051e8ce59a9c7f8eceac097ab3d0b Mon Sep 17 00:00:00 2001 From: Somebody Else Date: Sun, 7 Jun 2026 04:03:29 +0300 Subject: [PATCH 5/5] add open telemetry --- .env | 7 ++++ README.md | 50 ++++++++++++++++++---- docker-compose.otel.yml | 53 ++---------------------- observability/otel-collector-config.yaml | 2 +- pyproject.toml | 2 +- 5 files changed, 54 insertions(+), 60 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000..c67887c --- /dev/null +++ b/.env @@ -0,0 +1,7 @@ +# OpenTelemetry local export settings +ENABLE_OTEL=true +OTEL_EXPORTER_OTLP_ENDPOINT=localhost:4317 +OTEL_SERVICE_NAME=prompt-orchestrator +OTEL_SERVICE_NAMESPACE=prompt-stack +OTEL_DEPLOYMENT_ENVIRONMENT=dev +OTEL_UPSTREAM_OTLP_ENDPOINT=host.docker.internal:4317 diff --git a/README.md b/README.md index f5f9f71..4255c5f 100644 --- a/README.md +++ b/README.md @@ -36,22 +36,43 @@ pip install -e .[otel] OpenTelemetry is optional. If not installed or not enabled, PromptOrchestrator works as before. -Enable OTel from environment: +SigNoz is expected to run separately (for example, official SigNoz Docker deployment on `http://localhost:8080`). + +Enable OTel (host runtime): ```bash ENABLE_OTEL=true -OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 +OTEL_EXPORTER_OTLP_ENDPOINT=localhost:4317 OTEL_SERVICE_NAME=prompt-orchestrator OTEL_SERVICE_NAMESPACE=prompt-stack OTEL_DEPLOYMENT_ENVIRONMENT=dev ``` -Bring up OTel Collector + SigNoz (2 additional containers): +Required/optional flags summary: + +- Start telemetry export (required): set `ENABLE_OTEL=true` +- Stop telemetry export (required): set `ENABLE_OTEL=false` +- OTLP destination (optional, used when enabled): `OTEL_EXPORTER_OTLP_ENDPOINT` +- Resource labels (optional): `OTEL_SERVICE_NAME`, `OTEL_SERVICE_NAMESPACE`, `OTEL_DEPLOYMENT_ENVIRONMENT`, `OTEL_SERVICE_VERSION` + +Run local OTel Collector (1 additional container): ```bash docker compose -f docker-compose.otel.yml up -d ``` +Disable OTel (host runtime): + +```bash +ENABLE_OTEL=false +``` + +Stop local OTel Collector: + +```bash +docker compose -f docker-compose.otel.yml down +``` + Files used: - `docker-compose.otel.yml` @@ -59,15 +80,26 @@ Files used: Default endpoints: -- SigNoz UI: `http://localhost:8080` -- OTLP gRPC ingest: `http://localhost:4317` -- OTLP HTTP ingest: `http://localhost:4318` +- SigNoz UI (external): `http://localhost:8080` +- OTLP gRPC ingest (local collector): `http://localhost:4317` +- OTLP HTTP ingest (local collector): `http://localhost:4318` Exposed telemetry (when enabled): -- traces: `prompt_orchestrator.build_for_request` -- metrics: build requests/errors/latency, prompt token+char volume, RAG chunk count, warnings count, safety events, summary-call latency -- logs: summary/build error events exported through OTLP logs pipeline +| Telemetry signal name | Description | +| --- | --- | +| `prompt_orchestrator.build_for_request` | Trace span for one prompt build request. Includes attribute `session.id`. | +| `prompt_build_requests_total` | Counter of prompt build attempts. Attributes include `operation=build_for_request` and `status` (`ok`/`error`). | +| `prompt_errors_total` | Counter of errors by operation and error type. Attributes include `operation` and `error.type`. | +| `prompt_build_latency_ms` | Histogram of prompt build latency in milliseconds. | +| `prompt_total_tokens` | Histogram of total token count in the built prompt payload. | +| `prompt_total_chars` | Histogram of total character count in the built prompt payload. | +| `prompt_rag_chunks_count` | Histogram of retrieved RAG chunks used in the prompt. | +| `prompt_warnings_count` | Histogram of analyzer warnings count per build. | +| `prompt_safety_events_total` | Counter of safety events. Attributes include `severity` and `status`. | +| `prompt_summary_calls_total` | Counter of summary calls. Attributes include `operation=summary`, `provider`, and `status`. | +| `prompt_summary_latency_ms` | Histogram of summary call latency in milliseconds. | +| `prompt.error operation={operation} error_type={error_type}` | OTLP log message emitted on errors (for example in `build_for_request` or `summary`). | Dashboard template blueprint: diff --git a/docker-compose.otel.yml b/docker-compose.otel.yml index 6429267..d03f010 100644 --- a/docker-compose.otel.yml +++ b/docker-compose.otel.yml @@ -2,58 +2,13 @@ services: otel-collector: image: otel/opentelemetry-collector-contrib:0.103.0 command: ["--config=/etc/otelcol/config.yaml"] + environment: + OTEL_UPSTREAM_OTLP_ENDPOINT: "${OTEL_UPSTREAM_OTLP_ENDPOINT:-host.docker.internal:4317}" volumes: - ./observability/otel-collector-config.yaml:/etc/otelcol/config.yaml:ro + extra_hosts: + - "host.docker.internal:host-gateway" ports: - "4317:4317" - "4318:4318" - depends_on: - - signoz - restart: unless-stopped - - signoz: - image: signoz/signoz-standalone:latest - privileged: true - ports: - - "8080:8080" - - "3301:3301" - restart: unless-stopped - - telemetrygen-traces: - image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest - command: - - "traces" - - "--otlp-endpoint=otel-collector:4317" - - "--otlp-insecure" - - "--duration=inf" - - "--rate=2" - - "--service=seed-traces" - depends_on: - - otel-collector - restart: unless-stopped - - telemetrygen-metrics: - image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest - command: - - "metrics" - - "--otlp-endpoint=otel-collector:4317" - - "--otlp-insecure" - - "--duration=inf" - - "--rate=2" - - "--service=seed-metrics" - depends_on: - - otel-collector - restart: unless-stopped - - telemetrygen-logs: - image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest - command: - - "logs" - - "--otlp-endpoint=otel-collector:4317" - - "--otlp-insecure" - - "--duration=inf" - - "--rate=2" - - "--service=seed-logs" - depends_on: - - otel-collector restart: unless-stopped diff --git a/observability/otel-collector-config.yaml b/observability/otel-collector-config.yaml index 4df54f6..4700ad3 100644 --- a/observability/otel-collector-config.yaml +++ b/observability/otel-collector-config.yaml @@ -9,7 +9,7 @@ processors: exporters: otlp: - endpoint: signoz:4317 + endpoint: ${OTEL_UPSTREAM_OTLP_ENDPOINT} tls: insecure: true diff --git a/pyproject.toml b/pyproject.toml index e8ea08c..1e16a45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "prompt-orchestrator" -version = "0.1.3" +version = "0.1.4" description = "Structured prompt orchestration with cache, safety, and analyzer layers" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.10"